diff --git a/AGENTS.md b/AGENTS.md index 7776917bb7..3831a4f3ef 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -204,7 +204,7 @@ ## Rust and Native Builds - Python imports the native extension built from `crates/sqlbuild-python`. After changing any - Rust, rebuild it incrementally with `uv run --no-sync maturin develop --release --skip-install` + Rust, rebuild it incrementally with `make fast-native` (release optimisation, incremental) before running Python tests. Use one profile consistently within a worktree. - Always run Python commands with `uv run --no-sync`. A syncing `uv run` or `uv sync` can replace the locally built extension with a packaged one; if that happens, rebuild with maturin rather diff --git a/Cargo.toml b/Cargo.toml index b4f5016969..cb65e46fc8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -126,3 +126,11 @@ debug = false [profile.release] strip = "symbols" + +# Local iteration: release-level optimisation with incremental builds, so rebuilding after a +# focused change does not recompile the whole binding crate. CI and wheels use `release`. +[profile.fast] +inherits = "release" +incremental = true +codegen-units = 256 +strip = "none" diff --git a/Makefile b/Makefile index 52be76b8d4..ce32a1d665 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,10 @@ type: dev-native: uv run --no-sync maturin develop --profile dev --skip-install +.PHONY: fast-native +fast-native: + uv run --no-sync maturin develop --profile fast --skip-install + rust-check: rust-lint uv run fensu check cargo test --workspace --all-features diff --git a/crates/sqlbuild-analysis/src/explicit_references/_helpers/python_sql.rs b/crates/sqlbuild-analysis/src/explicit_references/_helpers/python_sql.rs index 382c3a3af7..97c479d155 100644 --- a/crates/sqlbuild-analysis/src/explicit_references/_helpers/python_sql.rs +++ b/crates/sqlbuild-analysis/src/explicit_references/_helpers/python_sql.rs @@ -5,10 +5,10 @@ use std::collections::HashSet; use crate::explicit_references::_helpers::relations::{ ProjectRelation, matched, project_relations, }; -use crate::explicit_references::constants::{ - HARD_CODED_RELATION_CODE, HOOK_OWNER, LOADER_OWNER, SOURCE_TYPE, -}; +use crate::explicit_references::constants::HARD_CODED_RELATION_CODE; use crate::explicit_references::errors::ReferenceFailure; +use crate::explicit_references::main::old_name_message::old_name_message; +use crate::explicit_references::main::relation_remedy::relation_remedy; use crate::explicit_references::models::{ LiteralSql, ReferenceDiagnostic, ReferenceReport, ReferenceRequest, Relation, SqlOwner, UnmatchedRelation, @@ -154,12 +154,7 @@ impl OwnerScan<'_> { fn message(&self, literal: &LiteralSql, written: &str, found: &ProjectRelation) -> String { match &found.compatibility_for { - Some(model) => format!( - "{} names '{written}' in SQL passed to ctx.{}(); it is the old name of \ - model:{model}, kept only as a compatibility view for consumers outside the \ - project", - self.owner.label, literal.method - ), + Some(model) => old_name_message(&self.owner.label, written, &literal.method, model), None => format!( "{} names {}:{} as '{written}' in SQL passed to ctx.{}()", self.owner.label, found.target.0, found.target.1, literal.method @@ -178,43 +173,14 @@ impl OwnerScan<'_> { }; format!( "{}{qualify}\n = help: {}", - self.remedy(&found.target), + relation_remedy( + &self.owner.kind, + &found.target, + &self.owner.upstream_loaders + ), self.request.explicit_help ) } - - /// `hard_coded_relation_remedy_impl`. - fn remedy(&self, (kind, name): &ResourceKey) -> String { - let typed: String = format!("{kind}(\"{name}\")"); - if self.owner.kind == LOADER_OWNER { - if kind != SOURCE_TYPE { - return format!( - "loaders run before models and seeds and cannot read them; read \ - {kind}:{name} from a task or asset instead" - ); - } - return match self - .owner - .upstream_loaders - .iter() - .find(|(source, _)| source == name) - { - Some((_, loader)) => { - format!("use ctx.loader({loader}) instead of the relation name") - } - None => format!("use ctx.source(\"{name}\") instead of the relation name"), - }; - } - let declaration: String = if self.owner.kind == HOOK_OWNER { - format!("@hook(reads={typed})") - } else { - format!("depends_on={typed}") - }; - format!( - "declare it with {declaration} and use ctx.relation({typed}) instead of the relation \ - name" - ) - } } fn extracted( diff --git a/crates/sqlbuild-analysis/src/explicit_references/main/mod.rs b/crates/sqlbuild-analysis/src/explicit_references/main/mod.rs index 9e7e305311..3b3871c48a 100644 --- a/crates/sqlbuild-analysis/src/explicit_references/main/mod.rs +++ b/crates/sqlbuild-analysis/src/explicit_references/main/mod.rs @@ -1,2 +1,4 @@ pub mod check_explicit_references; pub mod execution_gate_edges; +pub mod old_name_message; +pub mod relation_remedy; diff --git a/crates/sqlbuild-analysis/src/explicit_references/main/old_name_message.rs b/crates/sqlbuild-analysis/src/explicit_references/main/old_name_message.rs new file mode 100644 index 0000000000..fcdb49f902 --- /dev/null +++ b/crates/sqlbuild-analysis/src/explicit_references/main/old_name_message.rs @@ -0,0 +1,15 @@ +//! The P008 message for SQL naming a relation kept only as a compatibility view. + +/// `old_name_reference_message`: `owner_label` wrote `written` in SQL passed to `ctx.()`. +#[must_use] +pub fn old_name_message( + owner_label: &str, + written: &str, + method: &str, + model_name: &str, +) -> String { + format!( + "{owner_label} names '{written}' in SQL passed to ctx.{method}(); it is the old name of \ + model:{model_name}, kept only as a compatibility view for consumers outside the project" + ) +} diff --git a/crates/sqlbuild-analysis/src/explicit_references/main/relation_remedy.rs b/crates/sqlbuild-analysis/src/explicit_references/main/relation_remedy.rs new file mode 100644 index 0000000000..43db711343 --- /dev/null +++ b/crates/sqlbuild-analysis/src/explicit_references/main/relation_remedy.rs @@ -0,0 +1,34 @@ +//! How Python code should read a project resource instead of naming its relation in SQL. + +use crate::explicit_references::constants::{HOOK_OWNER, LOADER_OWNER, SOURCE_TYPE}; +use crate::explicit_references::types::ResourceKey; + +/// `hard_coded_relation_remedy`; `upstream_loaders` maps sources to the loaders writing them. +#[must_use] +pub fn relation_remedy( + owner_kind: &str, + (kind, name): &ResourceKey, + upstream_loaders: &[(String, String)], +) -> String { + let typed: String = format!("{kind}(\"{name}\")"); + if owner_kind == LOADER_OWNER { + if kind != SOURCE_TYPE { + return format!( + "loaders run before models and seeds and cannot read them; read \ + {kind}:{name} from a task or asset instead" + ); + } + return match upstream_loaders.iter().find(|(source, _)| source == name) { + Some((_, loader)) => format!("use ctx.loader({loader}) instead of the relation name"), + None => format!("use ctx.source(\"{name}\") instead of the relation name"), + }; + } + let declaration: String = if owner_kind == HOOK_OWNER { + format!("@hook(reads={typed})") + } else { + format!("depends_on={typed}") + }; + format!( + "declare it with {declaration} and use ctx.relation({typed}) instead of the relation name" + ) +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/main/mod.rs b/crates/sqlbuild-analysis/src/semantic_checks/main/mod.rs index 95cdc4c65b..b04c2c7848 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/main/mod.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/main/mod.rs @@ -3,3 +3,4 @@ pub mod complete_semantic_diagnostics; pub mod finish_type_recovery; pub mod plan_type_recovery; pub mod resource_sql_diagnostics; +pub mod semantic_help_catalogue; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/main/semantic_help_catalogue.rs b/crates/sqlbuild-analysis/src/semantic_checks/main/semantic_help_catalogue.rs new file mode 100644 index 0000000000..0ae8e0741d --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/main/semantic_help_catalogue.rs @@ -0,0 +1,9 @@ +//! The code-owned remediation of each semantic diagnostic code. + +use crate::semantic_checks::constants::SEMANTIC_HELP; + +/// Each semantic diagnostic code and its help, in catalogue order. +#[must_use] +pub fn semantic_help_catalogue() -> &'static [(&'static str, &'static str)] { + SEMANTIC_HELP +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs index c578031f6e..503d409342 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs @@ -1,56 +1,187 @@ //! Compile's explicit-reference checks over plain project facts, run natively. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; -use serde_json::{Value, json}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_analysis::explicit_references::errors::ReferenceFailure; use sqlbuild_analysis::explicit_references::main::check_explicit_references::check_explicit_references; use sqlbuild_analysis::explicit_references::main::execution_gate_edges::execution_gate_edges; +use sqlbuild_analysis::explicit_references::main::old_name_message::old_name_message; +use sqlbuild_analysis::explicit_references::main::relation_remedy::relation_remedy; use sqlbuild_analysis::explicit_references::models::{ - ExecutionGateEdges, ExecutionGateRequest, GateEdge, HookEdge, HookType, ReferenceReport, - ReferenceRequest, + ExecutionGateEdges, ExecutionGateRequest, GateEdge, HookEdge, HookType, LiteralSql, + ReferenceReport, ReferenceRequest, SqlOwner, }; use sqlbuild_analysis::explicit_references::types::ResourceKey; use sqlbuild_core::panics::main::native_failure::native_failure; use sqlbuild_pysource::literal_sql::main::literal_sql_calls::literal_sql_calls; use sqlbuild_pysource::literal_sql::models::FunctionSource; +use crate::bindings::_helpers::analysis::reference_facts::{ProjectFacts, project_facts, ref_keys}; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::types::CompilerDetach; const EXPLICIT_REFERENCES_CONTEXT: &str = "native explicit-reference checks"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_INPUT_ERROR: &str = "CompileInputError"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const REFERENCE_MODELS_MODULE: &str = "sqlbuild.compiler.references.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const DIAGNOSTIC_PHASE: &str = "compile"; +const DIAGNOSTIC_SEVERITY: &str = "error"; + +/// One Python SQL owner as Python collects it: the owner, its path order and literal SQL calls. +type OwnerInput<'py> = (Bound<'py, PyAny>, usize, Vec<(String, String, usize)>); + +/// The project-wide settings of one explicit-reference check. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +struct ReportSettings { + /// Whether the project keeps old-name views unless a model disables them. + retains_old_names: bool, + enforce_explicit: bool, + explicit_help: String, + /// The text Python substitutes for f-string values in literal SQL. + placeholder: String, +} -/// Python's audit-gate, hook-read and Python-SQL checks: `{report}` or `{raise}` JSON. +/// Python's audit-gate, hook-read and Python-SQL checks as a `PythonSqlReferenceReport`. #[pyfunction] -fn check_native_explicit_references(py: Python<'_>, request_json: &str) -> PyResult { - let request: ReferenceRequest = serde_json::from_str(request_json).map_err(|error| { - compiler_error(native_failure( - EXPLICIT_REFERENCES_CONTEXT, - &error.to_string(), - )) - })?; - let outcome: Value = match py +fn explicit_reference_report<'py>( + py: Python<'py>, + project: &Bound<'py, PyAny>, + settings: ReportSettings, + owners: Vec>, +) -> PyResult> { + let facts: ProjectFacts = project_facts(project, settings.retains_old_names)?; + let request = ReferenceRequest { + enforce_explicit: settings.enforce_explicit, + dialect: project.getattr("sql_analysis_dialect")?.extract()?, + explicit_help: settings.explicit_help, + placeholder: settings.placeholder, + resources: facts.resources, + models: facts.models, + sources: facts.sources, + seeds: facts.seeds, + audits: facts.audits, + hooks: facts.hooks, + owners: owners + .iter() + .map(|(owner, path_order, literals)| sql_owner(owner, *path_order, literals)) + .collect::>()?, + }; + match py .compiler_detach(|| Ok(check_explicit_references(&request))) .map_err(compiler_error)? { - Ok(report) => json!({ "report": report_value(&report)? }), - Err(ReferenceFailure::CompileInput(message)) => json!({ "raise": message }), - Err(ReferenceFailure::UnknownDialect(message)) => return Err(compiler_error(message)), - Err(ReferenceFailure::Internal(reason)) => { - return Err(compiler_error(native_failure( - EXPLICIT_REFERENCES_CONTEXT, - &reason, - ))); - } - }; - serde_json::to_string(&outcome).map_err(|error| { - compiler_error(native_failure( + Ok(report) => python_report(py, &report, &owners), + Err(ReferenceFailure::CompileInput(message)) => Err(PyErr::from_value( + py.import(COMPILE_ERRORS_MODULE)? + .getattr(COMPILE_INPUT_ERROR)? + .call1((message,))?, + )), + Err(ReferenceFailure::UnknownDialect(message)) => Err(compiler_error(message)), + Err(ReferenceFailure::Internal(reason)) => Err(compiler_error(native_failure( EXPLICIT_REFERENCES_CONTEXT, - &error.to_string(), - )) + &reason, + ))), + } +} + +fn sql_owner( + owner: &Bound<'_, PyAny>, + path_order: usize, + literals: &[(String, String, usize)], +) -> PyResult { + Ok(SqlOwner { + label: owner.getattr("label")?.extract()?, + kind: owner.getattr("kind")?.str()?.extract()?, + path_order, + own_refs: ref_keys(&owner.getattr("own_refs")?)?, + upstream_loaders: owner + .getattr("upstream_loader_by_source")? + .call_method0("items")? + .try_iter()? + .map(|item| item?.extract()) + .collect::>()?, + literals: literals + .iter() + .map(|(method, sql, line)| LiteralSql { + method: method.clone(), + sql: sql.clone(), + line: *line, + }) + .collect(), }) } +/// The report's diagnostics and unmatched relations as Python's compile models. +fn python_report<'py>( + py: Python<'py>, + report: &ReferenceReport, + owners: &[OwnerInput<'py>], +) -> PyResult> { + let compile_models = py.import(COMPILE_MODELS_MODULE)?; + let reference_models = py.import(REFERENCE_MODELS_MODULE)?; + let source_location: Bound<'py, PyAny> = py + .import(CONTRACT_MODELS_MODULE)? + .getattr("SourceLocation")?; + let path_class: Bound<'py, PyAny> = py.import("pathlib")?.getattr("Path")?; + let compiler_diagnostic: Bound<'py, PyAny> = compile_models.getattr("CompilerDiagnostic")?; + let mut diagnostics: Vec> = Vec::with_capacity(report.diagnostics.len()); + for row in &report.diagnostics { + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("phase", DIAGNOSTIC_PHASE)?; + fields.set_item("severity", DIAGNOSTIC_SEVERITY)?; + fields.set_item("code", row.code)?; + fields.set_item("message", &row.message)?; + fields.set_item("resource_type", row.resource.as_ref().map(|key| &key.0))?; + fields.set_item("resource_name", row.resource.as_ref().map(|key| &key.1))?; + match &row.path { + Some(path) => fields.set_item("path", path_class.call1((path,))?)?, + None => fields.set_item("path", py.None())?, + } + match row.owner { + Some(owner) => { + let location: Bound<'py, PyDict> = PyDict::new(py); + location.set_item("path", owners[owner].0.getattr("relative_path")?)?; + location.set_item("line", row.line)?; + location.set_item("column", 1)?; + fields.set_item("location", source_location.call((), Some(&location))?)?; + } + None => fields.set_item("location", py.None())?, + } + fields.set_item("help", &row.help)?; + diagnostics.push(compiler_diagnostic.call((), Some(&fields))?); + } + let literal_relation: Bound<'py, PyAny> = reference_models.getattr("LiteralSqlRelation")?; + let relation_name: Bound<'py, PyAny> = reference_models.getattr("RelationName")?; + let mut unmatched: Vec> = Vec::with_capacity(report.unmatched.len()); + for row in &report.unmatched { + let owner: &Bound<'py, PyAny> = &owners[row.owner].0; + let relation: Bound<'py, PyDict> = PyDict::new(py); + relation.set_item("name", &row.relation.name)?; + relation.set_item("schema", &row.relation.schema)?; + relation.set_item("database", &row.relation.database)?; + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("owner_label", owner.getattr("label")?)?; + fields.set_item("owner_kind", owner.getattr("kind")?)?; + fields.set_item("relative_path", owner.getattr("relative_path")?)?; + fields.set_item("line", row.line)?; + fields.set_item("method", &row.method)?; + fields.set_item("relation", relation_name.call((), Some(&relation))?)?; + unmatched.push(literal_relation.call((), Some(&fields))?); + } + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("diagnostics", PyTuple::new(py, diagnostics)?)?; + fields.set_item("unmatched", PyTuple::new(py, unmatched)?)?; + compile_models + .getattr("PythonSqlReferenceReport")? + .call((), Some(&fields)) +} + /// `(method, sql, line)` for each literal SQL call to the context in a dedented function source. #[pyfunction] fn python_literal_sql_calls( @@ -71,6 +202,27 @@ fn python_literal_sql_calls( .collect() } +/// How an owner of `owner_kind` should read `target` instead of naming its relation in SQL. +#[pyfunction] +fn hard_coded_relation_remedy( + owner_kind: &str, + target: ResourceKey, + upstream_loaders: Vec<(String, String)>, +) -> String { + relation_remedy(owner_kind, &target, &upstream_loaders) +} + +/// The P008 message for SQL naming a relation kept only as a compatibility view. +#[pyfunction] +fn old_name_reference_message( + owner_label: &str, + written: &str, + method: &str, + model_name: &str, +) -> String { + old_name_message(owner_label, written, method, model_name) +} + /// `(audit name, target, gated, read)` for one attached-audit ordering edge. type GateRow = (String, ResourceKey, ResourceKey, ResourceKey); /// `(hook name, gated, read, HookReadType value, path)` for one hook-read ordering edge. @@ -82,16 +234,18 @@ type HookRow = ( Option, ); -/// The attached-audit gate edges and hook-read edges of one project's facts, in Python's order. +/// The attached-audit gate edges and hook-read edges of one project, in Python's order. #[pyfunction] -fn native_execution_gate_edges(request_json: &str) -> PyResult<(Vec, Vec)> { - let request: ExecutionGateRequest = serde_json::from_str(request_json).map_err(|error| { - compiler_error(native_failure( - EXPLICIT_REFERENCES_CONTEXT, - &error.to_string(), - )) - })?; - let edges: ExecutionGateEdges = execution_gate_edges(&request); +fn native_execution_gate_edges( + project: &Bound<'_, PyAny>, +) -> PyResult<(Vec, Vec)> { + let facts: ProjectFacts = project_facts(project, false)?; + let edges: ExecutionGateEdges = execution_gate_edges(&ExecutionGateRequest { + resources: facts.resources, + models: facts.models, + audits: facts.audits, + hooks: facts.hooks, + }); Ok(( edges.audit_gates.into_iter().map(gate_row).collect(), edges.hook_reads.into_iter().map(hook_row).collect(), @@ -111,18 +265,11 @@ fn hook_row(edge: HookEdge) -> HookRow { (edge.hook_name, edge.gated, edge.read, hook_type, edge.path) } -fn report_value(report: &ReferenceReport) -> PyResult { - serde_json::to_value(report).map_err(|error| { - compiler_error(native_failure( - EXPLICIT_REFERENCES_CONTEXT, - &error.to_string(), - )) - }) -} - pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(check_native_explicit_references, module)?)?; + module.add_function(wrap_pyfunction!(explicit_reference_report, module)?)?; module.add_function(wrap_pyfunction!(python_literal_sql_calls, module)?)?; module.add_function(wrap_pyfunction!(native_execution_gate_edges, module)?)?; + module.add_function(wrap_pyfunction!(hard_coded_relation_remedy, module)?)?; + module.add_function(wrap_pyfunction!(old_name_reference_message, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs index 62d4a083e3..0177fd8277 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs @@ -6,6 +6,7 @@ pub(crate) mod normalization; pub(crate) mod normalization_results; mod positions; pub(crate) mod queries; +pub(crate) mod reference_facts; pub(crate) mod relation_names; mod required_columns; pub(crate) mod sql_tests; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/reference_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/reference_facts.rs new file mode 100644 index 0000000000..d03eb26660 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/reference_facts.rs @@ -0,0 +1,269 @@ +//! The explicit-reference facts of a compiled project, read from its Python views. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyList, PyTuple}; +use sqlbuild_analysis::explicit_references::models::{ + AuditGate, GraphResource, ModelReferences, NamedRelation, PythonHook, Relation, SourceRelation, + SqlHook, +}; +use sqlbuild_analysis::explicit_references::types::ResourceKey; + +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const PYTHON_HOOK_ENTRY: &str = "PythonHookEntry"; +const SQL_HOOK_ENTRY: &str = "SqlHookEntry"; +const MODEL_HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; +const MIGRATE_FROM_KEY: &str = "migrate_from"; +const OLD_NAME_VIEW_KEY: &str = "old_name_view"; + +/// The project facts the explicit-reference checks and execution-gate edges read. +pub(crate) struct ProjectFacts { + /// Models, then sources, seeds and functions, with their dependencies. + pub(crate) resources: Vec, + pub(crate) models: Vec, + pub(crate) sources: Vec, + pub(crate) seeds: Vec, + pub(crate) audits: Vec, + pub(crate) hooks: Vec, +} + +/// The facts of `project`; `retains_old_names` is whether the project keeps old-name views. +pub(crate) fn project_facts( + project: &Bound<'_, PyAny>, + retains_old_names: bool, +) -> PyResult { + let py: Python<'_> = project.py(); + let entries: HookEntries<'_> = HookEntries::imported(py)?; + let mut resources: Vec = Vec::new(); + let mut models: Vec = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + let model: Bound<'_, PyAny> = model?; + resources.push(GraphResource { + key: model.getattr("key_row")?.extract()?, + deps: model.getattr("dep_rows")?.extract()?, + }); + models.push(model_references(&model, &entries, retains_old_names)?); + } + let mut sources: Vec = Vec::new(); + for source in project.getattr("sources")?.try_iter()? { + let source: Bound<'_, PyAny> = source?; + resources.push(graph_resource(&source)?); + let entry: Bound<'_, PyAny> = source.getattr("source_entry")?; + sources.push(SourceRelation { + name: source.getattr("name")?.extract()?, + table: entry.getattr("table")?.extract()?, + schema: entry.getattr("schema")?.extract()?, + database: entry.getattr("database")?.extract()?, + expression: !entry.getattr("expression")?.is_none(), + }); + } + let mut seeds: Vec = Vec::new(); + for seed in project.getattr("seeds")?.try_iter()? { + let seed: Bound<'_, PyAny> = seed?; + resources.push(graph_resource(&seed)?); + let destination: Bound<'_, PyAny> = seed.getattr("destination")?; + seeds.push(NamedRelation { + name: seed.getattr("name")?.extract()?, + relation: Relation { + name: destination.getattr("name")?.extract()?, + schema: destination.getattr("schema")?.extract()?, + database: destination.getattr("database")?.extract()?, + }, + }); + } + for function in project.getattr("functions")?.try_iter()? { + resources.push(graph_resource(&function?)?); + } + let mut audits: Vec = Vec::new(); + for audit in project.getattr("audits")?.try_iter()? { + audits.push(audit_gate(&audit?)?); + } + let mut hooks: Vec = Vec::new(); + for hook in project.getattr("hook_functions")?.try_iter()? { + let hook: Bound<'_, PyAny> = hook?; + hooks.push(PythonHook { + name: hook.getattr("name")?.extract()?, + path: hook + .getattr("relative_path")? + .call_method0("as_posix")? + .extract()?, + reads: ref_keys(&hook.getattr("reads")?)?, + }); + } + Ok(ProjectFacts { + resources, + models, + sources, + seeds, + audits, + hooks, + }) +} + +/// `PythonHookEntry` and `SqlHookEntry`, which model hook entries are instances of. +struct HookEntries<'py> { + python: Bound<'py, PyAny>, + sql: Bound<'py, PyAny>, +} + +impl<'py> HookEntries<'py> { + fn imported(py: Python<'py>) -> PyResult { + let module = py.import(DISCOVERY_MODELS_MODULE)?; + Ok(Self { + python: module.getattr(PYTHON_HOOK_ENTRY)?, + sql: module.getattr(SQL_HOOK_ENTRY)?, + }) + } +} + +fn model_references( + model: &Bound<'_, PyAny>, + entries: &HookEntries<'_>, + retains_old_names: bool, +) -> PyResult { + let values: Bound<'_, PyAny> = model.getattr("config")?.getattr("values")?; + let (database, schema, name): (Option, Option, String) = + model.getattr("destination_row")?.extract()?; + let mut sql_hooks: Vec = Vec::new(); + for hook_key in MODEL_HOOK_KEYS { + for (index, entry) in hook_items(&values.call_method1("get", (hook_key,))?)? + .into_iter() + .enumerate() + { + if !entry.is_instance(&entries.sql)? || !entry.getattr("reads")?.is_truthy()? { + continue; + } + let path: Bound<'_, PyAny> = entry.getattr("relative_path")?; + sql_hooks.push(SqlHook { + hook_key: hook_key.to_owned(), + index, + name: entry.getattr("name")?.extract()?, + path: if path.is_none() { + None + } else { + Some(path.str()?.extract()?) + }, + reads: ref_keys(&entry.getattr("reads")?)?, + }); + } + } + Ok(ModelReferences { + name: model.getattr("name")?.extract()?, + path: model.getattr("relative_path")?.str()?.extract()?, + destination: Relation { + name, + schema, + database, + }, + python_hooks: python_hook_names(&values, entries)?, + sql_hooks, + migrate_from: migrate_from(&values, retains_old_names)?, + }) +} + +/// `model_python_hook_names`: the model's Python hooks in lifecycle order, each once. +fn python_hook_names( + values: &Bound<'_, PyAny>, + entries: &HookEntries<'_>, +) -> PyResult> { + let mut names: Vec = Vec::new(); + for hook_key in MODEL_HOOK_KEYS { + let value: Bound<'_, PyAny> = values.call_method1("get", (hook_key,))?; + let items: Vec> = if value.is_instance(&entries.python)? { + vec![value] + } else if is_sequence(&value) { + value.try_iter()?.collect::>()? + } else { + Vec::new() + }; + for item in items { + if !item.is_instance(&entries.python)? { + continue; + } + let name: String = item.getattr("name")?.extract()?; + if !names.contains(&name) { + names.push(name); + } + } + } + Ok(names) +} + +/// The entries of one hook key: a list or tuple's items, or the value itself. +fn hook_items<'py>(value: &Bound<'py, PyAny>) -> PyResult>> { + if is_sequence(value) { + return value.try_iter()?.collect(); + } + Ok(vec![value.clone()]) +} + +fn is_sequence(value: &Bound<'_, PyAny>) -> bool { + value.is_instance_of::() || value.is_instance_of::() +} + +/// The `migrate_from` string when the model keeps an old-name view, as Python resolves it. +fn migrate_from(values: &Bound<'_, PyAny>, retains_old_names: bool) -> PyResult> { + let raw: Bound<'_, PyAny> = values.call_method1("get", (MIGRATE_FROM_KEY,))?; + let Ok(text) = raw.extract::() else { + return Ok(None); + }; + let retained: bool = if values.contains(OLD_NAME_VIEW_KEY)? { + !values + .get_item(OLD_NAME_VIEW_KEY)? + .is(PyBool::new(values.py(), false)) + } else { + retains_old_names + }; + Ok(retained.then_some(text)) +} + +fn audit_gate(audit: &Bound<'_, PyAny>) -> PyResult { + let kind: Bound<'_, PyAny> = audit.getattr("attached_target_kind")?; + let name: Bound<'_, PyAny> = audit.getattr("attached_target_name")?; + let target: Option = if kind.is_none() || name.is_none() { + None + } else { + Some((kind.str()?.extract()?, name.extract()?)) + }; + Ok(AuditGate { + name: audit.getattr("name")?.extract()?, + path: audit + .getattr("audit_file")? + .getattr("relative_path")? + .str()? + .extract()?, + target, + scope_deps: object_keys(&audit.getattr("scope_deps")?)?, + }) +} + +fn graph_resource(resource: &Bound<'_, PyAny>) -> PyResult { + Ok(GraphResource { + key: object_key(&resource.getattr("key")?)?, + deps: object_keys(&resource.getattr("deps")?)?, + }) +} + +/// A `CompiledObjectKey` as its `(resource type, name)` pair. +fn object_key(key: &Bound<'_, PyAny>) -> PyResult { + Ok(( + key.getattr("resource_type")?.str()?.extract()?, + key.getattr("name")?.extract()?, + )) +} + +fn object_keys(keys: &Bound<'_, PyAny>) -> PyResult> { + keys.try_iter()?.map(|key| object_key(&key?)).collect() +} + +/// `SqlResourceRef`s as `(kind, name)` pairs. +pub(crate) fn ref_keys(refs: &Bound<'_, PyAny>) -> PyResult> { + refs.try_iter()? + .map(|reference| { + let reference: Bound<'_, PyAny> = reference?; + Ok(( + reference.getattr("kind")?.str()?.extract()?, + reference.getattr("name")?.extract()?, + )) + }) + .collect() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs index 11eaaf57b5..d212ed073a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs @@ -1,5 +1,9 @@ +pub(crate) mod model_analyses; pub(crate) mod nullability_rules; pub(crate) mod oracle; +pub(crate) mod outcome_objects; +pub(crate) mod profile_inputs; pub(crate) mod query_columns; pub(crate) mod registration; pub(crate) mod session; +pub(crate) mod stage_functions; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs new file mode 100644 index 0000000000..965951e4aa --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -0,0 +1,318 @@ +//! One compile's native model analysis session, with Python's catalog and metric bookkeeping. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::analysis_session::profile_inputs::{ + case_sensitive_shapes, nullability_rules, profile_dialect, return_types, shape_rows, +}; +use crate::bindings::_helpers::analysis_session::stage_functions::family_rows; +use crate::bindings::_helpers::boundary::keyword_request::Request; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; +const PYTHON_ANALYSIS_MODULE: &str = "sqlbuild.compiler.compile.classes.python_model_analysis"; +const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; +const CODE_IDENTITY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiled_code_identity"; +const LOG_DEBUG_MODULE: &str = "sqlbuild.diagnostics.main.log_debug_event"; +const LOGGING_MODULE: &str = "logging"; +const DEBUG_LOGGER: &str = "sqlbuild.compile"; +const ANALYSIS_FAILURE_MESSAGE: &str = "native compact query analysis failed"; +const STORE_FAILURE_MESSAGE: &str = "native model analysis cache not read or saved"; +const STORE_FILE_NAME: &str = "model-analyses.bin"; +const STORE_VERSION: &str = "native-model-analysis-store-v1"; + +/// Each model's analysis by name and the finished session; a native internal failure raises. +#[pyfunction] +#[pyo3(signature = (**request))] +fn analyze_native_model_sql<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("analyze_native_model_sql", request)?; + let py: Python<'py> = request.py(); + let request: Bound<'py, PyAny> = request.get("request")?; + let analyses_class: Bound<'py, PyAny> = py + .import(ANALYSIS_MODELS_MODULE)? + .getattr("NativeModelAnalyses")?; + let model_inputs: Bound<'py, PyAny> = request.getattr("model_inputs")?; + if !model_inputs.is_truthy()? { + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("positions", PyDict::new(py))?; + arguments.set_item("succeeded", PyFrozenSet::empty(py)?)?; + arguments.set_item("session", py.None())?; + return analyses_class.call((), Some(&arguments)); + } + let profile: Bound<'py, PyAny> = request.getattr("inference_profile")?; + let catalog: Bound<'py, PyAny> = profile.getattr("binding_catalog")?; + let python: Bound<'py, PyAny> = { + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("model_inputs", &model_inputs)?; + arguments.set_item("inference_profile", &profile)?; + arguments.set_item( + "complete_binding_schemas", + request.getattr("complete_binding_schemas")?, + )?; + py.import(PYTHON_ANALYSIS_MODULE)? + .getattr("PythonModelAnalysis")? + .call((), Some(&arguments))? + }; + let (rules, adapter_rules) = nullability_rules(&profile)?; + let session_request: Bound<'py, PyAny> = session_request(&request, &python, &catalog, rules)?; + let (cache, adapter): (Bound<'py, PyAny>, Bound<'py, PyAny>) = match adapter_rules { + Some((adapter, nullability)) => ( + py.None().into_bound(py), + PyTuple::new( + py, + [ + adapter.into_any().into_bound(py), + nullability.into_bound(py), + ], + )? + .into_any(), + ), + None => (analysis_store(&request)?, py.None().into_bound(py)), + }; + let session: Bound<'py, PyAny> = py + .import(NATIVE_MODULE)? + .getattr("start_model_analysis_session")? + .call1((catalog.getattr("native")?, session_request, cache, adapter))?; + run_session(&request, (&python, &catalog), &session, &analyses_class) +} + +/// The session request row, in the order the native session reads it. +fn session_request<'py>( + request: &Bound<'py, PyAny>, + python: &Bound<'py, PyAny>, + catalog: &Bound<'py, PyAny>, + rules: Vec<(String, String)>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let profile: Bound<'py, PyAny> = request.getattr("inference_profile")?; + let mut families: Vec<(String, Vec<_>)> = Vec::new(); + for item in request + .getattr("dynamic_families_by_table")? + .call_method0("items")? + .try_iter()? + { + let (name, declared): (String, Bound<'py, PyAny>) = item?.extract()?; + families.push((name, family_rows(&declared)?)); + } + let items: Vec> = vec![ + profile_dialect(&profile)?.into_bound_py_any(py)?, + case_sensitive_shapes(&profile, &profile.getattr("sql_analysis_dialect")?)? + .into_bound_py_any(py)?, + return_types(&profile)?.into_bound_py_any(py)?, + rules.into_bound_py_any(py)?, + request.getattr("rich_type_inference")?, + shape_rows(&request.getattr("column_types_by_table")?)?.into_bound_py_any(py)?, + shape_rows(&request.getattr("column_nullability_by_table")?)?.into_bound_py_any(py)?, + shape_rows(&request.getattr("complete_binding_schemas")?)?.into_bound_py_any(py)?, + shape_rows(&catalog.getattr("schemas")?)?.into_bound_py_any(py)?, + families.into_bound_py_any(py)?, + python.call_method0("model_rows")?, + ]; + Ok(PyTuple::new(py, items)?.into_any()) +} + +/// The analysis store's path and environment, or None where Python bypasses its cache. +fn analysis_store<'py>(request: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = request.py(); + let context: Bound<'py, PyAny> = request.getattr("analysis_cache")?; + if context.is_none() { + return Ok(py.None().into_bound(py)); + } + let path: Bound<'py, PyAny> = context + .getattr("root")? + .div(STORE_FILE_NAME)? + .str()? + .into_any(); + let identity: Bound<'py, PyAny> = py + .import(CODE_IDENTITY_MODULE)? + .getattr("compiled_code_identity")? + .call0()?; + let parts: Bound<'py, PyList> = PyList::new( + py, + [ + PyString::new(py, STORE_VERSION).into_any(), + identity, + context.getattr("shared_fingerprint")?, + context.getattr("signature_namespace")?, + ], + )?; + let environment: Bound<'py, PyAny> = py + .import(NATIVE_MODULE)? + .getattr("content_digest")? + .call1((parts,))?; + Ok(PyTuple::new(py, [path, environment])?.into_any()) +} + +/// Run and finish the session, leaving Python's catalog and metrics as its analysis did. +fn run_session<'py>( + request: &Bound<'py, PyAny>, + (python, catalog): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + session: &Bound<'py, PyAny>, + analyses_class: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let types: Bound<'py, PyDict> = copied(&request.getattr("column_types_by_table")?)?; + let nullability: Bound<'py, PyDict> = copied(&request.getattr("column_nullability_by_table")?)?; + let (publications, failures): (Bound<'py, PyAny>, Bound<'py, PyAny>) = + session.call_method0("run")?.extract()?; + for failure in failures.try_iter()? { + log_debug(py, ANALYSIS_FAILURE_MESSAGE, &failure?)?; + } + let unknown: Bound<'py, PyAny> = py + .import(LINEAGE_TYPES_MODULE)? + .getattr("InferredNullability")? + .getattr("UNKNOWN")?; + for item in publications.try_iter()? { + let (name, shape): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + let published: Bound<'py, PyAny> = dict_of(&shape)?; + types.call_method1("setdefault", (&name, &published))?; + let defaults: Bound<'py, PyAny> = PyDict::new(py) + .into_any() + .getattr("fromkeys")? + .call1((&published, &unknown))?; + nullability.call_method1("setdefault", (&name, defaults))?; + } + let (additions, analysis_names, succeeded): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ) = session.call_method0("finish")?.extract()?; + record_catalog_changes(catalog, &additions, &analysis_names, (&types, &nullability))?; + record_cache(python, session)?; + let positions: Bound<'py, PyDict> = PyDict::new(py); + let mut names: Vec> = Vec::new(); + for analysed in python.getattr("requests")?.try_iter()? { + let name: Bound<'py, PyAny> = analysed? + .getattr("model_input")? + .getattr("model_file")? + .getattr("file_path")? + .getattr("stem")?; + positions.set_item(&name, names.len())?; + names.push(name); + } + let succeeded: Vec = succeeded.extract()?; + let succeeded_names: Vec<&Bound<'py, PyAny>> = names + .iter() + .zip(&succeeded) + .filter_map(|(name, success)| success.then_some(name)) + .collect(); + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("positions", positions)?; + arguments.set_item("succeeded", PyFrozenSet::new(py, succeeded_names)?)?; + arguments.set_item("session", session)?; + analyses_class.call((), Some(&arguments)) +} + +/// Leave Python's binding catalog as its own analysis would have. +fn record_catalog_changes<'py>( + catalog: &Bound<'py, PyAny>, + schema_additions: &Bound<'py, PyAny>, + analysis_names: &Bound<'py, PyAny>, + (types, nullability): (&Bound<'py, PyDict>, &Bound<'py, PyDict>), +) -> PyResult<()> { + let py: Python<'py> = catalog.py(); + let schemas: Bound<'py, PyAny> = catalog.getattr("schemas")?; + let additions: Bound<'py, PyDict> = PyDict::new(py); + for item in schema_additions.try_iter()? { + let (name, shape): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + if !schemas.contains(&name)? { + additions.set_item(name, dict_of(&shape)?)?; + } + } + schemas.call_method1("update", (&additions,))?; + if !additions.is_empty() { + catalog + .getattr("native")? + .call_method1("update_relations", (&additions,))?; + } + let selected_types: Bound<'py, PyDict> = PyDict::new(py); + let selected_nullability: Bound<'py, PyDict> = PyDict::new(py); + let names: Vec> = analysis_names.try_iter()?.collect::>()?; + for name in &names { + if let Some(shape) = types.get_item(name)? { + selected_types.set_item(name, shape)?; + } + } + for name in &names { + if let Some(shape) = nullability.get_item(name)? { + selected_nullability.set_item(name, shape)?; + } + } + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("types", selected_types)?; + arguments.set_item("nullability", selected_nullability)?; + catalog + .getattr("prepare_analysis")? + .call((), Some(&arguments))?; + Ok(()) +} + +fn record_cache(python: &Bound<'_, PyAny>, session: &Bound<'_, PyAny>) -> PyResult<()> { + let stats: Bound<'_, PyAny> = session.getattr("cache_stats")?; + if stats.is_none() { + python.call_method0("record_uncached")?; + return Ok(()); + } + let (hits, misses, _, failure): ( + Bound<'_, PyAny>, + Bound<'_, PyAny>, + Bound<'_, PyAny>, + Bound<'_, PyAny>, + ) = stats.extract()?; + if !failure.is_none() { + log_debug(python.py(), STORE_FAILURE_MESSAGE, &failure)?; + } + if hits.add(&misses)?.eq(0)? { + python.call_method0("record_uncached")?; + return Ok(()); + } + let arguments: Bound<'_, PyDict> = PyDict::new(python.py()); + arguments.set_item("hits", hits)?; + arguments.set_item("misses", misses)?; + python + .getattr("record_cached")? + .call((), Some(&arguments))?; + Ok(()) +} + +fn log_debug(py: Python<'_>, message: &str, failure: &Bound<'_, PyAny>) -> PyResult<()> { + let arguments: Bound<'_, PyDict> = PyDict::new(py); + arguments.set_item( + "logger", + py.import(LOGGING_MODULE)? + .getattr("getLogger")? + .call1((DEBUG_LOGGER,))?, + )?; + arguments.set_item("message", message)?; + arguments.set_item("sqlbuild_error", failure)?; + py.import(LOG_DEBUG_MODULE)? + .getattr("log_debug_event")? + .call((), Some(&arguments))?; + Ok(()) +} + +/// `dict(mapping)`. +fn copied<'py>(mapping: &Bound<'py, PyAny>) -> PyResult> { + let copy: Bound<'py, PyDict> = PyDict::new(mapping.py()); + copy.update(mapping.downcast()?)?; + Ok(copy) +} + +/// `dict(pairs)`. +fn dict_of<'py>(pairs: &Bound<'py, PyAny>) -> PyResult> { + let dict: Bound<'py, PyDict> = PyDict::new(pairs.py()); + for item in pairs.try_iter()? { + let (key, value): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + dict.set_item(key, value)?; + } + Ok(dict.into_any()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(analyze_native_model_sql, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs new file mode 100644 index 0000000000..b3d9bb9962 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs @@ -0,0 +1,295 @@ +//! Finished session outcomes and pivot proof rows as Python's analysis objects. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; + +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; +const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; +const LINEAGE_NATIVE: &str = "native"; + +type Object<'py> = Bound<'py, PyAny>; +/// `(succeeded, columns, lineage source, lineage rows, star, resolved, bindings, validated, SQL)`. +type OutcomeObjects<'py> = ( + Object<'py>, + Object<'py>, + String, + Object<'py>, + Object<'py>, + Object<'py>, + Object<'py>, + Object<'py>, + Object<'py>, +); +/// `(output proven, fixed columns, families, input relations, failure reason, bare pivot)`. +type ProofObjects<'py> = ( + Object<'py>, + Object<'py>, + Object<'py>, + Object<'py>, + Object<'py>, + Object<'py>, +); + +/// The outcome at `position` of a finished session as `ModelSqlAnalysis`, for hand-built inputs. +#[pyfunction] +#[pyo3(signature = (**request))] +fn session_outcome_analysis<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("session_outcome_analysis", request)?; + let py: Python<'py> = request.py(); + let (outcome, contract): (Bound<'py, PyAny>, Bound<'py, PyAny>) = request + .get("session")? + .call_method1("outcome_rows", (request.get("position")?,))? + .extract()?; + let (succeeded, columns, source, rows, has_star, star_resolved, raw, validated, cleaned): OutcomeObjects<'py> = outcome.extract()?; + let models = py.import(COMPILE_MODELS_MODULE)?; + let lineage: Bound<'py, PyAny> = if source != LINEAGE_NATIVE { + lineage_facts(&rows)? + } else if succeeded.is_truthy()? { + compact_lineage(&rows)? + } else { + PyTuple::empty(py).into_any() + }; + let binding_class: Bound<'py, PyAny> = py + .import(SQL_ANALYSIS_MODELS_MODULE)? + .getattr("SqlBindingDiagnostic")?; + let mut bindings: Vec> = Vec::new(); + for row in raw.try_iter()? { + let row: Bound<'py, PyTuple> = + PyTuple::new(py, row?.try_iter()?.collect::>>()?)?; + bindings.push(binding_class.call1(row)?); + } + let result: Bound<'py, PyAny> = call( + &models.getattr("PolyglotAnalysisResult")?, + &[ + ("analysis_succeeded", succeeded), + ("columns", inferred_columns(&columns)?), + ("lineage_columns", lineage), + ("has_star", has_star), + ("star_resolved", star_resolved), + ( + "binding_diagnostics", + PyTuple::new(py, bindings)?.into_any(), + ), + ("binding_validated", validated), + ], + )?; + call( + &models.getattr("ModelSqlAnalysis")?, + &[ + ("polyglot_analysis", result), + ("placeholders", request.get("placeholders")?), + ("fused_binding_validated", true.into_bound_py_any(py)?), + ("cleaned_sql", cleaned), + ("dynamic_column_contract", proof(&contract.get_item(1)?)?), + ], + ) +} + +/// A native dynamic pivot proof row as Python's proof, None when absent. +#[pyfunction] +fn dynamic_contract_proof<'py>(row: &Bound<'py, PyAny>) -> PyResult> { + proof(row) +} + +fn proof<'py>(row: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = row.py(); + if row.is_none() { + return Ok(row.clone()); + } + let (output_proven, columns, families, inputs, failure_reason, bare): ProofObjects<'py> = + row.extract()?; + let models = py.import(COMPILE_MODELS_MODULE)?; + let family_class: Bound<'py, PyAny> = models.getattr("DynamicColumnFamilyProof")?; + let mut proofs: Vec> = Vec::new(); + for family in families.try_iter()? { + let (name, inferred_type): (Bound<'py, PyAny>, Bound<'py, PyAny>) = family?.extract()?; + proofs.push(call( + &family_class, + &[("name", name), ("inferred_type", inferred_type)], + )?); + } + let fixed: Bound<'py, PyAny> = inferred_columns(&columns)?; + let fixed: Bound<'py, PyAny> = if fixed.is_truthy()? { + fixed + } else { + PyTuple::empty(py).into_any() + }; + call( + &models.getattr("DynamicColumnContractProof")?, + &[ + ("output_proven", output_proven), + ("fixed_columns", fixed), + ("families", PyTuple::new(py, proofs)?.into_any()), + ("input_relations", tuple_of(&inputs)?), + ("failure_reason", failure_reason), + ("bare_dynamic_pivot", bare), + ], + ) +} + +/// `(name, type, nullability)` rows as `InferredColumn`s, None for None. +fn inferred_columns<'py>(rows: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = rows.py(); + if rows.is_none() { + return Ok(rows.clone()); + } + let column_class: Bound<'py, PyAny> = py + .import(COMPILE_MODELS_MODULE)? + .getattr("InferredColumn")?; + let nullability_class: Bound<'py, PyAny> = py + .import(LINEAGE_TYPES_MODULE)? + .getattr("InferredNullability")?; + let mut columns: Vec> = Vec::new(); + for row in rows.try_iter()? { + let (name, data_type, nullability): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ) = row?.extract()?; + columns.push(call( + &column_class, + &[ + ("name", name), + ("type", data_type), + ("nullability", nullability_class.call1((nullability,))?), + ], + )?); + } + Ok(PyTuple::new(py, columns)?.into_any()) +} + +/// Native lineage rows as Python's compact facts over a per-model string pool. +fn compact_lineage<'py>(rows: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = rows.py(); + let pool: Bound<'py, PyDict> = PyDict::new(py); + let index = |value: Bound<'py, PyAny>| -> PyResult> { + if let Some(existing) = pool.get_item(&value)? { + return Ok(existing); + } + let position = pool.len().into_bound_py_any(py)?; + pool.set_item(&value, &position)?; + Ok(position) + }; + let mut compact: Vec> = Vec::new(); + for row in rows.try_iter()? { + let (output_column, transform_code, confidence_code, sources): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ) = row?.extract()?; + let mut source_indexes: Vec> = Vec::new(); + for source in sources.try_iter()? { + let (resource_type, resource_name, column_name): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ) = source?.extract()?; + let indexes = [ + index(resource_type)?, + index(resource_name)?, + index(column_name)?, + ]; + source_indexes.push(PyTuple::new(py, indexes)?.into_any()); + } + let output: Bound<'py, PyAny> = index(output_column)?; + compact.push( + PyTuple::new( + py, + [ + output, + transform_code, + confidence_code, + PyTuple::new(py, source_indexes)?.into_any(), + ], + )? + .into_any(), + ); + } + let keys: Bound<'py, PyAny> = PyTuple::new(py, pool.keys())?.into_any(); + call( + &py.import(COMPILE_MODELS_MODULE)? + .getattr("CompactLineageFacts")?, + &[ + ("string_pool", keys), + ("rows", PyTuple::new(py, compact)?.into_any()), + ], + ) +} + +/// Native lineage rows as the plain facts Python's re-analysis returns. +fn lineage_facts<'py>(rows: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = rows.py(); + let models = py.import(COMPILE_MODELS_MODULE)?; + let column_class: Bound<'py, PyAny> = models.getattr("CompiledLineageColumnFact")?; + let source_class: Bound<'py, PyAny> = models.getattr("CompiledLineageSourceFact")?; + let compact: Bound<'py, PyAny> = models.getattr("CompactLineageFacts")?; + let mut facts: Vec> = Vec::new(); + for row in rows.try_iter()? { + let (output_column, transform_code, confidence_code, sources): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ) = row?.extract()?; + let mut upstream: Vec> = Vec::new(); + for source in sources.try_iter()? { + let (resource_type, resource_name, column_name): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ) = source?.extract()?; + upstream.push(call( + &source_class, + &[ + ("resource_type", resource_type), + ("resource_name", resource_name), + ("column_name", column_name), + ], + )?); + } + facts.push(call( + &column_class, + &[ + ("output_column", output_column), + ("upstream_columns", PyTuple::new(py, upstream)?.into_any()), + ( + "transform_kind", + compact.call_method1("transform_kind", (transform_code,))?, + ), + ( + "confidence", + compact.call_method1("confidence", (confidence_code,))?, + ), + ], + )?); + } + Ok(PyTuple::new(py, facts)?.into_any()) +} + +fn tuple_of<'py>(items: &Bound<'py, PyAny>) -> PyResult> { + Ok(PyTuple::new(items.py(), items.try_iter()?.collect::>>()?)?.into_any()) +} + +fn call<'py>( + callable: &Bound<'py, PyAny>, + pairs: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + let arguments: Bound<'py, PyDict> = PyDict::new(callable.py()); + for (name, value) in pairs { + arguments.set_item(*name, value)?; + } + callable.call((), Some(&arguments)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(session_outcome_analysis, module)?)?; + module.add_function(wrap_pyfunction!(dynamic_contract_proof, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs new file mode 100644 index 0000000000..f9ca93d7c1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs @@ -0,0 +1,158 @@ +//! Inference-profile settings crossing into native analysis: rule ids, shapes and the catalog. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; + +const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; +const BINDING_CATALOG_MODULE: &str = "sqlbuild.compiler.sql_analysis.main._binding_catalog"; +const SQL_ANALYSIS_CONSTANTS_MODULE: &str = "sqlbuild.compiler.sql_analysis.constants"; +const GENERIC_DIALECT: &str = "generic"; +const ADAPTER_RULE_ID: &str = "python"; +/// `(module, function, native rule id)` of the nullability rules the native analysis knows. +const KNOWN_RULES: [(&str, &str, &str); 3] = [ + ( + "sqlbuild.adapter.type_system.main.first_arg_nullability", + "first_arg_nullability", + "first_arg", + ), + ( + "sqlbuild.adapter.type_system.main.conditional_result_nullability", + "conditional_result_nullability", + "conditional_result", + ), + ( + "sqlbuild.adapter.type_system.main.safe_cast_nullability", + "safe_cast_nullability", + "safe_cast", + ), +]; + +/// The adapter's own nullability rules and `InferredNullability`, which native analysis calls. +pub(crate) type AdapterRules = Option<(Py, Py)>; + +/// The adapter nullability rules as `(name, rule id)` rows, and the adapter's own rules. +pub(crate) fn nullability_rules(profile: &Bound<'_, PyAny>) -> PyResult<(Pairs, AdapterRules)> { + let py: Python<'_> = profile.py(); + let known: Bound<'_, PyDict> = PyDict::new(py); + for (module, function, id) in KNOWN_RULES { + known.set_item(py.import(module)?.getattr(function)?, id)?; + } + let adapter: Bound<'_, PyDict> = PyDict::new(py); + let mut rows: Pairs = Vec::new(); + for item in profile + .getattr("function_nullability_rules")? + .call_method0("items")? + .try_iter()? + { + let (name, rule): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; + let id: String = match known.get_item(&rule)? { + Some(id) => id.extract()?, + None => { + adapter.set_item(&name, &rule)?; + ADAPTER_RULE_ID.to_owned() + } + }; + rows.push((name.extract()?, id)); + } + if adapter.is_empty() { + return Ok((rows, None)); + } + let nullability: Bound<'_, PyAny> = py + .import(LINEAGE_TYPES_MODULE)? + .getattr("InferredNullability")?; + Ok((rows, Some((adapter.unbind(), nullability.unbind())))) +} + +/// `{relation: {column: value}}` as ordered rows, each value as `str(value)`. +pub(crate) fn shape_rows(shapes: &Bound<'_, PyAny>) -> PyResult { + let mut rows: Shapes = Vec::new(); + for item in shapes.call_method0("items")?.try_iter()? { + let (name, shape): (String, Bound<'_, PyAny>) = item?.extract()?; + let mut pairs: Pairs = Vec::new(); + for column in shape.call_method0("items")?.try_iter()? { + let (column, value): (String, Bound<'_, PyAny>) = column?.extract()?; + pairs.push((column, value.str()?.extract()?)); + } + rows.push((name, pairs)); + } + Ok(rows) +} + +/// A `{key: value}` text mapping's items in order. +pub(crate) fn pairs(mapping: &Bound<'_, PyAny>) -> PyResult { + mapping + .call_method0("items")? + .try_iter()? + .map(|item| item?.extract()) + .collect() +} + +/// `profile.sql_analysis_dialect or "generic"`. +pub(crate) fn profile_dialect(profile: &Bound<'_, PyAny>) -> PyResult { + dialect_or_generic(&profile.getattr("sql_analysis_dialect")?) +} + +/// `dialect or "generic"`. +pub(crate) fn dialect_or_generic(dialect: &Bound<'_, PyAny>) -> PyResult { + if dialect.is_truthy()? { + return dialect.extract(); + } + Ok(GENERIC_DIALECT.to_owned()) +} + +/// Python's `NATIVE_DIALECT_ALIASES.get(dialect, dialect)`. +pub(crate) fn aliased_dialect(py: Python<'_>, dialect: String) -> PyResult { + let aliases: Bound<'_, PyAny> = py + .import(SQL_ANALYSIS_CONSTANTS_MODULE)? + .getattr("NATIVE_DIALECT_ALIASES")?; + let key: Bound<'_, PyString> = PyString::new(py, &dialect); + aliases.call_method1("get", (&key, &key))?.extract() +} + +/// Python's `inferred_binding_shape` test for keeping authored identifier quoting. +pub(crate) fn case_sensitive_shapes( + profile: &Bound<'_, PyAny>, + dialect: &Bound<'_, PyAny>, +) -> PyResult { + if profile + .getattr("quoted_identifiers_ignore_case")? + .is_truthy()? + { + return Ok(false); + } + profile + .py() + .import(SQL_ANALYSIS_CONSTANTS_MODULE)? + .getattr("CASE_SENSITIVE_BINDING_DIALECTS")? + .contains(dialect) +} + +/// `profile.binding_catalog or create_binding_catalog(...)` with no functions, types or relations. +pub(crate) fn profile_catalog<'py>( + profile: &Bound<'py, PyAny>, + dialect: &str, +) -> PyResult> { + let catalog: Bound<'py, PyAny> = profile.getattr("binding_catalog")?; + if catalog.is_truthy()? { + return Ok(catalog); + } + let py: Python<'py> = profile.py(); + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("dialect", dialect)?; + arguments.set_item( + "quoted_ignore_case", + profile.getattr("quoted_identifiers_ignore_case")?, + )?; + arguments.set_item("known_functions", ())?; + arguments.set_item("known_types", ())?; + arguments.set_item("relations", PyDict::new(py))?; + py.import(BINDING_CATALOG_MODULE)? + .getattr("create_binding_catalog")? + .call((), Some(&arguments)) +} + +/// `profile.function_return_types` items in order. +pub(crate) fn return_types(profile: &Bound<'_, PyAny>) -> PyResult { + pairs(&profile.getattr("function_return_types")?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs index 2bd572c62a..13b2eecf79 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs @@ -24,7 +24,7 @@ const QUERY_COLUMNS_CONTEXT: &str = "native query column analysis"; const PROJECTION_FACTS_CONTEXT: &str = "native fixture projection facts"; /// `(sql, placeholders, analysis names, lineage references, recover CTE facts, mode)`. -type QueryRow = ( +pub(crate) type QueryRow = ( String, Pairs, Vec, @@ -37,12 +37,12 @@ type RequestRow = (String, Pairs, Pairs, Shapes, Shapes, Vec, String); /// `([(name, type)], NULL literal names, quoted names)`. type ProjectionFactsRow = (Vec<(String, String)>, Vec, Vec); /// `(succeeded, [(name, type, nullability)] or None, has star)`. -type ColumnsRow = (bool, Option, String)>>, bool); +pub(crate) type ColumnsRow = (bool, Option, String)>>, bool); /// Each query's columns, as Python's analysis outside the session inferred them. #[pyfunction] #[pyo3(signature = (catalog, request, adapter_rules=None))] -fn infer_query_columns( +pub(crate) fn infer_query_columns( py: Python<'_>, catalog: Option>, request: RequestRow, @@ -141,7 +141,6 @@ fn fixture_projection_facts( } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(infer_query_columns, module)?)?; module.add_function(wrap_pyfunction!(fixture_projection_facts, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs index 6e72b49d20..b0a64e9832 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs @@ -2,10 +2,15 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::analysis_session::{oracle, query_columns, session}; +use crate::bindings::_helpers::analysis_session::{ + model_analyses, oracle, outcome_objects, query_columns, session, stage_functions, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { session::register(module)?; + model_analyses::register(module)?; + outcome_objects::register(module)?; query_columns::register(module)?; + stage_functions::register(module)?; oracle::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs index d7cbfd8d38..df8973e549 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs @@ -46,7 +46,7 @@ type ColumnRow = (String, Option, String); /// `(output column, transform code, confidence code, [(resource type, resource, column)])`. type LineageItem = (String, u8, u8, Vec<(String, String, String)>); /// `(name, pivot column, value column, aggregate, type, name pattern)`. -type FamilyRow = (String, String, String, String, String, Option); +pub(crate) type FamilyRow = (String, String, String, String, String, Option); /// A model's name, SQL, placeholders, references, lineage, required names, flags and pivot. type ModelRow = ( String, @@ -95,7 +95,7 @@ type ProofRow = ( bool, ); /// One model's pivot SQL and declared families. -type PivotModelRow = (String, Vec); +pub(crate) type PivotModelRow = (String, Vec); /// The dialect, relation facts and families by table every proof reads, and the models. type PivotRequestRow = ( String, @@ -106,7 +106,7 @@ type PivotRequestRow = ( Vec, ); /// `("absent" | "deferred" | "proof", proof)`. -type ContractRow = (&'static str, Option); +pub(crate) type ContractRow = (&'static str, Option); /// `(hits, misses, stored, why the store was not read or saved)`. type CacheStatsRow = (usize, usize, usize, Option); /// `(catalog schema additions, analysis-shape names, whether each model's analysis succeeded)`. @@ -164,7 +164,7 @@ fn internal_error(reason: &str) -> PyErr { impl NativeModelAnalysisSession { /// Models whose output names and lineage the finished session keeps for later stages. #[getter] - fn fact_models(&self) -> Vec { + pub(crate) fn fact_models(&self) -> Vec { self.finished .as_ref() .map(finished_fact_models) @@ -241,7 +241,7 @@ impl NativeModelAnalysisSession { } /// Each model's dynamic pivot proof from the finished session's tables. - fn prove_dynamic_contracts( + pub(crate) fn prove_dynamic_contracts( &self, py: Python<'_>, models: Vec, @@ -307,7 +307,7 @@ fn start_model_analysis_session( /// Each expression's shape, `None` where it has none; adapter rules run as in the session. #[pyfunction] #[pyo3(signature = (catalog, request, adapter_rules=None))] -fn infer_expression_source_shapes( +pub(crate) fn infer_expression_source_shapes( py: Python<'_>, catalog: PyRef<'_, ProjectCatalog>, request: (String, bool, Pairs, Option, Vec), @@ -482,7 +482,7 @@ fn outcome_row(outcome: ModelOutcome, cleaned_sql: String) -> OutcomeRow { /// Each model's dynamic pivot proof outside any session. #[pyfunction] -fn prove_dynamic_column_contracts( +pub(crate) fn prove_dynamic_column_contracts( py: Python<'_>, request: PivotRequestRow, ) -> PyResult> { @@ -515,7 +515,5 @@ fn pivot_model(row: PivotModelRow) -> PivotModel { pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_function(wrap_pyfunction!(start_model_analysis_session, module)?)?; - module.add_function(wrap_pyfunction!(infer_expression_source_shapes, module)?)?; - module.add_function(wrap_pyfunction!(prove_dynamic_column_contracts, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs new file mode 100644 index 0000000000..0414970f55 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs @@ -0,0 +1,363 @@ +//! Query columns, expression-source shapes and dynamic pivot proofs read from Python objects. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::assembly::analysis_session::types::Pairs; + +use crate::bindings::_helpers::analysis_session::profile_inputs::{ + aliased_dialect, case_sensitive_shapes, dialect_or_generic, nullability_rules, profile_catalog, + profile_dialect, return_types, shape_rows, +}; +use crate::bindings::_helpers::analysis_session::query_columns::{ + ColumnsRow, QueryRow, infer_query_columns, +}; +use crate::bindings::_helpers::analysis_session::session::{ + ContractRow, FamilyRow, NativeModelAnalysisSession, PivotModelRow, + infer_expression_source_shapes, prove_dynamic_column_contracts, +}; +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::models::ProjectCatalog; + +const BATCH_MODE: &str = "batch"; +const TABLE_FUNCTION_KIND: &str = "table_fn"; +const TABLE_FUNCTION_ANALYSIS_PREFIX: &str = "__sqlbuild_table_function_"; +/// Python's `_LINEAGE_RESOURCE_TYPES`: reference kind to compiled resource type value. +const LINEAGE_RESOURCE_TYPES: [(&str, &str); 4] = [ + ("ref", "model"), + ("source", "source"), + ("seed", "seed"), + ("table_fn", "table_fn"), +]; +const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; + +/// Each query's `(succeeded, column rows, has star)`; a native internal failure raises. +#[pyfunction] +#[pyo3(signature = (**request))] +fn infer_native_query_column_rows<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("infer_native_query_column_rows", request)?; + let empty: Bound<'py, PyAny> = PyDict::new(request.py()).into_any(); + let nullability: Bound<'py, PyAny> = + or_empty(request.optional("column_nullability_by_table")?, &empty)?; + query_column_rows( + &request.get("queries")?, + &request.get("profile")?, + (&empty, &nullability), + ) +} + +/// Each query's columns as `NativeQueryColumns`; a native internal failure raises. +#[pyfunction] +#[pyo3(signature = (**request))] +fn infer_native_query_columns<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("infer_native_query_columns", request)?; + let py: Python<'py> = request.py(); + let empty: Bound<'py, PyAny> = PyDict::new(py).into_any(); + let types: Bound<'py, PyAny> = or_empty(request.optional("column_types_by_table")?, &empty)?; + let nullability: Bound<'py, PyAny> = + or_empty(request.optional("column_nullability_by_table")?, &empty)?; + let rows: Vec = query_column_rows( + &request.get("queries")?, + &request.get("profile")?, + (&types, &nullability), + )?; + let query_columns: Bound<'py, PyAny> = py + .import(ANALYSIS_MODELS_MODULE)? + .getattr("NativeQueryColumns")?; + let column_class: Bound<'py, PyAny> = py + .import(COMPILE_MODELS_MODULE)? + .getattr("InferredColumn")?; + let nullability_class: Bound<'py, PyAny> = py + .import(LINEAGE_TYPES_MODULE)? + .getattr("InferredNullability")?; + let mut analyses: Vec> = Vec::with_capacity(rows.len()); + for (succeeded, columns, has_star) in rows { + let columns: Bound<'py, PyAny> = match columns { + None => py.None().into_bound(py), + Some(columns) => { + let mut built: Vec> = Vec::with_capacity(columns.len()); + for (name, data_type, nullability) in columns { + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("name", name)?; + arguments.set_item("type", data_type)?; + arguments.set_item("nullability", nullability_class.call1((nullability,))?)?; + built.push(column_class.call((), Some(&arguments))?); + } + PyTuple::new(py, built)?.into_any() + } + }; + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("succeeded", succeeded)?; + arguments.set_item("columns", columns)?; + arguments.set_item("has_star", has_star)?; + analyses.push(query_columns.call((), Some(&arguments))?); + } + PyTuple::new(py, analyses) +} + +/// One shape per expression, cached on the profile's binding catalog; a failure raises. +#[pyfunction] +#[pyo3(signature = (**request))] +fn infer_native_expression_source_shapes<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("infer_native_expression_source_shapes", request)?; + let py: Python<'py> = request.py(); + let expressions: Bound<'py, PyAny> = request.get("expressions")?; + let profile: Bound<'py, PyAny> = request.get("profile")?; + let dialect: String = aliased_dialect(py, profile_dialect(&profile)?)?; + let catalog: Bound<'py, PyAny> = profile_catalog(&profile, &dialect)?; + let cache: Bound<'py, PyAny> = catalog.getattr("expression_shapes")?; + let pending: Bound<'py, PyDict> = PyDict::new(py); + for expression in expressions.try_iter()? { + let expression: Bound<'py, PyAny> = expression?; + if !cache.contains(&expression)? { + pending.set_item(expression, py.None())?; + } + } + if !pending.is_empty() { + let (rules, adapter_rules) = nullability_rules(&profile)?; + let native: PyRef<'py, ProjectCatalog> = catalog.getattr("native")?.extract()?; + let pending_names: Vec = pending + .keys() + .iter() + .map(|name| name.extract()) + .collect::>()?; + let case_sensitive: bool = + case_sensitive_shapes(&profile, &dialect.clone().into_bound_py_any(py)?)?; + let shapes: Vec> = infer_expression_source_shapes( + py, + native, + ( + dialect, + case_sensitive, + return_types(&profile)?, + Some(rules), + pending_names, + ), + adapter_rules, + )?; + for (expression, shape) in pending.keys().iter().zip(shapes) { + let value: Bound<'py, PyAny> = match shape { + Some(shape) => shape.into_py_dict_ordered(py)?, + None => py.None().into_bound(py), + }; + cache.set_item(expression, value)?; + } + } + let mut shapes: Vec> = Vec::new(); + for expression in expressions.try_iter()? { + let cached: Bound<'py, PyAny> = cache.get_item(expression?)?; + shapes.push(if cached.is_none() { + cached + } else { + let copied: Bound<'py, PyDict> = PyDict::new(py); + copied.update(cached.downcast::()?.as_mapping())?; + copied.into_any() + }); + } + PyTuple::new(py, shapes) +} + +/// The proof row per `(sql, families)` model from the session or `tables`, else None. +#[pyfunction] +#[pyo3(signature = (**request))] +fn prove_native_dynamic_contracts<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("prove_native_dynamic_contracts", request)?; + let py: Python<'py> = request.py(); + let session: Bound<'py, PyAny> = request.get("session")?; + let tables: Bound<'py, PyAny> = request.get("tables")?; + let models: Bound<'py, PyAny> = request.get("models")?; + if !models.is_truthy()? { + return Ok(PyTuple::empty(py)); + } + let mut rows: Vec = Vec::new(); + for model in models.try_iter()? { + let (sql, families): (String, Bound<'py, PyAny>) = model?.extract()?; + rows.push((sql, family_rows(&families)?)); + } + let contracts: Vec = if session.is_none() { + let by_table: Bound<'py, PyAny> = tables.getattr("dynamic_families_by_table")?; + let mut families: Vec<(String, Vec)> = Vec::new(); + for item in by_table.call_method0("items")?.try_iter()? { + let (name, declared): (String, Bound<'py, PyAny>) = item?.extract()?; + families.push((name, family_rows(&declared)?)); + } + prove_dynamic_column_contracts( + py, + ( + dialect_or_generic(&tables.getattr("dialect")?)?, + shape_rows(&tables.getattr("column_types_by_table")?)?, + shape_rows(&tables.getattr("authoritative_column_types_by_table")?)?, + shape_rows(&tables.getattr("column_nullability_by_table")?)?, + families, + rows, + ), + )? + } else { + let session: PyRef<'py, NativeModelAnalysisSession> = session.extract()?; + session.prove_dynamic_contracts(py, rows)? + }; + let mut proofs: Vec> = Vec::with_capacity(contracts.len()); + for (_, proof) in contracts { + proofs.push(proof.into_bound_py_any(py)?); + } + PyTuple::new(py, proofs) +} + +/// Declared dynamic families as native request rows. +pub(crate) fn family_rows(families: &Bound<'_, PyAny>) -> PyResult> { + let mut rows: Vec = Vec::new(); + for family in families.try_iter()? { + let family: Bound<'_, PyAny> = family?; + rows.push(( + family.getattr("name")?.extract()?, + family.getattr("pivot_column")?.extract()?, + family.getattr("value_column")?.extract()?, + family.getattr("aggregate")?.extract()?, + family.getattr("type")?.extract()?, + family.getattr("name_pattern")?.extract()?, + )); + } + Ok(rows) +} + +fn or_empty<'py>( + value: Option>, + empty: &Bound<'py, PyAny>, +) -> PyResult> { + match value { + Some(value) if value.is_truthy()? => Ok(value), + _ => Ok(empty.clone()), + } +} + +fn query_column_rows( + queries: &Bound<'_, PyAny>, + profile: &Bound<'_, PyAny>, + (types, nullability): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult> { + let py: Python<'_> = profile.py(); + if !queries.is_truthy()? { + return Ok(Vec::new()); + } + let queries: Vec> = queries.try_iter()?.collect::>()?; + let normalization_dialect: String = profile_dialect(profile)?; + let dialect: String = aliased_dialect(py, normalization_dialect.clone())?; + let mut batch: bool = false; + for query in &queries { + if query.getattr("mode")?.eq(BATCH_MODE)? { + batch = true; + break; + } + } + let catalog: Option> = if batch { + Some(profile_catalog(profile, &dialect)?.getattr("native")?) + } else { + None + }; + let (rules, adapter_rules) = nullability_rules(profile)?; + let mut rows: Vec = Vec::with_capacity(queries.len()); + for query in &queries { + rows.push(query_row(query)?); + } + let catalog: Option> = + catalog.map(|native| native.extract()).transpose()?; + infer_query_columns( + py, + catalog, + ( + dialect, + return_types(profile)?, + rules, + shape_rows(types)?, + shape_rows(nullability)?, + rows, + normalization_dialect, + ), + adapter_rules, + ) +} + +fn query_row(query: &Bound<'_, PyAny>) -> PyResult { + let placeholders: Bound<'_, PyAny> = query.getattr("placeholders")?; + let placeholders: Pairs = if placeholders.is_truthy()? { + placeholders + .call_method0("items")? + .try_iter()? + .map(|item| item?.extract()) + .collect::>()? + } else { + Vec::new() + }; + let mut analysis_names: Vec = Vec::new(); + let mut lineage: Vec<(String, String, String)> = Vec::new(); + for reference in query.getattr("references")?.try_iter()? { + let reference: Bound<'_, PyAny> = reference?; + let kind: String = reference.getattr("ref_kind")?.extract()?; + let name: String = reference.getattr("ref_name")?.extract()?; + let analysis_name: String = if kind == TABLE_FUNCTION_KIND { + format!("{TABLE_FUNCTION_ANALYSIS_PREFIX}{name}") + } else { + name.clone() + }; + analysis_names.push(analysis_name.clone()); + if let Some((_, resource_type)) = LINEAGE_RESOURCE_TYPES + .iter() + .find(|(reference_kind, _)| *reference_kind == kind) + { + let entry = (analysis_name, (*resource_type).to_owned(), name); + match lineage + .iter_mut() + .find(|(existing, _, _)| *existing == entry.0) + { + Some(existing) => *existing = entry, + None => lineage.push(entry), + } + } + } + Ok(( + query.getattr("sql")?.extract()?, + placeholders, + analysis_names, + lineage, + query.getattr("recover_cte_facts")?.extract()?, + query.getattr("mode")?.extract()?, + )) +} + +trait OrderedDict { + fn into_py_dict_ordered<'py>(self, py: Python<'py>) -> PyResult>; +} + +impl OrderedDict for Pairs { + fn into_py_dict_ordered<'py>(self, py: Python<'py>) -> PyResult> { + let dict: Bound<'py, PyDict> = PyDict::new(py); + for (key, value) in self { + dict.set_item(key, value)?; + } + Ok(dict.into_any()) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(infer_native_query_column_rows, module)?)?; + module.add_function(wrap_pyfunction!(infer_native_query_columns, module)?)?; + module.add_function(wrap_pyfunction!( + infer_native_expression_source_shapes, + module + )?)?; + module.add_function(wrap_pyfunction!(prove_native_dynamic_contracts, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/boundary/keyword_request.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/keyword_request.rs new file mode 100644 index 0000000000..944c60fcac --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/boundary/keyword_request.rs @@ -0,0 +1,58 @@ +//! Keyword-only native function arguments read from `**request`. + +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; + +/// Keyword arguments of one call, each required. +pub(crate) struct Request<'a, 'py> { + function: &'static str, + arguments: Option<&'a Bound<'py, PyDict>>, + py: Python<'py>, +} + +impl<'a, 'py> Request<'a, 'py> { + pub(crate) fn new( + function: &'static str, + arguments: Option<&'a Bound<'py, PyDict>>, + ) -> PyResult { + let py: Python<'py> = match arguments { + Some(arguments) => arguments.py(), + None => { + return Err(PyTypeError::new_err(format!( + "{function}() missing required keyword-only arguments" + ))); + } + }; + Ok(Self { + function, + arguments, + py, + }) + } + + pub(crate) fn py(&self) -> Python<'py> { + self.py + } + + pub(crate) fn get(&self, name: &str) -> PyResult> { + let value: Option> = match self.arguments { + Some(arguments) => arguments.get_item(name)?, + None => None, + }; + value.ok_or_else(|| { + PyTypeError::new_err(format!( + "{}() missing 1 required keyword-only argument: '{name}'", + self.function + )) + }) + } + + /// The argument, or `None` when the caller left it out. + pub(crate) fn optional(&self, name: &str) -> PyResult>> { + match self.arguments { + Some(arguments) => arguments.get_item(name), + None => Ok(None), + } + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs index 0a21932dc9..7c0e808fc6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs @@ -1,4 +1,5 @@ mod allocator; pub(crate) mod config_values; +pub(crate) mod keyword_request; pub(crate) mod oracles; pub(crate) mod panics; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs new file mode 100644 index 0000000000..cb2e2c978e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs @@ -0,0 +1,202 @@ +//! The process-local declaration scope of one compile invocation. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyTuple; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const SCAN_CACHE_MODULE: &str = "sqlbuild.compiler.compile.classes.sql_test_scan_cache"; +const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const DECLARATIONS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.declarations"; +const SCOPE_ERRORS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; +const OPEN_INDEX_MODULE: &str = "sqlbuild.compiler.scopes.main._open_native_scope_index"; +const VALIDATE_INDEX_MODULE: &str = "sqlbuild.compiler.scopes.main._validate_scope_index"; + +fn compile_error(py: Python<'_>, message: Bound<'_, PyAny>) -> PyResult { + Ok(PyErr::from_value( + imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")?.call1((message,))?, + )) +} + +/// The declaration scope of `index`, resolved for `loaded_macros`. +fn scope_build<'py>( + discovered: &Bound<'py, PyAny>, + loaded_macros: &Bound<'py, PyAny>, + (index, lookup, scan_cache): (Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let resolver: Bound<'py, PyAny> = call_with( + &imported(py, DECLARATIONS_MODULE, "build_declaration_scope_resolver")?, + &[ + ("discovered_inputs", discovered.clone()), + ("scope_index", index.clone()), + ("loaded_macros", loaded_macros.clone()), + ("lookup", lookup), + ], + )?; + call_with( + &imported(py, COMPILE_MODELS_MODULE, "DeclarationScopeBuild")?, + &[ + ("loaded_macros", loaded_macros.clone()), + ("index", index), + ("resolver", resolver), + ("sql_test_scans", scan_cache), + ], + ) +} + +/// Build one canonical index and validate it before SQL expansion. +#[pyfunction] +#[pyo3(signature = (*, discovered_inputs, loaded_macros, sql_lexical_syntax, compile_cache_dir=None))] +pub(crate) fn build_declaration_scope<'py>( + discovered_inputs: &Bound<'py, PyAny>, + loaded_macros: &Bound<'py, PyAny>, + sql_lexical_syntax: &Bound<'py, PyAny>, + compile_cache_dir: Option>, +) -> PyResult> { + let py: Python<'py> = discovered_inputs.py(); + let scan_cache: Bound<'py, PyAny> = call_with( + &imported(py, SCAN_CACHE_MODULE, "SqlTestScanCache")?, + &[( + "cache_dir", + compile_cache_dir.unwrap_or_else(|| py.None().into_bound(py)), + )], + )?; + let test_files: Bound<'py, PyAny> = discovered_inputs.getattr("test_files")?; + let scenario_files: Bound<'py, PyAny> = discovered_inputs.getattr("scenario_files")?; + imported(py, NATIVE_MODULE, "validate_authored_sql_test_ctes")?.call1(( + &test_files, + &scenario_files, + sql_lexical_syntax.getattr("native_mapping")?, + ))?; + let native: Bound<'py, PyAny> = call_with( + &imported(py, OPEN_INDEX_MODULE, "open_native_scope_index")?, + &[ + ("discovered_inputs", discovered_inputs.clone()), + ("loaded_macros", loaded_macros.clone()), + ], + )?; + let validated: PyResult> = call_with( + &imported(py, VALIDATE_INDEX_MODULE, "validate_scope_index")?, + &[("index", native.getattr("index")?)], + ); + if let Err(error) = validated { + if !error.is_instance( + py, + &imported(py, SCOPE_ERRORS_MODULE, "ScopeValidationError")?, + ) { + return Err(error); + } + let raised: PyErr = compile_error(py, error.value(py).str()?.into_any())?; + raised.set_cause(py, Some(error)); + return Err(raised); + } + if native + .getattr("has_scoped_relationship_declarations")? + .is_truthy()? + && (test_files.is_truthy()? || scenario_files.is_truthy()?) + { + let faults: Bound<'py, PyAny> = call_with( + &native.getattr("grant_relationships")?, + &[ + ("discovered_inputs", discovered_inputs.clone()), + ("syntax", sql_lexical_syntax.clone()), + ( + "scan_store", + PyTuple::new( + py, + [ + scan_cache.call_method0("opened")?, + imported( + py, + COMPILE_CONSTANTS_MODULE, + "SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM", + )?, + ], + )? + .into_any(), + ), + ], + )?; + if faults.is_truthy()? { + return Err(compile_error(py, faults.get_item(0)?.get_item(1)?)?); + } + } + let index: Bound<'py, PyAny> = native.call_method0("index_with_relationships")?; + let lookup: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "build_scope_lookup")?.call1((&index,))?; + scope_build( + discovered_inputs, + loaded_macros, + (index, lookup, scan_cache), + ) +} + +/// Each loaded macro's metadata that the built index recorded. +fn indexed_macro_metadata<'py>(loaded_macros: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = loaded_macros.py(); + let code_type: Bound<'py, PyAny> = imported(py, "types", "CodeType")?; + let signature: Bound<'py, PyAny> = imported(py, "inspect", "signature")?; + let mut rows: Vec> = Vec::new(); + for pair in loaded_macros.call_method0("items")?.try_iter()? { + let (key, loaded): (Bound<'py, PyAny>, Bound<'py, PyAny>) = pair?.extract()?; + let function: Bound<'py, PyAny> = loaded.getattr("function")?; + let code: Bound<'py, PyAny> = + imported(py, "builtins", "getattr")?.call1((&function, "__code__", py.None()))?; + let first_line: Bound<'py, PyAny> = if code.is_instance(&code_type)? { + code.getattr("co_firstlineno")? + } else { + py.None().into_bound(py) + }; + let parameters: Bound<'py, PyTuple> = PyTuple::new( + py, + signature + .call1((&function,))? + .getattr("parameters")? + .try_iter()? + .collect::>>()?, + )?; + rows.push(PyTuple::new( + py, + [ + key, + loaded.getattr("name")?, + loaded.getattr("relative_path")?, + loaded.getattr("raw_source")?, + loaded.getattr("dependencies")?, + first_line, + parameters.into_any(), + ], + )?); + } + PyTuple::new(py, rows) +} + +/// Pair a built index with private macro instances, or return None if their metadata differs. +#[pyfunction] +#[pyo3(signature = (*, scope, discovered_inputs, loaded_macros))] +pub(crate) fn rebind_declaration_scope<'py>( + scope: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, + loaded_macros: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = scope.py(); + let current: Bound<'py, PyTuple> = indexed_macro_metadata(loaded_macros)?; + let indexed: Bound<'py, PyTuple> = indexed_macro_metadata(&scope.getattr("loaded_macros")?)?; + if current.as_any().ne(indexed.as_any())? { + return Ok(py.None().into_bound(py)); + } + scope_build( + discovered_inputs, + loaded_macros, + ( + scope.getattr("index")?, + scope.getattr("resolver")?.getattr("lookup")?, + scope.getattr("sql_test_scans")?, + ), + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs new file mode 100644 index 0000000000..bff1441a51 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs @@ -0,0 +1,105 @@ +//! The effective connection, settings and vars of a project, its local config and target. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyBool, PyDict, PyList, PyString}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const TEMPLATING_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.templating"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const CONNECTION_LABEL: &str = "effective connection"; + +/// Merge base project connection with the selected environment overrides. +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config, target_config, effective_vars))] +pub(crate) fn build_effective_connection<'py>( + project_config: &Bound<'py, PyAny>, + local_config: &Bound<'py, PyAny>, + target_config: &Bound<'py, PyAny>, + effective_vars: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_config.py(); + let (connection, error): (Bound<'py, PyAny>, Option) = + imported(py, NATIVE_MODULE, "effective_config_connection")? + .call1((project_config, local_config, target_config))? + .extract()?; + if connection.is_none() { + let raised: Bound<'py, PyAny> = imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")? + .call1((error.unwrap_or_default(),))?; + return Err(PyErr::from_value(raised)); + } + let falsy = || PyBool::new(py, false).to_owned().into_any(); + call_with( + &imported(py, TEMPLATING_MODULE, "expand_template_data")?, + &[ + ("value", connection), + ("variables", effective_vars.clone()), + ("context_values", PyDict::new(py).into_any()), + ( + "context_label", + PyString::new(py, CONNECTION_LABEL).into_any(), + ), + ("allow_context", falsy()), + ("preserve_context_tokens", falsy()), + ("preserve_unknown_context", falsy()), + ], + ) +} + +/// Merge project settings with local developer overrides. +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config))] +pub(crate) fn build_effective_settings<'py>( + project_config: &Bound<'py, PyAny>, + local_config: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_config.py(); + let settings_class: Bound<'py, PyAny> = imported(py, CONTRACT_MODELS_MODULE, "SettingsConfig")?; + let names: Vec> = imported(py, "dataclasses", "fields")? + .call1((&settings_class,))? + .try_iter()? + .map(|field| field?.getattr("name")) + .collect::>()?; + let values: Bound<'py, PyDict> = imported(py, NATIVE_MODULE, "effective_config_settings")? + .call1(( + ( + project_config.getattr("settings")?, + local_config.getattr("settings")?, + ), + local_config.getattr("setting_overrides")?, + PyList::new(py, names)?, + ))? + .downcast_into()?; + settings_class.call((), Some(&values)) +} + +/// Merge effective vars using the locked precedence order. +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config, target_config, cli_vars))] +pub(crate) fn build_effective_vars<'py>( + project_config: &Bound<'py, PyAny>, + local_config: &Bound<'py, PyAny>, + target_config: &Bound<'py, PyAny>, + cli_vars: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_config.py(); + let merged: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "effective_config_vars")? + .call1(((project_config, local_config), target_config, cli_vars))?; + imported(py, TEMPLATING_MODULE, "expand_effective_vars")?.call1((merged,)) +} + +/// `build_effective_vars` for a discovered project and target, without CLI vars. +pub(crate) fn project_vars<'py>( + discovered: &Bound<'py, PyAny>, + target_config: &Bound<'py, PyAny>, +) -> PyResult> { + build_effective_vars( + &discovered.getattr("project_config")?, + &discovered.getattr("local_config")?, + target_config, + PyDict::new(discovered.py()).as_any(), + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs new file mode 100644 index 0000000000..bb3a0161fb --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod declaration_scope; +pub(crate) mod effective_config; +pub(crate) mod model_config; +pub(crate) mod model_inputs; +pub(crate) mod registration; +pub(crate) mod target; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs new file mode 100644 index 0000000000..8e98074ca2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs @@ -0,0 +1,96 @@ +//! The native config builder, validator and inherited storage policies of one compile. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::compiled_project::loop_classes::table_backed; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const UNICODE_ENVIRONMENT_MODULE: &str = "sqlbuild.compiler.compile.classes.unicode_environment"; +const RETENTION_MODULE: &str = "sqlbuild.compiler.compile._helpers.config.retention"; +const TABLE_TYPE_MODULE: &str = "sqlbuild.compiler.compile._helpers.config.table_type"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; + +/// Capture the project inputs every model's native config build and validation read. +#[pyfunction] +#[pyo3(signature = (*, inputs, validator))] +pub(crate) fn native_model_config_session<'py>( + inputs: &Bound<'py, PyAny>, + validator: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let project_config: Bound<'py, PyAny> = inputs.getattr("project_config")?; + let target: Bound<'py, PyAny> = inputs.getattr("target_config")?; + let declared: Bound<'py, PyAny> = project_config.getattr("materialization_defaults")?; + let materialization_defaults: Bound<'py, PyAny> = if declared.is_truthy()? { + declared + } else { + imported(py, CONTRACT_MODELS_MODULE, "MaterializationDefaultsConfig")?.call0()? + }; + let hook_entries: Bound<'py, PyTuple> = PyTuple::new( + py, + ["SqlHookEntry", "NamedSqlHookEntry", "PythonHookEntry"] + .iter() + .map(|name| imported(py, DISCOVERY_MODELS_MODULE, name)) + .collect::>>()?, + )?; + let target_namespace: Bound<'py, PyAny> = if target.is_none() { + target.clone() + } else { + PyTuple::new(py, [target.getattr("database")?, target.getattr("schema")?])?.into_any() + }; + let builder: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "NativeModelConfigBuilder")? + .call1(( + ( + project_config.getattr("defaults")?, + project_config.getattr("path_defaults")?, + hook_entries, + ), + ( + inputs.getattr("effective_vars")?, + imported(py, UNICODE_ENVIRONMENT_MODULE, "UnicodeEnvironment")?.call0()?, + ), + ( + ( + inputs.getattr("effective_target_name")?, + inputs.getattr("run_id")?, + ), + target_namespace, + ), + ))?; + let inherited: Bound<'py, PyDict> = PyDict::new(py); + let empty_name: Bound<'py, PyAny> = PyString::new(py, "").into_any(); + let mut materializations: Vec> = vec![py.None().into_bound(py)]; + materializations.extend(table_backed(py)?); + for materialized in materializations { + let policy = |module: &str, name: &str| -> PyResult> { + call_with( + &imported(py, module, name)?, + &[ + ("materialized", materialized.clone()), + ("model_value", py.None().into_bound(py)), + ("materialization_defaults", materialization_defaults.clone()), + ("target_config", target.clone()), + ("model_name", empty_name.clone()), + ], + ) + }; + let retention: Bound<'py, PyAny> = + policy(RETENTION_MODULE, "resolve_time_travel_retention")?; + let table_type: Bound<'py, PyAny> = policy(TABLE_TYPE_MODULE, "resolve_table_type")?; + inherited.set_item(&materialized, (retention, table_type))?; + } + call_with( + &imported(py, COMPILE_MODELS_MODULE, "NativeModelConfigSession")?, + &[ + ("path_defaults", project_config.getattr("path_defaults")?), + ("builder", builder), + ("validator", validator.clone()), + ("inherited_storage", inherited.into_any()), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs new file mode 100644 index 0000000000..a150be18c6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -0,0 +1,318 @@ +//! Every discovered model file attached as a model input by the native model loop. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyFrozenSet, PyList, PyTuple}; + +use crate::bindings::_helpers::compile_attachment::model_config::native_model_config_session; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::compiled_project::loop_classes::model_loop_classes; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; +const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const VALIDATION_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.validation"; +const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const CURSOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.cursor_intrinsics"; +const DECLARATIONS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.declarations"; +const MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.macros"; +const TEMPLATING_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.templating"; +const UNICODE_ENVIRONMENT_MODULE: &str = "sqlbuild.compiler.compile.classes.unicode_environment"; +const PROJECT_VARS_MODULE: &str = "sqlbuild.compiler.authored_values.main._project_var_values"; +const MACRO_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.active_macro_bridge"; +const CONTRACT_CLASSES_MODULE: &str = + "sqlbuild.compiler.model_config.main._native_contract_classes"; +const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; +const PYTHON_NODE_MODELS_MODULE: &str = "sqlbuild.python_nodes.models"; + +fn dict<'py>( + py: Python<'py>, + entries: Vec<(&str, Bound<'py, PyAny>)>, +) -> PyResult> { + let built: Bound<'py, PyDict> = PyDict::new(py); + for (key, value) in entries { + built.set_item(key, value)?; + } + Ok(built) +} + +/// `sql_analysis_opt_out_help(model_owned=...)`. +fn opt_out_help<'py>(py: Python<'py>, model_owned: bool) -> PyResult> { + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("model_owned", model_owned)?; + imported(py, VALIDATION_MODULE, "sql_analysis_opt_out_help")?.call((), Some(&keywords)) +} + +/// The host the loop calls back into: `constant_text(*, declaration, file_path)`. +fn loop_host<'py>(context: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = context.py(); + let context: Py = context.clone().unbind(); + let constant_text = move |arguments: &Bound<'_, PyTuple>, + keywords: Option<&Bound<'_, PyDict>>| + -> PyResult> { + let py: Python<'_> = arguments.py(); + let context: &Bound<'_, PyAny> = context.bind(py); + let passed: Bound<'_, PyDict> = + keywords.map_or_else(|| PyDict::new(py), |keywords| keywords.clone()); + call_with( + &imported(py, DECLARATIONS_MODULE, "render_constant_declaration")?, + &[ + ("declaration", passed.as_any().get_item("declaration")?), + ("value_renderer", context.getattr("value_renderer")?), + ( + "collection_rendering", + context.getattr("collection_rendering")?, + ), + ("file_path", passed.as_any().get_item("file_path")?), + ], + ) + .map(Bound::unbind) + }; + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item( + "constant_text", + PyCFunction::new_closure(py, None, None, constant_text)?, + )?; + imported(py, "types", "SimpleNamespace")?.call((), Some(&keywords)) +} + +fn macro_inputs<'py>(context: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = context.py(); + dict( + py, + vec![ + ("loaded_macros", context.getattr("loaded_macros")?), + ("macro_context", context.getattr("macro_context")?), + ( + "bridge", + imported(py, MACRO_BRIDGE_MODULE, "active_macro_bridge")?.call0()?, + ), + ( + "scoped", + pyo3::types::PyBool::new(py, !context.getattr("declaration_resolver")?.is_none()) + .to_owned() + .into_any(), + ), + ( + "declaration_context", + imported(py, COMPILE_MODELS_MODULE, "DeclarationResolutionContext")?, + ), + ( + "macro_argument_error", + imported(py, COMPILE_ERRORS_MODULE, "MacroArgumentError")?, + ), + ( + "expansion", + imported(py, MACROS_MODULE, "macro_expansion_classes")?.call0()?, + ), + ( + "generic_syntax", + imported(py, SQL_ANALYSIS_MODELS_MODULE, "SqlLexicalSyntax")? + .call0()? + .getattr("native_mapping")?, + ), + ], + ) +} + +fn sql_hook_inputs<'py>( + discovered: &Bound<'py, PyAny>, + context: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = context.py(); + dict( + py, + vec![ + ( + "named_sql_hook_entry", + imported(py, DISCOVERY_MODELS_MODULE, "NamedSqlHookEntry")?, + ), + ( + "python_hook_entry", + imported(py, DISCOVERY_MODELS_MODULE, "PythonHookEntry")?, + ), + ("hook_files", discovered.getattr("sql_hook_files")?), + ("effective_vars", context.getattr("effective_vars")?), + ( + "environment", + imported(py, UNICODE_ENVIRONMENT_MODULE, "UnicodeEnvironment")?.call0()?, + ), + ( + "render_project_var", + imported(py, PROJECT_VARS_MODULE, "render_project_var_text")?, + ), + ( + "sql_resource_ref", + imported(py, PYTHON_NODE_MODELS_MODULE, "SqlResourceRef")?, + ), + ( + "explicit_kinds", + imported(py, COMPILE_CONSTANTS_MODULE, "EXPLICIT_REFERENCE_KINDS")?, + ), + ( + "declaration_identity", + imported(py, SCOPE_MODELS_MODULE, "DeclarationIdentity")?, + ), + ( + "sql_hook_kind", + imported(py, SCOPE_TYPES_MODULE, "DeclarationKind")?.getattr("SQL_HOOK")?, + ), + ], + ) +} + +/// Attach schema metadata to discovered model files. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn build_model_inputs<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_model_inputs", + options, + ( + &["discovered_inputs", "context", "native_project"], + &[ + "no_sql_validation", + "defer_model_sql_validation", + "external_sql_reference_resolver", + ], + ), + )?; + let discovered_inputs: &Bound<'py, PyAny> = &arguments.value("discovered_inputs")?; + let context: &Bound<'py, PyAny> = &arguments.value("context")?; + let native_project: &Bound<'py, PyAny> = &arguments.value("native_project")?; + let no_sql_validation: bool = arguments.flag("no_sql_validation")?; + let defer_model_sql_validation: bool = arguments.flag("defer_model_sql_validation")?; + let external: Bound<'py, PyAny> = arguments.value("external_sql_reference_resolver")?; + let settings: Bound<'py, PyAny> = context.getattr("effective_settings")?; + let validator: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "NativeModelValidator")? + .call_method1( + "from_inputs", + ( + discovered_inputs, + settings.getattr("microbatch_concurrency")?, + ), + )?; + if !external.is_none() { + call_with( + &external.getattr("validate_model_names")?, + &[("known_model_names", validator.call_method0("model_names")?)], + )?; + } + let project_dir: Bound<'py, PyAny> = discovered_inputs.getattr("project_dir")?; + let root: Bound<'py, PyAny> = if project_dir.is_truthy()? { + project_dir + } else { + imported(py, "pathlib", "Path")?.call0()? + }; + let project_config_path: Bound<'py, PyAny> = root.div(imported( + py, + DISCOVERY_CONSTANTS_MODULE, + "PROJECT_CONFIG_FILENAME", + )?)?; + let config_inputs: Bound<'py, PyAny> = call_with( + &imported(py, COMPILE_MODELS_MODULE, "NativeModelConfigInputs")?, + &[ + ( + "project_config", + discovered_inputs.getattr("project_config")?, + ), + ("target_config", context.getattr("target_config")?), + ("effective_vars", context.getattr("effective_vars")?), + ( + "effective_target_name", + context.getattr("effective_target_name")?, + ), + ("run_id", context.getattr("run_id")?), + ( + "microbatch_concurrency", + settings.getattr("microbatch_concurrency")?, + ), + ], + )?; + let session: Bound<'py, PyAny> = native_model_config_session(&config_inputs, &validator)?; + let provider_names: Vec> = discovered_inputs + .getattr("providers")? + .try_iter()? + .map(|provider| provider?.getattr("name")) + .collect::>()?; + let host_inputs: Bound<'py, PyDict> = dict( + py, + vec![ + ( + "report_diagnostic", + imported(py, COLLECTOR_MODULE, "report_compile_diagnostic")?, + ), + ( + "record_template_reads", + imported(py, TEMPLATING_MODULE, "record_template_reads")?, + ), + ( + "visible_declarations", + imported(py, COMPILE_MODELS_MODULE, "VisibleModelDeclarations")?, + ), + ("config_session", session), + ("syntax_help", opt_out_help(py, true)?), + ("source_syntax_help", opt_out_help(py, false)?), + ("macros", macro_inputs(context)?.into_any()), + ( + "sql_hooks", + sql_hook_inputs(discovered_inputs, context)?.into_any(), + ), + ( + "hook_functions", + discovered_inputs.getattr("hook_functions")?, + ), + ( + "provider_names", + PyFrozenSet::new(py, &provider_names)?.into_any(), + ), + ], + )?; + let classes: Bound<'py, PyDict> = model_loop_classes( + &discovered_inputs.getattr("audit_factories")?, + context, + &external, + &host_inputs, + )?; + let syntax: Bound<'py, PyAny> = context.getattr("sql_lexical_syntax")?; + let native_loop: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "NativeModelLoop")?.call1(( + ( + validator, + imported(py, CONTRACT_CLASSES_MODULE, "native_contract_classes")?.call0()?, + classes, + ), + syntax.getattr("native_mapping")?, + imported(py, CURSOR_MODULE, "cursor_intrinsic_semantics")?.call0()?, + ( + ( + settings.getattr("sql_analysis")?, + no_sql_validation, + defer_model_sql_validation, + settings.getattr("require_sql_analysis")?, + ), + project_config_path.str()?, + ), + ))?; + let files: Bound<'py, PyList> = PyList::new( + py, + discovered_inputs + .getattr("model_files")? + .try_iter()? + .collect::>>()?, + )?; + let inputs: Bound<'py, PyAny> = + native_loop.call_method1("run", (native_project, loop_host(context)?, files))?; + PyTuple::new(py, inputs.try_iter()?.collect::>>()?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs new file mode 100644 index 0000000000..ba7d88e1d6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs @@ -0,0 +1,45 @@ +//! Register the compile-input attachment functions. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +use crate::bindings::_helpers::compile_attachment::{ + declaration_scope, effective_config, model_config, model_inputs, target, +}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + effective_config::build_effective_connection, + module + )?)?; + module.add_function(wrap_pyfunction!( + effective_config::build_effective_settings, + module + )?)?; + module.add_function(wrap_pyfunction!( + effective_config::build_effective_vars, + module + )?)?; + module.add_function(wrap_pyfunction!( + model_config::native_model_config_session, + module + )?)?; + module.add_function(wrap_pyfunction!(model_inputs::build_model_inputs, module)?)?; + module.add_function(wrap_pyfunction!( + target::build_compile_target_context, + module + )?)?; + module.add_function(wrap_pyfunction!( + target::build_effective_target_namespace, + module + )?)?; + module.add_function(wrap_pyfunction!( + declaration_scope::build_declaration_scope, + module + )?)?; + module.add_function(wrap_pyfunction!( + declaration_scope::rebind_declaration_scope, + module + )?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs new file mode 100644 index 0000000000..cabc68551f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs @@ -0,0 +1,247 @@ +//! The effective target, its namespace, and the shared project-local compile-cache directory. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; + +use crate::bindings::_helpers::compile_attachment::effective_config::project_vars; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; + +const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; +const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; +const FRONTIER_CONSTANTS_MODULE: &str = "sqlbuild.compiler.frontier.constants"; +const CACHE_DIRECTORY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiler_cache_directory"; +const CONTEXT_TEMPLATES_MODULE: &str = + "sqlbuild.compiler.compile._helpers.render.context_templates"; +const DATABASE_KEY: &str = "database"; +const SCHEMA_KEY: &str = "schema"; + +/// The selected target's name and merged config, or `(None, None)`. +fn resolved_target<'py>( + discovered: &Bound<'py, PyAny>, + selected_target: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py: Python<'py> = discovered.py(); + let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; + let local_config: Bound<'py, PyAny> = discovered.getattr("local_config")?; + let target_name: Bound<'py, PyAny> = call_with( + &imported(py, TARGET_NAME_MODULE, "resolve_target_name")?, + &[ + ("project_config", project_config.clone()), + ("local_config", local_config.clone()), + ("selected_target", selected_target.clone()), + ], + )?; + if target_name.is_none() { + return Ok((target_name, py.None().into_bound(py))); + } + let target_config: Bound<'py, PyAny> = call_with( + &imported(py, TARGET_CONFIG_MODULE, "resolve_target_config")?, + &[ + ("project_config", project_config), + ("local_config", local_config), + ("target_name", target_name.clone()), + ], + )?; + Ok((target_name, target_config)) +} + +/// Resolve the effective target and shared project-local compile-cache directory. +#[pyfunction] +#[pyo3(signature = (*, discovered_inputs, selected_target, no_cache))] +pub(crate) fn build_compile_target_context<'py>( + discovered_inputs: &Bound<'py, PyAny>, + selected_target: &Bound<'py, PyAny>, + no_cache: bool, +) -> PyResult> { + let py: Python<'py> = discovered_inputs.py(); + let (target_name, target_config) = resolved_target(discovered_inputs, selected_target)?; + let cache_dir: Bound<'py, PyAny> = compile_cache_root( + &discovered_inputs.getattr("project_dir")?, + &target_config, + no_cache, + )?; + PyTuple::new(py, [target_name, target_config, cache_dir]) +} + +/// The shared compile-cache directory, or `None` when caching is disabled. +fn compile_cache_root<'py>( + project_dir: &Bound<'py, PyAny>, + target_config: &Bound<'py, PyAny>, + no_cache: bool, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let constant = |name: &str| imported(py, COMPILE_CONSTANTS_MODULE, name); + let disabled_value: String = constant("COMPILE_CACHE_DISABLE_VALUE")?.extract()?; + let disable_env: String = constant("COMPILE_CACHE_DISABLE_ENV_VAR")?.extract()?; + let disabled: bool = py + .import("os")? + .getattr("environ")? + .call_method1("get", (disable_env,))? + .extract::>()? + .is_some_and(|value| value == disabled_value); + if no_cache + || project_dir.is_none() + || (!target_config.is_none() + && target_config + .getattr("compile_cache")? + .is(pyo3::types::PyBool::new(py, false))) + || disabled + { + return Ok(py.None().into_bound(py)); + } + let root: Bound<'py, PyAny> = + imported(py, CACHE_DIRECTORY_MODULE, "compiler_cache_directory")?.call1((project_dir,))?; + remove_retired_caches(&root)?; + Ok(root) +} + +/// Delete the fact and reference caches older releases kept under each retired cache root. +fn remove_retired_caches(root: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = root.py(); + let directory_name: String = imported( + py, + FRONTIER_CONSTANTS_MODULE, + "COMPILER_CACHE_DIRECTORY_NAME", + )? + .extract()?; + let names: [Bound<'_, PyAny>; 2] = [ + imported( + py, + COMPILE_CONSTANTS_MODULE, + "RETIRED_FACT_CACHE_DIRECTORY_NAME", + )?, + imported( + py, + COMPILE_CONSTANTS_MODULE, + "RETIRED_REFERENCE_CACHE_DIRECTORY_NAME", + )?, + ]; + let rmtree: Bound<'_, PyAny> = imported(py, "shutil", "rmtree")?; + let ignore: Bound<'_, PyDict> = PyDict::new(py); + ignore.set_item("ignore_errors", true)?; + for suffix in imported( + py, + FRONTIER_CONSTANTS_MODULE, + "RETIRED_CACHE_NAMESPACE_SUFFIXES", + )? + .try_iter()? + { + let suffix: String = suffix?.extract()?; + for name in &names { + let retired: Bound<'_, PyAny> = root + .getattr("parent")? + .div(format!("{directory_name}{suffix}"))? + .div(name)?; + if retired.call_method0("is_dir")?.is_truthy()? { + rmtree.call((retired,), Some(&ignore))?; + } + } + } + Ok(()) +} + +/// `value if isinstance(value, str) and value else None`. +fn optional_string<'py>( + value: Option>, + py: Python<'py>, +) -> PyResult> { + Ok(match value { + Some(value) if value.is_instance_of::() && value.is_truthy()? => value, + _ => py.None().into_bound(py), + }) +} + +/// `first or second or third`. +fn first_truthy<'py>(values: [Bound<'py, PyAny>; 3]) -> PyResult> { + let [first, second, third] = values; + if first.is_truthy()? { + return Ok(first); + } + if second.is_truthy()? { + return Ok(second); + } + Ok(third) +} + +/// Resolve the target and its effective database/schema without compiling resources. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn build_effective_target_namespace<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_effective_target_namespace", + options, + ( + &[ + "discovered_inputs", + "selected_target", + "connection_database", + "connection_schema", + "default_database", + "default_schema", + ], + &[], + ), + )?; + let discovered_inputs: &Bound<'py, PyAny> = &arguments.value("discovered_inputs")?; + let selected_target: &Bound<'py, PyAny> = &arguments.value("selected_target")?; + let connection_database: Bound<'py, PyAny> = arguments.value("connection_database")?; + let connection_schema: Bound<'py, PyAny> = arguments.value("connection_schema")?; + let default_database: Bound<'py, PyAny> = arguments.value("default_database")?; + let default_schema: Bound<'py, PyAny> = arguments.value("default_schema")?; + let (target_name, target_config) = resolved_target(discovered_inputs, selected_target)?; + let effective_vars: Bound<'py, PyAny> = project_vars(discovered_inputs, &target_config)?; + let target_value = |key: &str| -> PyResult> { + if target_config.is_none() { + return Ok(py.None().into_bound(py)); + } + target_config.getattr(key) + }; + let values: Bound<'py, PyDict> = PyDict::new(py); + values.set_item( + DATABASE_KEY, + first_truthy([ + target_value(DATABASE_KEY)?, + connection_database, + default_database, + ])?, + )?; + values.set_item( + SCHEMA_KEY, + first_truthy([target_value(SCHEMA_KEY)?, connection_schema, default_schema])?, + )?; + let run_id: Bound<'py, PyAny> = call_with( + &imported(py, CONTEXT_TEMPLATES_MODULE, "resolve_run_id")?, + &[("selected_run_id", py.None().into_bound(py))], + )?; + let resolved: Bound<'py, PyDict> = call_with( + &imported( + py, + CONTEXT_TEMPLATES_MODULE, + "resolve_early_model_templates", + )?, + &[ + ("values", values.into_any()), + ("effective_vars", effective_vars), + ("effective_target_name", target_name.clone()), + ("run_id", run_id), + ], + )? + .downcast_into()?; + PyTuple::new( + py, + [ + target_name, + target_config, + optional_string(resolved.get_item(DATABASE_KEY)?, py)?, + optional_string(resolved.get_item(SCHEMA_KEY)?, py)?, + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs index 29a3595ccb..807c9a2003 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs @@ -11,9 +11,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; -const PROJECT_WITH_MODULE: &str = "sqlbuild.compiler.compiled_project.main.project_with"; -const RECORD_ANALYSES_MODULE: &str = - "sqlbuild.compiler.compiled_project.main._record_model_analyses"; +const PROJECT_WITH_MODULE: &str = "sqlbuild._native"; +const RECORD_ANALYSES_MODULE: &str = "sqlbuild._native"; const EXPLICIT_REFERENCES_MODULE: &str = "sqlbuild.compiler.compile._helpers.explicit_references.checks"; const ACTIVE_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.active_macro_bridge"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs index 96aa7a4797..582675569c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs @@ -14,8 +14,6 @@ const SYNTAX_CHECKS_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis. const CURSOR_INTRINSICS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.cursor_intrinsics"; const MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.macros"; -const INFER_ROWS_MODULE: &str = - "sqlbuild.compiler.analysis_session.main._infer_native_query_column_rows"; const HAND_BUILT_MODULE: &str = "sqlbuild.compiler.compile._helpers.assembly.hand_built_models"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const NATIVE_FAILURE_PREFIX: &str = "NativeCompilerError: "; @@ -218,7 +216,7 @@ fn fallback_columns<'py>( ], )?; call_with( - &imported(py, INFER_ROWS_MODULE, "infer_native_query_column_rows")?, + &imported(py, NATIVE_MODULE, "infer_native_query_column_rows")?, &[ ("queries", PyTuple::new(py, [query])?.into_any()), ("profile", context.profile.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs index 376a3f1339..5f71574f50 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -16,10 +16,10 @@ use crate::bindings::_helpers::compile_driver::assembly_models::{ binding_rows, compiled_models, project_binding_diagnostics, }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::scopes::scope_errors::report_scope_index_errors; const NATIVE_MODULE: &str = "sqlbuild._native"; const SCOPE_USAGES_MODULE: &str = "sqlbuild.compiler.compile.main._scope_index_with_compile_usages"; -const SCOPE_DIAGNOSTICS_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.scope"; const CONTEXT_TEMPLATES_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.context_templates"; pub(crate) const ADAPTER_NAME_MODULE: &str = @@ -110,10 +110,7 @@ pub(crate) fn assembled_project<'py>( &imported(py, SCOPE_USAGES_MODULE, "scope_index_with_compile_usages")?, &[("inputs", inputs.clone())], )?; - call_with( - &imported(py, SCOPE_DIAGNOSTICS_MODULE, "report_scope_index_errors")?, - &[("index", scope_index.clone())], - )?; + report_scope_index_errors(&scope_index)?; let target_values: Bound<'py, PyAny> = effective_target_values(inputs)?; let models: Bound<'py, PyTuple> = compiled_models(inputs, &context, (&native.getattr("models")?, names))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs index fc76e5b89e..ceb29e8a99 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs @@ -9,18 +9,17 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; const CALL_STORE_MODULE: &str = "sqlbuild.compiler.compile._helpers.macro_bridge.call_store"; -const ATTACHMENT_MODULE: &str = "sqlbuild.compiler.compile._helpers.attachment.core"; -const DECLARATION_SCOPE_MODULE: &str = - "sqlbuild.compiler.compile._helpers.attachment.declaration_scope"; +const ATTACHMENT_MODULE: &str = "sqlbuild._native"; +const DECLARATION_SCOPE_MODULE: &str = "sqlbuild._native"; const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; const CONTEXT_TEMPLATES_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.context_templates"; const MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.macros"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const ATTACH_CHECKS_MODULE: &str = "sqlbuild.compiler.compiled_project.main._attach_checks"; -const ATTACH_RESOURCES_MODULE: &str = "sqlbuild.compiler.compiled_project.main._attach_resources"; -const INPUTS_VIEW_MODULE: &str = "sqlbuild.compiler.compiled_project.main._compile_inputs_view"; -const INPUTS_WITH_MODULE: &str = "sqlbuild.compiler.compiled_project.main.inputs_with"; +const ATTACH_CHECKS_MODULE: &str = "sqlbuild._native"; +const ATTACH_RESOURCES_MODULE: &str = "sqlbuild._native"; +const INPUTS_VIEW_MODULE: &str = "sqlbuild._native"; +const INPUTS_WITH_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; const NATIVE_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index 797e9b18bc..4ed96a7291 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -14,7 +14,7 @@ const FRONTIER_TYPES_MODULE: &str = "sqlbuild.compiler.frontier.types"; const BUILD_INPUTS_MODULE: &str = "sqlbuild._native"; const ASSEMBLE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const PROJECT_WITH_MODULE: &str = "sqlbuild.compiler.compiled_project.main.project_with"; +const PROJECT_WITH_MODULE: &str = "sqlbuild._native"; const PROMOTION_MODULE: &str = "sqlbuild.compiler.contracts.main.promotion_conflicts"; const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs new file mode 100644 index 0000000000..4ac75aaedf --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs @@ -0,0 +1,637 @@ +//! Seeds, functions, sources, SQL tests, scenarios and audits attached from discovered inputs. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use std::collections::HashMap; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, within, +}; +use crate::bindings::_helpers::compiled_project::audit_loop::{AuditBlockRow, AuditFileRow}; +use crate::bindings::_helpers::compiled_project::function_loop::FunctionRow; +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::resource_diagnostics::{ + DiagnosticSettings, LoaderRow, resource_diagnostics, +}; +use crate::bindings::_helpers::compiled_project::scenario_loop::ScenarioRow; +use crate::bindings::_helpers::compiled_project::source_loop::SourceRow; +use crate::bindings::_helpers::compiled_project::sql_test_loop::{ + TestBlockRow, TestCaseRow, TestFileRow, +}; + +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; +const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; +const BUILTIN_AUDITS_MODULE: &str = "sqlbuild.compiler.auditing.main._builtins"; +const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main.record"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const SQL_VALUE_ERRORS_MODULE: &str = "sqlbuild.sql_values.exceptions"; +const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; +const PYTHON_RESOURCES_MODULE: &str = + "sqlbuild.compiler.compile._helpers.diagnostics.python_resources"; +const PATHLIB_MODULE: &str = "pathlib"; +const TEST_TIMING_KEY: &str = "test_input_compile_ms"; +const DESCRIPTION_KEY: &str = "description"; +const MEASUREMENT_MODE: &str = "MEASUREMENT"; + +/// `getattr(value, name)` extracted. +fn attr<'py, T: pyo3::FromPyObject<'py>>(value: &Bound<'py, PyAny>, name: &str) -> PyResult { + value.getattr(name)?.extract() +} + +/// `str(value.)`. +fn text(value: &Bound<'_, PyAny>, name: &str) -> PyResult { + Ok(value.getattr(name)?.str()?.to_string()) +} + +/// `value..stem`. +fn stem(value: &Bound<'_, PyAny>, name: &str) -> PyResult { + attr(&value.getattr(name)?, "stem") +} + +/// Each item of the iterable `value.`. +fn items<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult>> { + value.getattr(name)?.try_iter()?.collect() +} + +fn enum_member<'py>( + py: Python<'py>, + (module, class, member): (&str, &str, &str), +) -> PyResult> { + imported(py, module, class)?.getattr(member) +} + +/// `{member.value: member for member in enum_class}`. +fn by_value<'py>(py: Python<'py>, module: &str, class: &str) -> PyResult> { + let members: Bound<'py, PyDict> = PyDict::new(py); + for member in imported(py, module, class)?.try_iter()? { + let member: Bound<'py, PyAny> = member?; + members.set_item(member.getattr("value")?, member)?; + } + Ok(members) +} + +fn seed_declarations<'py>(discovered: &Bound<'py, PyAny>) -> PyResult, Py)>> { + let mut declarations: Vec<(Py, Py)> = Vec::new(); + for schema_file in items(discovered, "schema_files")? { + for entry in items(&schema_file, "seed_entries")? { + declarations.push((entry.unbind(), schema_file.clone().unbind())); + } + } + Ok(declarations) +} + +/// The function rows of `field`; Python function files have no authored SQL contents. +fn function_rows( + discovered: &Bound<'_, PyAny>, + (field, body): (&str, &str), + python: bool, +) -> PyResult> { + items(discovered, field)? + .into_iter() + .map(|file| { + Ok(( + text(&file, "file_path")?, + text(&file, "relative_path")?, + stem(&file, "file_path")?, + file.clone().unbind(), + attr(&file, "header_values")?, + attr(&file, body)?, + if python { + None + } else { + Some(attr(&file, "contents")?) + }, + )) + }) + .collect() +} + +/// `override if override is not None else fallback`. +fn either<'py>( + defaults: &Bound<'py, PyAny>, + (preferred, fallback): (&str, &str), +) -> PyResult> { + let value: Bound<'py, PyAny> = defaults.getattr(preferred)?; + if value.is_none() { + return attr(defaults, fallback); + } + value.extract() +} + +/// `(database, schema)` of a target config, or `None`. +fn target_namespace( + target: &Bound<'_, PyAny>, +) -> PyResult, Option)>> { + if target.is_none() { + return Ok(None); + } + Ok(Some((attr(target, "database")?, attr(target, "schema")?))) +} + +/// Pair every seed, then attach every function and source, in that order. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn attach_resources<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "attach_resources", + options, + ( + &[ + "discovered_inputs", + "native_project", + "settings", + "no_sql_validation", + "python_functions_inherit_default_namespace", + ], + &[], + ), + )?; + let discovered_inputs: &Bound<'py, PyAny> = &arguments.value("discovered_inputs")?; + let native_project: &Bound<'py, NativeCompiledProject> = + &arguments.value("native_project")?.downcast_into()?; + let (effective_settings, target_config, adapter_name): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + String, + ) = arguments.value("settings")?.extract()?; + let no_sql_validation: bool = arguments.flag("no_sql_validation")?; + let python_functions_inherit_default_namespace: bool = + arguments.flag("python_functions_inherit_default_namespace")?; + let validate: bool = attr::(&effective_settings, "sql_analysis")? && !no_sql_validation; + let seeds = NativeCompiledProject::attach_seeds( + native_project, + seed_declarations(discovered_inputs)?, + items(discovered_inputs, "seed_files")? + .into_iter() + .map(Bound::unbind) + .collect(), + &imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")?, + )?; + let defaults: Bound<'py, PyAny> = discovered_inputs + .getattr("project_config")? + .getattr("defaults")?; + let function_kind: Bound<'py, PyAny> = + enum_member(py, (SCOPE_TYPES_MODULE, "ResourceKind", "FUNCTION"))?; + let functions = NativeCompiledProject::attach_functions( + native_project, + ( + function_rows(discovered_inputs, ("sql_function_files", "body_sql"), false)?, + function_rows( + discovered_inputs, + ("python_function_files", "body_python"), + true, + )?, + ), + ( + adapter_name, + validate, + ( + either(&defaults, ("function_database", "database"))?, + either(&defaults, ("function_schema", "schema"))?, + ), + target_namespace(&target_config)?, + python_functions_inherit_default_namespace, + function_kind.unbind(), + ), + )?; + let mut source_rows: Vec = Vec::new(); + for source_file in items(discovered_inputs, "source_files")? { + let file_path: String = text(&source_file, "file_path")?; + for entry in items(&source_file, "source_entries")? { + source_rows.push(( + file_path.clone(), + source_file.clone().unbind(), + entry.unbind(), + )); + } + } + let sources = NativeCompiledProject::attach_sources( + native_project, + source_rows, + ( + attr(&defaults, "contract")?, + validate, + enum_member(py, (SCOPE_TYPES_MODULE, "ResourceKind", "SOURCE"))?.unbind(), + ), + )?; + PyTuple::new( + py, + [ + PyTuple::new(py, seeds)?.into_any(), + PyTuple::new(py, functions)?.into_any(), + PyTuple::new(py, sources)?.into_any(), + ], + ) +} + +fn test_case_row(case: Bound<'_, PyAny>) -> PyResult { + Ok(( + case.clone().unbind(), + attr(&case, "name")?, + items(&case, "values")? + .into_iter() + .map(|pair| pair.extract()) + .collect::>()?, + )) +} + +fn test_block_row(block: Bound<'_, PyAny>) -> PyResult { + Ok(( + block.clone().unbind(), + attr(&block, "name")?, + attr(&block, "sql_body")?, + attr(&block.getattr("mode")?, "value")?, + items(&block, "parameters")? + .iter() + .map(|parameter| attr(parameter, "name")) + .collect::>()?, + items(&block, "cases")? + .into_iter() + .map(test_case_row) + .collect::>()?, + attr(&block, "sql_body_span")?, + )) +} + +fn test_file_row(file: Bound<'_, PyAny>) -> PyResult { + Ok(( + text(&file, "file_path")?, + text(&file, "relative_path")?, + stem(&file, "relative_path")?, + file.clone().unbind(), + attr(&file, "contents")?, + items(&file, "blocks")? + .into_iter() + .map(test_block_row) + .collect::>()?, + )) +} + +fn scenario_row(scenario: Bound<'_, PyAny>) -> PyResult { + Ok(( + text(&scenario, "file_path")?, + text(&scenario, "relative_path")?, + scenario.clone().unbind(), + attr(&scenario, "contents")?, + attr(&scenario, "sql_body")?, + attr(&scenario, "name")?, + )) +} + +/// The adapter's typed scalar renderer as `(text, None)`, or `(None, error)` for a bad value. +fn parameter_renderer<'py>(value_renderer: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value_renderer.py(); + let renderer: Py = value_renderer.clone().unbind(); + let errors: Py = PyTuple::new( + py, + [ + imported(py, SQL_VALUE_ERRORS_MODULE, "SqlValueRenderingError")?, + imported(py, SQL_VALUE_ERRORS_MODULE, "SqlValueValidationError")?, + ], + )? + .into_any() + .unbind(); + let render = move |arguments: &Bound<'_, PyTuple>, + _: Option<&Bound<'_, PyDict>>| + -> PyResult<(Option, Option)> { + let py: Python<'_> = arguments.py(); + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("value", arguments.get_item(0)?)?; + match renderer + .bind(py) + .call_method("render_typed_scalar", (), Some(&keywords)) + { + Ok(rendered) => Ok((Some(rendered.extract()?), None)), + Err(error) if error.matches(py, errors.bind(py))? => { + Ok((None, Some(error.value(py).str()?.to_string()))) + } + Err(error) => Err(error), + } + }; + Ok(PyCFunction::new_closure(py, None, None, render)?.into_any()) +} + +fn sql_test_settings<'py>( + py: Python<'py>, + (value_renderer, syntax, scan_cache): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), +) -> PyResult> { + let settings: Bound<'py, PyDict> = PyDict::new(py); + for (key, class) in [ + ("test_input", "CompileSqlTestInput"), + ("model_payload", "CompileModelSqlTestInputPayload"), + ("direct_payload", "CompileDirectLogicSqlTestInputPayload"), + ("test_cte", "CompileSqlTestCte"), + ("scenario_input", "CompileSqlScenarioInput"), + ("scenario_cte", "CompileSqlScenarioCte"), + ] { + settings.set_item(key, imported(py, COMPILE_MODELS_MODULE, class)?)?; + } + settings.set_item("modes", by_value(py, COMPILE_TYPES_MODULE, "SqlTestMode")?)?; + settings.set_item( + "test_kind", + enum_member(py, (SCOPE_TYPES_MODULE, "ResourceKind", "TEST"))?, + )?; + settings.set_item( + "scenario_kind", + enum_member(py, (SCOPE_TYPES_MODULE, "ResourceKind", "SCENARIO"))?, + )?; + settings.set_item( + "runtime_usage", + enum_member(py, (SCOPE_TYPES_MODULE, "UsageKind", "RUNTIME"))?, + )?; + settings.set_item("render_parameter", parameter_renderer(value_renderer)?)?; + settings.set_item("adapter_name", value_renderer.getattr("adapter_name")?)?; + settings.set_item( + "source_location", + imported(py, CONTRACT_MODELS_MODULE, "SourceLocation")?, + )?; + settings.set_item("scan_store", scan_cache.call_method0("opened")?)?; + settings.set_item( + "scan_key", + ( + imported(py, COMPILE_CONSTANTS_MODULE, "SQL_TEST_CTE_SCAN_ALGORITHM")?, + syntax.getattr("cache_key")?, + ), + )?; + Ok(settings) +} + +fn audit_block_row(block: Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = block.py(); + let measurement: Bound<'_, PyAny> = enum_member( + py, + ( + AUDITING_TYPES_MODULE, + "AuditEvaluationMode", + MEASUREMENT_MODE, + ), + )?; + Ok(( + block.clone().unbind(), + attr(&block, "name")?, + attr(&block, "sql_body")?, + attr(&block, "evidence_sql")?, + block.getattr("evaluation_mode")?.eq(measurement)?, + attr(&block, "header_values")?, + )) +} + +fn audit_file_row(file: Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = file.py(); + let generic: Bound<'_, PyAny> = + enum_member(py, (SCOPE_TYPES_MODULE, "DeclarationKind", "AUDIT"))?; + Ok(( + text(&file, "file_path")?, + text(&file, "relative_path")?, + stem(&file, "file_path")?, + file.clone().unbind(), + file.getattr("declaration_kind")?.is(&generic), + attr(&file, "contents")?, + items(&file, "blocks")? + .into_iter() + .map(audit_block_row) + .collect::>()?, + )) +} + +fn audit_settings<'py>( + py: Python<'py>, + effective_settings: &Bound<'py, PyAny>, +) -> PyResult> { + let settings: Bound<'py, PyDict> = PyDict::new(py); + let severity: Bound<'py, PyAny> = effective_settings.getattr("default_audit_severity")?; + settings.set_item( + "default_severity", + if severity.is_none() { + None + } else { + Some(severity.str()?) + }, + )?; + settings.set_item( + "default_run_scope", + effective_settings.getattr("default_audit_run_scope")?, + )?; + let builtins: Vec = + imported(py, BUILTIN_AUDITS_MODULE, "builtin_audit_definitions")? + .call0()? + .try_iter()? + .map(|file| audit_file_row(file?)) + .collect::>()?; + settings.set_item("builtins", builtins)?; + settings.set_item( + "declaration_identity", + imported(py, SCOPE_MODELS_MODULE, "DeclarationIdentity")?, + )?; + settings.set_item( + "singular_kind", + enum_member( + py, + (SCOPE_TYPES_MODULE, "DeclarationKind", "SINGULAR_AUDIT"), + )?, + )?; + settings.set_item( + "audit_kind", + enum_member(py, (SCOPE_TYPES_MODULE, "DeclarationKind", "AUDIT"))?, + )?; + settings.set_item( + "resource_kinds", + by_value(py, SCOPE_TYPES_MODULE, "ResourceKind")?, + )?; + Ok(settings) +} + +/// Each loader's description, relative path and whether that path is a YAML file. +fn loader_rows<'py>(discovered: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = discovered.py(); + let suffixes: Bound<'py, PyAny> = + imported(py, DISCOVERY_CONSTANTS_MODULE, "YAML_FILE_SUFFIXES")?; + let mut rows: HashMap = HashMap::new(); + for loader in items(discovered, "loader_functions")? { + let relative_path: Bound<'py, PyAny> = loader.getattr("relative_path")?; + rows.insert( + attr(&loader, "name")?, + ( + attr(&loader, DESCRIPTION_KEY)?, + relative_path.str()?.to_string(), + suffixes.contains(relative_path.getattr("suffix")?)?, + ), + ); + } + Ok(rows) +} + +fn diagnostic_settings<'py>( + discovered: &Bound<'py, PyAny>, + source_inputs: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let settings: Bound<'py, PyDict> = PyDict::new(py); + settings.set_item( + "compiler_diagnostic", + imported(py, COMPILE_MODELS_MODULE, "CompilerDiagnostic")?, + )?; + settings.set_item( + "compile_error", + imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")?, + )?; + settings.set_item( + "source_location", + imported(py, CONTRACT_MODELS_MODULE, "SourceLocation")?, + )?; + settings.set_item("path_class", imported(py, PATHLIB_MODULE, "Path")?)?; + settings.set_item("model_files", items(discovered, "model_files")?)?; + let factories: Vec<(String, String, usize)> = items(discovered, "audit_factories")? + .iter() + .map(|factory| { + Ok(( + attr(factory, "name")?, + text(factory, "relative_path")?, + attr(factory, "line")?, + )) + }) + .collect::>()?; + settings.set_item("audit_factories", factories)?; + let scenarios: Vec> = items(discovered, "scenario_files")? + .iter() + .map(|scenario| { + PyTuple::new( + py, + [ + scenario.getattr("name")?, + text(scenario, "relative_path")?.into_bound_py_any(py)?, + scenario.getattr("contents")?, + scenario + .getattr("header_values")? + .call_method1("get", (DESCRIPTION_KEY,))?, + ], + ) + }) + .collect::>()?; + settings.set_item("scenarios", scenarios)?; + let hooks: Vec> = items(discovered, "sql_hook_files")? + .iter() + .map(|hook| { + PyTuple::new( + py, + [ + hook.getattr("name")?, + text(hook, "relative_path")?.into_bound_py_any(py)?, + hook.getattr("contents")?, + hook.getattr(DESCRIPTION_KEY)?, + ], + ) + }) + .collect::>()?; + settings.set_item("sql_hooks", hooks)?; + settings.set_item("loaders", loader_rows(discovered)?)?; + settings.set_item( + "python_resources", + call_with( + &imported(py, PYTHON_RESOURCES_MODULE, "python_resource_rows")?, + &[ + ("discovered_inputs", discovered.clone()), + ("source_inputs", source_inputs.clone()), + ], + )?, + )?; + Ok(settings) +} + +/// Attach every SQL test case, scenario and audit, then report every resource diagnostic. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn attach_checks<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "attach_checks", + options, + ( + &[ + "discovered_inputs", + "native_project", + "settings", + "scan_cache", + "source_inputs", + ], + &[], + ), + )?; + let discovered_inputs: &Bound<'py, PyAny> = &arguments.value("discovered_inputs")?; + let native_project: &Bound<'py, NativeCompiledProject> = + &arguments.value("native_project")?.downcast_into()?; + let (effective_settings, value_renderer, syntax): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ) = arguments.value("settings")?.extract()?; + let scan_cache: &Bound<'py, PyAny> = &arguments.value("scan_cache")?; + let source_inputs: &Bound<'py, PyAny> = &arguments.value("source_inputs")?; + let timing: Bound<'py, PyAny> = + imported(py, PROFILING_MODULE, "record_compile_timing")?.call1((TEST_TIMING_KEY,))?; + let (tests, scenarios) = within(&timing, |_| { + let attached = NativeCompiledProject::attach_sql_tests( + native_project, + items(discovered_inputs, "test_files")? + .into_iter() + .map(test_file_row) + .collect::>()?, + items(discovered_inputs, "scenario_files")? + .into_iter() + .map(scenario_row) + .collect::>()?, + sql_test_settings(py, (&value_renderer, &syntax, scan_cache))?.extract()?, + )?; + scan_cache.call_method0("save")?; + Ok(attached) + })?; + let (audits, audit_diagnostics) = NativeCompiledProject::attach_audits( + native_project, + items(discovered_inputs, "audit_files")? + .into_iter() + .map(audit_file_row) + .collect::>()?, + audit_settings(py, &effective_settings)?.extract()?, + )?; + let settings: DiagnosticSettings = + diagnostic_settings(discovered_inputs, source_inputs)?.extract()?; + let (orphans, descriptions, cursors) = + resource_diagnostics(&native_project.borrow(), py, &settings)?; + let diagnostics: Vec> = audit_diagnostics + .into_iter() + .chain(orphans) + .chain(descriptions) + .chain(cursors) + .collect(); + PyTuple::new( + py, + [ + PyTuple::new(py, tests)?.into_any(), + PyTuple::new(py, scenarios)?.into_any(), + PyTuple::new(py, audits)?.into_any(), + PyTuple::new(py, diagnostics)?.into_any(), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(attach_resources, module)?)?; + module.add_function(wrap_pyfunction!(attach_checks, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs new file mode 100644 index 0000000000..219cf1d045 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs @@ -0,0 +1,488 @@ +//! The Python classes, project declarations and callbacks the native model loop builds with. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; +const CONFIG_ERROR_MODULE: &str = "sqlbuild.compiler.model_config.main._native_config_error"; +const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; +const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; +const SCOPE_ERRORS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; +const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; +const NATIVE_MODULE: &str = "sqlbuild._native"; +/// The materializations a model's storage policies are inherited for, besides none. +const TABLE_BACKED_MATERIALIZATIONS: [&str; 3] = ["TABLE", "INCREMENTAL", "SNAPSHOT"]; +/// The declaration kinds whose project-owned records the loop names. +const NAMED_KINDS: [&str; 4] = ["AUDIT", "SCHEMA", "SQL_HOOK", "PYTHON_HOOK"]; + +fn class<'py>(py: Python<'py>, module: &str, name: &str) -> PyResult> { + imported(py, module, name) +} + +fn member<'py>( + py: Python<'py>, + module: &str, + (enum_class, name): (&str, &str), +) -> PyResult> { + imported(py, module, enum_class)?.getattr(name) +} + +fn dict<'py>( + py: Python<'py>, + entries: Vec<(&str, Bound<'py, PyAny>)>, +) -> PyResult> { + let built: Bound<'py, PyDict> = PyDict::new(py); + for (key, value) in entries { + built.set_item(key, value)?; + } + Ok(built) +} + +/// The values of a table-backed `MaterializationType`, as text. +pub(crate) fn table_backed<'py>(py: Python<'py>) -> PyResult>> { + TABLE_BACKED_MATERIALIZATIONS + .iter() + .map(|name| member(py, PLANNER_TYPES_MODULE, ("MaterializationType", name))) + .collect() +} + +/// The builder, storage defaults and classes a model's `CompileModelConfig` is made of. +pub(crate) fn native_config_classes<'py>( + session: &Bound<'py, PyAny>, + record_template_reads: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = session.py(); + let table_backed: Vec> = table_backed(py)? + .iter() + .map(|materialization| Ok(materialization.str()?.into_any())) + .collect::>()?; + dict( + py, + vec![ + ("builder", session.getattr("builder")?), + ("inherited_storage", session.getattr("inherited_storage")?), + ("table_backed", PyTuple::new(py, table_backed)?.into_any()), + ( + "compile_model_config", + class(py, COMPILE_MODELS_MODULE, "CompileModelConfig")?, + ), + ( + "resolved_retention", + class(py, CONTRACT_MODELS_MODULE, "ResolvedTimeTravelRetention")?, + ), + ( + "retention_source", + member( + py, + CONTRACT_TYPES_MODULE, + ("TimeTravelRetentionSource", "MODEL"), + )?, + ), + ( + "resolved_table_type", + class(py, CONTRACT_MODELS_MODULE, "ResolvedTableType")?, + ), + ("table_type", class(py, CONTRACT_TYPES_MODULE, "TableType")?), + ( + "table_type_source", + member(py, CONTRACT_TYPES_MODULE, ("TableTypeSource", "MODEL"))?, + ), + ("record_template_reads", record_template_reads.clone()), + ], + ) +} + +/// Each project-owned generic audit's, schema's and hook's first record, by `(kind, name)`. +fn named_records<'py>(resolver: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = resolver.py(); + let records: Bound<'py, PyDict> = PyDict::new(py); + if resolver.is_none() { + return Ok(records); + } + let kinds: Vec> = NAMED_KINDS + .iter() + .map(|name| member(py, SCOPE_TYPES_MODULE, ("DeclarationKind", name))) + .collect::>()?; + let declarations: Bound<'py, PyAny> = resolver.getattr("lookup")?.getattr("declarations")?; + for pair in declarations.call_method0("items")?.try_iter()? { + let (identity, records_of): (Bound<'py, PyAny>, Bound<'py, PyAny>) = pair?.extract()?; + let kind: Bound<'py, PyAny> = identity.getattr("kind")?; + if kinds.iter().any(|named| kind.is(named)) + && identity.getattr("owner")?.is_none() + && records_of.is_truthy()? + { + records.set_item( + (kind.getattr("value")?, identity.getattr("name")?), + records_of.get_item(0)?, + )?; + } + } + Ok(records) +} + +/// Project-owned enums by name, as `model_schema_enum_declarations` resolves column types. +fn schema_enums<'py>(resolver: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = resolver.py(); + let enums: Bound<'py, PyDict> = PyDict::new(py); + if resolver.is_none() { + return Ok(enums); + } + let enum_kind: Bound<'py, PyAny> = member(py, SCOPE_TYPES_MODULE, ("DeclarationKind", "ENUM"))?; + let enum_class: Bound<'py, PyAny> = class(py, DISCOVERY_MODELS_MODULE, "EnumDeclaration")?; + let declarations: Bound<'py, PyAny> = + resolver.getattr("projection")?.getattr("declarations")?; + for pair in declarations.call_method0("items")?.try_iter()? { + let (identity, value): (Bound<'py, PyAny>, Bound<'py, PyAny>) = pair?.extract()?; + if identity.getattr("kind")?.is(&enum_kind) + && identity.getattr("owner")?.is_none() + && value.is_instance(&enum_class)? + { + enums.set_item(identity.getattr("name")?, value)?; + } + } + Ok(enums) +} + +/// A hook function's parameters as `inspect.signature` reads them: name, kind, default. +#[pyfunction] +fn hook_signature<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = function.py(); + let inspect: Bound<'py, PyAny> = py.import("inspect")?.into_any(); + let empty: Bound<'py, PyAny> = inspect.getattr("Parameter")?.getattr("empty")?; + let mut parameters: Vec<(String, i64, bool)> = Vec::new(); + for parameter in inspect + .call_method1("signature", (function,))? + .getattr("parameters")? + .call_method0("values")? + .try_iter()? + { + let parameter: Bound<'py, PyAny> = parameter?; + parameters.push(( + parameter.getattr("name")?.extract()?, + parameter + .getattr("kind")? + .call_method0("__int__")? + .extract()?, + !parameter.getattr("default")?.is(&empty), + )); + } + PyList::new(py, parameters) +} + +/// What the loop resolves each model's visible declarations from. +fn visibility_classes<'py>( + context: &Bound<'py, PyAny>, + visible_declarations: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = context.py(); + let resolver: Bound<'py, PyAny> = context.getattr("declaration_resolver")?; + let (projects, resource_specific) = if resolver.is_none() { + (false, PyFrozenSet::empty(py)?) + } else { + let lookup: Bound<'py, PyAny> = resolver.getattr("lookup")?; + let declarations: Bound<'py, PyAny> = lookup.getattr("index")?.getattr("declarations")?; + let private: Bound<'py, PyAny> = member(py, SCOPE_TYPES_MODULE, ("ScopeKind", "PRIVATE"))?; + let mut owners: Vec> = Vec::new(); + for declaration in declarations.try_iter()? { + let declaration: Bound<'py, PyAny> = declaration?; + let owner: Bound<'py, PyAny> = declaration.getattr("identity")?.getattr("owner")?; + if declaration.getattr("scope")?.is(&private) && !owner.is_none() { + owners.push(owner); + } + } + for resource in lookup.getattr("grants_by_resource")?.try_iter()? { + owners.push(resource?); + } + (declarations.is_truthy()?, PyFrozenSet::new(py, &owners)?) + }; + dict( + py, + vec![ + ("resolver", resolver), + ("projects", projects.into_bound_py_any(py)?), + ("resource_specific", resource_specific.into_any()), + ("public_enums", context.getattr("public_enums")?), + ("public_constants", context.getattr("public_constants")?), + ("visible_declarations", visible_declarations.clone()), + ( + "resource_identity", + class(py, SCOPE_MODELS_MODULE, "ResourceIdentity")?, + ), + ( + "declaration_identity", + class(py, SCOPE_MODELS_MODULE, "DeclarationIdentity")?, + ), + ( + "model_kind", + member(py, SCOPE_TYPES_MODULE, ("ResourceKind", "MODEL"))?, + ), + ( + "enum_kind", + member(py, SCOPE_TYPES_MODULE, ("DeclarationKind", "ENUM"))?, + ), + ( + "constant_kind", + member(py, SCOPE_TYPES_MODULE, ("DeclarationKind", "CONSTANT"))?, + ), + ( + "private_owner", + member( + py, + SCOPE_TYPES_MODULE, + ("VisibilityReason", "PRIVATE_OWNER"), + )?, + ), + ( + "visibility_record", + class(py, SCOPE_MODELS_MODULE, "VisibilityRecord")?, + ), + ], + ) +} + +fn input_view_classes<'py>(py: Python<'py>) -> PyResult> { + let compiled: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ( + "object_key", + class(py, COMPILE_MODELS_MODULE, "CompiledObjectKey")?, + ), + ( + "relation_location", + class(py, COMPILE_MODELS_MODULE, "CompiledRelationLocation")?, + ), + ( + "inferred_column", + class(py, COMPILE_MODELS_MODULE, "InferredColumn")?, + ), + ( + "nullability", + class(py, LINEAGE_TYPES_MODULE, "InferredNullability")?, + ), + ( + "compiler_diagnostic", + class(py, COMPILE_MODELS_MODULE, "CompilerDiagnostic")?, + ), + ( + "source_location", + class(py, CONTRACT_MODELS_MODULE, "SourceLocation")?, + ), + ( + "compact_lineage_facts", + class(py, COMPILE_MODELS_MODULE, "CompactLineageFacts")?, + ), + ( + "lineage_column_fact", + class(py, COMPILE_MODELS_MODULE, "CompiledLineageColumnFact")?, + ), + ( + "lineage_source_fact", + class(py, COMPILE_MODELS_MODULE, "CompiledLineageSourceFact")?, + ), + ( + "contract_proof", + class(py, COMPILE_MODELS_MODULE, "DynamicColumnContractProof")?, + ), + ( + "family_proof", + class(py, COMPILE_MODELS_MODULE, "DynamicColumnFamilyProof")?, + ), + ]; + dict( + py, + vec![ + ( + "sql_reference", + class(py, COMPILE_MODELS_MODULE, "CompileSqlReference")?, + ), + ( + "reference_kind", + class(py, REFERENCE_TYPES_MODULE, "SqlReferenceKind")?, + ), + ( + "schema_model_entry", + class(py, CONTRACT_MODELS_MODULE, "SchemaModelEntry")?, + ), + ( + "sql_expansion", + class(py, COMPILE_MODELS_MODULE, "CompiledSqlExpansion")?, + ), + ( + "expansion_span", + class(py, COMPILE_MODELS_MODULE, "ExpansionSpan")?, + ), + ( + "audits", + dict( + py, + vec![ + ( + "audit_severity", + class(py, AUDITING_TYPES_MODULE, "AuditSeverity")?, + ), + ( + "attached_target_kind", + class(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")?, + ), + ], + )? + .into_any(), + ), + ( + "functions", + dict( + py, + vec![ + ( + "function_argument", + class(py, COMPILE_MODELS_MODULE, "FunctionArgument")?, + ), + ( + "function_return_column", + class(py, COMPILE_MODELS_MODULE, "FunctionReturnColumn")?, + ), + ( + "function_language", + class(py, COMPILE_TYPES_MODULE, "FunctionLanguage")?, + ), + ], + )? + .into_any(), + ), + ("compiled", dict(py, compiled)?.into_any()), + ], + ) +} + +/// `{item.: item for item in items}`. +fn by_attribute<'py>(items: &Bound<'py, PyAny>, attribute: &str) -> PyResult> { + let keyed: Bound<'py, PyDict> = PyDict::new(items.py()); + for item in items.try_iter()? { + let item: Bound<'py, PyAny> = item?; + keyed.set_item(item.getattr(attribute)?, item)?; + } + Ok(keyed) +} + +/// The classes, error factories and project declarations the model loop reads. +pub(crate) fn model_loop_classes<'py>( + audit_factories: &Bound<'py, PyAny>, + context: &Bound<'py, PyAny>, + external_resolver: &Bound<'py, PyAny>, + host_inputs: &Bound<'py, PyDict>, +) -> PyResult> { + let py: Python<'py> = context.py(); + let host = |key: &str| -> PyResult> { host_inputs.as_any().get_item(key) }; + let resolver: Bound<'py, PyAny> = context.getattr("declaration_resolver")?; + let mut provider_names: Vec = host("provider_names")? + .try_iter()? + .map(|name| name?.extract()) + .collect::>()?; + provider_names.sort(); + let hooks: Bound<'py, PyDict> = dict( + py, + vec![ + ( + "functions", + by_attribute(&host("hook_functions")?, "name")?.into_any(), + ), + ("provider_names", provider_names.into_bound_py_any(py)?), + ( + "python_hook_entry", + class(py, DISCOVERY_MODELS_MODULE, "PythonHookEntry")?, + ), + ( + "signature", + wrap_pyfunction!(hook_signature, py)?.into_any(), + ), + ], + )?; + let record: Bound<'py, PyDict> = dict( + py, + vec![ + ("replace", imported(py, "dataclasses", "replace")?), + ( + "declaration_rows", + imported(py, NATIVE_MODULE, "declaration_rows")?, + ), + ], + )?; + dict( + py, + vec![ + ( + "usage_record", + class(py, SCOPE_MODELS_MODULE, "UsageRecord")?, + ), + ( + "compile_error", + class(py, COMPILE_ERRORS_MODULE, "CompileInputError")?, + ), + ("syntax_help", host("syntax_help")?), + ("source_syntax_help", host("source_syntax_help")?), + ("macros", host("macros")?), + ("sql_hooks", host("sql_hooks")?), + ( + "scope_path_error", + class(py, SCOPE_ERRORS_MODULE, "InvalidScopePathError")?, + ), + ( + "sql_hook_entry", + class(py, DISCOVERY_MODELS_MODULE, "SqlHookEntry")?, + ), + ( + "config_error", + imported(py, CONFIG_ERROR_MODULE, "native_config_error")?, + ), + ( + "audit_factories", + by_attribute(audit_factories, "name")?.into_any(), + ), + ( + "generated_usage", + member(py, SCOPE_TYPES_MODULE, ("UsageKind", "GENERATED"))?, + ), + ( + "contract_enforced", + member(py, PLANNER_TYPES_MODULE, ("ContractPolicy", "ENFORCED"))?, + ), + ("model_schemas", context.getattr("public_model_schemas")?), + ("named_records", named_records(&resolver)?.into_any()), + ("schema_enums", schema_enums(&resolver)?.into_any()), + ( + "compiler_diagnostic", + class(py, COMPILE_MODELS_MODULE, "CompilerDiagnostic")?, + ), + ("report_diagnostic", host("report_diagnostic")?), + ("path_class", imported(py, "pathlib", "Path")?), + ("external_resolver", external_resolver.clone()), + ( + "reference_kind", + class(py, REFERENCE_TYPES_MODULE, "SqlReferenceKind")?, + ), + ("partial", imported(py, "functools", "partial")?), + ( + "config", + native_config_classes(&host("config_session")?, &host("record_template_reads")?)? + .into_any(), + ), + ("hooks", hooks.into_any()), + ( + "visibility", + visibility_classes(context, &host("visible_declarations")?)?.into_any(), + ), + ("record", record.into_any()), + ("input_view", input_view_classes(py)?.into_any()), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs index aaf6b3a972..29c1c178b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod attachment; pub(crate) mod audit_loop; pub(crate) mod audit_views; pub(crate) mod compiled_model_analysis; @@ -5,6 +6,7 @@ pub(crate) mod compiled_model_view; pub(crate) mod function_loop; pub(crate) mod function_views; pub(crate) mod inputs_view; +pub(crate) mod loop_classes; pub(crate) mod model_input_view; pub(crate) mod model_loop; pub(crate) mod model_loop_calls; @@ -23,6 +25,7 @@ pub(crate) mod model_loop_schema; pub(crate) mod model_loop_visibility; pub(crate) mod project; pub(crate) mod project_view; +pub(crate) mod records; pub(crate) mod registration; pub(crate) mod resource_diagnostics; pub(crate) mod scenario_loop; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs index bd519a1933..e74ac57b62 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs @@ -334,7 +334,7 @@ impl NativeCompiledProject { } /// Retain one model's compile-input facts; a name recorded again is replaced in place. - fn record_model(&mut self, model: ModelInputPy<'_>) -> PyResult<()> { + pub(crate) fn record_model(&mut self, model: ModelInputPy<'_>) -> PyResult<()> { compiler_guard(|| { let facts: CompiledModelFacts = model_facts(model)?; record_model(&mut self.facts, facts); @@ -343,7 +343,7 @@ impl NativeCompiledProject { } /// Attach a recorded model's typed analysis; `False` when that model was never recorded. - fn record_model_analysis( + pub(crate) fn record_model_analysis( &mut self, name: &str, columns: AnalysisPy, @@ -373,7 +373,7 @@ impl NativeCompiledProject { } /// Record each assembled native model view's final typed analysis; return how many. - fn record_view_analyses( + pub(crate) fn record_view_analyses( slf: &Bound<'_, Self>, views: Vec>, ) -> PyResult { @@ -391,12 +391,15 @@ impl NativeCompiledProject { } /// The compile inputs view of this owner's model inputs with these input fields. - fn inputs_view(slf: Bound<'_, Self>, fields: Bound<'_, PyDict>) -> PyResult { + pub(crate) fn inputs_view( + slf: Bound<'_, Self>, + fields: Bound<'_, PyDict>, + ) -> PyResult { NativeInputsView::new(slf.unbind(), &fields) } /// The compiled project view of this owner's models with these project fields. - fn project_view( + pub(crate) fn project_view( slf: Bound<'_, Self>, fields: Bound<'_, PyDict>, ) -> PyResult { @@ -404,7 +407,7 @@ impl NativeCompiledProject { } /// Pair seed declarations with their CSV files and keep them; return each seed's view. - fn attach_seeds( + pub(crate) fn attach_seeds( slf: &Bound<'_, Self>, declarations: Vec<(Py, Py)>, seed_files: Vec>, @@ -430,7 +433,7 @@ impl NativeCompiledProject { } /// Attach every discovered source in order with the kept loop; return each source's view. - fn attach_sources( + pub(crate) fn attach_sources( slf: &Bound<'_, Self>, rows: Vec, settings: SourceSettings, @@ -447,7 +450,7 @@ impl NativeCompiledProject { } /// Attach every SQL, then Python, function with the kept loop; return each one's view. - fn attach_functions( + pub(crate) fn attach_functions( slf: &Bound<'_, Self>, rows: (Vec, Vec), settings: FunctionSettings, @@ -495,7 +498,7 @@ impl NativeCompiledProject { } /// Attach every SQL test case, then every scenario, with the kept loop. - fn attach_sql_tests( + pub(crate) fn attach_sql_tests( slf: &Bound<'_, Self>, files: Vec, scenarios: Vec, @@ -524,7 +527,7 @@ impl NativeCompiledProject { } /// Attach singular, then resource-attached, audits; return views and override warnings. - fn attach_audits<'py>( + pub(crate) fn attach_audits<'py>( slf: &Bound<'py, Self>, files: Vec, settings: AuditSettings, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs index 8d7cfae0eb..88bc987e5c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs @@ -274,7 +274,7 @@ impl NativeProjectView { /// Every field by name, for an explicit `CompiledProject` projection. #[pyo3(signature = (include_sql_expansions = true))] - fn fields<'py>( + pub(crate) fn fields<'py>( &self, py: Python<'py>, include_sql_expansions: bool, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs new file mode 100644 index 0000000000..86cb146d1e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs @@ -0,0 +1,619 @@ +//! Compiled records and views: projecting native views as records, and recording later stages. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyFrozenSetMethods, PyList, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::compiled_project::compiled_model_view::NativeCompiledModel; +use crate::bindings::_helpers::compiled_project::inputs_view::NativeInputsView; +use crate::bindings::_helpers::compiled_project::model_input_view::NativeModelInput; +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::project_view::NativeProjectView; + +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; +const DATACLASSES_MODULE: &str = "dataclasses"; +const DECIMAL_KIND: &str = "decimal"; +const COLLECTION_KINDS: [&str; 2] = ["list", "set"]; +const SCALAR_KINDS: [&str; 5] = ["string", "integer", "boolean", "float", "null"]; + +fn replace<'py>( + value: &Bound<'py, PyAny>, + changes: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + let py: Python<'py> = value.py(); + let keywords: Bound<'py, PyDict> = PyDict::new(py); + for (name, change) in changes { + keywords.set_item(*name, change)?; + } + imported(py, DATACLASSES_MODULE, "replace")?.call((value,), Some(&keywords)) +} + +/// `value` when it already is a `class` record, else `class(**{field: getattr(value, field)})`. +fn record<'py>(value: &Bound<'py, PyAny>, class: &str) -> PyResult> { + let py: Python<'py> = value.py(); + let class: Bound<'py, PyAny> = imported(py, COMPILE_MODELS_MODULE, class)?; + if value.is_instance(&class)? { + return Ok(value.clone()); + } + let fields: Bound<'py, PyDict> = PyDict::new(py); + for field in imported(py, DATACLASSES_MODULE, "fields")? + .call1((&class,))? + .try_iter()? + { + let name: Bound<'py, PyAny> = field?.getattr("name")?; + fields.set_item( + &name, + value.getattr(name.downcast::()?)?, + )?; + } + class.call((), Some(&fields)) +} + +/// The model as a `CompiledModel`, projected from its owner when it is a native view. +#[pyfunction] +pub(crate) fn compiled_model_record<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + record(model, "CompiledModel") +} + +/// The audit as a `CompiledAudit`, projected from its owner when it is a native view. +#[pyfunction] +pub(crate) fn compiled_audit_record<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + record(audit, "CompiledAudit") +} + +/// The function as a `CompiledFunction`, projected from its owner when it is a native view. +#[pyfunction] +pub(crate) fn compiled_function_record<'py>( + function: &Bound<'py, PyAny>, +) -> PyResult> { + record(function, "CompiledFunction") +} + +/// The source as a `CompiledSource`, projected from its owner when it is a native view. +#[pyfunction] +pub(crate) fn compiled_source_record<'py>( + source: &Bound<'py, PyAny>, +) -> PyResult> { + record(source, "CompiledSource") +} + +/// The seed as a `CompiledSeed`, projected from its owner when it is a native view. +#[pyfunction] +pub(crate) fn compiled_seed_record<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { + record(seed, "CompiledSeed") +} + +/// Each item of `fields[name]` as a `class` record. +fn records_of<'py>(fields: &Bound<'py, PyDict>, (name, class): (&str, &str)) -> PyResult<()> { + let py: Python<'py> = fields.py(); + let items: Vec> = fields + .as_any() + .get_item(name)? + .try_iter()? + .map(|item| record(&item?, class)) + .collect::>()?; + fields.set_item(name, PyTuple::new(py, items)?) +} + +/// The project as a `CompiledProject`, projected from its owner when it is a native view. +#[pyfunction] +#[pyo3(signature = (project, *, include_sql_expansions=true))] +pub(crate) fn compiled_project_record<'py>( + project: &Bound<'py, PyAny>, + include_sql_expansions: bool, +) -> PyResult> { + let py: Python<'py> = project.py(); + let class: Bound<'py, PyAny> = imported(py, COMPILE_MODELS_MODULE, "CompiledProject")?; + if project.is_instance(&class)? { + if include_sql_expansions { + return Ok(project.clone()); + } + return replace(project, &[("sql_expansions", PyDict::new(py).into_any())]); + } + let view: PyRef<'py, NativeProjectView> = project.extract()?; + let fields: Bound<'py, PyDict> = view.fields(py, include_sql_expansions)?; + drop(view); + for pair in [ + ("seeds", "CompiledSeed"), + ("sources", "CompiledSource"), + ("functions", "CompiledFunction"), + ("audits", "CompiledAudit"), + ] { + records_of(&fields, pair)?; + } + class.call((), Some(&fields)) +} + +/// The model with these final binding diagnostics. +#[pyfunction] +#[pyo3(signature = (*, model, diagnostics))] +pub(crate) fn with_binding_diagnostics<'py>( + model: &Bound<'py, PyAny>, + diagnostics: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = model.py(); + if model.is_instance(&imported(py, COMPILE_MODELS_MODULE, "CompiledModel")?)? { + return replace(model, &[("binding_diagnostics", diagnostics.clone())]); + } + model.call_method1("with_binding_diagnostics", (diagnostics,)) +} + +/// The model with its recovered bindings and its poisoned outputs' types forgotten. +#[pyfunction] +#[pyo3(signature = (*, model, diagnostics, poisoned))] +pub(crate) fn with_type_recovery<'py>( + model: &Bound<'py, PyAny>, + diagnostics: &Bound<'py, PyAny>, + poisoned: &Bound<'py, PyFrozenSet>, +) -> PyResult> { + let py: Python<'py> = model.py(); + let name: String = model.getattr("name")?.extract()?; + let mut unchecked: Vec = Vec::new(); + for pair in poisoned.iter() { + let (model_name, column): (String, String) = pair.extract()?; + if model_name == name { + unchecked.push(column); + } + } + if !model.is_instance(&imported(py, COMPILE_MODELS_MODULE, "CompiledModel")?)? { + return model.call_method1("with_type_recovery", (diagnostics, unchecked)); + } + let inferred: Bound<'py, PyAny> = model.getattr("inferred_columns")?; + let columns: Bound<'py, PyAny> = if inferred.is_none() { + inferred + } else { + let mut columns: Vec> = Vec::new(); + for column in inferred.try_iter()? { + let column: Bound<'py, PyAny> = column?; + let key = (name.as_str(), column.getattr("name")?); + columns.push(if poisoned.contains(key)? { + replace(&column, &[("type", py.None().into_bound(py))])? + } else { + column + }); + } + PyTuple::new(py, columns)?.into_any() + }; + replace( + model, + &[ + ("binding_diagnostics", diagnostics.clone()), + ("inferred_columns", columns), + ( + "unchecked_output_columns", + PyFrozenSet::new(py, &unchecked)?.into_any(), + ), + ], + ) +} + +/// The project with `changes`; a native view shares its owner's models. +#[pyfunction] +#[pyo3(signature = (*, project, **changes))] +pub(crate) fn project_with<'py>( + project: &Bound<'py, PyAny>, + changes: Option>, +) -> PyResult> { + with_changes(project, "CompiledProject", changes) +} + +/// The compile inputs with `changes`; a native view shares its owner's model inputs. +#[pyfunction] +#[pyo3(signature = (*, inputs, **changes))] +pub(crate) fn inputs_with<'py>( + inputs: &Bound<'py, PyAny>, + changes: Option>, +) -> PyResult> { + with_changes(inputs, "CompileProjectInputs", changes) +} + +/// `replace(value, **changes)` for a record, else the native view's `with_changes(**changes)`. +fn with_changes<'py>( + value: &Bound<'py, PyAny>, + class: &str, + changes: Option>, +) -> PyResult> { + let py: Python<'py> = value.py(); + let changes: Bound<'py, PyDict> = changes.unwrap_or_else(|| PyDict::new(py)); + if value.is_instance(&imported(py, COMPILE_MODELS_MODULE, class)?)? { + return imported(py, DATACLASSES_MODULE, "replace")?.call((value,), Some(&changes)); + } + value.call_method("with_changes", (), Some(&changes)) +} + +/// The native owner's inputs view when it holds every model input; else a record. +#[pyfunction] +#[pyo3(signature = (*, fields, native_project))] +pub(crate) fn compile_inputs_view<'py>( + fields: &Bound<'py, PyDict>, + native_project: Option>, +) -> PyResult> { + let py: Python<'py> = fields.py(); + if let Some(project) = &native_project { + let mut native: bool = true; + for model in fields.as_any().get_item("model_inputs")?.try_iter()? { + if !model?.is_instance_of::() { + native = false; + break; + } + } + if native { + let view: NativeInputsView = + NativeCompiledProject::inputs_view(project.clone(), fields.clone())?; + return view.into_bound_py_any(py); + } + } + let keywords: Bound<'py, PyDict> = fields.copy()?; + keywords.set_item("native_project", native_project)?; + imported(py, COMPILE_MODELS_MODULE, "CompileProjectInputs")?.call((), Some(&keywords)) +} + +/// A typed SQL value as the plain value native consumers read. +#[pyfunction] +pub(crate) fn plain_typed_value<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + let kind: String = value.getattr("kind")?.getattr("value")?.extract()?; + let inner: Bound<'py, PyAny> = value.getattr("value")?; + if kind == DECIMAL_KIND { + return Ok(inner.str()?.into_any()); + } + if SCALAR_KINDS.contains(&kind.as_str()) { + return Ok(inner); + } + if COLLECTION_KINDS.contains(&kind.as_str()) { + let items: Vec> = inner + .try_iter()? + .map(|item| plain_typed_value(&item?)) + .collect::>()?; + return Ok(PyList::new(py, items)?.into_any()); + } + let mapping: Bound<'py, PyDict> = PyDict::new(py); + for pair in inner.try_iter()? { + let (key, item): (Bound<'py, PyAny>, Bound<'py, PyAny>) = pair?.extract()?; + mapping.set_item(key, plain_typed_value(&item)?)?; + } + Ok(mapping.into_any()) +} + +/// An enum declaration as the native project row. +fn enum_row<'py>(declaration: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = declaration.py(); + let members: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = declaration + .getattr("members")? + .try_iter()? + .map(|member| { + let member: Bound<'py, PyAny> = member?; + Ok((member.getattr("name")?, member.getattr("value")?)) + }) + .collect::>()?; + PyTuple::new( + py, + [ + declaration.getattr("name")?, + declaration + .getattr("relative_path")? + .call_method0("as_posix")?, + members.into_bound_py_any(py)?, + py.None().into_bound(py), + py.None().into_bound(py), + py.None().into_bound(py), + ], + ) +} + +/// A constant declaration as the native project row, its value made JSON-safe. +fn constant_row<'py>(declaration: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = declaration.py(); + let render_as: Bound<'py, PyAny> = declaration.getattr("render_as")?; + PyTuple::new( + py, + [ + declaration.getattr("name")?, + declaration + .getattr("relative_path")? + .call_method0("as_posix")?, + PyList::empty(py).into_any(), + plain_typed_value(&declaration.getattr("value")?)?, + declaration + .getattr("logical_type")? + .getattr("display_name")?, + if render_as.is_none() { + render_as + } else { + render_as.getattr("value")? + }, + ], + ) +} + +/// Enum and constant declarations as native rows, in the given order. +#[pyfunction] +#[pyo3(signature = (*, enums, constants))] +pub(crate) fn declaration_rows<'py>( + enums: &Bound<'py, PyAny>, + constants: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyList>, Bound<'py, PyList>)> { + let py: Python<'py> = enums.py(); + let enum_rows: Vec> = enums + .try_iter()? + .map(|declaration| enum_row(&declaration?)) + .collect::>()?; + let constant_rows: Vec> = constants + .try_iter()? + .map(|declaration| constant_row(&declaration?)) + .collect::>()?; + Ok((PyList::new(py, enum_rows)?, PyList::new(py, constant_rows)?)) +} + +/// `(name, type)` of each dynamic family; `casefold` keys the family by its folded name. +fn family_types<'py>( + families: &Bound<'py, PyAny>, + casefold: bool, +) -> PyResult)>> { + families + .try_iter()? + .map(|family| { + let family: Bound<'py, PyAny> = family?; + let name: Bound<'py, PyAny> = family.getattr("name")?; + let name: String = if casefold { + name.call_method0("casefold")?.extract()? + } else { + name.extract()? + }; + Ok((name, family.getattr("inferred_type")?.extract()?)) + }) + .collect() +} + +/// Transfer each assembled model's typed analysis to its retained facts; return how many. +#[pyfunction] +#[pyo3(signature = (*, project, models))] +pub(crate) fn record_model_analyses<'py>( + project: &Bound<'py, NativeCompiledProject>, + models: &Bound<'py, PyAny>, +) -> PyResult { + let py: Python<'py> = project.py(); + let models: Vec> = models.try_iter()?.collect::>()?; + let views: Vec> = models + .iter() + .filter(|model| model.is_instance_of::()) + .map(|model| model.extract::>()) + .collect::>()?; + let mut transferred: usize = NativeCompiledProject::record_view_analyses(project, views)?; + let nullable: Bound<'py, PyAny> = + imported(py, LINEAGE_TYPES_MODULE, "InferredNullability")?.getattr("NULLABLE")?; + for model in &models { + if model.is_instance_of::() { + continue; + } + let inferred: Bound<'py, PyAny> = model.getattr("inferred_rows")?; + let rows: Vec<(String, Option, Bound<'py, PyAny>)> = if inferred.is_none() { + Vec::new() + } else { + inferred.extract()? + }; + let proof: Bound<'py, PyAny> = model.getattr("dynamic_column_contract")?; + let proof = if proof.is_none() { + None + } else { + let families: Bound<'py, PyAny> = proof.getattr("families")?; + Some(( + proof.getattr("output_proven")?.extract()?, + proof.getattr("bare_dynamic_pivot")?.extract()?, + family_types(&families, true)?, + proof.getattr("failure_reason")?.extract()?, + family_types(&families, false)?, + )) + }; + let analysis = ( + !inferred.is_none(), + rows.iter() + .map(|(name, column_type, _)| (name.clone(), column_type.clone())) + .collect(), + rows.iter() + .map(|(_, _, nullability)| nullability.eq(&nullable)) + .collect::>()?, + model.getattr("fast_lineage_has_star")?.is_truthy()?, + model + .getattr("unchecked_output_columns")? + .try_iter()? + .map(|column| column?.extract()) + .collect::>()?, + ); + let name: String = model.getattr("name")?.extract()?; + transferred += usize::from( + project + .borrow_mut() + .record_model_analysis(&name, analysis, proof), + ); + } + Ok(transferred) +} + +/// A native project for a `CompiledProject` built without the compile-input stage. +fn hand_built_project<'py>( + project: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let native: Bound<'py, NativeCompiledProject> = + Bound::new(py, NativeCompiledProject::default())?; + let models: Bound<'py, PyAny> = project.getattr("models")?; + for model in models.try_iter()? { + let model: Bound<'py, PyAny> = model?; + native + .borrow_mut() + .record_model(model_row(&model)?.as_any().extract()?)?; + } + let _ = record_model_analyses(&native, &models)?; + Ok(native) +} + +/// The compile's retained project, or one recorded from a hand-built project. +#[pyfunction] +pub(crate) fn compiled_project_facts<'py>( + project: &Bound<'py, PyAny>, +) -> PyResult> { + let retained: Bound<'py, PyAny> = project.getattr("native_project")?; + if !retained.is_none() { + return Ok(retained.downcast_into()?); + } + hand_built_project(project) +} + +fn model_row<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + let relative_path: Bound<'py, PyAny> = model.getattr("relative_path")?; + let config: Bound<'py, PyAny> = model.getattr("config")?; + let references: Vec> = model + .getattr("references")? + .try_iter()? + .map(|reference| { + let reference: Bound<'py, PyAny> = reference?; + PyTuple::new( + py, + [ + reference.getattr("ref_kind")?.str()?.into_any(), + reference.getattr("ref_name")?, + reference.getattr("ref_package")?, + ], + ) + }) + .collect::>()?; + let (enums, constants) = declaration_rows( + &model.getattr("enum_declarations")?, + &model.getattr("constant_declarations")?, + )?; + PyTuple::new( + py, + [ + PyTuple::new( + py, + [ + model.getattr("name")?, + relative_path.call_method0("as_posix")?, + model.getattr("query_sql")?, + model.getattr("authored_query_sql")?, + model.getattr("authored_sql")?, + ], + )? + .into_any(), + PyTuple::new( + py, + [ + config.getattr("values")?, + PyList::new( + py, + config + .getattr("model_header_keys")? + .try_iter()? + .collect::>>()?, + )? + .into_any(), + config.getattr("layer_schema")?, + ], + )? + .into_any(), + PyList::new(py, references)?.into_any(), + schema_row(&model.getattr("schema_entry")?, &relative_path)?, + PyList::new( + py, + model + .getattr("enum_columns")? + .try_iter()? + .collect::>>()?, + )? + .into_any(), + PyTuple::new(py, [enums.into_any(), constants.into_any()])?.into_any(), + ], + ) +} + +fn schema_row<'py>( + entry: &Bound<'py, PyAny>, + relative_path: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = entry.py(); + if entry.is_none() { + return Ok(entry.clone()); + } + let columns: Vec> = entry + .getattr("columns")? + .try_iter()? + .map(|column| { + let column: Bound<'py, PyAny> = column?; + let location: Bound<'py, PyAny> = column.getattr("location")?; + let elsewhere: bool = + !location.is_none() && location.getattr("path")?.ne(relative_path)?; + PyTuple::new( + py, + [ + column.getattr("name")?, + column.getattr("type")?, + column.getattr("nullable")?, + column.getattr("audits")?.len()?.into_bound_py_any(py)?, + elsewhere.into_bound_py_any(py)?, + ], + ) + }) + .collect::>()?; + let families: Vec> = entry + .getattr("dynamic_columns")? + .try_iter()? + .map(|family| { + let family: Bound<'py, PyAny> = family?; + PyTuple::new( + py, + [ + family.getattr("name")?.call_method0("casefold")?, + family.getattr("name")?, + family.getattr("pivot_column")?, + family.getattr("value_column")?, + family.getattr("aggregate")?, + family.getattr("type")?, + family.getattr("name_pattern")?, + ], + ) + }) + .collect::>()?; + Ok(PyTuple::new( + py, + [ + entry.getattr("audits")?.len()?.into_bound_py_any(py)?, + PyList::new(py, columns)?.into_any(), + PyList::new(py, families)?.into_any(), + ( + entry.getattr("type_enforcement")?.is_truthy()?, + !entry.getattr("model_schema")?.is_none(), + ) + .into_bound_py_any(py)?, + ], + )? + .into_any()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + for function in [ + wrap_pyfunction!(compiled_model_record, module)?, + wrap_pyfunction!(compiled_audit_record, module)?, + wrap_pyfunction!(compiled_function_record, module)?, + wrap_pyfunction!(compiled_source_record, module)?, + wrap_pyfunction!(compiled_seed_record, module)?, + wrap_pyfunction!(compiled_project_record, module)?, + wrap_pyfunction!(with_binding_diagnostics, module)?, + wrap_pyfunction!(with_type_recovery, module)?, + wrap_pyfunction!(project_with, module)?, + wrap_pyfunction!(inputs_with, module)?, + wrap_pyfunction!(compile_inputs_view, module)?, + wrap_pyfunction!(plain_typed_value, module)?, + wrap_pyfunction!(declaration_rows, module)?, + wrap_pyfunction!(record_model_analyses, module)?, + wrap_pyfunction!(compiled_project_facts, module)?, + ] { + module.add_function(function)?; + } + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs index 4b36f9b0eb..93b6bc7006 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs @@ -2,5 +2,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { crate::bindings::_helpers::compiled_project::project::register(module)?; - crate::bindings::_helpers::compiled_project::model_loop::register(module) + crate::bindings::_helpers::compiled_project::model_loop::register(module)?; + crate::bindings::_helpers::compiled_project::attachment::register(module)?; + crate::bindings::_helpers::compiled_project::records::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs index c7f9412a8a..4fa99fa92f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs @@ -32,7 +32,7 @@ const CURSOR_INPUTS_KEY: &str = "cursor_inputs"; const PYTHON_UDF_DECORATOR: &str = "udf"; /// A loader's description, relative path and whether that path is a YAML file. -type LoaderRow = (Option, String, bool); +pub(crate) type LoaderRow = (Option, String, bool); /// A Python resource's kind, name, description (a `str` that may hold surrogates), location, help. type PythonResourceRow = (String, String, Option>, Py, String); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/tree_render.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/tree_render.rs index 6007889188..724458e623 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/tree_render.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/tree_render.rs @@ -2,7 +2,6 @@ use std::collections::HashSet; -use pyo3::exceptions::PyTypeError; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; @@ -13,6 +12,8 @@ use sqlbuild_analysis::lineage_graph::main::tree_lines::tree_lines; use sqlbuild_analysis::lineage_graph::models::TreeLine; use sqlbuild_analysis::lineage_graph::types::TreeVisitor; +use crate::bindings::_helpers::boundary::keyword_request::Request; + const BUILTINS_MODULE: &str = "builtins"; /// Box-drawing dependency tree lines for one focus node, expanding each node once. @@ -111,48 +112,6 @@ fn serialize_column_edge<'py>( Ok(payload) } -/// Keyword arguments of one call, each required. -struct Request<'a, 'py> { - function: &'static str, - arguments: Option<&'a Bound<'py, PyDict>>, - py: Python<'py>, -} - -impl<'a, 'py> Request<'a, 'py> { - fn new(function: &'static str, arguments: Option<&'a Bound<'py, PyDict>>) -> PyResult { - let py: Python<'py> = match arguments { - Some(arguments) => arguments.py(), - None => { - return Err(PyTypeError::new_err(format!( - "{function}() missing required keyword-only arguments" - ))); - } - }; - Ok(Self { - function, - arguments, - py, - }) - } - - fn py(&self) -> Python<'py> { - self.py - } - - fn get(&self, name: &str) -> PyResult> { - let value: Option> = match self.arguments { - Some(arguments) => arguments.get_item(name)?, - None => None, - }; - value.ok_or_else(|| { - PyTypeError::new_err(format!( - "{}() missing 1 required keyword-only argument: '{name}'", - self.function - )) - }) - } -} - /// `str(value)`, as an f-string renders a callback's text. fn text(value: &Bound<'_, PyAny>) -> PyResult { value.str()?.extract() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 8e36750bba..cb760a85cc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod analysis_session; pub(crate) mod attachments; pub(crate) mod boundary; pub(crate) mod cache; +pub(crate) mod compile_attachment; pub(crate) mod compile_driver; pub(crate) mod compile_outputs; pub(crate) mod compiled_project; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs index 7773c54f72..33705c8763 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs @@ -3,7 +3,10 @@ use std::collections::HashMap; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_scopes::compile_usages::main::distinct_positions::distinct_positions; use sqlbuild_scopes::compile_usages::main::merged_dependencies::merged_dependencies; @@ -17,6 +20,13 @@ use sqlbuild_scopes::scope_index::models::{ use crate::bindings::_helpers::compiled_project::model_loop_named::scope_path_message; +const SCOPE_ERRORS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; +const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; +const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const INACCESSIBLE_DECLARATION_CODE: &str = "S006"; +const SCHEMA_USAGE_KINDS: [&str; 2] = ["schema", "enum"]; + type IdentityRow = (String, String); /// A declaration's kind, name and private owner; a resource's kind and name with no owner. type QualifiedRow = (String, String, Option); @@ -75,8 +85,135 @@ fn diagnostic_row(diagnostic: InaccessibleDiagnostic) -> DiagnosticRow { ) } -/// Each schema's parent and enum usages over `records` by `(kind, name)` handle, with S006s. +/// `model_schema_scope_usages`: each schema's parent and enum `UsageRecord`s; S006s are reported. #[pyfunction] +fn model_schema_scope_usages<'py>( + schema_files: &Bound<'py, PyAny>, + resolver: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = schema_files.py(); + if resolver.is_none() { + return Ok(PyTuple::empty(py)); + } + let mut records: Vec> = Vec::new(); + let mut record_rows: Vec<(IdentityRow, RecordRow)> = Vec::new(); + let declarations = resolver.getattr("lookup")?.getattr("declarations")?; + for item in declarations.call_method0("items")?.try_iter()? { + let (identity, found): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + if !found.is_truthy()? + || !identity.getattr("owner")?.is_none() + || !SCHEMA_USAGE_KINDS.contains( + &identity + .getattr("kind")? + .getattr("value")? + .extract::()? + .as_str(), + ) + { + continue; + } + let record: Bound<'py, PyAny> = found.get_item(0)?; + let recorded: Bound<'py, PyAny> = record.getattr("identity")?; + record_rows.push(( + ( + recorded.getattr("kind")?.getattr("value")?.extract()?, + recorded.getattr("name")?.extract()?, + ), + record_row(&record)?, + )); + records.push(record); + } + let mut schemas: Vec = Vec::new(); + for file in schema_files.try_iter()? { + for declaration in file?.getattr("declarations")?.try_iter()? { + let declaration: Bound<'py, PyAny> = declaration?; + let mut columns: Vec<(String, Option)> = Vec::new(); + for column in declaration.getattr("columns")?.try_iter()? { + let column: Bound<'py, PyAny> = column?; + columns.push(( + column.getattr("name")?.extract()?, + column.getattr("type")?.extract()?, + )); + } + schemas.push(( + declaration.getattr("name")?.extract()?, + declaration.getattr("extends")?.extract()?, + columns, + )); + } + } + let (usages, diagnostics, error) = scope_schema_usages(schemas, record_rows)?; + if let Some(message) = error { + return Err(PyErr::from_value( + py.import(SCOPE_ERRORS_MODULE)? + .getattr("InvalidScopePathError")? + .call1((message,))?, + )); + } + report_inaccessible(py, diagnostics)?; + let usage_record = py.import(SCOPE_MODELS_MODULE)?.getattr("UsageRecord")?; + let mut built: Vec> = Vec::with_capacity(usages.len()); + for (consumer, used) in usages { + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("consumer", records[consumer].getattr("identity")?)?; + fields.set_item("declaration", records[used].getattr("identity")?)?; + built.push(usage_record.call((), Some(&fields))?); + } + PyTuple::new(py, built) +} + +/// A declaration record's path, scope, private owner, owning path and ownership root path. +fn record_row(record: &Bound<'_, PyAny>) -> PyResult { + let identity: Bound<'_, PyAny> = record.getattr("identity")?; + let owner: Bound<'_, PyAny> = identity.getattr("owner")?; + Ok(( + record.getattr("path")?.extract()?, + record.getattr("scope")?.getattr("value")?.extract()?, + if owner.is_none() { + None + } else { + Some(( + owner.getattr("kind")?.getattr("value")?.extract()?, + owner.getattr("name")?.extract()?, + )) + }, + record.getattr("owning_path")?.extract()?, + record + .getattr("ownership_root")? + .getattr("path")? + .extract()?, + )) +} + +/// Report each S006 through the active compile diagnostic collector. +fn report_inaccessible(py: Python<'_>, diagnostics: Vec) -> PyResult<()> { + if diagnostics.is_empty() { + return Ok(()); + } + let report = py + .import(COLLECTOR_MODULE)? + .getattr("report_compile_diagnostic")?; + let compiler_diagnostic = py + .import(COMPILE_MODELS_MODULE)? + .getattr("CompilerDiagnostic")?; + let path_class = py.import("pathlib")?.getattr("Path")?; + for (key, message, path, help) in diagnostics { + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("phase", "compile")?; + fields.set_item("severity", "error")?; + fields.set_item("code", INACCESSIBLE_DECLARATION_CODE)?; + fields.set_item("message", message)?; + fields.set_item("path", path_class.call1((path,))?)?; + fields.set_item("help", help)?; + let arguments: Bound<'_, PyDict> = PyDict::new(py); + arguments.set_item("key", PyTuple::new(py, key)?)?; + arguments.set_item("diagnostic", compiler_diagnostic.call((), Some(&fields))?)?; + report.call((), Some(&arguments))?; + } + Ok(()) +} + +/// Each schema's parent and enum usages over `records` by `(kind, name)` handle, with S006s. fn scope_schema_usages( schemas: Vec, records: Vec<(IdentityRow, RecordRow)>, @@ -157,6 +294,6 @@ pub(crate) fn merged_usage_rows( } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(scope_schema_usages, module)?)?; + module.add_function(wrap_pyfunction!(model_schema_scope_usages, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs index 88f94269f2..c4023fd1a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs @@ -6,6 +6,7 @@ pub(crate) mod registration; pub(crate) mod relationship_names; pub(crate) mod relationship_scan; pub(crate) mod scope_cache; +pub(crate) mod scope_errors; pub(crate) mod scope_index; pub(crate) mod scope_lookup; pub(crate) mod scope_objects; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs new file mode 100644 index 0000000000..40b2b87e1c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs @@ -0,0 +1,99 @@ +//! Report a scope index's error findings as individual compile diagnostics. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; + +const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const ERROR_SEVERITY: &str = "error"; + +/// `report_scope_index_errors`: each error-severity finding of `index`, keyed for the collector. +pub(crate) fn report_scope_index_errors(index: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = index.py(); + let report = py + .import(COLLECTOR_MODULE)? + .getattr("report_compile_diagnostic")?; + let compiler_diagnostic = py + .import(COMPILE_MODELS_MODULE)? + .getattr("CompilerDiagnostic")?; + let path_class = py.import("pathlib")?.getattr("Path")?; + for item in index.getattr("diagnostics")?.try_iter()? { + let item: Bound<'_, PyAny> = item?; + if item.getattr("severity")?.str()?.extract::()? != ERROR_SEVERITY { + continue; + } + let code: Bound<'_, PyAny> = item.getattr("code")?.getattr("value")?; + let message: Bound<'_, PyAny> = item.getattr("message")?; + let path: Bound<'_, PyAny> = item.getattr("path")?; + let declaration: Bound<'_, PyAny> = item.getattr("declaration")?; + let resource: Bound<'_, PyAny> = item.getattr("resource")?; + let key: Bound<'_, PyTuple> = PyTuple::new( + py, + [ + code.clone(), + or_empty(&path)?, + or_empty(&declaration)?.str()?.into_any(), + or_empty(&resource)?.str()?.into_any(), + message.clone(), + ], + )?; + let (resource_type, resource_name) = owning_resource(&resource, &declaration)?; + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("phase", "compile")?; + fields.set_item("severity", ERROR_SEVERITY)?; + fields.set_item("code", code)?; + fields.set_item("message", message)?; + fields.set_item("resource_type", resource_type)?; + fields.set_item("resource_name", resource_name)?; + if path.is_none() { + fields.set_item("path", py.None())?; + } else { + fields.set_item("path", path_class.call1((path,))?)?; + } + fields.set_item("line", item.getattr("line")?)?; + fields.set_item("column", item.getattr("column")?)?; + let diagnostic: Bound<'_, PyAny> = compiler_diagnostic.call((), Some(&fields))?; + let arguments: Bound<'_, PyDict> = PyDict::new(py); + arguments.set_item("key", key)?; + arguments.set_item("diagnostic", diagnostic)?; + report.call((), Some(&arguments))?; + } + Ok(()) +} + +/// `value or ""`. +fn or_empty<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + if value.is_truthy()? { + return Ok(value.clone()); + } + Ok(PyString::new(value.py(), "").into_any()) +} + +/// The compiled resource a finding names: its resource, else an audit declaration. +fn owning_resource( + resource: &Bound<'_, PyAny>, + declaration: &Bound<'_, PyAny>, +) -> PyResult<(Option<&'static str>, Option)> { + if !resource.is_none() { + let kind: String = resource.getattr("kind")?.str()?.extract()?; + let resource_type: Option<&'static str> = match kind.as_str() { + "model" => Some("model"), + "source" => Some("source"), + "seed" => Some("seed"), + "function" => Some("udf"), + "test" => Some("sql_test"), + "scenario" => Some("sql_scenario"), + _ => None, + }; + if resource_type.is_some() { + return Ok((resource_type, Some(resource.getattr("name")?.extract()?))); + } + } + if !declaration.is_none() { + let kind: String = declaration.getattr("kind")?.str()?.extract()?; + if matches!(kind.as_str(), "audit" | "singular_audit") { + return Ok((Some("audit"), Some(declaration.getattr("name")?.extract()?))); + } + } + Ok((None, None)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs index 8829c70ce4..64e06a54a3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs @@ -1,189 +1,250 @@ -//! Native diagnostic recovery, explanations and opt-out rejection. +//! Native diagnostic recovery, explanations and opt-out rejection after the metadata checks. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyRef, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyString, PyTuple}; use sqlbuild_analysis::semantic_checks::main::complete_semantic_diagnostics::complete_semantic_diagnostics; use sqlbuild_analysis::semantic_checks::models::{ - CompletedDiagnostic, CompletionModel, CompletionRequest, FinalDiagnostic, SemanticDiagnostic, - SemanticLocation, + CompletedDiagnostic, CompletionModel, CompletionRequest, SemanticDiagnostic, SemanticLocation, }; -use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::semantic_checks::failures::semantic_error; +use crate::bindings::_helpers::semantic_checks::metadata::{ShapeRows, shape_rows}; use crate::bindings::_helpers::semantic_checks::session_facts::{ column_names, session_model_facts, }; -use crate::bindings::_helpers::semantic_checks::type_recovery::{LineageInput, lineage_outputs}; -use crate::bindings::models::ProjectCatalog; +use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; +use crate::bindings::_helpers::semantic_checks::type_recovery::{ + LineageInput, StageShared, diagnostic_ids, id_of, inferred_names, items, lineage_outputs, + lineage_payload, +}; use crate::bindings::types::CompilerDetach; -/// `(line, column, end_line, end_column)`. -type LocationRow = (i64, i64, Option, Option); -/// `(id, code, message, resource_type, resource_name, line, column, location, help, notes)`. -type DiagnosticInput = ( - usize, - String, - String, - Option, - Option, - Option, - Option, - Option, - Option, - Vec, -); -/// `(name, query_sql, authored_sql, inferred names, lineage, binding codes, opt-out file)`. -type ModelInput = ( - String, - String, - String, - Option>, - Option>, - Vec, - Option, -); -/// `(dialect, diagnostics, models, shapes)`. -type RequestInput = ( - Option, - Vec, - Vec, - Vec<(String, Vec<(String, String)>)>, -); -/// `(source, message, help, notes, location, line, column, changed)`. -type CompletedRow = ( - usize, - String, - Option, - Vec, - Option, - Option, - Option, - bool, -); -/// `(completed position, (model, message, note, help))`: exactly one side is set. -type OrderRow = (Option, Option<(usize, String, String, String)>); -/// `(completed, model binding positions, order)`. -type OutcomeRow = (Vec, Option>>, Vec); +const RECOVERED_CODE: &str = "B002"; -/// Recover, explain and reject opt-outs, reading facts the payload omits from `session`. -#[pyfunction] -#[pyo3(signature = (catalog, request, session=None))] -fn complete_semantic_checks( - py: Python<'_>, - catalog: PyRef<'_, ProjectCatalog>, - request: RequestInput, - session: Option>, -) -> PyResult { - let request = completion_request(request, session.as_deref()).map_err(compiler_error)?; - let catalog = &catalog.inner; - let (diagnostics, model_bindings, order) = py - .compiler_detach(|| Ok(complete_semantic_diagnostics(&request, catalog))) +/// The recovered, explained and opt-out-checked project; a native internal failure raises. +pub(crate) fn completed_diagnostics<'py>( + classes: &StageClasses<'py>, + project: &Bound<'py, PyAny>, + shared: &StageShared<'_>, +) -> PyResult> { + let py: Python<'py> = classes.py(); + let models: Vec> = items(&project.getattr("models")?)?; + let ids: Bound<'py, PyDict> = diagnostic_ids(project, &models)?; + let diagnostics: Vec> = items(&project.getattr("diagnostics")?)?; + let mut recovers: bool = false; + for item in &diagnostics { + if item.getattr("code")?.eq(RECOVERED_CODE)? { + recovers = true; + break; + } + } + let shapes: ShapeRows = if diagnostics.is_empty() { + Vec::new() + } else { + shape_rows(&classes.inputs.call_method1("shapes", (project,))?)? + }; + let mut semantic: Vec = Vec::with_capacity(diagnostics.len()); + for item in &diagnostics { + semantic.push(semantic_diagnostic(item, &ids)?); + } + let mut completion_models: Vec = Vec::with_capacity(models.len()); + for model in &models { + completion_models.push(completion_model(model, recovers, shared)?); + } + let request = CompletionRequest { + dialect: project.getattr("sql_analysis_dialect")?.extract()?, + diagnostics: semantic, + models: completion_models, + shapes, + }; + let (completed, model_bindings, order) = py + .compiler_detach(|| Ok(complete_semantic_diagnostics(&request, shared.catalog))) .map_err(compiler_error)? .map_err(|failure| semantic_error(py, &failure))? .into_parts(); - Ok(( - diagnostics.into_iter().map(completed_row).collect(), - model_bindings, - order.into_iter().map(order_row).collect(), - )) -} - -fn location(row: LocationRow) -> SemanticLocation { - SemanticLocation { - line: row.0, - column: row.1, - end_line: row.2, - end_column: row.3, + let mut rebuilt: Vec> = Vec::with_capacity(completed.len()); + for row in completed { + let original: Bound<'py, PyAny> = diagnostics[row.source].clone(); + rebuilt.push(completed_diagnostic(classes, original, row)?); } -} - -fn location_row(location: SemanticLocation) -> LocationRow { - ( - location.line, - location.column, - location.end_line, - location.end_column, + let models_value: Bound<'py, PyAny> = match model_bindings { + None => project.getattr("models")?, + Some(model_bindings) => { + let mut updated: Vec> = Vec::with_capacity(models.len()); + for (model, positions) in models.iter().zip(model_bindings) { + let selected: Vec> = positions + .into_iter() + .map(|position| rebuilt[position].clone()) + .collect(); + updated.push(classes.call( + &classes.with_binding_diagnostics, + &[ + ("model", model), + ("diagnostics", &PyTuple::new(py, selected)?.into_any()), + ], + )?); + } + PyTuple::new(py, updated)?.into_any() + } + }; + let mut ordered: Vec> = Vec::with_capacity(order.len()); + for entry in order { + let (position, opt_out) = entry.into_parts(); + if let Some(position) = position { + ordered.push(rebuilt[position].clone()); + } else if let Some(opt_out) = opt_out { + let model: &Bound<'py, PyAny> = &models[opt_out.model]; + let notes: Bound<'py, PyAny> = PyTuple::new(py, [opt_out.note])?.into_any(); + ordered.push(classes.call( + &classes.diagnostic, + &[ + ("phase", &classes.compile_phase), + ("severity", &classes.error), + ("code", &classes.opt_out_code), + ("message", &PyString::new(py, &opt_out.message).into_any()), + ("resource_type", &classes.model_resource), + ("resource_name", &model.getattr("name")?), + ("location", &model.getattr("rejected_sql_analysis_opt_out")?), + ("notes", ¬es), + ("help", &PyString::new(py, &opt_out.help).into_any()), + ], + )?); + } + } + classes.project_with( + project, + &[ + ("models", &models_value), + ("diagnostics", &PyTuple::new(py, ordered)?.into_any()), + ], ) } -fn completion_request( - (dialect, diagnostics, models, shapes): RequestInput, - session: Option<&NativeModelAnalysisSession>, -) -> Result { - Ok(CompletionRequest { - dialect, - diagnostics: diagnostics.into_iter().map(semantic_diagnostic).collect(), - models: models - .into_iter() - .map(|model| completion_model(model, session)) - .collect::>()?, - shapes, - }) +fn optional_location(location: &Bound<'_, PyAny>) -> PyResult> { + if location.is_none() { + return Ok(None); + } + Ok(Some(SemanticLocation { + line: location.getattr("line")?.extract()?, + column: location.getattr("column")?.extract()?, + end_line: location.getattr("end_line")?.extract()?, + end_column: location.getattr("end_column")?.extract()?, + })) } fn semantic_diagnostic( - (id, code, message, resource_type, resource_name, line, column, place, help, notes): DiagnosticInput, -) -> SemanticDiagnostic { - SemanticDiagnostic { - id, - code, - message, - resource_type, - resource_name, - line, - column, - location: place.map(location), - help, - notes, - } + item: &Bound<'_, PyAny>, + ids: &Bound<'_, PyDict>, +) -> PyResult { + let resource_type: Bound<'_, PyAny> = item.getattr("resource_type")?; + Ok(SemanticDiagnostic { + id: id_of(ids, item)?, + code: item.getattr("code")?.extract()?, + message: item.getattr("message")?.extract()?, + resource_type: if resource_type.is_none() { + None + } else { + Some(resource_type.str()?.extract()?) + }, + resource_name: item.getattr("resource_name")?.extract()?, + line: item.getattr("line")?.extract()?, + column: item.getattr("column")?.extract()?, + location: optional_location(&item.getattr("location")?)?, + help: item.getattr("help")?.extract()?, + notes: items(&item.getattr("notes")?)? + .iter() + .map(PyAnyMethods::extract) + .collect::>()?, + }) } fn completion_model( - (name, query_sql, authored_sql, inferred_columns, lineage, binding_codes, opt_out): ModelInput, - session: Option<&NativeModelAnalysisSession>, -) -> Result { - let inferred_columns: Vec = match inferred_columns { - Some(names) => names, - None => column_names(session_model_facts(session, &name)?).unwrap_or_default(), + model: &Bound<'_, PyAny>, + recovers: bool, + shared: &StageShared<'_>, +) -> PyResult { + let name: String = model.getattr("name")?.extract()?; + let from_session: bool = shared.session_models.contains(&name); + let inferred_columns: Vec = if from_session { + column_names(session_model_facts(shared.session, &name).map_err(compiler_error)?) + .unwrap_or_default() + } else { + inferred_names(model)?.unwrap_or_default() }; - let lineage: Vec = match lineage { - Some(lineage) => lineage, - None => session_model_facts(session, &name)?.lineage.clone(), + let lineage: Vec = if !recovers { + Vec::new() + } else if from_session { + session_model_facts(shared.session, &name) + .map_err(compiler_error)? + .lineage + .clone() + } else { + lineage_payload(model)? }; + let opt_out: Bound<'_, PyAny> = model.getattr("rejected_sql_analysis_opt_out")?; + let mut binding_codes: Vec = Vec::new(); + for item in model.getattr("binding_diagnostics")?.try_iter()? { + binding_codes.push(item?.getattr("code")?.extract()?); + } Ok(CompletionModel { name, - query_sql, - authored_sql, + query_sql: model.getattr("query_sql")?.extract()?, + authored_sql: model.getattr("authored_sql")?.extract()?, inferred_columns, lineage: lineage_outputs(lineage), binding_codes, - rejected_opt_out_file: opt_out, + rejected_opt_out_file: if opt_out.is_none() { + None + } else { + Some(opt_out.getattr("path")?.getattr("name")?.extract()?) + }, }) } -fn completed_row(diagnostic: CompletedDiagnostic) -> CompletedRow { - ( - diagnostic.source, - diagnostic.message, - diagnostic.help, - diagnostic.notes, - diagnostic.location.map(location_row), - diagnostic.line, - diagnostic.column, - diagnostic.changed, - ) -} - -fn order_row(entry: FinalDiagnostic) -> OrderRow { - let (position, opt_out) = entry.into_parts(); - ( - position, - opt_out.map(|opt_out| (opt_out.model, opt_out.message, opt_out.note, opt_out.help)), +fn completed_diagnostic<'py>( + classes: &StageClasses<'py>, + original: Bound<'py, PyAny>, + row: CompletedDiagnostic, +) -> PyResult> { + if !row.changed { + return Ok(original); + } + let py: Python<'py> = classes.py(); + let mut current: Bound<'py, PyAny> = original.getattr("location")?; + if let (Some(existing), Some(location)) = (optional_location(¤t)?, row.location.as_ref()) + && ( + existing.line, + existing.column, + existing.end_line, + existing.end_column, + ) != ( + location.line, + location.column, + location.end_line, + location.end_column, + ) + { + current = classes.replaced( + ¤t, + &[ + ("line", &location.line.into_bound_py_any(py)?), + ("column", &location.column.into_bound_py_any(py)?), + ("end_line", &location.end_line.into_bound_py_any(py)?), + ("end_column", &location.end_column.into_bound_py_any(py)?), + ], + )?; + } + classes.replaced( + &original, + &[ + ("message", &PyString::new(py, &row.message).into_any()), + ("help", &row.help.into_bound_py_any(py)?), + ("notes", &PyTuple::new(py, row.notes)?.into_any()), + ("location", ¤t), + ("line", &row.line.into_bound_py_any(py)?), + ("column", &row.column.into_bound_py_any(py)?), + ], ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(complete_semantic_checks, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/help_catalogue.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/help_catalogue.rs new file mode 100644 index 0000000000..8061ca065b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/help_catalogue.rs @@ -0,0 +1,15 @@ +//! The semantic diagnostic help catalogue, for Python's tests and corpus tooling. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::semantic_checks::main::semantic_help_catalogue::semantic_help_catalogue as catalogue; + +/// Each semantic diagnostic code and its help. +#[pyfunction] +fn semantic_help_catalogue() -> Vec<(&'static str, &'static str)> { + catalogue().to_vec() +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(semantic_help_catalogue, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs index b6f265a8da..d155831cc8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs @@ -1,225 +1,557 @@ //! Native semantic metadata checks: config references, cursors, function calls and SQL tests. -use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyRef, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; +use std::sync::Arc; + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyString, PyTuple}; use sqlbuild_analysis::semantic_checks::main::check_semantic_metadata::check_semantic_metadata; use sqlbuild_analysis::semantic_checks::models::{ AuditValue, MetadataAudit, MetadataFinding, MetadataFunction, MetadataModel, MetadataRequest, MetadataSource, MetadataSqlTest, ModelMetadataFindings, }; -use std::sync::Arc; +use sqlbuild_analysis::semantic_validation::models as validation; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::semantic_checks::failures::semantic_error; -use crate::bindings::models::ProjectCatalog; +use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; +use crate::bindings::_helpers::semantic_checks::type_recovery::items; use crate::bindings::types::CompilerDetach; +const METADATA_CONFIG_KEYS: [&str; 4] = [ + "unique_key", + "cursor", + "partition_column", + "row_diff_exclude_columns", +]; + +/// `[(relation, [(column, value)])]` in Python's order. +pub(crate) type ShapeRows = Vec<(String, Vec<(String, String)>)>; /// `(key, names)` pairs in Python's order. -type NamesInput = Vec<(String, Vec)>; -/// `(column, definition, expression, to, field, values as (kind, text))`. -type AuditInput = ( - Option, - String, - Option, - String, - String, - Option>, -); -/// `(name, query, authored, checked, functions, references, cursor inputs, cursor, type, audits)`. -type ModelInput = ( - String, - String, - String, - bool, - bool, - NamesInput, - NamesInput, - Option, - Option, - Vec, -); -/// `(dialect, quoted case, return types, functions, shapes, models, files, sources, SQL tests)`. -type RequestInput = ( - Option, - bool, - Vec<(String, String)>, - Vec<(String, Vec<(String, String)>)>, - Vec<(String, Vec<(String, String)>)>, - Vec, - Vec, - Vec<(String, Option, usize)>, - Vec<(usize, NamesInput)>, -); -/// `(code, message, line, column)`. -type ErrorRow = (&'static str, String, i64, i64); -/// `(code, message, line, column, is error)`. -type AuditRow = (String, String, i64, i64, bool); -/// `([(function, audit, reference findings)], source, SQL test findings, fallback types)`. -type OutcomeRow = ( - Vec<(Vec, Vec, Vec)>, - Vec<(usize, ErrorRow)>, - Vec<(usize, ErrorRow, i64)>, - Vec, -); - -/// Run the metadata checks on `catalog`'s analysis pool. -#[pyfunction] -fn check_semantic_metadata_rows( - py: Python<'_>, - catalog: PyRef<'_, ProjectCatalog>, - request: RequestInput, -) -> PyResult { - let request = metadata_request(request)?; - let catalog = &catalog.inner; +type NamesRow = (String, Vec); + +/// The relations the metadata request reads, kept for building its diagnostics. +struct MetadataInputs<'py> { + shapes: Bound<'py, PyAny>, + models: Vec>, + sources: Vec>, + tests: Vec>, +} + +/// The project's semantic metadata diagnostics, then its resource SQL diagnostics when asked. +pub(crate) fn metadata_diagnostics<'py>( + classes: &StageClasses<'py>, + (project, profile): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + resource_sql_analysis: bool, + catalog: &validation::ProjectCatalog, +) -> PyResult>> { + let py: Python<'py> = classes.py(); + if !project + .getattr("settings")? + .getattr("sql_analysis")? + .is_truthy()? + { + return Ok(Vec::new()); + } + let (inputs, request) = metadata_request(classes, project, profile)?; let outcome = py .compiler_detach(|| Ok(check_semantic_metadata(&request, catalog))) .map_err(compiler_error)? .map_err(|failure| semantic_error(py, &failure))?; - Ok(( - outcome.models.into_iter().map(model_rows).collect(), - outcome - .sources - .into_iter() - .map(|(index, error)| (index, error_row(error))) - .collect(), - outcome - .sql_tests - .into_iter() - .map(|(index, error, end)| (index, error_row(error), end)) - .collect(), - outcome.fallback_types, - )) -} - -fn error_row(error: MetadataFinding) -> ErrorRow { - (error.code, error.message, error.line, error.column) -} - -fn model_rows(errors: ModelMetadataFindings) -> (Vec, Vec, Vec) { - ( - errors.functions.into_iter().map(error_row).collect(), - errors - .audits - .into_iter() - .map(|finding| { - ( - finding.code, - finding.message, - finding.line, - finding.column, - finding.is_error, - ) - }) - .collect(), - errors.references.into_iter().map(error_row).collect(), - ) + let dialect: Bound<'py, PyAny> = profile.getattr("sql_analysis_dialect")?; + for type_sql in &outcome.fallback_types { + let _ = classes.call( + &classes.normalize_type, + &[ + ("type_sql", &PyString::new(py, type_sql).into_any()), + ("dialect", &dialect), + ], + )?; + } + let mut diagnostics: Vec> = Vec::new(); + for (model, findings) in inputs.models.iter().zip(outcome.models) { + if checked(classes, model)? { + diagnostics.extend(model_diagnostics(classes, model, findings)?); + } + } + for (index, finding) in outcome.sources { + let source: &Bound<'py, PyAny> = &inputs.sources[index]; + let path: Bound<'py, PyAny> = source.getattr("source_file")?.getattr("relative_path")?; + diagnostics.push(diagnostic( + classes, + finding, + (&classes.source_resource, &source.getattr("name")?), + (&path, None), + )?); + } + for (index, finding, end_column) in outcome.sql_tests { + diagnostics.push(sql_test_diagnostic( + classes, + &inputs.tests[index], + finding, + end_column, + )?); + } + if resource_sql_analysis { + let resource: Bound<'py, PyAny> = classes.call( + &classes.inputs.getattr("resource_sql_diagnostics")?, + &[ + ("project", project), + ("shapes", &inputs.shapes), + ("profile", profile), + ], + )?; + diagnostics.extend(items(&resource)?); + } + Ok(diagnostics) } -fn metadata_request( - ( - dialect, - quoted_ignore_case, - return_types, +fn metadata_request<'py>( + classes: &StageClasses<'py>, + project: &Bound<'py, PyAny>, + profile: &Bound<'py, PyAny>, +) -> PyResult<(MetadataInputs<'py>, MetadataRequest)> { + let shapes: Bound<'py, PyAny> = classes.call( + &classes.inputs.getattr("metadata_shapes")?, + &[("project", project), ("profile", profile)], + )?; + let mut tests: Vec> = Vec::new(); + for test in project.getattr("sql_tests")?.try_iter()? { + let test: Bound<'py, PyAny> = test?; + if test + .getattr("payload")? + .is_instance(&classes.model_test_payload)? + { + tests.push(test); + } + } + let columns_by_sql: Bound<'py, PyAny> = classes.call( + &classes.inputs.getattr("sql_test_columns")?, + &[("project", project), ("profile", profile)], + )?; + let functions: Vec = metadata_functions(project)?; + let sources: Vec> = items(&project.getattr("sources")?)?; + let (files, contents) = file_contents(classes.py(), &sources, &tests)?; + let models: Vec> = items(&project.getattr("models")?)?; + let mut metadata_models: Vec = Vec::with_capacity(models.len()); + for model in &models { + metadata_models.push(metadata_model(classes, model)?); + } + let mut metadata_sources: Vec = Vec::with_capacity(sources.len()); + for source in &sources { + let cursor: Bound<'py, PyAny> = source.getattr("source_entry")?.getattr("cursor_column")?; + let index: usize = file_index(&files, &source.getattr("source_file")?)?; + metadata_sources.push(MetadataSource { + name: source.getattr("name")?.extract()?, + cursor_column: if cursor.is_truthy()? { + Some(cursor.extract()?) + } else { + None + }, + contents: contents[index].clone(), + }); + } + let mut metadata_tests: Vec = Vec::with_capacity(tests.len()); + for test in &tests { + let index: usize = file_index(&files, &test.getattr("test_file")?)?; + metadata_tests.push(MetadataSqlTest { + contents: contents[index].clone(), + ctes: test_ctes(&test.getattr("payload")?, &columns_by_sql)?, + }); + } + let request = MetadataRequest { + dialect: profile.getattr("sql_analysis_dialect")?.extract()?, + quoted_ignore_case: profile + .getattr("quoted_identifiers_ignore_case")? + .extract()?, + return_types: pairs(&profile.getattr("function_return_types")?)?, functions, + shapes: shape_rows(&shapes)?, + models: metadata_models, + sources: metadata_sources, + sql_tests: metadata_tests, + }; + let inputs = MetadataInputs { shapes, models, - files, sources, - sql_tests, - ): RequestInput, -) -> PyResult { - let files: Vec> = files.into_iter().map(Arc::from).collect(); - let file = |index: usize| -> PyResult> { - files - .get(index) - .cloned() - .ok_or_else(|| PyValueError::new_err("a metadata row names no file text")) + tests, }; - Ok(MetadataRequest { - dialect, - quoted_ignore_case, - return_types, - functions: functions - .into_iter() - .map(|(key, arguments)| MetadataFunction { key, arguments }) - .collect(), - shapes, - models: models.into_iter().map(metadata_model).collect(), - sources: sources - .into_iter() - .map(|(name, cursor_column, index)| { - Ok(MetadataSource { - name, - cursor_column, - contents: file(index)?, - }) - }) - .collect::>()?, - sql_tests: sql_tests - .into_iter() - .map(|(index, ctes)| { - Ok(MetadataSqlTest { - contents: file(index)?, - ctes, - }) - }) - .collect::>()?, - }) + Ok((inputs, request)) } -fn metadata_model( - ( - name, - query_sql, - authored_sql, - checked, - calls_functions, - references, - cursor_inputs, - cursor, - cursor_type, - audits, - ): ModelInput, -) -> MetadataModel { - MetadataModel { - name, - query_sql, - authored_sql, - checked, - calls_functions, - references, - cursor_inputs, - cursor, - cursor_type, - audits: audits.into_iter().map(metadata_audit).collect(), +/// Declared functions by `name.casefold()`, the last declaration winning in the first's place. +fn metadata_functions(project: &Bound<'_, PyAny>) -> PyResult> { + let functions: Bound<'_, PyDict> = PyDict::new(project.py()); + for function in project.getattr("functions")?.try_iter()? { + let function: Bound<'_, PyAny> = function?; + functions.set_item( + function.getattr("name")?.call_method0("casefold")?, + &function, + )?; } + let mut metadata: Vec = Vec::with_capacity(functions.len()); + for (key, function) in functions.iter() { + let mut arguments: Vec<(String, String)> = Vec::new(); + for argument in function.getattr("arguments")?.try_iter()? { + let argument: Bound<'_, PyAny> = argument?; + arguments.push(( + argument.getattr("name")?.extract()?, + argument.getattr("type")?.extract()?, + )); + } + metadata.push(MetadataFunction { + key: key.extract()?, + arguments, + }); + } + Ok(metadata) } -fn audit_value((kind, text): (String, String)) -> AuditValue { - match kind.as_str() { - "null" => AuditValue::Null, - "bool" => AuditValue::Bool(!text.is_empty()), - "number" => AuditValue::Number(text), - "text" => AuditValue::Text(text), - _ => AuditValue::Other, +/// Each distinct file path's index and the text its first resource carries. +fn file_contents<'py>( + py: Python<'py>, + sources: &[Bound<'py, PyAny>], + tests: &[Bound<'py, PyAny>], +) -> PyResult<(Bound<'py, PyDict>, Vec>)> { + let file_texts: Bound<'py, PyDict> = PyDict::new(py); + for file in sources + .iter() + .map(|source| source.getattr("source_file")) + .chain(tests.iter().map(|test| test.getattr("test_file"))) + { + let file: Bound<'py, PyAny> = file?; + let path: Bound<'py, PyAny> = file.getattr("file_path")?; + if !file_texts.contains(&path)? { + file_texts.set_item(path, file.getattr("contents")?)?; + } + } + let files: Bound<'py, PyDict> = PyDict::new(py); + let mut contents: Vec> = Vec::with_capacity(file_texts.len()); + for (index, (path, text)) in file_texts.iter().enumerate() { + files.set_item(path, index)?; + contents.push(Arc::from(text.extract::()?)); } + Ok((files, contents)) +} + +fn file_index(files: &Bound<'_, PyDict>, file: &Bound<'_, PyAny>) -> PyResult { + files + .as_any() + .get_item(file.getattr("file_path")?)? + .extract() +} + +/// Each authored and expected CTE's name with the columns its body infers. +fn test_ctes( + payload: &Bound<'_, PyAny>, + columns_by_sql: &Bound<'_, PyAny>, +) -> PyResult> { + let mut ctes: Vec = Vec::new(); + for cte in payload + .getattr("authored_ctes")? + .try_iter()? + .chain(payload.getattr("expected_ctes")?.try_iter()?) + { + let cte: Bound<'_, PyAny> = cte?; + ctes.push(( + cte.getattr("name")?.extract()?, + columns_by_sql + .get_item(cte.getattr("sql_body")?)? + .try_iter()? + .map(|name| name?.extract()) + .collect::>()?, + )); + } + Ok(ctes) +} + +/// A checked model's function errors, then its audit findings, then its reference errors. +fn model_diagnostics<'py>( + classes: &StageClasses<'py>, + model: &Bound<'py, PyAny>, + findings: ModelMetadataFindings, +) -> PyResult>> { + let mut diagnostics: Vec> = Vec::new(); + for finding in findings.functions { + diagnostics.push(model_error( + classes, + model, + &finding_row(&finding), + &classes.error, + )?); + } + for audit in findings.audits { + let severity: &Bound<'py, PyAny> = if audit.is_error { + &classes.error + } else { + &classes.warning + }; + let row: FindingRow<'_> = (&audit.code, &audit.message, audit.line, audit.column); + diagnostics.push(model_error(classes, model, &row, severity)?); + } + for finding in findings.references { + diagnostics.push(model_error( + classes, + model, + &finding_row(&finding), + &classes.error, + )?); + } + Ok(diagnostics) +} + +fn sql_test_diagnostic<'py>( + classes: &StageClasses<'py>, + test: &Bound<'py, PyAny>, + finding: MetadataFinding, + end_column: i64, +) -> PyResult> { + let py: Python<'py> = classes.py(); + let path: Bound<'py, PyAny> = test.getattr("test_file")?.getattr("relative_path")?; + let location: Bound<'py, PyAny> = classes.call( + &classes.location, + &[ + ("path", &path), + ("line", &finding.line.into_bound_py_any(py)?), + ("column", &finding.column.into_bound_py_any(py)?), + ("end_line", &finding.line.into_bound_py_any(py)?), + ("end_column", &end_column.into_bound_py_any(py)?), + ], + )?; + diagnostic( + classes, + finding, + (&classes.sql_test_resource, &test.getattr("name")?), + (&path, Some(&location)), + ) } -fn metadata_audit( - (column, definition, expression, target, field, values): AuditInput, -) -> MetadataAudit { - MetadataAudit { - column, - definition, - expression, - target, - field, - values: values.map(|values| values.into_iter().map(audit_value).collect()), +/// `{name: {column: value}}` as ordered rows. +pub(crate) fn shape_rows(shapes: &Bound<'_, PyAny>) -> PyResult { + let mut rows: ShapeRows = Vec::new(); + for item in shapes.call_method0("items")?.try_iter()? { + let (name, columns): (String, Bound<'_, PyAny>) = item?.extract()?; + rows.push((name, pairs(&columns)?)); } + Ok(rows) } -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(check_semantic_metadata_rows, module)?)?; - Ok(()) +/// A `{key: value}` text mapping's items in order. +fn pairs(mapping: &Bound<'_, PyAny>) -> PyResult> { + mapping + .call_method0("items")? + .try_iter()? + .map(|item| item?.extract()) + .collect() +} + +/// `(code, message, line, column)`. +type FindingRow<'a> = (&'a str, &'a str, i64, i64); + +fn finding_row(finding: &MetadataFinding) -> FindingRow<'_> { + (finding.code, &finding.message, finding.line, finding.column) +} + +fn model_error<'py>( + classes: &StageClasses<'py>, + model: &Bound<'py, PyAny>, + &(code, message, line, column): &FindingRow<'_>, + severity: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = classes.py(); + classes.call( + &classes.diagnostic, + &[ + ("phase", &classes.compile_phase), + ("severity", severity), + ("code", &PyString::new(py, code).into_any()), + ("message", &PyString::new(py, message).into_any()), + ("resource_type", &classes.model_resource), + ("resource_name", &model.getattr("name")?), + ("path", &model.getattr("relative_path")?), + ("line", &line.into_bound_py_any(py)?), + ("column", &column.into_bound_py_any(py)?), + ], + ) +} + +fn diagnostic<'py>( + classes: &StageClasses<'py>, + finding: MetadataFinding, + (resource_type, resource_name): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (path, location): (&Bound<'py, PyAny>, Option<&Bound<'py, PyAny>>), +) -> PyResult> { + let py: Python<'py> = classes.py(); + let code: Bound<'py, PyAny> = PyString::new(py, finding.code).into_any(); + let message: Bound<'py, PyAny> = PyString::new(py, &finding.message).into_any(); + let line: Bound<'py, PyAny> = finding.line.into_bound_py_any(py)?; + let column: Bound<'py, PyAny> = finding.column.into_bound_py_any(py)?; + let mut arguments: Vec<(&str, &Bound<'py, PyAny>)> = vec![ + ("phase", &classes.compile_phase), + ("severity", &classes.error), + ("code", &code), + ("message", &message), + ("resource_type", resource_type), + ("resource_name", resource_name), + ("path", path), + ("line", &line), + ("column", &column), + ]; + if let Some(location) = location { + arguments.push(("location", location)); + } + classes.call(&classes.diagnostic, &arguments) +} + +/// Python's `config.get(sql_analysis) is not False and model.binding_validated`. +fn checked(classes: &StageClasses<'_>, model: &Bound<'_, PyAny>) -> PyResult { + let opted: Bound<'_, PyAny> = model + .getattr("config")? + .getattr("values")? + .call_method1("get", (&classes.sql_analysis_key,))?; + if opted.is(PyBool::new(model.py(), false)) { + return Ok(false); + } + model.getattr("binding_validated")?.is_truthy() +} + +fn text(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_instance_of::() { + return Ok(Some(value.extract()?)); + } + Ok(None) +} + +fn column_names(classes: &StageClasses<'_>, value: &Bound<'_, PyAny>) -> PyResult> { + classes + .inputs + .call_method1("column_names", (value,))? + .try_iter()? + .map(|name| name?.extract()) + .collect() +} + +fn metadata_model(classes: &StageClasses<'_>, model: &Bound<'_, PyAny>) -> PyResult { + let values: Bound<'_, PyAny> = model.getattr("config")?.getattr("values")?; + let get = |key: &str| values.call_method1("get", (key,)); + let mut references: Vec = Vec::new(); + for key in METADATA_CONFIG_KEYS { + references.push((key.to_owned(), column_names(classes, &get(key)?)?)); + } + let custom: Bound<'_, PyAny> = get("config")?; + if custom.is_instance_of::() { + let partition: Bound<'_, PyAny> = custom.call_method1("get", ("partition_column",))?; + references.push(( + "partition_column".to_owned(), + column_names(classes, &partition)?, + )); + } + let tolerances: Bound<'_, PyAny> = get("row_diff_tolerances")?; + if tolerances.is_instance_of::() { + let by_column: Bound<'_, PyAny> = tolerances.call_method1("get", ("by_column",))?; + if by_column.is_instance_of::() { + let names: Vec = by_column + .try_iter()? + .map(|name| name?.str()?.extract()) + .collect::>()?; + references.push(("row_diff_tolerances".to_owned(), names)); + } + } + let cursor_inputs: Bound<'_, PyAny> = get("cursor_inputs")?; + let mut upstreams: Vec = Vec::new(); + if cursor_inputs.is_instance_of::() { + for item in cursor_inputs.call_method0("items")?.try_iter()? { + let (upstream, columns): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; + upstreams.push((upstream.str()?.extract()?, column_names(classes, &columns)?)); + } + } + let mut calls_functions: bool = false; + for reference in model.getattr("references")?.try_iter()? { + let kind: Bound<'_, PyAny> = reference?.getattr("ref_kind")?; + if kind.eq(&classes.udf_kind)? || kind.eq(&classes.table_function_kind)? { + calls_functions = true; + break; + } + } + Ok(MetadataModel { + name: model.getattr("name")?.extract()?, + query_sql: model.getattr("query_sql")?.extract()?, + authored_sql: model.getattr("authored_sql")?.extract()?, + checked: checked(classes, model)?, + calls_functions, + references, + cursor_inputs: upstreams, + cursor: text(&get("cursor")?)?, + cursor_type: text(&get("cursor_type")?)?, + audits: metadata_audits(model)?, + }) +} + +fn metadata_audits(model: &Bound<'_, PyAny>) -> PyResult> { + let entry: Bound<'_, PyAny> = model.getattr("schema_entry")?; + if entry.is_none() { + return Ok(Vec::new()); + } + let py: Python<'_> = model.py(); + let mut attached: Vec<(Option, Bound<'_, PyAny>)> = Vec::new(); + for audit in entry.getattr("audits")?.try_iter()? { + attached.push((None, audit?)); + } + for column in entry.getattr("columns")?.try_iter()? { + let column: Bound<'_, PyAny> = column?; + let name: String = column.getattr("name")?.extract()?; + for audit in column.getattr("audits")?.try_iter()? { + attached.push((Some(name.clone()), audit?)); + } + } + let empty: Bound<'_, PyAny> = PyString::new(py, "").into_any(); + let mut audits: Vec = Vec::with_capacity(attached.len()); + for (column, audit) in attached { + let arguments: Bound<'_, PyAny> = audit.getattr("arguments")?; + let values: Bound<'_, PyAny> = arguments.call_method1("get", ("values",))?; + let values: Option> = + if values.is_instance_of::() || values.is_instance_of::() { + Some( + values + .try_iter()? + .map(|value| audit_value(&value?)) + .collect::>()?, + ) + } else { + None + }; + audits.push(MetadataAudit { + column, + definition: audit.getattr("definition_name")?.extract()?, + expression: text(&arguments.call_method1("get", ("expression",))?)?, + target: arguments + .call_method1("get", ("to", &empty))? + .str()? + .extract()?, + field: arguments + .call_method1("get", ("field", &empty))? + .str()? + .extract()?, + values, + }); + } + Ok(audits) +} + +/// An accepted value by Python type: null, bool, number, text or other. +fn audit_value(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_none() { + return Ok(AuditValue::Null); + } + if value.is_instance_of::() { + return Ok(AuditValue::Bool(value.is_truthy()?)); + } + if value.is_instance_of::() || value.is_instance_of::() { + return Ok(AuditValue::Number(value.str()?.extract()?)); + } + if value.is_instance_of::() { + return Ok(AuditValue::Text(value.extract()?)); + } + Ok(AuditValue::Other) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs index 9ce91706e6..c5c67b1a80 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs @@ -1,7 +1,10 @@ pub(crate) mod completion; pub(crate) mod failures; +pub(crate) mod help_catalogue; pub(crate) mod metadata; pub(crate) mod registration; pub(crate) mod resource_sql; pub(crate) mod session_facts; +pub(crate) mod stage; +pub(crate) mod stage_objects; pub(crate) mod type_recovery; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs index 10b484a3e1..487d03d3ea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs @@ -2,13 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::semantic_checks::{ - completion, metadata, resource_sql, type_recovery, -}; +use crate::bindings::_helpers::semantic_checks::{help_catalogue, resource_sql, stage}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - type_recovery::register(module)?; - metadata::register(module)?; resource_sql::register(module)?; - completion::register(module) + help_catalogue::register(module)?; + stage::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs index 3ef89e711a..5fb50715ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs @@ -3,8 +3,9 @@ use std::sync::Arc; use pyo3::prelude::{ - Bound, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python, + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python, }; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::semantic_checks::main::resource_sql_diagnostics::resource_sql_diagnostics; use sqlbuild_analysis::semantic_checks::models::{ @@ -17,156 +18,315 @@ use crate::bindings::models::ProjectCatalog; use crate::bindings::types::CompilerDetach; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const SQL_ANALYSIS_CONFIG_KEY: &str = "sql_analysis"; +const MODEL_HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; +const BUILT_IN_AUDIT_ROOT: &str = ""; +const UDF_KIND: &str = "udf"; +const TABLE_FUNCTION_KIND: &str = "table_fn"; +const TABLE_FUNCTION_STUB_PREFIX: &str = "__sqlbuild_table_function_"; -type Shape = (String, Vec<(String, String)>); -/// `(name, definition, (target kind, target), column, built-in, path, contents, bodies, opts out, relations)`. -type AuditRow = ( - String, - String, - Option<(String, String)>, - Option, - bool, - String, - String, - Vec<(Option, Option)>, - bool, - Vec, -); -/// `(name, expected models, path, contents, compiled body, authored body, opts out)`. -type TestRow = (String, Vec, String, String, String, String, bool); -/// `(key, index, statement, hook path, definition, relations)`. -type HookRow = ( - String, - usize, - String, - Option, - Option, - Vec, -); -/// `(name, path, authored SQL, analysis disabled, hooks)`. -type ModelRow = (String, String, String, bool, Vec); -/// `(dialect, quoted ignore case, require analysis, shapes, audits, tests, models, hook files)`. -type RequestInput = ( - Option, - bool, - bool, - Vec, - Vec, - Vec, - Vec, - Vec<(String, String)>, -); -/// `(kind, name, is error, code, message, path, span, note, help)`. -type DiagnosticRow = ( - &'static str, - String, - bool, - String, - String, - String, - (i64, i64, Option, Option), - Option, - Option, -); - -fn audit(row: AuditRow) -> AuditSqlInput { - let ( - name, - definition_name, - attached_target, - attached_column, - built_in, - path, - contents, - bodies, - opts_out, - relations, - ) = row; - AuditSqlInput { - name, - definition_name, +/// The compile models a resource SQL diagnostic is built from. +struct DiagnosticClasses<'py> { + compiler_diagnostic: Bound<'py, PyAny>, + source_location: Bound<'py, PyAny>, + path: Bound<'py, PyAny>, +} + +/// Bind the project's audit, SQL test and SQL hook bodies on `catalog` against `shapes`. +#[pyfunction] +fn resource_sql_diagnostics_of<'py>( + catalog: PyRef<'_, ProjectCatalog>, + project: &Bound<'py, PyAny>, + shapes: &Bound<'py, PyDict>, + profile: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let request: ResourceSqlRequest = request(project, shapes, profile)?; + let catalog = &catalog.inner; + let outcome = py + .compiler_detach(|| Ok(resource_sql_diagnostics(&request, catalog))) + .map_err(compiler_error)?; + let diagnostics: Vec = match outcome { + Ok(diagnostics) => diagnostics, + Err(failure) => return Err(failure_error(py, failure)?), + }; + let classes = DiagnosticClasses { + compiler_diagnostic: py + .import(COMPILE_MODELS_MODULE)? + .getattr("CompilerDiagnostic")?, + source_location: py + .import(CONTRACT_MODELS_MODULE)? + .getattr("SourceLocation")?, + path: py.import("pathlib")?.getattr("Path")?, + }; + let built: Vec> = diagnostics + .into_iter() + .map(|diagnostic| python_diagnostic(py, &classes, diagnostic)) + .collect::>()?; + PyTuple::new(py, built) +} + +fn request( + project: &Bound<'_, PyAny>, + shapes: &Bound<'_, PyDict>, + profile: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = project.py(); + let reference_class: Bound<'_, PyAny> = py + .import(COMPILE_MODELS_MODULE)? + .getattr("CompileSqlReference")?; + let mut audits: Vec = Vec::new(); + for audit in project.getattr("audits")?.try_iter()? { + audits.push(audit_input(&audit?, &reference_class)?); + } + let mut sql_tests: Vec = Vec::new(); + for test in project.getattr("sql_tests")?.try_iter()? { + sql_tests.push(sql_test_input(&test?)?); + } + let mut models: Vec = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + models.push(model_input(&model?, &reference_class)?); + } + let mut hook_files: Vec<(String, Arc)> = Vec::new(); + for file in project.getattr("sql_hook_files")?.try_iter()? { + let file: Bound<'_, PyAny> = file?; + hook_files.push(( + file.getattr("relative_path")?.str()?.extract()?, + Arc::from(file.getattr("contents")?.extract::()?), + )); + } + Ok(ResourceSqlRequest { + dialect: profile.getattr("sql_analysis_dialect")?.extract()?, + quoted_ignore_case: profile + .getattr("quoted_identifiers_ignore_case")? + .extract()?, + require_sql_analysis: project + .getattr("settings")? + .getattr("require_sql_analysis")? + .extract()?, + shapes: shapes + .iter() + .map(|(name, columns)| shape(&name, &columns)) + .collect::>()?, + audits, + sql_tests, + models, + hook_files, + }) +} + +/// One closed relation shape: its name and `(column, type)` pairs in order. +fn shape( + name: &Bound<'_, PyAny>, + columns: &Bound<'_, PyAny>, +) -> PyResult<(String, Vec<(String, String)>)> { + let mut pairs: Vec<(String, String)> = Vec::new(); + for pair in columns.call_method0("items")?.try_iter()? { + pairs.push(pair?.extract()?); + } + Ok((name.extract()?, pairs)) +} + +fn audit_input( + audit: &Bound<'_, PyAny>, + reference_class: &Bound<'_, PyAny>, +) -> PyResult { + let file: Bound<'_, PyAny> = audit.getattr("audit_file")?; + let block: Bound<'_, PyAny> = audit.getattr("audit_block")?; + let target_name: Bound<'_, PyAny> = audit.getattr("attached_target_name")?; + let attached_target: Option<(String, String)> = if target_name.is_none() { + None + } else { + Some(( + audit.getattr("attached_target_kind")?.str()?.extract()?, + target_name.extract()?, + )) + }; + let first_part: Option = file + .getattr("file_path")? + .getattr("parts")? + .try_iter()? + .next() + .transpose()? + .map(|part| part.extract()) + .transpose()?; + let mut bodies: Vec<(Option, Option)> = Vec::new(); + for attribute in ["sql_body", "measure_sql", "evidence_sql"] { + bodies.push(( + audit.getattr(attribute)?.extract()?, + block.getattr(attribute)?.extract()?, + )); + } + Ok(AuditSqlInput { + name: audit.getattr("name")?.extract()?, + definition_name: audit.getattr("definition_name")?.extract()?, attached_target, - attached_column, - built_in, - path, - contents: Arc::from(contents), + attached_column: audit.getattr("attached_column_name")?.extract()?, + built_in: first_part.as_deref() == Some(BUILT_IN_AUDIT_ROOT), + path: file.getattr("relative_path")?.str()?.extract()?, + contents: Arc::from(file.getattr("contents")?.extract::()?), bodies, - opts_out, - relations, + opts_out: opts_out(&block.getattr("header_values")?)?, + relations: relations(&audit.getattr("references")?, reference_class)?, + }) +} + +fn sql_test_input(test: &Bound<'_, PyAny>) -> PyResult { + let file: Bound<'_, PyAny> = test.getattr("test_file")?; + let block: Bound<'_, PyAny> = test.getattr("test_block")?; + Ok(SqlTestSqlInput { + name: test.getattr("name")?.extract()?, + expected_model_names: test + .getattr("expected_model_names")? + .try_iter()? + .map(|name| name?.extract()) + .collect::>()?, + path: file.getattr("relative_path")?.str()?.extract()?, + contents: Arc::from(file.getattr("contents")?.extract::()?), + sql_body: test.getattr("sql_body")?.extract()?, + authored_body: block.getattr("sql_body")?.extract()?, + opts_out: opts_out(&block.getattr("header_values")?)?, + }) +} + +fn model_input( + model: &Bound<'_, PyAny>, + reference_class: &Bound<'_, PyAny>, +) -> PyResult { + let values: Bound<'_, PyAny> = model.getattr("config")?.getattr("values")?; + let mut hooks: Vec = Vec::new(); + for key in MODEL_HOOK_KEYS { + let entries: Bound<'_, PyAny> = values.call_method1("get", (key,))?; + if !(entries.is_instance_of::() || entries.is_instance_of::()) { + continue; + } + for (index, entry) in entries.try_iter()?.enumerate() { + let entry: Bound<'_, PyAny> = entry?; + let Some(statement) = text(&optional_attribute(&entry, "statement")?)? else { + continue; + }; + let hook_path: Bound<'_, PyAny> = optional_attribute(&entry, "relative_path")?; + let reads: Bound<'_, PyAny> = optional_attribute(&entry, "reads")?; + hooks.push(HookSqlInput { + key: key.to_owned(), + index, + statement, + hook_path: if hook_path.is_none() { + None + } else { + Some(hook_path.str()?.extract()?) + }, + definition: text(&optional_attribute(&entry, "definition_sql")?)?, + relations: if reads.is_truthy()? { + relations(&reads, reference_class)? + } else { + Vec::new() + }, + }); + } } + Ok(HookModelInput { + name: model.getattr("name")?.extract()?, + path: model.getattr("relative_path")?.str()?.extract()?, + authored_sql: Arc::from(model.getattr("authored_sql")?.extract::()?), + analysis_disabled: opts_out(&values)?, + hooks, + }) } -fn sql_test(row: TestRow) -> SqlTestSqlInput { - let (name, expected_model_names, path, contents, sql_body, authored_body, opts_out) = row; - SqlTestSqlInput { - name, - expected_model_names, - path, - contents: Arc::from(contents), - sql_body, - authored_body, - opts_out, +/// The value when it is a `str`, otherwise `None`. +fn text(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_instance_of::() { + return value.extract().map(Some); } + Ok(None) } -fn model(row: ModelRow) -> HookModelInput { - let (name, path, authored_sql, analysis_disabled, hooks) = row; - HookModelInput { - name, - path, - authored_sql: Arc::from(authored_sql), - analysis_disabled, - hooks: hooks - .into_iter() - .map( - |(key, index, statement, hook_path, definition, relations)| HookSqlInput { - key, - index, - statement, - hook_path, - definition, - relations, - }, - ) - .collect(), +/// `getattr(value, name, None)`. +fn optional_attribute<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult> { + if value.hasattr(name)? { + value.getattr(name) + } else { + Ok(value.py().None().into_bound(value.py())) } } -fn request(input: RequestInput) -> ResourceSqlRequest { - let (dialect, quoted_ignore_case, require_sql_analysis, shapes, audits, tests, models, files) = - input; - ResourceSqlRequest { - dialect, - quoted_ignore_case, - require_sql_analysis, - shapes, - audits: audits.into_iter().map(audit).collect(), - sql_tests: tests.into_iter().map(sql_test).collect(), - models: models.into_iter().map(model).collect(), - hook_files: files - .into_iter() - .map(|(path, contents)| (path, Arc::from(contents))) - .collect(), +/// Whether the values set `sql_analysis = false`. +fn opts_out(values: &Bound<'_, PyAny>) -> PyResult { + Ok(values + .call_method1("get", (SQL_ANALYSIS_CONFIG_KEY,))? + .is(PyBool::new(values.py(), false))) +} + +/// `sorted(binding_relation_names(...))` of the `CompileSqlReference`s among `references`. +fn relations( + references: &Bound<'_, PyAny>, + reference_class: &Bound<'_, PyAny>, +) -> PyResult> { + let mut names: Vec = Vec::new(); + for reference in references.try_iter()? { + let reference: Bound<'_, PyAny> = reference?; + if !reference.is_instance(reference_class)? { + continue; + } + let kind: String = reference.getattr("ref_kind")?.str()?.extract()?; + let name: String = reference.getattr("ref_name")?.extract()?; + if kind == UDF_KIND { + continue; + } + names.push(if kind == TABLE_FUNCTION_KIND { + format!("{TABLE_FUNCTION_STUB_PREFIX}{name}") + } else { + name + }); } + names.sort_unstable(); + names.dedup(); + Ok(names) } -fn row(diagnostic: ResourceSqlDiagnostic) -> DiagnosticRow { - let kind = match diagnostic.kind { - ResourceSqlKind::Audit => "audit", - ResourceSqlKind::SqlTest => "sql_test", - ResourceSqlKind::Model => "model", - }; - ( - kind, - diagnostic.resource_name, - diagnostic.is_error, - diagnostic.code, - diagnostic.message, - diagnostic.path, - diagnostic.span, - diagnostic.note, - diagnostic.help, - ) +fn python_diagnostic<'py>( + py: Python<'py>, + classes: &DiagnosticClasses<'py>, + diagnostic: ResourceSqlDiagnostic, +) -> PyResult> { + let (line, column, end_line, end_column) = diagnostic.span; + let location: Bound<'py, PyDict> = PyDict::new(py); + location.set_item("path", classes.path.call1((&diagnostic.path,))?)?; + location.set_item("line", line)?; + location.set_item("column", column)?; + location.set_item("end_line", end_line)?; + location.set_item("end_column", end_column)?; + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("phase", "compile")?; + fields.set_item( + "severity", + if diagnostic.is_error { + "error" + } else { + "warning" + }, + )?; + fields.set_item("code", diagnostic.code)?; + fields.set_item("message", diagnostic.message)?; + fields.set_item( + "resource_type", + match diagnostic.kind { + ResourceSqlKind::Audit => "audit", + ResourceSqlKind::SqlTest => "sql_test", + ResourceSqlKind::Model => "model", + }, + )?; + fields.set_item("resource_name", diagnostic.resource_name)?; + fields.set_item( + "location", + classes.source_location.call((), Some(&location))?, + )?; + fields.set_item("notes", PyTuple::new(py, diagnostic.note)?)?; + fields.set_item("help", diagnostic.help)?; + classes.compiler_diagnostic.call((), Some(&fields)) } fn failure_error(py: Python<'_>, failure: ResourceSqlFailure) -> PyResult { @@ -182,24 +342,6 @@ fn failure_error(py: Python<'_>, failure: ResourceSqlFailure) -> PyResult }) } -/// Bind audit, SQL test and SQL hook bodies on `catalog` as Python's resource SQL checks do. -#[pyfunction] -fn resource_sql_diagnostic_rows( - py: Python<'_>, - catalog: PyRef<'_, ProjectCatalog>, - input: RequestInput, -) -> PyResult> { - let request = request(input); - let catalog = &catalog.inner; - let outcome = py - .compiler_detach(|| Ok(resource_sql_diagnostics(&request, catalog))) - .map_err(compiler_error)?; - match outcome { - Ok(diagnostics) => Ok(diagnostics.into_iter().map(row).collect()), - Err(failure) => Err(failure_error(py, failure)?), - } -} - pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resource_sql_diagnostic_rows, module)?) + module.add_function(wrap_pyfunction!(resource_sql_diagnostics_of, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs new file mode 100644 index 0000000000..9cd3ca1d7b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs @@ -0,0 +1,121 @@ +//! The native semantic completion stage: type recovery, metadata checks, then completion. + +use std::collections::HashSet; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult}; +use pyo3::types::{PyDict, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::boundary::panics::NativeCompilerError; +use crate::bindings::_helpers::semantic_checks::completion::completed_diagnostics; +use crate::bindings::_helpers::semantic_checks::metadata::metadata_diagnostics; +use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; +use crate::bindings::_helpers::semantic_checks::type_recovery::{ + StageShared, items, recovered_types, +}; +use crate::bindings::models::ProjectCatalog; + +const NO_CATALOG_MESSAGE: &str = + "NativeCompilerError: native semantic completion: the project has no analysis catalog"; + +/// The completed project from keyword arguments; an internal failure raises `NativeCompilerError`. +#[pyfunction] +#[pyo3(signature = (**request))] +fn complete_semantic_project<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("complete_semantic_project", request)?; + let project: Bound<'py, PyAny> = request.get("project")?; + let profile: Bound<'py, PyAny> = request.get("profile")?; + let binding_results: Bound<'py, PyAny> = request.get("binding_results")?; + let resource_sql_analysis: bool = request.get("resource_sql_analysis")?.extract()?; + let session: Option> = + match request.optional("session")? { + Some(session) if !session.is_none() => Some(session.extract()?), + _ => None, + }; + let catalog: PyRef<'py, ProjectCatalog> = project_catalog(&project)?; + let classes = StageClasses::new(request.py())?; + let shared = StageShared { + catalog: &catalog.inner, + session: session.as_deref(), + session_models: session_fact_models(&project, session.as_deref())?, + }; + let recovered: Bound<'py, PyAny> = + recovered_types(&classes, &project, &binding_results, &shared)?; + let metadata: Vec> = metadata_diagnostics( + &classes, + (&recovered, &profile), + resource_sql_analysis, + &catalog.inner, + )?; + let mut diagnostics: Vec> = items(&recovered.getattr("diagnostics")?)?; + diagnostics.extend(metadata); + let combined: Bound<'py, PyAny> = classes.project_with( + &recovered, + &[( + "diagnostics", + &PyTuple::new(request.py(), diagnostics)?.into_any(), + )], + )?; + completed_diagnostics(&classes, &combined, &shared) +} + +/// The metadata and resource SQL diagnostics alone, as the stage appends them. +#[pyfunction] +#[pyo3(signature = (*, project, profile, resource_sql_analysis))] +fn semantic_metadata_diagnostics<'py>( + project: &Bound<'py, PyAny>, + profile: &Bound<'py, PyAny>, + resource_sql_analysis: bool, +) -> PyResult> { + let catalog: PyRef<'py, ProjectCatalog> = project_catalog(project)?; + let classes = StageClasses::new(project.py())?; + let diagnostics: Vec> = metadata_diagnostics( + &classes, + (project, profile), + resource_sql_analysis, + &catalog.inner, + )?; + PyTuple::new(project.py(), diagnostics) +} + +fn project_catalog<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let binding_catalog: Bound<'py, PyAny> = project.getattr("binding_catalog")?; + if binding_catalog.is_none() { + return Err(NativeCompilerError::new_err(NO_CATALOG_MESSAGE)); + } + binding_catalog.getattr("native")?.extract() +} + +/// Models whose output names and lineage are the finished session's, so need no payload. +fn session_fact_models( + project: &Bound<'_, PyAny>, + session: Option<&NativeModelAnalysisSession>, +) -> PyResult> { + let Some(session) = session else { + return Ok(HashSet::new()); + }; + let kept: HashSet = session.fact_models().into_iter().collect(); + let mut models: HashSet = HashSet::new(); + for model in project.getattr("models")?.try_iter()? { + let model: Bound<'_, PyAny> = model?; + let name: String = model.getattr("name")?.extract()?; + if !kept.contains(&name) { + continue; + } + let proof: Bound<'_, PyAny> = model.getattr("dynamic_column_contract")?; + if !proof.is_none() && proof.getattr("output_proven")?.is_truthy()? { + continue; + } + models.insert(name); + } + Ok(models) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(complete_semantic_project, module)?)?; + module.add_function(wrap_pyfunction!(semantic_metadata_diagnostics, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs new file mode 100644 index 0000000000..1fd931636b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs @@ -0,0 +1,134 @@ +//! The Python models, enums and compile helpers the semantic completion stage reads and builds. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; + +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const PROJECT_WITH_MODULE: &str = "sqlbuild._native"; +const TYPE_RECOVERY_MODULE: &str = "sqlbuild._native"; +const BINDING_DIAGNOSTICS_MODULE: &str = "sqlbuild._native"; +const COMPLETION_INPUTS_MODULE: &str = + "sqlbuild.compiler.compile.classes.semantic_completion_inputs"; +const SCHEMA_VALIDATION_MODULE: &str = "sqlbuild.compiler.sql_analysis.main._schema_validation"; +const NORMALIZE_TYPE_MODULE: &str = "sqlbuild.adapter.type_system.main.normalize_type"; +const DATACLASSES_MODULE: &str = "dataclasses"; +const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; +const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; + +/// Imported once per stage run. +pub(crate) struct StageClasses<'py> { + pub(crate) diagnostic: Bound<'py, PyAny>, + pub(crate) location: Bound<'py, PyAny>, + pub(crate) model_test_payload: Bound<'py, PyAny>, + pub(crate) compile_phase: Bound<'py, PyAny>, + pub(crate) error: Bound<'py, PyAny>, + pub(crate) warning: Bound<'py, PyAny>, + pub(crate) model_resource: Bound<'py, PyAny>, + pub(crate) source_resource: Bound<'py, PyAny>, + pub(crate) sql_test_resource: Bound<'py, PyAny>, + pub(crate) opt_out_code: Bound<'py, PyAny>, + pub(crate) replace: Bound<'py, PyAny>, + pub(crate) project_with: Bound<'py, PyAny>, + pub(crate) with_type_recovery: Bound<'py, PyAny>, + pub(crate) with_binding_diagnostics: Bound<'py, PyAny>, + pub(crate) inputs: Bound<'py, PyAny>, + pub(crate) schema_validations: Bound<'py, PyAny>, + pub(crate) normalize_type: Bound<'py, PyAny>, + pub(crate) sql_analysis_key: Bound<'py, PyAny>, + pub(crate) udf_kind: Bound<'py, PyAny>, + pub(crate) table_function_kind: Bound<'py, PyAny>, +} + +impl<'py> StageClasses<'py> { + pub(crate) fn new(py: Python<'py>) -> PyResult { + let models = py.import(COMPILE_MODELS_MODULE)?; + let types = py.import(COMPILE_TYPES_MODULE)?; + let severity = types.getattr("DiagnosticSeverity")?; + let resource = types.getattr("CompiledResourceType")?; + let kinds = py + .import(REFERENCE_TYPES_MODULE)? + .getattr("SqlReferenceKind")?; + Ok(Self { + diagnostic: models.getattr("CompilerDiagnostic")?, + location: py + .import(CONTRACT_MODELS_MODULE)? + .getattr("SourceLocation")?, + model_test_payload: models.getattr("CompiledModelSqlTestPayload")?, + compile_phase: types.getattr("DiagnosticPhase")?.getattr("COMPILE")?, + error: severity.getattr("ERROR")?, + warning: severity.getattr("WARNING")?, + model_resource: resource.getattr("MODEL")?, + source_resource: resource.getattr("SOURCE")?, + sql_test_resource: resource.getattr("SQL_TEST")?, + opt_out_code: py + .import(COMPILE_CONSTANTS_MODULE)? + .getattr("UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE")?, + replace: py.import(DATACLASSES_MODULE)?.getattr("replace")?, + project_with: py.import(PROJECT_WITH_MODULE)?.getattr("project_with")?, + with_type_recovery: py + .import(TYPE_RECOVERY_MODULE)? + .getattr("with_type_recovery")?, + with_binding_diagnostics: py + .import(BINDING_DIAGNOSTICS_MODULE)? + .getattr("with_binding_diagnostics")?, + inputs: py + .import(COMPLETION_INPUTS_MODULE)? + .getattr("SemanticCompletionInputs")?, + schema_validations: py + .import(SCHEMA_VALIDATION_MODULE)? + .getattr("get_schema_validations")?, + normalize_type: py + .import(NORMALIZE_TYPE_MODULE)? + .getattr("normalize_type")?, + sql_analysis_key: py + .import(DISCOVERY_CONSTANTS_MODULE)? + .getattr("SQL_ANALYSIS_CONFIG_KEY")?, + udf_kind: kinds.getattr("UDF")?, + table_function_kind: kinds.getattr("TABLE_FUNCTION")?, + }) + } + + pub(crate) fn py(&self) -> Python<'py> { + self.diagnostic.py() + } + + /// `callable(**pairs)`. + pub(crate) fn call( + &self, + callable: &Bound<'py, PyAny>, + pairs: &[(&str, &Bound<'py, PyAny>)], + ) -> PyResult> { + let kwargs: Bound<'py, PyDict> = PyDict::new(self.py()); + for (name, value) in pairs { + kwargs.set_item(*name, *value)?; + } + callable.call((), Some(&kwargs)) + } + + /// `dataclasses.replace(value, **pairs)`. + pub(crate) fn replaced( + &self, + value: &Bound<'py, PyAny>, + pairs: &[(&str, &Bound<'py, PyAny>)], + ) -> PyResult> { + let kwargs: Bound<'py, PyDict> = PyDict::new(self.py()); + for (name, value) in pairs { + kwargs.set_item(*name, *value)?; + } + self.replace.call((value,), Some(&kwargs)) + } + + /// `project_with(project=project, **pairs)`. + pub(crate) fn project_with( + &self, + project: &Bound<'py, PyAny>, + pairs: &[(&str, &Bound<'py, PyAny>)], + ) -> PyResult> { + let mut all: Vec<(&str, &Bound<'py, PyAny>)> = vec![("project", project)]; + all.extend_from_slice(pairs); + self.call(&self.project_with, &all) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs index c95f6030ed..85df3f79b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs @@ -1,14 +1,16 @@ -//! Native output-type recovery for failed models, with Python's revalidation in between. +//! Native output-type recovery for failed models, revalidating planned models in Python. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyRef, PyResult, Python}; -use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use std::collections::HashSet; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyString, PyTuple}; use sqlbuild_analysis::semantic_checks::main::finish_type_recovery::finish_type_recovery; use sqlbuild_analysis::semantic_checks::main::plan_type_recovery::plan_type_recovery; use sqlbuild_analysis::semantic_checks::models::{ - DiagnosticOwner, LineageOutput, ModelBinding, RawBinding, RecoveryModel, TypeRecoveryPlan, - TypeRecoveryRequest, TypeRecoveryStep, + DiagnosticOwner, LineageOutput, ModelBinding, RawBinding, RecoveryModel, TypeRecoveryRequest, }; use sqlbuild_analysis::semantic_checks::types::RevisedBinding; +use sqlbuild_analysis::semantic_validation::models as validation; use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; use crate::bindings::_helpers::boundary::panics::compiler_error; @@ -16,91 +18,170 @@ use crate::bindings::_helpers::semantic_checks::failures::{internal_error, seman use crate::bindings::_helpers::semantic_checks::session_facts::{ column_names, session_model_facts, }; -use crate::bindings::models::ProjectCatalog; +use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; use crate::bindings::types::CompilerDetach; +const UNCHANGED: &str = "unchanged"; +const UNKNOWN_RECOVERED_TYPE: &str = "UNKNOWN"; + /// `(output_column, [(resource_name, column_name)])`. pub(crate) type LineageInput = (String, Vec<(String, String)>); -/// `(name, query_sql, inferred names, reference names, lineage, bindings, raw bindings)`. -type ModelInput = ( - String, - String, - Option>, - Vec, - Option>, - Vec<(usize, String, String, bool)>, - Vec<(usize, String, String, Option, Option)>, -); -/// `(dialect, models, [(id, code, is_model, resource_name)])`. -type RequestInput = ( - Option, - Vec, - Vec<(usize, String, bool, Option)>, -); -/// `(kept [(diagnostic index, appended note)], model binding positions)`. -type OutcomeRow = (Vec<(usize, Option)>, Vec>); - -/// One planned type recovery, held between the native plan and Python's revalidation. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct SemanticTypeRecovery { - request: TypeRecoveryRequest, - step: TypeRecoveryStep, -} -#[pymethods] -impl SemanticTypeRecovery { - /// `unchanged` or `planned`. - #[getter] - fn status(&self) -> &'static str { - self.step.status() - } - - /// Poisoned `(model, column)` outputs in Python's dict order. - #[getter] - fn poisoned(&self) -> Vec<(String, String)> { - self.step - .plan() - .map(TypeRecoveryPlan::poisoned_outputs) - .unwrap_or_default() - } - - /// Model indexes Python revalidates with unknown poisoned types, in order. - #[getter] - fn revalidated(&self) -> Vec { - self.step - .plan() - .map(TypeRecoveryPlan::revalidated_models) - .unwrap_or_default() - } - - /// Retained diagnostics and model binding positions; an internal failure raises. - fn finish(&self, revised: Vec>) -> PyResult { - let plan = self - .step - .plan() - .ok_or_else(|| internal_error("finishing a type recovery that has no plan"))?; - finish_type_recovery(&self.request, plan, &revised) - .map(|outcome| (outcome.kept, outcome.model_bindings)) - .map_err(|reason| internal_error(&reason)) - } +/// The analysis state every semantic completion step shares. +pub(crate) struct StageShared<'a> { + pub(crate) catalog: &'a validation::ProjectCatalog, + pub(crate) session: Option<&'a NativeModelAnalysisSession>, + /// Models whose output names and lineage the finished session supplies. + pub(crate) session_models: HashSet, } -/// Plan type recovery, reading names and lineage from `session` for models with no lineage. -#[pyfunction] -#[pyo3(signature = (catalog, request, session=None))] -fn plan_semantic_type_recovery( - py: Python<'_>, - catalog: PyRef<'_, ProjectCatalog>, - request: RequestInput, - session: Option>, -) -> PyResult { - let request = recovery_request(request, session.as_deref()).map_err(compiler_error)?; - let catalog = &catalog.inner; +/// The project with recovered output types; unchanged when no model has a blocking binding. +pub(crate) fn recovered_types<'py>( + classes: &StageClasses<'py>, + project: &Bound<'py, PyAny>, + binding_results: &Bound<'py, PyAny>, + shared: &StageShared<'_>, +) -> PyResult> { + let py: Python<'py> = classes.py(); + let models: Vec> = items(&project.getattr("models")?)?; + if !blocking_binding_found(&models)? { + return Ok(project.clone()); + } + let ids: Bound<'py, PyDict> = diagnostic_ids(project, &models)?; + let raw_ids: Bound<'py, PyDict> = PyDict::new(py); + for rows in binding_results.call_method0("values")?.try_iter()? { + for row in rows?.try_iter()? { + let row: Bound<'py, PyAny> = row?; + if !raw_ids.contains(&row)? { + raw_ids.set_item(&row, raw_ids.len())?; + } + } + } + let diagnostics: Vec> = items(&project.getattr("diagnostics")?)?; + let mut owners: Vec = Vec::with_capacity(diagnostics.len()); + for item in &diagnostics { + owners.push(DiagnosticOwner { + id: id_of(&ids, item)?, + code: item.getattr("code")?.extract()?, + is_model: item.getattr("resource_type")?.eq(&classes.model_resource)?, + resource_name: item.getattr("resource_name")?.extract()?, + }); + } + let mut recovery_models: Vec = Vec::with_capacity(models.len()); + for model in &models { + recovery_models.push(recovery_model( + model, + binding_results, + (&ids, &raw_ids), + shared, + )?); + } + let request = TypeRecoveryRequest { + dialect: project.getattr("sql_analysis_dialect")?.extract()?, + models: recovery_models, + diagnostics: owners, + }; let step = py - .compiler_detach(|| Ok(plan_type_recovery(&request, catalog))) + .compiler_detach(|| Ok(plan_type_recovery(&request, shared.catalog))) .map_err(compiler_error)? .map_err(|failure| semantic_error(py, &failure))?; - Ok(SemanticTypeRecovery { request, step }) + if step.status() == UNCHANGED { + return Ok(project.clone()); + } + let plan = step + .plan() + .ok_or_else(|| internal_error("finishing a type recovery that has no plan"))?; + let poisoned: Vec<(String, String)> = plan.poisoned_outputs(); + let revised: Vec> = + revalidated(classes, project, &plan.revalidated_models(), &poisoned)?; + let outcome = + finish_type_recovery(&request, plan, &revised).map_err(|reason| internal_error(&reason))?; + let mut kept: Vec> = Vec::with_capacity(outcome.kept.len()); + for (index, note) in outcome.kept { + let original: &Bound<'py, PyAny> = &diagnostics[index]; + let mut notes: Vec> = items(&original.getattr("notes")?)?; + if let Some(note) = note { + notes.push(PyString::new(py, ¬e).into_any()); + } + kept.push(classes.replaced(original, &[("notes", &PyTuple::new(py, notes)?.into_any())])?); + } + let poisoned_outputs: Bound<'py, PyAny> = PyFrozenSet::new(py, &poisoned)?.into_any(); + let mut recovered: Vec> = Vec::with_capacity(models.len()); + for (model, positions) in models.iter().zip(outcome.model_bindings) { + let selected: Vec> = positions + .into_iter() + .map(|position| kept[position].clone()) + .collect(); + recovered.push(classes.call( + &classes.with_type_recovery, + &[ + ("model", model), + ("diagnostics", &PyTuple::new(py, selected)?.into_any()), + ("poisoned", &poisoned_outputs), + ], + )?); + } + classes.project_with( + project, + &[ + ("models", &PyTuple::new(py, recovered)?.into_any()), + ("diagnostics", &PyTuple::new(py, kept)?.into_any()), + ], + ) +} + +/// Every item of a Python iterable. +pub(crate) fn items<'py>(value: &Bound<'py, PyAny>) -> PyResult>> { + value.try_iter()?.collect() +} + +/// One id per distinct diagnostic value, project diagnostics first, as Python's dicts see them. +pub(crate) fn diagnostic_ids<'py>( + project: &Bound<'py, PyAny>, + models: &[Bound<'py, PyAny>], +) -> PyResult> { + let ids: Bound<'py, PyDict> = PyDict::new(project.py()); + for item in project.getattr("diagnostics")?.try_iter()? { + let item: Bound<'py, PyAny> = item?; + if !ids.contains(&item)? { + ids.set_item(&item, ids.len())?; + } + } + for model in models { + for item in model.getattr("binding_diagnostics")?.try_iter()? { + let item: Bound<'py, PyAny> = item?; + if !ids.contains(&item)? { + ids.set_item(&item, ids.len())?; + } + } + } + Ok(ids) +} + +pub(crate) fn id_of(ids: &Bound<'_, PyDict>, item: &Bound<'_, PyAny>) -> PyResult { + ids.as_any().get_item(item)?.extract() +} + +/// Each output column with the `(resource, column)` pairs it reads. +pub(crate) fn lineage_payload(model: &Bound<'_, PyAny>) -> PyResult> { + let columns: Bound<'_, PyAny> = model.getattr("fast_lineage_columns")?; + if !columns.is_truthy()? { + return Ok(Vec::new()); + } + let mut lineage: Vec = Vec::new(); + for output in columns.try_iter()? { + let output: Bound<'_, PyAny> = output?; + let mut upstream: Vec<(String, String)> = Vec::new(); + for source in output.getattr("upstream_columns")?.try_iter()? { + let source: Bound<'_, PyAny> = source?; + upstream.push(( + source.getattr("resource_name")?.extract()?, + source.getattr("column_name")?.extract()?, + )); + } + lineage.push((output.getattr("output_column")?.extract()?, upstream)); + } + Ok(lineage) } pub(crate) fn lineage_outputs(lineage: Vec) -> Vec { @@ -113,69 +194,137 @@ pub(crate) fn lineage_outputs(lineage: Vec) -> Vec .collect() } -fn recovery_request( - (dialect, models, diagnostics): RequestInput, - session: Option<&NativeModelAnalysisSession>, -) -> Result { - Ok(TypeRecoveryRequest { - dialect, - models: models - .into_iter() - .map(|model| recovery_model(model, session)) - .collect::>()?, - diagnostics: diagnostics - .into_iter() - .map(|(id, code, is_model, resource_name)| DiagnosticOwner { - id, - code, - is_model, - resource_name, - }) - .collect(), - }) +/// The names of a model's inferred columns, or `None` when its shape is unknown. +pub(crate) fn inferred_names(model: &Bound<'_, PyAny>) -> PyResult>> { + let columns: Bound<'_, PyAny> = model.getattr("inferred_columns")?; + if columns.is_none() { + return Ok(None); + } + columns + .try_iter()? + .map(|column| column?.getattr("name")?.extract()) + .collect::>() + .map(Some) +} + +fn blocking_binding_found(models: &[Bound<'_, PyAny>]) -> PyResult { + for model in models { + for item in model.getattr("binding_diagnostics")?.try_iter()? { + if item?.getattr("is_error")?.is_truthy()? { + return Ok(true); + } + } + } + Ok(false) } fn recovery_model( - (name, query_sql, inferred_columns, references, lineage, bindings, raw_bindings): ModelInput, - session: Option<&NativeModelAnalysisSession>, -) -> Result { - let (inferred_columns, lineage) = match lineage { - Some(lineage) => (inferred_columns, lineage), - None => { - let facts = session_model_facts(session, &name)?; - (column_names(facts), facts.lineage.clone()) + model: &Bound<'_, PyAny>, + binding_results: &Bound<'_, PyAny>, + (ids, raw_ids): (&Bound<'_, PyDict>, &Bound<'_, PyDict>), + shared: &StageShared<'_>, +) -> PyResult { + let name: String = model.getattr("name")?.extract()?; + let from_session: bool = shared.session_models.contains(&name); + let bindings: Vec> = items(&model.getattr("binding_diagnostics")?)?; + let mut model_bindings: Vec = Vec::with_capacity(bindings.len()); + for item in &bindings { + model_bindings.push(ModelBinding { + id: id_of(ids, item)?, + code: item.getattr("code")?.extract()?, + message: item.getattr("message")?.extract()?, + is_error: item.getattr("is_error")?.extract()?, + }); + } + let mut raw_bindings: Vec = Vec::new(); + if !bindings.is_empty() { + let raws: Bound<'_, PyAny> = + binding_results.call_method1("get", (&name, PyList::empty(model.py())))?; + for raw in raws.try_iter()? { + let raw: Bound<'_, PyAny> = raw?; + raw_bindings.push(RawBinding { + id: id_of(raw_ids, &raw)?, + code: raw.get_item(0)?.extract()?, + message: raw.get_item(1)?.extract()?, + start: raw.get_item(4)?.extract()?, + end: raw.get_item(5)?.extract()?, + }); } + } + let (inferred_columns, lineage) = if from_session { + let facts = session_model_facts(shared.session, &name).map_err(compiler_error)?; + (column_names(facts), facts.lineage.clone()) + } else { + (inferred_names(model)?, lineage_payload(model)?) }; + let mut references: Vec = Vec::new(); + for reference in model.getattr("references")?.try_iter()? { + references.push(reference?.getattr("ref_name")?.extract()?); + } Ok(RecoveryModel { name, - query_sql, + query_sql: model.getattr("query_sql")?.extract()?, inferred_columns, references, lineage: lineage_outputs(lineage), - bindings: bindings - .into_iter() - .map(|(id, code, message, is_error)| ModelBinding { - id, - code, - message, - is_error, - }) - .collect(), - raw_bindings: raw_bindings - .into_iter() - .map(|(id, code, message, start, end)| RawBinding { - id, - code, - message, - start, - end, - }) - .collect(), + bindings: model_bindings, + raw_bindings, }) } -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(plan_semantic_type_recovery, module)?)?; - Ok(()) +/// Python's revalidation of each planned model with unknown types for poisoned outputs. +fn revalidated<'py>( + classes: &StageClasses<'py>, + project: &Bound<'py, PyAny>, + models: &[usize], + poisoned: &[(String, String)], +) -> PyResult>> { + if models.is_empty() { + return Ok(Vec::new()); + } + let py: Python<'py> = classes.py(); + let shapes: Bound<'py, PyAny> = classes.inputs.call_method1("shapes", (project,))?; + let unknown: Bound<'py, PyDict> = PyDict::new(py); + for item in shapes.call_method0("items")?.try_iter()? { + let (name, columns): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + let copied: Bound<'py, PyDict> = PyDict::new(py); + copied.update(columns.downcast::()?.as_mapping())?; + unknown.set_item(name, copied)?; + } + for (name, column) in poisoned { + if let Some(columns) = unknown.get_item(name)? + && columns.contains(column)? + { + columns.set_item(column, UNKNOWN_RECOVERED_TYPE)?; + } + } + let project_models: Bound<'py, PyAny> = project.getattr("models")?; + let mut revised: Vec> = Vec::with_capacity(models.len()); + for index in models { + let request: Bound<'py, PyAny> = classes.call( + &classes.inputs.getattr("revalidation_request")?, + &[ + ("project", project), + ("model", &project_models.get_item(*index)?), + ("shapes", &unknown.clone().into_any()), + ], + )?; + let result: Bound<'py, PyAny> = classes + .call( + &classes.schema_validations, + &[("requests", &PyTuple::new(py, [request])?.into_any())], + )? + .get_item(0)?; + let mut rows: Vec = Vec::new(); + for item in result.getattr("diagnostics")?.try_iter()? { + let item: Bound<'py, PyAny> = item?; + rows.push(( + item.getattr("code")?.extract()?, + item.getattr("start")?.extract()?, + item.getattr("end")?.extract()?, + )); + } + revised.push(rows); + } + Ok(revised) } diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 86bb63b8e9..b1e9de0921 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -4,10 +4,10 @@ use sqlbuild_rules::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::{ - analysis, analysis_session, attachments, boundary, cache, compile_driver, compile_outputs, - compiled_project, contracts, dag_artifact, discovery, fingerprints, graph, lineage, manifest, - model_config, project_assembly, refactor, render, rules, scopes, semantic_checks, - sql_test_glue, sqltext, type_system, + analysis, analysis_session, attachments, boundary, cache, compile_attachment, compile_driver, + compile_outputs, compiled_project, contracts, dag_artifact, discovery, fingerprints, graph, + lineage, manifest, model_config, project_assembly, refactor, render, rules, scopes, + semantic_checks, sql_test_glue, sqltext, type_system, }; use crate::bindings::models; @@ -58,6 +58,7 @@ fn register_analysis_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { compile_outputs::registration::register(module)?; graph::registration::register(module)?; compiled_project::registration::register(module)?; + compile_attachment::registration::register(module)?; compile_driver::registration::register(module)?; dag_artifact::registration::register(module)?; manifest::registration::register(module)?; diff --git a/fensu.toml b/fensu.toml index a4d7b2e642..58169a47a0 100644 --- a/fensu.toml +++ b/fensu.toml @@ -23,11 +23,6 @@ exclude = ["tests/e2e/fixtures/**"] [targets.python.thresholds] max_positional_args = 1 -[[targets.python.rule_exceptions]] -rule = "FFS120" -path = "src/sqlbuild/compiler/compiled_project/main/compiled_project_record.py" -reason = "Preserve the existing positional project argument while adding an optional keyword-only projection flag for custom-rule hosts." - [[targets.python.rule_exceptions]] rule = "FFR001" path = "src/sqlbuild/cli/commands/models.py" @@ -78,6 +73,11 @@ rule = "FFR309" path = "src/sqlbuild/compiler/lineage/__init__.py" reason = "Column lineage is built natively; the package keeps only the lineage models, enums and errors the native graph and its callers share." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/compiler/analysis_session/__init__.py" +reason = "Model analysis runs natively; the package keeps only the request and result models, row types and constants the native session and its callers share." + [[targets.python.rule_exceptions]] rule = "FFR601" path = "src/sqlbuild/adapter/contract/classes/base_adapter.py" diff --git a/python-boundary.toml b/python-boundary.toml index 3bef235801..b7e53da950 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 130215 +remaining_lines = 125499 [[stays]] path = "executor" diff --git a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py index 5ea544538f..fcdfecef9c 100644 --- a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py +++ b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py @@ -6,6 +6,7 @@ from concurrent.futures import ThreadPoolExecutor from pathlib import Path +import sqlbuild._native as _native from scripts.compiler_differential._helpers.comparing.comparison import ( diagnostic_codes, first_diagnostic, @@ -39,7 +40,6 @@ ) from sqlbuild.compiler.auditing.constants import BUILT_IN_AUDIT_SHADOW_CODE from sqlbuild.compiler.compile import constants as compile_constants -from sqlbuild.compiler.compile._helpers.diagnostics.help import semantic_help_catalogue from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.discovery.exceptions import DiscoveryError from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError @@ -118,7 +118,7 @@ def render_error_codes() -> frozenset[str]: def analysis_error_codes(compiler_root: Path) -> frozenset[str]: """Return the help-catalogue codes plus every code in the analysis-stage modules and crates.""" - found: set[str] = set(semantic_help_catalogue()) + found: set[str] = {code for code, _ in _native.semantic_help_catalogue()} for root in ANALYSIS_CODE_SCAN_ROOTS: path: Path = compiler_root / root for module in sorted(path.rglob("*.py")) if path.is_dir() else (path,): diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 208b7c1233..42d2504d8b 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -661,7 +661,6 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: ANALYSIS_CODE_SCAN_ROOTS: tuple[str, ...] = ( "compile/_helpers/assembly", "compile/_helpers/analysis", - "compile/_helpers/attachment/model_config.py", "compile/_helpers/diagnostics", "contracts", "pipeline/_helpers/target_validation.py", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 6b8d8c4dd1..899d0a65d9 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -15,6 +15,7 @@ from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt from sqlbuild.cli.entry.models import CompileCommandRequest, CompileProfileFlags from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity +from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache from sqlbuild.compiler.compile.models import ( CompileAdapterContext, CompileAnalysisSelection, @@ -22,7 +23,9 @@ from sqlbuild.compiler.compile.models import ( CompiledAudit, CompiledFunction, CompiledLineageColumnFact, + CompiledModel, CompiledObjectKey, + CompiledProject, CompiledRelationLocation, CompiledSeed, CompiledSource, @@ -38,21 +41,36 @@ from sqlbuild.compiler.compile.models import ( CompileSqlScenarioInput, CompileSqlTestInput, DeclarationScopeBuild, + DeclarationScopeResolver, DynamicColumnContractProof, FunctionArgument, FunctionReturnColumn, InferredColumn, LoadedMacro, MacroContext, + ModelInputBuildContext, + NativeModelConfigInputs, + NativeModelConfigSession, + PythonSqlReferenceReport, StaticMacroExport, StaticMacroFault, ) from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, + AuditInput, + CompiledAuditView, + CompiledFunctionView, CompiledModelView, CompiledProjectView, + CompiledSeedView, + CompiledSourceView, CompileInputsView, + FunctionInput, FunctionLanguage, + ModelInput, + SeedInput, + SourceInput, + TypedSqlValueRenderer, ) from sqlbuild.compiler.discovery.models import ( ConstantDeclaration, @@ -140,6 +158,7 @@ from sqlbuild.spec.contracts.models import ( SourceLocation, TargetConfig, ) +from sqlbuild.sql_values.models import SqlValue BUILD_IDENTITY: str @@ -768,6 +787,112 @@ class NativeCompiledProject: def graph_builds(self) -> int: ... def __eq__(self, other: object) -> bool: ... +def compiled_model_record(model: CompiledModelView) -> CompiledModel: ... +def compiled_audit_record(audit: CompiledAuditView) -> CompiledAudit: ... +def compiled_function_record(function: CompiledFunctionView) -> CompiledFunction: ... +def compiled_source_record(source: CompiledSourceView) -> CompiledSource: ... +def compiled_seed_record(seed: CompiledSeedView) -> CompiledSeed: ... +def compiled_project_record( + project: CompiledProjectView, *, include_sql_expansions: bool = True +) -> CompiledProject: ... +def with_binding_diagnostics( + *, model: CompiledModelView, diagnostics: tuple[CompilerDiagnostic, ...] +) -> CompiledModelView: ... +def with_type_recovery( + *, + model: CompiledModelView, + diagnostics: tuple[CompilerDiagnostic, ...], + poisoned: frozenset[tuple[str, str]], +) -> CompiledModelView: ... +def project_with(*, project: CompiledProjectView, **changes: Any) -> CompiledProjectView: ... +def inputs_with(*, inputs: CompileInputsView, **changes: Any) -> CompileInputsView: ... +def compile_inputs_view( + *, fields: dict[str, Any], native_project: NativeCompiledProject | None +) -> CompileInputsView: ... +def plain_typed_value(value: SqlValue) -> object: ... +def declaration_rows( + *, enums: Sequence[EnumDeclaration], constants: Sequence[ConstantDeclaration] +) -> tuple[list[tuple[object, ...]], list[tuple[object, ...]]]: ... +def record_model_analyses( + *, project: NativeCompiledProject, models: Sequence[CompiledModelView] +) -> int: ... +def compiled_project_facts(project: CompiledProjectView) -> NativeCompiledProject: ... +def attach_resources( + *, + discovered_inputs: DiscoveredProjectInputs, + native_project: NativeCompiledProject, + settings: tuple[SettingsConfig, TargetConfig | None, str], + no_sql_validation: bool, + python_functions_inherit_default_namespace: bool, +) -> tuple[tuple[SeedInput, ...], tuple[FunctionInput, ...], tuple[SourceInput, ...]]: ... +def attach_checks( + *, + discovered_inputs: DiscoveredProjectInputs, + native_project: NativeCompiledProject, + settings: tuple[SettingsConfig, TypedSqlValueRenderer, SqlLexicalSyntax], + scan_cache: SqlTestScanCache, + source_inputs: tuple[SourceInput, ...], +) -> tuple[ + tuple[CompileSqlTestInput, ...], + tuple[CompileSqlScenarioInput, ...], + tuple[AuditInput, ...], + tuple[CompilerDiagnostic, ...], +]: ... +def build_effective_connection( + *, + project_config: ProjectConfig, + local_config: LocalConfig, + target_config: TargetConfig | None, + effective_vars: dict[str, object], +) -> dict[str, object]: ... +def build_effective_settings( + *, project_config: ProjectConfig, local_config: LocalConfig +) -> SettingsConfig: ... +def build_effective_vars( + *, + project_config: ProjectConfig, + local_config: LocalConfig, + target_config: TargetConfig | None, + cli_vars: dict[str, object], +) -> dict[str, object]: ... +def native_model_config_session( + *, inputs: NativeModelConfigInputs, validator: NativeModelValidator +) -> NativeModelConfigSession: ... +def build_model_inputs( + *, + discovered_inputs: DiscoveredProjectInputs, + context: ModelInputBuildContext, + no_sql_validation: bool = False, + defer_model_sql_validation: bool = False, + external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, + native_project: NativeCompiledProject, +) -> tuple[ModelInput, ...]: ... +def build_compile_target_context( + *, discovered_inputs: DiscoveredProjectInputs, selected_target: str | None, no_cache: bool +) -> tuple[str | None, TargetConfig | None, Path | None]: ... +def build_effective_target_namespace( + *, + discovered_inputs: DiscoveredProjectInputs, + selected_target: str | None, + connection_database: object | None, + connection_schema: object | None, + default_database: str | None, + default_schema: str | None, +) -> tuple[str | None, TargetConfig | None, str | None, str | None]: ... +def build_declaration_scope( + *, + discovered_inputs: DiscoveredProjectInputs, + loaded_macros: dict[str, LoadedMacro], + sql_lexical_syntax: SqlLexicalSyntax, + compile_cache_dir: Path | None = None, +) -> DeclarationScopeBuild: ... +def rebind_declaration_scope( + *, + scope: DeclarationScopeBuild, + discovered_inputs: DiscoveredProjectInputs, + loaded_macros: dict[str, LoadedMacro], +) -> DeclarationScopeBuild | None: ... + class NativeMacroExpansion: def __init__( self, @@ -1308,7 +1433,14 @@ def unknown_mapping_keys( keys: list[tuple[str, bool]], allowed: list[str], / ) -> tuple[str, str] | None: ... def unsupported_keys_help(keys: list[str], supported_keys: list[str], /) -> str: ... -def resource_sql_diagnostic_rows(catalog: object, request: object, /) -> list[object]: ... +def semantic_help_catalogue() -> list[tuple[str, str]]: ... +def resource_sql_diagnostics_of( + catalog: object, + project: Any, + shapes: dict[str, dict[str, str]], + profile: ExpressionInferenceProfile, + /, +) -> tuple[CompilerDiagnostic, ...]: ... def compiler_query_hash(sql: str, dialect: str | None, /) -> str: ... def function_definition_hash(sql: str, language: str, dialect: str | None, /) -> str: ... def normalize_query_text(sql: str, /) -> str: ... @@ -1377,10 +1509,11 @@ def read_scope_index_cache(path: Path, fingerprint: str, /) -> ScopeIndex | None def write_scope_index_cache( location: tuple[Path, str], fingerprint: str, index: ScopeIndex, / ) -> bool: ... -def scope_schema_usages( - schemas: list[tuple[str, str | None, list[tuple[str, str | None]]]], - records: list[tuple[tuple[str, str], tuple[str, str, tuple[str, str] | None, str | None, str]]], -) -> tuple[list[tuple[int, int]], list[tuple[list[str], str, str, str]], str | None]: ... +def model_schema_scope_usages( + schema_files: Sequence[DiscoveredModelSchemaFile], + resolver: DeclarationScopeResolver | None, + /, +) -> tuple[UsageRecord, ...]: ... class NativeConfigError: class_name: str @@ -1605,35 +1738,33 @@ def start_model_analysis_session( cache: tuple[str, str] | None = None, adapter_rules: tuple[Mapping[str, Callable[..., object]], type] | None = None, ) -> NativeModelAnalysisSession: ... -def prove_dynamic_column_contracts( - request: tuple[ - str, - list[tuple[str, list[tuple[str, str]]]], - list[tuple[str, list[tuple[str, str]]]], - list[tuple[str, list[tuple[str, str]]]], - list[tuple[str, list[tuple[str, str, str, str, str, str | None]]]], - list[tuple[str, list[tuple[str, str, str, str, str, str | None]]]], - ], - /, -) -> list[_PivotContractRow]: ... -def infer_query_columns( - catalog: object | None, - request: tuple[ - str, - list[tuple[str, str]], - list[tuple[str, str]], - list[tuple[str, list[tuple[str, str]]]], - list[tuple[str, list[tuple[str, str]]]], - list[tuple[object, ...]], - str, - ], - adapter_rules: tuple[Mapping[str, Callable[..., object]], type] | None = None, +def analyze_native_model_sql(*, request: Any) -> Any: ... +def session_outcome_analysis( + *, session: Any, position: int, placeholders: dict[str, str] | None +) -> Any: ... +def dynamic_contract_proof(row: Any, /) -> Any: ... +def infer_native_query_column_rows( + *, + queries: Sequence[Any], + profile: Any, + column_nullability_by_table: Mapping[str, Mapping[str, Any]] | None = None, ) -> list[tuple[bool, list[tuple[str, str | None, str]] | None, bool]]: ... -def infer_expression_source_shapes( - catalog: object, - request: tuple[str, bool, list[tuple[str, str]], list[tuple[str, str]], list[str]], - adapter_rules: tuple[Mapping[str, Callable[..., object]], type] | None = None, -) -> list[list[tuple[str, str]] | None]: ... +def infer_native_query_columns( + *, + queries: Sequence[Any], + profile: Any, + column_types_by_table: Mapping[str, Mapping[str, str]] | None = None, + column_nullability_by_table: Mapping[str, Mapping[str, Any]] | None = None, +) -> tuple[Any, ...]: ... +def infer_native_expression_source_shapes( + *, expressions: tuple[str, ...], profile: Any +) -> tuple[dict[str, str] | None, ...]: ... +def prove_native_dynamic_contracts( + *, + session: NativeModelAnalysisSession | None, + tables: Any, + models: tuple[tuple[str, tuple[Any, ...]], ...], +) -> tuple[Any, ...]: ... def _oracle_names_set_operation(sql: str, /) -> bool: ... def _oracle_cte_fact_recovery( request: tuple[ @@ -1649,112 +1780,19 @@ def _oracle_cte_fact_recovery( ) -> tuple[str | None, list[tuple[str, str]], list[tuple[str, str]], list[str], list[str]]: ... # Native analysis: semantic completion. -class SemanticTypeRecovery: - @property - def status(self) -> str: ... - @property - def poisoned(self) -> list[tuple[str, str]]: ... - @property - def revalidated(self) -> list[int]: ... - def finish( - self, revised: list[list[tuple[str, int | None, int | None]]], / - ) -> tuple[list[tuple[int, str | None]], list[list[int]]]: ... - -def plan_semantic_type_recovery( - catalog: object, - request: tuple[ - str | None, - list[ - tuple[ - str, - str, - list[str] | None, - list[str], - list[tuple[str, list[tuple[str, str]]]] | None, - list[tuple[int, str, str, bool]], - list[tuple[int, str, str, int | None, int | None]], - ] - ], - list[tuple[int, str, bool, str | None]], - ], - session: NativeModelAnalysisSession | None = None, - /, -) -> SemanticTypeRecovery: ... -def check_semantic_metadata_rows( - catalog: object, - request: tuple[ - str | None, - bool, - list[tuple[str, str]], - list[tuple[str, list[tuple[str, str]]]], - list[tuple[str, list[tuple[str, str]]]], - list[tuple[object, ...]], - list[str], - list[tuple[str, str | None, int]], - list[tuple[int, list[tuple[str, list[str]]]]], - ], - /, -) -> tuple[ - list[ - tuple[ - list[tuple[str, str, int, int]], - list[tuple[str, str, int, int, bool]], - list[tuple[str, str, int, int]], - ] - ], - list[tuple[int, tuple[str, str, int, int]]], - list[tuple[int, tuple[str, str, int, int], int]], - list[str], -]: ... -def complete_semantic_checks( - catalog: object, - request: tuple[ - str | None, - list[ - tuple[ - int, - str, - str, - str | None, - str | None, - int | None, - int | None, - tuple[int, int, int | None, int | None] | None, - str | None, - list[str], - ] - ], - list[ - tuple[ - str, - str, - str, - list[str] | None, - list[tuple[str, list[tuple[str, str]]]] | None, - list[str], - str | None, - ] - ], - list[tuple[str, list[tuple[str, str]]]], +def complete_semantic_project( + *, + project: Any, + profile: Any, + binding_results: dict[ + str, list[tuple[str, str, int | None, int | None, int | None, int | None, str]] ], + resource_sql_analysis: bool, session: NativeModelAnalysisSession | None = None, - /, -) -> tuple[ - list[ - tuple[ - int, - str, - str | None, - list[str], - tuple[int, int, int | None, int | None] | None, - int | None, - int | None, - bool, - ] - ], - list[list[int]] | None, - list[tuple[int | None, tuple[int, str, str, str] | None]], -]: ... +) -> Any: ... +def semantic_metadata_diagnostics( + *, project: Any, profile: Any, resource_sql_analysis: bool +) -> tuple[Any, ...]: ... # Native analysis: contracts. type _ContractDiagnosticRow = tuple[ @@ -1909,9 +1947,20 @@ def assemble_project_resource_facts( list[bool], tuple[str, int, str, str, str | None, int | None] | None, ]: ... -def check_native_explicit_references(request_json: str, /) -> str: ... +def explicit_reference_report( + project: Any, + settings: dict[str, object], + owners: list[tuple[Any, int, list[tuple[str, str, int]]]], + /, +) -> PythonSqlReferenceReport: ... +def hard_coded_relation_remedy( + owner_kind: str, target: tuple[str, str], upstream_loaders: list[tuple[str, str]], / +) -> str: ... +def old_name_reference_message( + owner_label: str, written: str, method: str, model_name: str, / +) -> str: ... def native_execution_gate_edges( - request_json: str, / + project: Any, / ) -> tuple[ list[tuple[str, tuple[str, str], tuple[str, str], tuple[str, str]]], list[tuple[str, tuple[str, str], tuple[str, str], str, str | None]], diff --git a/src/sqlbuild/compiler/analysis_session/_helpers/__init__.py b/src/sqlbuild/compiler/analysis_session/_helpers/__init__.py deleted file mode 100644 index 64fb62aa62..0000000000 --- a/src/sqlbuild/compiler/analysis_session/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Native model analysis session helpers.""" diff --git a/src/sqlbuild/compiler/analysis_session/_helpers/dynamic_contracts.py b/src/sqlbuild/compiler/analysis_session/_helpers/dynamic_contracts.py deleted file mode 100644 index 6699417d25..0000000000 --- a/src/sqlbuild/compiler/analysis_session/_helpers/dynamic_contracts.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Prove dynamic column contracts natively for models outside the analysis session.""" - -from __future__ import annotations - -from typing import Any - -import sqlbuild._native as _native -from sqlbuild.compiler.analysis_session._helpers.profile_rows import shape_rows -from sqlbuild.compiler.analysis_session._helpers.session_rows import family_rows -from sqlbuild.compiler.analysis_session.models import NativePivotTables -from sqlbuild.compiler.analysis_session.types import ContractRow, ProofRow -from sqlbuild.compiler.compile.classes.python_model_analysis import PythonModelAnalysis -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily - -type _ModelRow = tuple[str, list[tuple[str, str, str, str, str, str | None]]] - - -def native_dynamic_contracts( - *, - session: Any | None, - tables: NativePivotTables, - models: tuple[tuple[str, tuple[SchemaDynamicColumnFamily, ...]], ...], -) -> tuple[ProofRow | None, ...]: - """Each model's proof row in order; a native internal failure raises.""" - - if not models: - return () - rows: list[_ModelRow] = [_model_row(sql=sql, families=families) for sql, families in models] - contracts: list[ContractRow] = ( - session.prove_dynamic_contracts(rows) - if session is not None - else _native.prove_dynamic_column_contracts( - ( - tables.dialect or "generic", - shape_rows(tables.column_types_by_table), - shape_rows(tables.authoritative_column_types_by_table), - shape_rows(tables.column_nullability_by_table), - family_rows(tables.dynamic_families_by_table), - rows, - ) - ) - ) - return tuple(proof for _, proof in contracts) - - -def _model_row(*, sql: str, families: tuple[SchemaDynamicColumnFamily, ...]) -> _ModelRow: - return (sql, list(map(PythonModelAnalysis.family_row, families))) diff --git a/src/sqlbuild/compiler/analysis_session/_helpers/expression_shapes.py b/src/sqlbuild/compiler/analysis_session/_helpers/expression_shapes.py deleted file mode 100644 index 880ce5d100..0000000000 --- a/src/sqlbuild/compiler/analysis_session/_helpers/expression_shapes.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Infer expression-source shapes natively, filling the binding catalog's shape cache.""" - -from __future__ import annotations - -from typing import Any - -import sqlbuild._native as _native -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.adapter.contract.types import FunctionNullabilityRule -from sqlbuild.compiler.analysis_session._helpers.profile_rows import ( - adapter_nullability_rules, - case_sensitive_shapes, - nullability_rule_rows, -) -from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.sql_analysis.constants import NATIVE_DIALECT_ALIASES -from sqlbuild.compiler.sql_analysis.main._binding_catalog import create_binding_catalog - - -def native_expression_source_shapes( - *, expressions: tuple[str, ...], profile: ExpressionInferenceProfile -) -> tuple[dict[str, str] | None, ...]: - """One shape per expression; a native internal failure raises `NativeCompilerError`.""" - - dialect: str = profile.sql_analysis_dialect or "generic" - dialect = NATIVE_DIALECT_ALIASES.get(dialect, dialect) - catalog: Any = profile.binding_catalog or create_binding_catalog( - dialect=dialect, - quoted_ignore_case=profile.quoted_identifiers_ignore_case, - known_functions=(), - known_types=(), - relations={}, - ) - cache: dict[str, dict[str, str] | None] = catalog.expression_shapes - pending: tuple[str, ...] = tuple( - dict.fromkeys(expression for expression in expressions if expression not in cache) - ) - if pending: - adapter_rules: dict[str, FunctionNullabilityRule] = adapter_nullability_rules(profile) - shapes: list[list[tuple[str, str]] | None] = _native.infer_expression_source_shapes( - catalog.native, - ( - dialect, - case_sensitive_shapes(profile=profile, dialect=dialect), - list(profile.function_return_types.items()), - nullability_rule_rows(profile), - list(pending), - ), - (adapter_rules, InferredNullability) if adapter_rules else None, - ) - cache.update( - (expression, dict(shape) if shape is not None else None) - for expression, shape in zip(pending, shapes, strict=True) - ) - return tuple( - None if cache[expression] is None else dict(cache[expression] or {}) - for expression in expressions - ) diff --git a/src/sqlbuild/compiler/analysis_session/_helpers/model_analysis.py b/src/sqlbuild/compiler/analysis_session/_helpers/model_analysis.py deleted file mode 100644 index a6d659465f..0000000000 --- a/src/sqlbuild/compiler/analysis_session/_helpers/model_analysis.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Run the native model analysis session.""" - -from __future__ import annotations - -from typing import Any, cast - -import sqlbuild._native as _native -from sqlbuild.adapter.contract.types import FunctionNullabilityRule -from sqlbuild.compiler.analysis_session._helpers.profile_rows import adapter_nullability_rules -from sqlbuild.compiler.analysis_session._helpers.session_rows import session_request -from sqlbuild.compiler.analysis_session.classes.native_model_analysis import NativeModelAnalysis -from sqlbuild.compiler.analysis_session.constants import ( - NATIVE_ANALYSIS_STORE_FILE_NAME, - NATIVE_ANALYSIS_STORE_VERSION, -) -from sqlbuild.compiler.analysis_session.models import ( - NativeModelAnalyses, - NativeModelAnalysisRequest, -) -from sqlbuild.compiler.compile.classes.python_model_analysis import PythonModelAnalysis -from sqlbuild.compiler.compile.models import AnalysisCacheContext -from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity -from sqlbuild.compiler.lineage.types import InferredNullability - - -def native_model_analyses(*, request: NativeModelAnalysisRequest) -> NativeModelAnalyses: - """Each model's analysis by name and the finished session; a native internal failure raises.""" - - if not request.model_inputs: - return NativeModelAnalyses(positions={}, succeeded=frozenset(), session=None) - catalog: Any = cast(Any, request.inference_profile.binding_catalog) - python: PythonModelAnalysis = PythonModelAnalysis( - model_inputs=request.model_inputs, - inference_profile=request.inference_profile, - complete_binding_schemas=request.complete_binding_schemas, - ) - adapter_rules: dict[str, FunctionNullabilityRule] = adapter_nullability_rules( - request.inference_profile - ) - session: _native.NativeModelAnalysisSession = _native.start_model_analysis_session( - catalog.native, - session_request(request=request, python=python, schemas=catalog.schemas), - None if adapter_rules else _analysis_store(request), - (adapter_rules, InferredNullability) if adapter_rules else None, - ) - return NativeModelAnalysis(request=request, python=python, catalog=catalog).analyses(session) - - -def _analysis_store(request: NativeModelAnalysisRequest) -> tuple[str, str] | None: - """The analysis store's path and environment, or None where Python bypasses its cache.""" - - context: AnalysisCacheContext | None = request.analysis_cache - if context is None: - return None - return ( - str(context.root / NATIVE_ANALYSIS_STORE_FILE_NAME), - _native.content_digest( - [ - NATIVE_ANALYSIS_STORE_VERSION, - compiled_code_identity(), - context.shared_fingerprint, - context.signature_namespace, - ] - ), - ) diff --git a/src/sqlbuild/compiler/analysis_session/_helpers/profile_rows.py b/src/sqlbuild/compiler/analysis_session/_helpers/profile_rows.py deleted file mode 100644 index e4cb8eb2a5..0000000000 --- a/src/sqlbuild/compiler/analysis_session/_helpers/profile_rows.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Inference-profile settings crossing into native analysis: rule ids and shape case.""" - -from __future__ import annotations - -from collections.abc import Mapping - -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.adapter.contract.types import FunctionNullabilityRule -from sqlbuild.adapter.type_system.main.conditional_result_nullability import ( - conditional_result_nullability, -) -from sqlbuild.adapter.type_system.main.first_arg_nullability import first_arg_nullability -from sqlbuild.adapter.type_system.main.safe_cast_nullability import safe_cast_nullability -from sqlbuild.compiler.analysis_session.constants import ( - NULLABILITY_RULE_ADAPTER, - NULLABILITY_RULE_CONDITIONAL_RESULT, - NULLABILITY_RULE_FIRST_ARG, - NULLABILITY_RULE_SAFE_CAST, -) -from sqlbuild.compiler.analysis_session.types import ShapeRows -from sqlbuild.compiler.sql_analysis.constants import CASE_SENSITIVE_BINDING_DIALECTS - -_NULLABILITY_RULE_IDS: dict[FunctionNullabilityRule, str] = { - first_arg_nullability: NULLABILITY_RULE_FIRST_ARG, - conditional_result_nullability: NULLABILITY_RULE_CONDITIONAL_RESULT, - safe_cast_nullability: NULLABILITY_RULE_SAFE_CAST, -} - - -def nullability_rule_rows(profile: ExpressionInferenceProfile) -> list[tuple[str, str]]: - """Adapter nullability rules as `(name, rule id)`; an adapter's own rule is `python`.""" - - return [ - (name, _NULLABILITY_RULE_IDS.get(rule, NULLABILITY_RULE_ADAPTER)) - for name, rule in profile.function_nullability_rules.items() - ] - - -def adapter_nullability_rules( - profile: ExpressionInferenceProfile, -) -> dict[str, FunctionNullabilityRule]: - """The adapter's own nullability rules, which the native session calls back into.""" - - return { - name: rule - for name, rule in profile.function_nullability_rules.items() - if rule not in _NULLABILITY_RULE_IDS - } - - -def case_sensitive_shapes(*, profile: ExpressionInferenceProfile, dialect: str | None) -> bool: - """Python's `inferred_binding_shape` test for keeping authored identifier quoting.""" - - return not profile.quoted_identifiers_ignore_case and dialect in CASE_SENSITIVE_BINDING_DIALECTS - - -def shape_rows(shapes: Mapping[str, Mapping[str, str]]) -> ShapeRows: - """`{relation: {column: value}}` as ordered rows; discovery rejects non-text names and types.""" - - return [(name, _shape_pairs(shape)) for name, shape in shapes.items()] - - -def _shape_pairs(shape: Mapping[str, str]) -> list[tuple[str, str]]: - return [(column, str(value)) for column, value in shape.items()] diff --git a/src/sqlbuild/compiler/analysis_session/_helpers/query_columns.py b/src/sqlbuild/compiler/analysis_session/_helpers/query_columns.py deleted file mode 100644 index 25d3b946e1..0000000000 --- a/src/sqlbuild/compiler/analysis_session/_helpers/query_columns.py +++ /dev/null @@ -1,146 +0,0 @@ -"""Python's per-query column analysis outside the session, run natively.""" - -from __future__ import annotations - -from collections.abc import Mapping, Sequence -from typing import Any - -import sqlbuild._native as _native -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.adapter.contract.types import FunctionNullabilityRule -from sqlbuild.compiler.analysis_session._helpers.profile_rows import ( - adapter_nullability_rules, - nullability_rule_rows, - shape_rows, -) -from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns -from sqlbuild.compiler.analysis_session.types import ColumnRow -from sqlbuild.compiler.compile.models import CompileSqlReference, InferredColumn -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.sql_analysis.constants import ( - NATIVE_DIALECT_ALIASES, - TABLE_FUNCTION_ANALYSIS_PREFIX, -) -from sqlbuild.compiler.sql_analysis.main._binding_catalog import create_binding_catalog - -_BATCH_MODE: str = "batch" -_LINEAGE_RESOURCE_TYPES: dict[str, CompiledResourceType] = { - SqlReferenceKind.REF: CompiledResourceType.MODEL, - SqlReferenceKind.SOURCE: CompiledResourceType.SOURCE, - SqlReferenceKind.SEED: CompiledResourceType.SEED, - SqlReferenceKind.TABLE_FUNCTION: CompiledResourceType.TABLE_FN, -} - - -def native_query_columns( - *, - queries: Sequence[NativeColumnQuery], - profile: ExpressionInferenceProfile, - column_types_by_table: Mapping[str, Mapping[str, str]], - column_nullability_by_table: Mapping[str, Mapping[str, InferredNullability]], -) -> tuple[NativeQueryColumns, ...]: - """Each query's columns; a native internal failure raises `NativeCompilerError`.""" - - return tuple( - NativeQueryColumns( - succeeded=succeeded, - columns=None if columns is None else _inferred_columns(columns), - has_star=has_star, - ) - for succeeded, columns, has_star in native_query_column_rows( - queries=queries, - profile=profile, - column_types_by_table=column_types_by_table, - column_nullability_by_table=column_nullability_by_table, - ) - ) - - -def native_query_column_rows( - *, - queries: Sequence[NativeColumnQuery], - profile: ExpressionInferenceProfile, - column_types_by_table: Mapping[str, Mapping[str, str]], - column_nullability_by_table: Mapping[str, Mapping[str, InferredNullability]], -) -> list[tuple[bool, list[ColumnRow] | None, bool]]: - """Each query's `(succeeded, column rows, has star)` as the native analysis returns it.""" - - if not queries: - return [] - dialect: str = profile.sql_analysis_dialect or "generic" - dialect = NATIVE_DIALECT_ALIASES.get(dialect, dialect) - catalog: Any | None = ( - ( - profile.binding_catalog - or create_binding_catalog( - dialect=dialect, - quoted_ignore_case=profile.quoted_identifiers_ignore_case, - known_functions=(), - known_types=(), - relations={}, - ) - ) - if any(query.mode == _BATCH_MODE for query in queries) - else None - ) - adapter_rules: dict[str, FunctionNullabilityRule] = adapter_nullability_rules(profile) - return _native.infer_query_columns( - None if catalog is None else catalog.native, - ( - dialect, - list(profile.function_return_types.items()), - nullability_rule_rows(profile), - shape_rows(column_types_by_table), - shape_rows(column_nullability_by_table), - [_query_row(query) for query in queries], - profile.sql_analysis_dialect or "generic", - ), - (adapter_rules, InferredNullability) if adapter_rules else None, - ) - - -def _inferred_columns( - columns: list[tuple[str, str | None, str]], -) -> tuple[InferredColumn, ...]: - return tuple( - InferredColumn(name=name, type=data_type, nullability=InferredNullability(nullability)) - for name, data_type, nullability in columns - ) - - -def _query_row(query: NativeColumnQuery) -> tuple[object, ...]: - references: tuple[CompileSqlReference, ...] = query.references - return ( - query.sql, - list((query.placeholders or {}).items()), - [_analysis_name(reference) for reference in references], - [ - (name, resource_type, resource_name) - for name, (resource_type, resource_name) in _lineage_references(references).items() - ], - query.recover_cte_facts, - query.mode, - ) - - -def _lineage_references( - references: tuple[CompileSqlReference, ...], -) -> dict[str, tuple[str, str]]: - """Each lineage reference's analysis name, resource type and name; the last wins in place.""" - - return { - _analysis_name(reference): ( - _LINEAGE_RESOURCE_TYPES[reference.ref_kind].value, - reference.ref_name, - ) - for reference in references - if reference.ref_kind in _LINEAGE_RESOURCE_TYPES - } - - -def _analysis_name(reference: CompileSqlReference) -> str: - if reference.ref_kind == SqlReferenceKind.TABLE_FUNCTION: - return f"{TABLE_FUNCTION_ANALYSIS_PREFIX}{reference.ref_name}" - return reference.ref_name diff --git a/src/sqlbuild/compiler/analysis_session/_helpers/session_rows.py b/src/sqlbuild/compiler/analysis_session/_helpers/session_rows.py deleted file mode 100644 index c0566c871f..0000000000 --- a/src/sqlbuild/compiler/analysis_session/_helpers/session_rows.py +++ /dev/null @@ -1,188 +0,0 @@ -"""Plain rows crossing into and out of the native model analysis session.""" - -from __future__ import annotations - -from collections.abc import Mapping, Sequence - -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session._helpers.profile_rows import ( - case_sensitive_shapes, - nullability_rule_rows, - shape_rows, -) -from sqlbuild.compiler.analysis_session.constants import LINEAGE_NATIVE -from sqlbuild.compiler.analysis_session.models import NativeModelAnalysisRequest -from sqlbuild.compiler.analysis_session.types import ( - ColumnRow, - DiagnosticRow, - FamilyRow, - LineageItem, - OutcomeRow, - ProofRow, - ShapeRows, -) -from sqlbuild.compiler.compile.classes.python_model_analysis import PythonModelAnalysis -from sqlbuild.compiler.compile.models import ( - CompactLineageFacts, - CompiledLineageColumnFact, - CompiledLineageSourceFact, - DynamicColumnContractProof, - DynamicColumnFamilyProof, - InferredColumn, - ModelSqlAnalysis, - PolyglotAnalysisResult, -) -from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily - - -def session_request( - *, - request: NativeModelAnalysisRequest, - python: PythonModelAnalysis, - schemas: Mapping[str, Mapping[str, str]], -) -> tuple[object, ...]: - """The session request.""" - - profile: ExpressionInferenceProfile = request.inference_profile - shapes: tuple[ShapeRows, ...] = ( - shape_rows(request.column_types_by_table), - shape_rows(request.column_nullability_by_table), - shape_rows(request.complete_binding_schemas), - shape_rows(schemas), - ) - return ( - profile.sql_analysis_dialect or "generic", - case_sensitive_shapes(profile=profile, dialect=profile.sql_analysis_dialect), - list(profile.function_return_types.items()), - nullability_rule_rows(profile), - request.rich_type_inference, - *shapes, - family_rows(request.dynamic_families_by_table), - python.model_rows(), - ) - - -def outcome_analysis( - *, outcome: OutcomeRow, contract: ProofRow | None, placeholders: dict[str, str] | None -) -> ModelSqlAnalysis: - """One finished session outcome as the `ModelSqlAnalysis` it replaced.""" - - succeeded, columns, source, rows, has_star, star_resolved, raw, validated, cleaned = outcome - return ModelSqlAnalysis( - polyglot_analysis=PolyglotAnalysisResult( - analysis_succeeded=succeeded, - columns=inferred_columns(columns), - lineage_columns=_lineage(succeeded=succeeded, source=source, rows=rows), - has_star=has_star, - star_resolved=star_resolved, - binding_diagnostics=binding_diagnostics(raw), - binding_validated=validated, - ), - placeholders=placeholders, - fused_binding_validated=True, - cleaned_sql=cleaned, - dynamic_column_contract=contract_proof(contract), - ) - - -def _lineage( - *, succeeded: bool, source: str, rows: list[LineageItem] -) -> Sequence[CompiledLineageColumnFact]: - if source != LINEAGE_NATIVE: - return lineage_facts(rows) - return compact_lineage(rows) if succeeded else () - - -def contract_proof(row: ProofRow | None) -> DynamicColumnContractProof | None: - """A native dynamic pivot proof as Python's, None when absent.""" - - if row is None: - return None - output_proven, columns, families, input_relations, failure_reason, bare = row - return DynamicColumnContractProof( - output_proven=output_proven, - fixed_columns=inferred_columns(columns) or (), - families=tuple( - DynamicColumnFamilyProof(name=name, inferred_type=inferred_type) - for name, inferred_type in families - ), - input_relations=tuple(input_relations), - failure_reason=failure_reason, - bare_dynamic_pivot=bare, - ) - - -def inferred_columns(rows: list[ColumnRow] | None) -> tuple[InferredColumn, ...] | None: - if rows is None: - return None - return tuple( - InferredColumn(name=name, type=data_type, nullability=InferredNullability(nullability)) - for name, data_type, nullability in rows - ) - - -def binding_diagnostics(rows: list[DiagnosticRow]) -> tuple[SqlBindingDiagnostic, ...]: - return tuple(SqlBindingDiagnostic(*row) for row in rows) - - -def compact_lineage(rows: list[LineageItem]) -> CompactLineageFacts: - """Native lineage rows as Python's compact facts over a per-model string pool.""" - - pool: dict[str, int] = {} - compact_rows: list[tuple[int, int, int, tuple[tuple[int, int, int], ...]]] = [] - for output_column, transform_code, confidence_code, sources in rows: - source_indexes: list[tuple[int, int, int]] = [] - for resource_type, resource_name, column_name in sources: - source_indexes.append( - ( - pool.setdefault(resource_type, len(pool)), - pool.setdefault(resource_name, len(pool)), - pool.setdefault(column_name, len(pool)), - ) - ) - compact_rows.append( - ( - pool.setdefault(output_column, len(pool)), - transform_code, - confidence_code, - tuple(source_indexes), - ) - ) - return CompactLineageFacts(string_pool=tuple(pool), rows=tuple(compact_rows)) - - -def lineage_facts(rows: list[LineageItem]) -> tuple[CompiledLineageColumnFact, ...]: - """Native lineage rows as the plain facts Python's re-analysis returns.""" - - return tuple( - CompiledLineageColumnFact( - output_column=output_column, - upstream_columns=_source_facts(sources), - transform_kind=CompactLineageFacts.transform_kind(transform_code), - confidence=CompactLineageFacts.confidence(confidence_code), - ) - for output_column, transform_code, confidence_code, sources in rows - ) - - -def _source_facts(sources: list[tuple[str, str, str]]) -> tuple[CompiledLineageSourceFact, ...]: - return tuple( - CompiledLineageSourceFact( - resource_type=resource_type, resource_name=resource_name, column_name=column_name - ) - for resource_type, resource_name, column_name in sources - ) - - -def family_rows( - families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]], -) -> list[tuple[str, list[FamilyRow]]]: - """Declared dynamic families by relation, as native request rows.""" - - return [(name, _family_rows(families)) for name, families in families_by_table.items()] - - -def _family_rows(families: tuple[SchemaDynamicColumnFamily, ...]) -> list[FamilyRow]: - return [PythonModelAnalysis.family_row(family) for family in families] diff --git a/src/sqlbuild/compiler/analysis_session/classes/__init__.py b/src/sqlbuild/compiler/analysis_session/classes/__init__.py deleted file mode 100644 index 74909fb82d..0000000000 --- a/src/sqlbuild/compiler/analysis_session/classes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Native model analysis session classes.""" diff --git a/src/sqlbuild/compiler/analysis_session/classes/native_model_analysis.py b/src/sqlbuild/compiler/analysis_session/classes/native_model_analysis.py deleted file mode 100644 index 6fdec2125d..0000000000 --- a/src/sqlbuild/compiler/analysis_session/classes/native_model_analysis.py +++ /dev/null @@ -1,116 +0,0 @@ -"""One compile's native model analysis session, with Python's catalog and metric bookkeeping.""" - -from __future__ import annotations - -import logging -from typing import Any - -import sqlbuild._native as _native -from sqlbuild.compiler.analysis_session.constants import ( - NATIVE_ANALYSIS_FAILURE_MESSAGE, - NATIVE_ANALYSIS_STORE_FAILURE_MESSAGE, -) -from sqlbuild.compiler.analysis_session.models import ( - NativeModelAnalyses, - NativeModelAnalysisRequest, -) -from sqlbuild.compiler.analysis_session.types import FinishRow, ShapeRows, StepRow -from sqlbuild.compiler.compile.classes.python_model_analysis import PythonModelAnalysis -from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.diagnostics.main.log_debug_event import log_debug_event - -_DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.compile") - - -class NativeModelAnalysis: - """Drive one native session; its outcomes stay native for compiled model assembly.""" - - def __init__( - self, - *, - request: NativeModelAnalysisRequest, - python: PythonModelAnalysis, - catalog: Any, - ) -> None: - self._python: PythonModelAnalysis = python - self._catalog: Any = catalog - self._types: dict[str, dict[str, str]] = dict(request.column_types_by_table) - self._nullability: dict[str, dict[str, InferredNullability]] = dict( - request.column_nullability_by_table - ) - - def analyses(self, session: _native.NativeModelAnalysisSession) -> NativeModelAnalyses: - """Every model's session position by name; a native internal failure raises.""" - - step: StepRow = session.run() - publications, failures = step - for failure in failures: - log_debug_event( - logger=_DEBUG_LOGGER, - message=NATIVE_ANALYSIS_FAILURE_MESSAGE, - sqlbuild_error=failure, - ) - self._publish(publications) - finished: FinishRow = session.finish() - schema_additions, analysis_names, succeeded = finished - self._record_catalog_changes( - schema_additions=schema_additions, analysis_names=analysis_names - ) - self._record_cache(session) - names: tuple[str, ...] = tuple( - request.model_input.model_file.file_path.stem for request in self._python.requests - ) - return NativeModelAnalyses( - positions={name: position for position, name in enumerate(names)}, - succeeded=frozenset( - name for name, success in zip(names, succeeded, strict=True) if success - ), - session=session, - ) - - def _record_cache(self, session: _native.NativeModelAnalysisSession) -> None: - stats: tuple[int, int, int, str | None] | None = session.cache_stats - if stats is None: - self._python.record_uncached() - return - hits, misses, _, failure = stats - if failure is not None: - log_debug_event( - logger=_DEBUG_LOGGER, - message=NATIVE_ANALYSIS_STORE_FAILURE_MESSAGE, - sqlbuild_error=failure, - ) - if hits + misses == 0: - self._python.record_uncached() - return - self._python.record_cached(hits=hits, misses=misses) - - def _publish(self, publications: ShapeRows) -> None: - for name, shape in publications: - published: dict[str, str] = dict(shape) - self._types.setdefault(name, published) - self._nullability.setdefault( - name, dict.fromkeys(published, InferredNullability.UNKNOWN) - ) - - def _record_catalog_changes( - self, *, schema_additions: ShapeRows, analysis_names: list[str] - ) -> None: - """Leave Python's binding catalog as its own analysis would have.""" - - additions: dict[str, dict[str, str]] = { - name: dict(shape) - for name, shape in schema_additions - if name not in self._catalog.schemas - } - self._catalog.schemas.update(additions) - if additions: - self._catalog.native.update_relations(additions) - self._catalog.prepare_analysis( - types={name: self._types[name] for name in analysis_names if name in self._types}, - nullability={ - name: self._nullability[name] - for name in analysis_names - if name in self._nullability - }, - ) diff --git a/src/sqlbuild/compiler/analysis_session/constants.py b/src/sqlbuild/compiler/analysis_session/constants.py index f785ed67fe..cc6670f3f8 100644 --- a/src/sqlbuild/compiler/analysis_session/constants.py +++ b/src/sqlbuild/compiler/analysis_session/constants.py @@ -1,12 +1,4 @@ """Native model analysis session constants.""" LINEAGE_NATIVE: str = "native" -LINEAGE_FACTS: str = "facts" -NATIVE_ANALYSIS_FAILURE_MESSAGE: str = "native compact query analysis failed" -NULLABILITY_RULE_FIRST_ARG: str = "first_arg" -NULLABILITY_RULE_CONDITIONAL_RESULT: str = "conditional_result" -NULLABILITY_RULE_SAFE_CAST: str = "safe_cast" -NULLABILITY_RULE_ADAPTER: str = "python" NATIVE_ANALYSIS_STORE_FILE_NAME: str = "model-analyses.bin" -NATIVE_ANALYSIS_STORE_VERSION: str = "native-model-analysis-store-v1" -NATIVE_ANALYSIS_STORE_FAILURE_MESSAGE: str = "native model analysis cache not read or saved" diff --git a/src/sqlbuild/compiler/analysis_session/main/__init__.py b/src/sqlbuild/compiler/analysis_session/main/__init__.py deleted file mode 100644 index 355f68e13c..0000000000 --- a/src/sqlbuild/compiler/analysis_session/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Native model analysis session entry points.""" diff --git a/src/sqlbuild/compiler/analysis_session/main/_analyze_native_model_sql.py b/src/sqlbuild/compiler/analysis_session/main/_analyze_native_model_sql.py deleted file mode 100644 index 5db2708717..0000000000 --- a/src/sqlbuild/compiler/analysis_session/main/_analyze_native_model_sql.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Analyze model SQL natively.""" - -from __future__ import annotations - -from sqlbuild.compiler.analysis_session._helpers.model_analysis import native_model_analyses -from sqlbuild.compiler.analysis_session.models import ( - NativeModelAnalyses, - NativeModelAnalysisRequest, -) - - -def analyze_native_model_sql(*, request: NativeModelAnalysisRequest) -> NativeModelAnalyses: - """Return each model's analysis (and pivot proof) and the session.""" - - return native_model_analyses(request=request) diff --git a/src/sqlbuild/compiler/analysis_session/main/_dynamic_contract_proof.py b/src/sqlbuild/compiler/analysis_session/main/_dynamic_contract_proof.py deleted file mode 100644 index 2e058baa95..0000000000 --- a/src/sqlbuild/compiler/analysis_session/main/_dynamic_contract_proof.py +++ /dev/null @@ -1,13 +0,0 @@ -"""A native dynamic pivot proof row as Python's proof, for hand-built model inputs only.""" - -from __future__ import annotations - -from sqlbuild.compiler.analysis_session._helpers.session_rows import contract_proof -from sqlbuild.compiler.analysis_session.types import ProofRow -from sqlbuild.compiler.compile.models import DynamicColumnContractProof - - -def dynamic_contract_proof(row: ProofRow | None) -> DynamicColumnContractProof | None: - """The proof, or None when the row is absent.""" - - return contract_proof(row) diff --git a/src/sqlbuild/compiler/analysis_session/main/_infer_native_expression_source_shapes.py b/src/sqlbuild/compiler/analysis_session/main/_infer_native_expression_source_shapes.py deleted file mode 100644 index 7509f3c4f5..0000000000 --- a/src/sqlbuild/compiler/analysis_session/main/_infer_native_expression_source_shapes.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Infer expression-source shapes natively.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session._helpers.expression_shapes import ( - native_expression_source_shapes, -) - - -def infer_native_expression_source_shapes( - *, expressions: tuple[str, ...], profile: ExpressionInferenceProfile -) -> tuple[dict[str, str] | None, ...]: - """One shape per expression; a native internal failure raises `NativeCompilerError`.""" - - return native_expression_source_shapes(expressions=expressions, profile=profile) diff --git a/src/sqlbuild/compiler/analysis_session/main/_infer_native_query_column_rows.py b/src/sqlbuild/compiler/analysis_session/main/_infer_native_query_column_rows.py deleted file mode 100644 index 2d44728cbf..0000000000 --- a/src/sqlbuild/compiler/analysis_session/main/_infer_native_query_column_rows.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Infer query output column rows natively outside the analysis session.""" - -from __future__ import annotations - -from collections.abc import Mapping, Sequence - -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session._helpers.query_columns import native_query_column_rows -from sqlbuild.compiler.analysis_session.models import NativeColumnQuery -from sqlbuild.compiler.analysis_session.types import ColumnRow -from sqlbuild.compiler.lineage.types import InferredNullability - - -def infer_native_query_column_rows( - *, - queries: Sequence[NativeColumnQuery], - profile: ExpressionInferenceProfile, - column_nullability_by_table: Mapping[str, Mapping[str, InferredNullability]] | None = None, -) -> list[tuple[bool, list[ColumnRow] | None, bool]]: - """Each query's `(succeeded, column rows, has star)`; a native internal failure raises.""" - - return native_query_column_rows( - queries=queries, - profile=profile, - column_types_by_table={}, - column_nullability_by_table=column_nullability_by_table or {}, - ) diff --git a/src/sqlbuild/compiler/analysis_session/main/_infer_native_query_columns.py b/src/sqlbuild/compiler/analysis_session/main/_infer_native_query_columns.py deleted file mode 100644 index b4837e9689..0000000000 --- a/src/sqlbuild/compiler/analysis_session/main/_infer_native_query_columns.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Infer the output columns of queries outside the native analysis session.""" - -from __future__ import annotations - -from collections.abc import Mapping, Sequence - -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session._helpers.query_columns import native_query_columns -from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns -from sqlbuild.compiler.lineage.types import InferredNullability - - -def infer_native_query_columns( - *, - queries: Sequence[NativeColumnQuery], - profile: ExpressionInferenceProfile, - column_types_by_table: Mapping[str, Mapping[str, str]] | None = None, - column_nullability_by_table: Mapping[str, Mapping[str, InferredNullability]] | None = None, -) -> tuple[NativeQueryColumns, ...]: - """Each query's columns; a native internal failure raises `NativeCompilerError`.""" - - return native_query_columns( - queries=queries, - profile=profile, - column_types_by_table=column_types_by_table or {}, - column_nullability_by_table=column_nullability_by_table or {}, - ) diff --git a/src/sqlbuild/compiler/analysis_session/main/_prove_native_dynamic_contracts.py b/src/sqlbuild/compiler/analysis_session/main/_prove_native_dynamic_contracts.py deleted file mode 100644 index 81c0e362ff..0000000000 --- a/src/sqlbuild/compiler/analysis_session/main/_prove_native_dynamic_contracts.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Prove dynamic column contracts natively.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.compiler.analysis_session._helpers.dynamic_contracts import native_dynamic_contracts -from sqlbuild.compiler.analysis_session.models import NativePivotTables -from sqlbuild.compiler.analysis_session.types import ProofRow -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily - - -def prove_native_dynamic_contracts( - *, - session: Any | None, - tables: NativePivotTables, - models: tuple[tuple[str, tuple[SchemaDynamicColumnFamily, ...]], ...], -) -> tuple[ProofRow | None, ...]: - """The proof row per `(sql, families)` model from the session or `tables`, else None.""" - - return native_dynamic_contracts(session=session, tables=tables, models=models) diff --git a/src/sqlbuild/compiler/analysis_session/main/_session_outcome_analysis.py b/src/sqlbuild/compiler/analysis_session/main/_session_outcome_analysis.py deleted file mode 100644 index 74342b9b6a..0000000000 --- a/src/sqlbuild/compiler/analysis_session/main/_session_outcome_analysis.py +++ /dev/null @@ -1,20 +0,0 @@ -"""A finished session outcome as Python's analysis, for hand-built model inputs only.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.compiler.analysis_session._helpers.session_rows import outcome_analysis -from sqlbuild.compiler.analysis_session.types import ContractRow, OutcomeRow -from sqlbuild.compiler.compile.models import ModelSqlAnalysis - - -def session_outcome_analysis( - *, session: Any, position: int, placeholders: dict[str, str] | None -) -> ModelSqlAnalysis: - """The outcome at `position` of a finished session as an explicit boundary conversion.""" - - outcome: OutcomeRow - contract: ContractRow - outcome, contract = session.outcome_rows(position) - return outcome_analysis(outcome=outcome, contract=contract[1], placeholders=placeholders) diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/fixture_columns.py b/src/sqlbuild/compiler/compile/_helpers/analysis/fixture_columns.py index 05b747a64d..1b7373d28d 100644 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/fixture_columns.py +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/fixture_columns.py @@ -3,10 +3,8 @@ from __future__ import annotations import sqlbuild._native as _native +from sqlbuild._native import infer_native_query_columns from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.main._infer_native_query_columns import ( - infer_native_query_columns, -) from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns from sqlbuild.compiler.compile.models import ( FixtureColumnInference, diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/hand_built_models.py b/src/sqlbuild/compiler/compile/_helpers/assembly/hand_built_models.py index 0bfa744cc6..7c80f25bcf 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/hand_built_models.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/hand_built_models.py @@ -6,10 +6,7 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild.compiler.analysis_session.main._dynamic_contract_proof import dynamic_contract_proof -from sqlbuild.compiler.analysis_session.main._session_outcome_analysis import ( - session_outcome_analysis, -) +from sqlbuild._native import dynamic_contract_proof, session_outcome_analysis from sqlbuild.compiler.compile._helpers.assembly.targets import build_model_relation_target from sqlbuild.compiler.compile.models import ( CompiledLineageColumnFact, diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/metadata_validation.py b/src/sqlbuild/compiler/compile/_helpers/assembly/metadata_validation.py index 6c496aef8d..96987854ef 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/metadata_validation.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/metadata_validation.py @@ -2,10 +2,8 @@ from __future__ import annotations +from sqlbuild._native import infer_native_query_columns from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.main._infer_native_query_columns import ( - infer_native_query_columns, -) from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns from sqlbuild.compiler.compile.models import CompiledModelSqlTestPayload from sqlbuild.compiler.compile.types import CompiledProjectView diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py b/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py index 77c5a9d064..31f71d945a 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py @@ -5,10 +5,8 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import infer_native_expression_source_shapes from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.main._infer_native_expression_source_shapes import ( - infer_native_expression_source_shapes, -) from sqlbuild.compiler.compile._helpers.analysis.reference_names import table_function_analysis_name from sqlbuild.compiler.compile.models import ( CompileSqlReference, diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/source_bindings.py b/src/sqlbuild/compiler/compile/_helpers/assembly/source_bindings.py index eb3c17763c..495f234713 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/source_bindings.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/source_bindings.py @@ -6,10 +6,8 @@ from graphlib import TopologicalSorter from typing import Any +from sqlbuild._native import infer_native_query_columns from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.main._infer_native_query_columns import ( - infer_native_query_columns, -) from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns from sqlbuild.compiler.compile._helpers.analysis.binding_requests import ( get_complete_schema_binding_request, diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py deleted file mode 100644 index 665ff09619..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ /dev/null @@ -1,311 +0,0 @@ -"""Attachment helpers for building pre-semantic compile inputs.""" - -from __future__ import annotations - -from dataclasses import dataclass, fields -from pathlib import Path -from typing import Any, cast - -import sqlbuild._native as _native -from sqlbuild.compiler.authored_values.main._project_var_values import render_project_var_text -from sqlbuild.compiler.compile._helpers.analysis.validation import sql_analysis_opt_out_help -from sqlbuild.compiler.compile._helpers.attachment.model_config import ( - native_model_config_session, -) -from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic -from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( - cursor_intrinsic_semantics, -) -from sqlbuild.compiler.compile._helpers.render.declarations import ( - render_constant_declaration, -) -from sqlbuild.compiler.compile._helpers.render.macros import ( - macro_expansion_classes, -) -from sqlbuild.compiler.compile._helpers.render.templating import ( - expand_effective_vars, - expand_template_data, - record_template_reads, -) -from sqlbuild.compiler.compile.classes.unicode_environment import UnicodeEnvironment -from sqlbuild.compiler.compile.constants import EXPLICIT_REFERENCE_KINDS -from sqlbuild.compiler.compile.exceptions import CompileInputError, MacroArgumentError -from sqlbuild.compiler.compile.models import ( - DeclarationResolutionContext, - LoadedMacro, - ModelInputBuildContext, - ModelValidatorContext, - NativeModelConfigInputs, -) -from sqlbuild.compiler.compile.types import ModelInput -from sqlbuild.compiler.compiled_project.main._model_loop_classes import model_loop_classes -from sqlbuild.compiler.discovery.constants import PROJECT_CONFIG_FILENAME -from sqlbuild.compiler.discovery.models import ( - ConstantDeclaration, - DiscoveredProjectInputs, - DiscoveredSqlModelFile, - EnumDeclaration, - NamedSqlHookEntry, - PythonHookEntry, -) -from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge -from sqlbuild.compiler.model_config.main._native_contract_classes import native_contract_classes -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationRecord, - VisibilityRecord, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, -) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.spec.contracts.models import ( - LocalConfig, - ProjectConfig, - SettingsConfig, - TargetConfig, -) - - -@dataclass(frozen=True) -class _VisibleModelDeclarations: - local_enums: dict[str, EnumDeclaration] - local_constants: dict[str, ConstantDeclaration] - enums: dict[str, EnumDeclaration] - constants: dict[str, ConstantDeclaration] - inaccessible_enums: dict[str, DeclarationRecord] - inaccessible_constants: dict[str, DeclarationRecord] - enum_visibility: dict[str, tuple[VisibilityRecord, ...]] - constant_visibility: dict[str, tuple[VisibilityRecord, ...]] - macros: dict[str, LoadedMacro] - macro_records: dict[str, DeclarationRecord] - inaccessible_macros: dict[str, DeclarationRecord] - - -@dataclass(frozen=True) -class _ModelValidationContext: - effective_settings: SettingsConfig - no_sql_validation: bool - project_config_path: Path - defer_model_sql_validation: bool - external_sql_reference_resolver: ExternalSqlReferenceResolver | None - sql_lexical_syntax: SqlLexicalSyntax - validators: ModelValidatorContext - - -@dataclass(frozen=True) -class _ModelInputLoop: - discovered_inputs: DiscoveredProjectInputs - context: ModelInputBuildContext - validation_context: _ModelValidationContext - - -@dataclass -class _ModelLoopHost: - """The attachment steps the native model loop calls back into, one model index at a time.""" - - loop: _ModelInputLoop - - def constant_text(self, *, declaration: ConstantDeclaration, file_path: Path) -> str: - return render_constant_declaration( - declaration=declaration, - value_renderer=self.loop.context.value_renderer, - collection_rendering=self.loop.context.collection_rendering, - file_path=file_path, - ) - - -def build_model_inputs( - *, - discovered_inputs: DiscoveredProjectInputs, - context: ModelInputBuildContext, - no_sql_validation: bool = False, - defer_model_sql_validation: bool = False, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, - native_project: _native.NativeCompiledProject, -) -> tuple[ModelInput, ...]: - """Attach schema metadata to discovered model files.""" - - return _build_model_inputs( - discovered_inputs=discovered_inputs, - context=context, - no_sql_validation=no_sql_validation, - defer_model_sql_validation=defer_model_sql_validation, - external_sql_reference_resolver=external_sql_reference_resolver, - native_project=native_project, - ) - - -def _build_model_inputs( - *, - discovered_inputs: DiscoveredProjectInputs, - context: ModelInputBuildContext, - no_sql_validation: bool, - defer_model_sql_validation: bool, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None, - native_project: _native.NativeCompiledProject, -) -> tuple[ModelInput, ...]: - effective_vars: dict[str, object] = context.effective_vars - effective_settings: SettingsConfig = context.effective_settings - validator: _native.NativeModelValidator = _native.NativeModelValidator.from_inputs( - discovered_inputs, effective_settings.microbatch_concurrency - ) - if external_sql_reference_resolver is not None: - external_sql_reference_resolver.validate_model_names( - known_model_names=validator.model_names() - ) - validation_context: _ModelValidationContext = _ModelValidationContext( - effective_settings=effective_settings, - no_sql_validation=no_sql_validation, - project_config_path=(discovered_inputs.project_dir or Path()) / PROJECT_CONFIG_FILENAME, - defer_model_sql_validation=defer_model_sql_validation, - external_sql_reference_resolver=external_sql_reference_resolver, - sql_lexical_syntax=context.sql_lexical_syntax, - validators=ModelValidatorContext( - settings=effective_settings, - external_sql_reference_resolver=external_sql_reference_resolver, - native_config=native_model_config_session( - inputs=NativeModelConfigInputs( - project_config=discovered_inputs.project_config, - target_config=context.target_config, - effective_vars=effective_vars, - effective_target_name=context.effective_target_name, - run_id=context.run_id, - microbatch_concurrency=effective_settings.microbatch_concurrency, - ), - validator=validator, - ), - ), - ) - loop: _ModelInputLoop = _ModelInputLoop( - discovered_inputs=discovered_inputs, - context=context, - validation_context=validation_context, - ) - return tuple(_run_native_model_loop(loop=loop, native_project=native_project)) - - -def _run_native_model_loop( - *, loop: _ModelInputLoop, native_project: _native.NativeCompiledProject -) -> list[ModelInput]: - """Run the model loop natively; models it cannot finish yet call back into the attachment.""" - - files: tuple[DiscoveredSqlModelFile, ...] = loop.discovered_inputs.model_files - settings: SettingsConfig = loop.validation_context.effective_settings - native_loop: _native.NativeModelLoop = _native.NativeModelLoop( - ( - loop.validation_context.validators.native_config.validator, - native_contract_classes(), - model_loop_classes( - audit_factories=loop.discovered_inputs.audit_factories, - context=loop.context, - external_resolver=loop.validation_context.external_sql_reference_resolver, - host_inputs={ - "report_diagnostic": report_compile_diagnostic, - "record_template_reads": record_template_reads, - "visible_declarations": _VisibleModelDeclarations, - "config_session": loop.validation_context.validators.native_config, - "syntax_help": sql_analysis_opt_out_help(model_owned=True), - "source_syntax_help": sql_analysis_opt_out_help(model_owned=False), - "macros": { - "loaded_macros": loop.context.loaded_macros, - "macro_context": loop.context.macro_context, - "bridge": active_macro_bridge(), - "scoped": loop.context.declaration_resolver is not None, - "declaration_context": DeclarationResolutionContext, - "macro_argument_error": MacroArgumentError, - "expansion": macro_expansion_classes(), - "generic_syntax": SqlLexicalSyntax().native_mapping, - }, - "sql_hooks": { - "named_sql_hook_entry": NamedSqlHookEntry, - "python_hook_entry": PythonHookEntry, - "hook_files": loop.discovered_inputs.sql_hook_files, - "effective_vars": loop.context.effective_vars, - "environment": UnicodeEnvironment(), - "render_project_var": render_project_var_text, - "sql_resource_ref": SqlResourceRef, - "explicit_kinds": EXPLICIT_REFERENCE_KINDS, - "declaration_identity": DeclarationIdentity, - "sql_hook_kind": DeclarationKind.SQL_HOOK, - }, - "hook_functions": loop.discovered_inputs.hook_functions, - "provider_names": frozenset( - provider.name for provider in loop.discovered_inputs.providers - ), - }, - ), - ), - loop.context.sql_lexical_syntax.native_mapping, - cursor_intrinsic_semantics(), - ( - ( - settings.sql_analysis, - loop.validation_context.no_sql_validation, - loop.validation_context.defer_model_sql_validation, - settings.require_sql_analysis, - ), - str(loop.validation_context.project_config_path), - ), - ) - inputs: list[_native.NativeModelInput] = native_loop.run( - native_project, _ModelLoopHost(loop=loop), list(files) - ) - return list(inputs) - - -def build_effective_connection( - *, - project_config: ProjectConfig, - local_config: LocalConfig, - target_config: TargetConfig | None, - effective_vars: dict[str, object], -) -> dict[str, object]: - """Merge base project connection with the selected environment overrides.""" - - connection, error = _native.effective_config_connection( - project_config, local_config, target_config - ) - if connection is None: - raise CompileInputError(error or "") - return cast( - dict[str, object], - expand_template_data( - value=connection, - variables=effective_vars, - context_values={}, - context_label="effective connection", - allow_context=False, - preserve_context_tokens=False, - preserve_unknown_context=False, - ), - ) - - -def build_effective_settings( - *, project_config: ProjectConfig, local_config: LocalConfig -) -> SettingsConfig: - """Merge project settings with local developer overrides.""" - - values: dict[str, object] = _native.effective_config_settings( - (project_config.settings, local_config.settings), - local_config.setting_overrides, - [field.name for field in fields(SettingsConfig)], - ) - return SettingsConfig(**cast(dict[str, Any], values)) - - -def build_effective_vars( - *, - project_config: ProjectConfig, - local_config: LocalConfig, - target_config: TargetConfig | None, - cli_vars: dict[str, object], -) -> dict[str, object]: - """Merge effective vars using the locked precedence order.""" - - return expand_effective_vars( - _native.effective_config_vars((project_config, local_config), target_config, cli_vars) - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py b/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py deleted file mode 100644 index 6874fa1934..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py +++ /dev/null @@ -1,131 +0,0 @@ -"""Build process-local declaration scope state for one compile invocation.""" - -from __future__ import annotations - -import inspect -from collections.abc import Mapping -from pathlib import Path -from types import CodeType - -import sqlbuild._native as _native -from sqlbuild.compiler.compile._helpers.render.declarations import ( - build_declaration_scope_resolver, -) -from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache -from sqlbuild.compiler.compile.constants import SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - DeclarationScopeBuild, - LoadedMacro, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.scopes.exceptions import ScopeValidationError -from sqlbuild.compiler.scopes.main._open_native_scope_index import open_native_scope_index -from sqlbuild.compiler.scopes.main._validate_scope_index import validate_scope_index -from sqlbuild.compiler.scopes.models import ScopeIndex, ScopeLookup -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - - -def build_declaration_scope( - *, - discovered_inputs: DiscoveredProjectInputs, - loaded_macros: dict[str, LoadedMacro], - sql_lexical_syntax: SqlLexicalSyntax, - compile_cache_dir: Path | None = None, -) -> DeclarationScopeBuild: - """Build one canonical index and validate it before SQL expansion.""" - - scan_cache: SqlTestScanCache = SqlTestScanCache(cache_dir=compile_cache_dir) - - _native.validate_authored_sql_test_ctes( - discovered_inputs.test_files, - discovered_inputs.scenario_files, - sql_lexical_syntax.native_mapping, - ) - return _build_native_declaration_scope( - discovered_inputs=discovered_inputs, - loaded_macros=loaded_macros, - sql_lexical_syntax=sql_lexical_syntax, - scan_cache=scan_cache, - ) - - -def _build_native_declaration_scope( - *, - discovered_inputs: DiscoveredProjectInputs, - loaded_macros: dict[str, LoadedMacro], - sql_lexical_syntax: SqlLexicalSyntax, - scan_cache: SqlTestScanCache, -) -> DeclarationScopeBuild: - """Build and validate the scope index natively, with its relationship grants and lookup.""" - - native: _native.NativeScopeIndex = open_native_scope_index( - discovered_inputs=discovered_inputs, loaded_macros=loaded_macros - ) - try: - validate_scope_index(index=native.index) - except ScopeValidationError as error: - raise CompileInputError(str(error)) from error - if native.has_scoped_relationship_declarations and ( - discovered_inputs.test_files or discovered_inputs.scenario_files - ): - faults: tuple[tuple[Path, str], ...] = native.grant_relationships( - discovered_inputs=discovered_inputs, - syntax=sql_lexical_syntax, - scan_store=(scan_cache.opened(), SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM), - ) - if faults: - raise CompileInputError(faults[0][1]) - index: ScopeIndex = native.index_with_relationships() - lookup: ScopeLookup = _native.build_scope_lookup(index) - return DeclarationScopeBuild( - loaded_macros=loaded_macros, - index=index, - resolver=build_declaration_scope_resolver( - discovered_inputs=discovered_inputs, - scope_index=index, - loaded_macros=loaded_macros, - lookup=lookup, - ), - sql_test_scans=scan_cache, - ) - - -def rebind_declaration_scope( - *, - scope: DeclarationScopeBuild, - discovered_inputs: DiscoveredProjectInputs, - loaded_macros: dict[str, LoadedMacro], -) -> DeclarationScopeBuild | None: - """Pair a built index with private macro instances, or return None if their metadata differs.""" - - if _indexed_macro_metadata(loaded_macros) != _indexed_macro_metadata(scope.loaded_macros): - return None - return DeclarationScopeBuild( - loaded_macros=loaded_macros, - index=scope.index, - resolver=build_declaration_scope_resolver( - discovered_inputs=discovered_inputs, - scope_index=scope.index, - loaded_macros=loaded_macros, - lookup=scope.resolver.lookup, - ), - sql_test_scans=scope.sql_test_scans, - ) - - -def _indexed_macro_metadata(loaded_macros: Mapping[str, LoadedMacro]) -> tuple[object, ...]: - return tuple( - ( - key, - macro.name, - macro.relative_path, - macro.raw_source, - macro.dependencies, - code.co_firstlineno - if isinstance(code := getattr(macro.function, "__code__", None), CodeType) - else None, - tuple(inspect.signature(macro.function).parameters), - ) - for key, macro in loaded_macros.items() - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py deleted file mode 100644 index f29b2e7697..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py +++ /dev/null @@ -1,153 +0,0 @@ -"""Attachment helpers for building pre-semantic compile inputs.""" - -from __future__ import annotations - -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.compile._helpers.config.retention import resolve_time_travel_retention -from sqlbuild.compiler.compile._helpers.config.table_type import resolve_table_type -from sqlbuild.compiler.compile._helpers.render.templating import ( - record_template_reads, -) -from sqlbuild.compiler.compile.classes.unicode_environment import UnicodeEnvironment -from sqlbuild.compiler.compile.models import ( - CompileModelConfig, - ModelValidationRequest, - NativeModelConfigInputs, - NativeModelConfigSession, -) -from sqlbuild.compiler.compiled_project.main._native_config_classes import native_config_classes -from sqlbuild.compiler.discovery.models import ( - DiscoveredSqlModelFile, - NamedSqlHookEntry, - PythonHookEntry, - SqlHookEntry, -) -from sqlbuild.compiler.model_config.main._native_config_error import native_config_error -from sqlbuild.compiler.planner.types import MaterializationType -from sqlbuild.spec.contracts.models import ( - MaterializationDefaultsConfig, - ProjectConfig, - TargetConfig, -) - -_MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hooks", "post_hooks"}) -_TABLE_BACKED_MATERIALIZATIONS: tuple[str, ...] = ( - MaterializationType.TABLE, - MaterializationType.INCREMENTAL, - MaterializationType.SNAPSHOT, -) -type _BuiltNamespace = tuple[str | None, bool, str | None] -type _StorageOverrides = tuple[tuple[int | None, bool] | None, str | None] -type _BuiltConfig = tuple[ - dict[str, object], - tuple[str, ...], - _BuiltNamespace, - _StorageOverrides, - list[tuple[str, str]], -] -_MODEL_DESCRIPTION_KEY: str = "description" -_MODEL_SCHEMA_KEY: str = "model_schema" - - -def native_model_config_session( - *, inputs: NativeModelConfigInputs, validator: _native.NativeModelValidator -) -> NativeModelConfigSession: - """Capture the project inputs every model's native config build and validation read.""" - - project_config: ProjectConfig = inputs.project_config - target: TargetConfig | None = inputs.target_config - materialization_defaults: MaterializationDefaultsConfig = ( - project_config.materialization_defaults or MaterializationDefaultsConfig() - ) - return NativeModelConfigSession( - path_defaults=project_config.path_defaults, - builder=_native.NativeModelConfigBuilder( - ( - project_config.defaults, - project_config.path_defaults, - (SqlHookEntry, NamedSqlHookEntry, PythonHookEntry), - ), - (inputs.effective_vars, UnicodeEnvironment()), - ( - (inputs.effective_target_name, inputs.run_id), - None if target is None else (target.database, target.schema), - ), - ), - validator=validator, - inherited_storage={ - materialized: ( - resolve_time_travel_retention( - materialized=materialized, - model_value=None, - materialization_defaults=materialization_defaults, - target_config=target, - model_name="", - ), - resolve_table_type( - materialized=materialized, - model_value=None, - materialization_defaults=materialization_defaults, - target_config=target, - model_name="", - ), - ) - for materialized in (None, *_TABLE_BACKED_MATERIALIZATIONS) - }, - ) - - -def build_native_model_config( - *, - session: NativeModelConfigSession, - model_file: DiscoveredSqlModelFile, - matched_path_default: str | None, -) -> CompileModelConfig: - """Build a model's effective config natively, raising its first config error.""" - - built: object = session.builder.model_config( - model_file.header_values, - (matched_path_default, model_file.file_path.stem), - native_config_classes(session=session, record_template_reads=record_template_reads), - ) - if isinstance(built, _native.NativeConfigError): - raise native_config_error(error=built) - return cast(CompileModelConfig, built) - - -def native_model_validation( - *, session: NativeModelConfigSession, request: ModelValidationRequest, rejected: list[bool] -) -> int | _native.NativeConfigError | None: - """Validate natively: None, the first error, or the index of a rejected dbt reference.""" - - config: CompileModelConfig = request.config - return session.validator.validate( - config.values, - ( - request.model_file.file_path.stem, - request.query_sql, - str(request.model_file.relative_path), - ), - ( - [ - (str(reference.ref_kind), reference.ref_name, reference_rejected) - for reference, reference_rejected in zip(request.references, rejected, strict=True) - ], - ( - None - if request.declared_columns is None - else [column.name for column in request.declared_columns] - ), - config.time_travel_retention.unmanaged, - config.table_type.declared, - ), - ) - - -def native_validation_error( - *, error: _native.NativeConfigError, values: dict[str, object] -) -> Exception: - """Return a native validator error, unmarked because validators read macro-expanded SQL.""" - - return native_config_error(error=error, values=values) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/target.py b/src/sqlbuild/compiler/compile/_helpers/attachment/target.py deleted file mode 100644 index 99b2964754..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/target.py +++ /dev/null @@ -1,141 +0,0 @@ -"""Effective target and compile-cache resolution helpers.""" - -from __future__ import annotations - -import os -import shutil -from pathlib import Path - -from sqlbuild.compiler.compile._helpers.attachment.core import build_effective_vars -from sqlbuild.compiler.compile._helpers.render.context_templates import ( - resolve_early_model_templates, - resolve_run_id, -) -from sqlbuild.compiler.compile.constants import ( - COMPILE_CACHE_DISABLE_ENV_VAR, - COMPILE_CACHE_DISABLE_VALUE, - RETIRED_FACT_CACHE_DIRECTORY_NAME, - RETIRED_REFERENCE_CACHE_DIRECTORY_NAME, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.frontier.constants import ( - COMPILER_CACHE_DIRECTORY_NAME, - RETIRED_CACHE_NAMESPACE_SUFFIXES, -) -from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name -from sqlbuild.spec.contracts.models import TargetConfig - - -def build_compile_target_context( - *, - discovered_inputs: DiscoveredProjectInputs, - selected_target: str | None, - no_cache: bool, -) -> tuple[str | None, TargetConfig | None, Path | None]: - """Resolve the effective target and shared project-local compile-cache directory.""" - - target_name: str | None = resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=selected_target, - ) - target_config: TargetConfig | None = ( - resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=target_name, - ) - if target_name is not None - else None - ) - cache_dir: Path | None = compile_cache_root( - project_dir=discovered_inputs.project_dir, - target_config=target_config, - no_cache=no_cache, - ) - return target_name, target_config, cache_dir - - -def compile_cache_root( - *, project_dir: Path | None, target_config: TargetConfig | None, no_cache: bool -) -> Path | None: - """Return the shared compile-cache directory, or None when caching is disabled.""" - - if ( - no_cache - or project_dir is None - or (target_config is not None and target_config.compile_cache is False) - or os.environ.get(COMPILE_CACHE_DISABLE_ENV_VAR) == COMPILE_CACHE_DISABLE_VALUE - ): - return None - root: Path = compiler_cache_directory(project_dir) - _remove_retired_caches(root=root) - return root - - -def _remove_retired_caches(*, root: Path) -> None: - """Delete the fact and reference caches older releases kept under each retired cache root.""" - - for suffix in RETIRED_CACHE_NAMESPACE_SUFFIXES: - for name in (RETIRED_FACT_CACHE_DIRECTORY_NAME, RETIRED_REFERENCE_CACHE_DIRECTORY_NAME): - retired: Path = root.parent / f"{COMPILER_CACHE_DIRECTORY_NAME}{suffix}" / name - if retired.is_dir(): - shutil.rmtree(retired, ignore_errors=True) - - -def build_effective_target_namespace( - *, - discovered_inputs: DiscoveredProjectInputs, - selected_target: str | None, - connection_database: object | None, - connection_schema: object | None, - default_database: str | None, - default_schema: str | None, -) -> tuple[str | None, TargetConfig | None, str | None, str | None]: - """Resolve the target and its effective database/schema without compiling resources.""" - - target_name: str | None = resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=selected_target, - ) - target_config: TargetConfig | None = ( - resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=target_name, - ) - if target_name is not None - else None - ) - effective_vars: dict[str, object] = build_effective_vars( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_config=target_config, - cli_vars={}, - ) - target_values: dict[str, object] = resolve_early_model_templates( - values={ - "database": (target_config.database if target_config is not None else None) - or connection_database - or default_database, - "schema": (target_config.schema if target_config is not None else None) - or connection_schema - or default_schema, - }, - effective_vars=effective_vars, - effective_target_name=target_name, - run_id=resolve_run_id(selected_run_id=None), - ) - return ( - target_name, - target_config, - _optional_string(target_values.get("database")), - _optional_string(target_values.get("schema")), - ) - - -def _optional_string(value: object | None) -> str | None: - return value if isinstance(value, str) and value else None diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/help.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/help.py deleted file mode 100644 index 1ccfd4bf25..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/help.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Code-owned semantic remediation; no catch-all help for unrelated diagnostics.""" - -_SEMANTIC_HELP: dict[str, str] = { - "B003": "qualify this column with the intended input alias", - "B004": "give each referenced relation a visible alias in this scope", - "B005": "match the relation alias list to its available output columns", - "B101": "check the function spelling or declare the project function before using it", - "B102": "pass the number of arguments required by this function's signature", - "B210": "use an expression whose inferred type satisfies this SQL construct", - "B211": "use a Boolean predicate in this condition", - "B212": "use operands supported by this arithmetic operator, or convert them explicitly", - "B213": "supply argument types supported by this function or conditional expression", - "B214": "make the assigned expression compatible with the destination type", - "B215": "align the corresponding types in each set-operation branch", - "B216": "return the same number of columns on both sides of this operation", - "B217": "compare compatible types using a correctly typed literal or explicit conversion", - "B218": "choose a supported source-to-target cast for this dialect", - "B219": "declare or cast this expression's unknown type so it can be checked", - "B230": "add this expression to GROUP BY or use it inside an aggregate", - "B231": "move this aggregate to SELECT or HAVING, or compute it in an input CTE", - "B232": "use this window function with a valid OVER clause and window context", - "B233": "use distinct names for relations or CTEs within this scope", - "B234": "use a valid constant bound for LIMIT or OFFSET", - "B300": "change the metadata column name to an existing output column", - "B301": "align this declared metadata or audit value with the column or argument type", - "B302": "align the SQL-test fixture or expected columns with the tested resource's output", -} - - -def semantic_help_catalogue() -> dict[str, str]: - """Return the code-owned remediation catalogue.""" - return dict(_SEMANTIC_HELP) diff --git a/src/sqlbuild/compiler/compiled_project/_helpers/resource_rows.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py similarity index 85% rename from src/sqlbuild/compiler/compiled_project/_helpers/resource_rows.py rename to src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py index d596ecbb38..18f136d0d9 100644 --- a/src/sqlbuild/compiler/compiled_project/_helpers/resource_rows.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py @@ -1,4 +1,4 @@ -"""Rows the native resource diagnostics read: loaders and user Python resources.""" +"""User Python resources as the rows the native missing-description diagnostics read.""" from __future__ import annotations @@ -7,27 +7,12 @@ from functools import partial from pathlib import Path -from sqlbuild.compiler.compile.types import SourceInput -from sqlbuild.compiler.compiled_project.types import PythonResourceRow -from sqlbuild.compiler.discovery.constants import YAML_FILE_SUFFIXES +from sqlbuild.compiler.compile.types import PythonResourceRow, SourceInput from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredProvider from sqlbuild.errors.setting_help.main.snippet_help import snippet_help from sqlbuild.spec.contracts.models import SourceLocation -def loader_rows(discovered_inputs: DiscoveredProjectInputs) -> dict[str, tuple[object, ...]]: - """Each loader's description, relative path and whether that path is a YAML file.""" - - return { - loader.name: ( - loader.description, - str(loader.relative_path), - loader.relative_path.suffix in YAML_FILE_SUFFIXES, - ) - for loader in discovered_inputs.loader_functions - } - - def python_resource_rows( *, discovered_inputs: DiscoveredProjectInputs, source_inputs: tuple[SourceInput, ...] ) -> list[PythonResourceRow]: diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py index d62407d202..907ed49485 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py @@ -4,12 +4,10 @@ from typing import Any +from sqlbuild._native import complete_semantic_project from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.analysis_session.types import DiagnosticRow from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.semantic_checks.main._complete_native_semantic_diagnostics import ( - complete_native_semantic_diagnostics, -) def complete_semantic_diagnostics( @@ -22,7 +20,7 @@ def complete_semantic_diagnostics( ) -> CompiledProjectView: """Recover type facts before metadata checks, then explain retained root diagnostics.""" - return complete_native_semantic_diagnostics( + return complete_semantic_project( project=project, profile=profile, binding_results=binding_results, diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql.py index 9712e43569..c04e76824c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql.py @@ -2,34 +2,12 @@ from __future__ import annotations -from pathlib import Path from typing import Any, cast import sqlbuild._native as _native from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import binding_relation_names -from sqlbuild.compiler.compile.models import CompilerDiagnostic, CompileSqlReference -from sqlbuild.compiler.compile.types import ( - CompiledAuditView, - CompiledModelView, - CompiledProjectView, - CompiledResourceType, - DiagnosticPhase, - DiagnosticSeverity, -) -from sqlbuild.compiler.discovery.constants import SQL_ANALYSIS_CONFIG_KEY -from sqlbuild.spec.contracts.models import SourceLocation - -_MODEL_HOOK_KEYS: tuple[str, ...] = ("pre_hooks", "post_hooks") -_BUILT_IN_AUDIT_ROOT: str = "" -_RESOURCE_TYPES: dict[str, CompiledResourceType] = { - "audit": CompiledResourceType.AUDIT, - "sql_test": CompiledResourceType.SQL_TEST, - "model": CompiledResourceType.MODEL, -} - -type _Span = tuple[int, int, int | None, int | None] -type _Row = tuple[str, str, bool, str, str, str, _Span, str | None, str | None] +from sqlbuild.compiler.compile.models import CompilerDiagnostic +from sqlbuild.compiler.compile.types import CompiledProjectView def get_resource_sql_diagnostics( @@ -40,126 +18,6 @@ def get_resource_sql_diagnostics( ) -> tuple[CompilerDiagnostic, ...]: """Bind audits, SQL tests and SQL hooks against the same closed shapes as models.""" - request: Any = ( - profile.sql_analysis_dialect, - profile.quoted_identifiers_ignore_case, - project.settings.require_sql_analysis, - [(name, list(columns.items())) for name, columns in shapes.items()], - [_audit_row(audit) for audit in project.audits], - [ - ( - test.name, - list(test.expected_model_names), - str(test.test_file.relative_path), - test.test_file.contents, - test.sql_body, - test.test_block.sql_body, - test.test_block.header_values.get(SQL_ANALYSIS_CONFIG_KEY) is False, - ) - for test in project.sql_tests - ], - [_model_row(model) for model in project.models], - [ - (str(hook_file.relative_path), hook_file.contents) - for hook_file in project.sql_hook_files - ], - ) - rows: list[_Row] = cast( - list[_Row], - _native.resource_sql_diagnostic_rows(cast(Any, project.binding_catalog).native, request), - ) - paths: dict[str, Path] = {str(path): path for path in _paths(project)} - return tuple(_diagnostic(row=row, paths=paths) for row in rows) - - -def _paths(project: CompiledProjectView) -> list[Path]: - """Every path a resource SQL diagnostic can name, so rows map back to the authored `Path`.""" - - paths: list[Path] = [audit.audit_file.relative_path for audit in project.audits] - paths.extend(test.test_file.relative_path for test in project.sql_tests) - paths.extend(model.relative_path for model in project.models) - paths.extend(hook_file.relative_path for hook_file in project.sql_hook_files) - return paths - - -def _relations(references: tuple[object, ...]) -> list[str]: - return sorted( - binding_relation_names( - tuple( - reference for reference in references if isinstance(reference, CompileSqlReference) - ) - ) - ) - - -def _audit_row(audit: CompiledAuditView) -> tuple[object, ...]: - target: tuple[str, str] | None = ( - None - if audit.attached_target_name is None - else (str(audit.attached_target_kind), audit.attached_target_name) - ) - return ( - audit.name, - audit.definition_name, - target, - audit.attached_column_name, - audit.audit_file.file_path.parts[:1] == (_BUILT_IN_AUDIT_ROOT,), - str(audit.audit_file.relative_path), - audit.audit_file.contents, - [ - (audit.sql_body, audit.audit_block.sql_body), - (audit.measure_sql, audit.audit_block.measure_sql), - (audit.evidence_sql, audit.audit_block.evidence_sql), - ], - audit.audit_block.header_values.get(SQL_ANALYSIS_CONFIG_KEY) is False, - _relations(audit.references), - ) - - -def _model_row(model: CompiledModelView) -> tuple[object, ...]: - hooks: list[tuple[object, ...]] = [] - for hook_key in _MODEL_HOOK_KEYS: - entries: object = model.config.values.get(hook_key) - if not isinstance(entries, list | tuple): - continue - for hook_index, entry in enumerate(cast(list[object] | tuple[object, ...], entries)): - statement: object = getattr(entry, "statement", None) - if not isinstance(statement, str): - continue - hook_path: Path | None = getattr(entry, "relative_path", None) - definition: object = getattr(entry, "definition_sql", None) - hooks.append( - ( - hook_key, - hook_index, - statement, - None if hook_path is None else str(hook_path), - definition if isinstance(definition, str) else None, - _relations(tuple(getattr(entry, "reads", ()) or ())), - ) - ) - return ( - model.name, - str(model.relative_path), - model.authored_sql, - model.config.values.get(SQL_ANALYSIS_CONFIG_KEY) is False, - hooks, - ) - - -def _diagnostic(*, row: _Row, paths: dict[str, Path]) -> CompilerDiagnostic: - kind, name, is_error, code, message, path, span, note, help_text = row - line, column, end_line, end_column = span - return CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR if is_error else DiagnosticSeverity.WARNING, - code=code, - message=message, - resource_type=_RESOURCE_TYPES[kind], - resource_name=name, - location=SourceLocation( - path=paths[path], line=line, column=column, end_line=end_line, end_column=end_column - ), - notes=() if note is None else (note,), - help=help_text, + return _native.resource_sql_diagnostics_of( + cast(Any, project.binding_catalog).native, project, shapes, profile ) diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/scope.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/scope.py deleted file mode 100644 index cc2e0e7731..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/scope.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Report declaration-scope index errors as individual compile diagnostics.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic -from sqlbuild.compiler.compile.models import CompilerDiagnostic -from sqlbuild.compiler.compile.types import ( - CompiledResourceType, - DiagnosticPhase, - DiagnosticSeverity, -) -from sqlbuild.compiler.scopes.models import ScopeDiagnostic, ScopeIndex -from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind -from sqlbuild.compiler.scopes.types import DiagnosticSeverity as ScopeSeverity - -_RESOURCE_TYPES: dict[ResourceKind, CompiledResourceType] = { - ResourceKind.MODEL: CompiledResourceType.MODEL, - ResourceKind.SOURCE: CompiledResourceType.SOURCE, - ResourceKind.SEED: CompiledResourceType.SEED, - ResourceKind.FUNCTION: CompiledResourceType.UDF, - ResourceKind.TEST: CompiledResourceType.SQL_TEST, - ResourceKind.SCENARIO: CompiledResourceType.SQL_SCENARIO, -} -_DECLARATION_RESOURCE_TYPES: dict[DeclarationKind, CompiledResourceType] = { - DeclarationKind.AUDIT: CompiledResourceType.AUDIT, - DeclarationKind.SINGULAR_AUDIT: CompiledResourceType.AUDIT, -} - - -def report_scope_index_errors(*, index: ScopeIndex) -> None: - """Report each error-severity scope finding as its own compile diagnostic.""" - - for item in index.diagnostics: - if item.severity is not ScopeSeverity.ERROR: - continue - report_compile_diagnostic( - key=( - item.code.value, - item.path or "", - str(item.declaration or ""), - str(item.resource or ""), - item.message, - ), - diagnostic=_compile_diagnostic(item), - ) - - -def _compile_diagnostic(item: ScopeDiagnostic) -> CompilerDiagnostic: - resource_type: CompiledResourceType | None = None - resource_name: str | None = None - if item.resource is not None and item.resource.kind in _RESOURCE_TYPES: - resource_type = _RESOURCE_TYPES[item.resource.kind] - resource_name = item.resource.name - elif item.declaration is not None and item.declaration.kind in _DECLARATION_RESOURCE_TYPES: - resource_type = _DECLARATION_RESOURCE_TYPES[item.declaration.kind] - resource_name = item.declaration.name - return CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=item.code.value, - message=item.message, - resource_type=resource_type, - resource_name=resource_name, - path=Path(item.path) if item.path is not None else None, - line=item.line, - column=item.column, - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py index 8c65c7639c..d4c081a7bc 100644 --- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py +++ b/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py @@ -1,4 +1,4 @@ -"""Compile's explicit-reference checks: Python collects the facts, native decides.""" +"""Compile's explicit-reference checks: Python collects user-code owners, native decides.""" from __future__ import annotations @@ -6,46 +6,19 @@ import textwrap from collections.abc import Callable, Mapping from dataclasses import dataclass, field -from itertools import chain from pathlib import Path -from typing import Any - -import orjson import sqlbuild._native as _native -from sqlbuild.compiler.compile.constants import MIGRATE_FROM_CONFIG_KEY -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, - CompilerDiagnostic, - PythonSqlReferenceReport, -) -from sqlbuild.compiler.compile.types import ( - AttachedAuditTargetKind, - CompiledAuditView, - CompiledModelView, - CompiledProjectView, - CompiledSourceView, - DiagnosticPhase, - DiagnosticSeverity, -) +from sqlbuild.compiler.compile.models import PythonSqlReferenceReport +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.main.explicit_references_help import explicit_references_help -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, SqlHookEntry +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.graph.main._model_python_hook_names import model_python_hook_names -from sqlbuild.compiler.migrations.main._resolve_old_name_view_retention import ( - resolve_old_name_view_retention, -) -from sqlbuild.compiler.references.models import LiteralSqlRelation, RelationName from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind from sqlbuild.python_nodes.main.read_loader_definition import read_loader_definition from sqlbuild.python_nodes.models import LoaderDefinition, SqlResourceRef from sqlbuild.python_nodes.types import SqlResourceRefKind -from sqlbuild.spec.contracts.models import SourceEntry, SourceLocation -_MODEL_HOOK_KEYS: tuple[str, ...] = ("pre_hooks", "post_hooks") -_RAISE_KEY: str = "raise" -_REPORT_KEY: str = "report" _VALUE_PLACEHOLDER: str = "__sqlbuild_python_value_" @@ -59,13 +32,6 @@ class _PythonSqlOwner: upstream_loader_by_source: Mapping[str, str] = field(default_factory=dict) -@dataclass(frozen=True) -class _LiteralSql: - method: str - sql: str - line: int - - def explicit_reference_report( *, project: CompiledProjectView, @@ -77,222 +43,26 @@ def explicit_reference_report( owners: tuple[_PythonSqlOwner, ...] = ( _owners(project=project, discovered_inputs=discovered_inputs) if enforce_explicit else () ) - literals: tuple[tuple[_LiteralSql, ...], ...] = tuple( - _literal_sql_calls(owner.function) for owner in owners - ) paths: list[Path] = sorted({owner.relative_path for owner in owners}) path_order: dict[Path, int] = {path: index for index, path in enumerate(paths)} - outcome: dict[str, Any] = orjson.loads( - _native.check_native_explicit_references( - orjson.dumps( - { - "enforce_explicit": enforce_explicit, - "dialect": project.sql_analysis_dialect, - "explicit_help": ( - explicit_references_help(allowed="hard-coded relation names") - if enforce_explicit - else "" - ), - "placeholder": _VALUE_PLACEHOLDER, - "resources": _resource_rows(project), - "models": [ - _model_row( - model=model, - old_name_retention=( - discovered_inputs.project_config.migrations.old_name_views - ), - ) - for model in project.models - ], - "sources": [_source_row(source) for source in project.sources], - "seeds": [ - { - "name": seed.name, - "relation": _location_row(seed.destination), - } - for seed in project.seeds - ], - "audits": [_audit_row(audit) for audit in project.audits], - "hooks": [ - { - "name": hook.name, - "path": hook.relative_path.as_posix(), - "reads": _ref_rows(hook.reads), - } - for hook in project.hook_functions - ], - "owners": [ - _owner_row( - owner=owner, - literals=owner_literals, - path_order=path_order[owner.relative_path], - ) - for owner, owner_literals in zip(owners, literals, strict=True) - ], - } - ).decode() - ) - ) - if _RAISE_KEY in outcome: - raise CompileInputError(outcome[_RAISE_KEY]) - report: dict[str, Any] = outcome[_REPORT_KEY] - return PythonSqlReferenceReport( - diagnostics=tuple(_diagnostic(row=row, owners=owners) for row in report["diagnostics"]), - unmatched=tuple( - LiteralSqlRelation( - owner_label=owners[row["owner"]].label, - owner_kind=owners[row["owner"]].kind, - relative_path=owners[row["owner"]].relative_path, - line=row["line"], - method=row["method"], - relation=RelationName( - name=row["relation"]["name"], - schema=row["relation"]["schema"], - database=row["relation"]["database"], - ), - ) - for row in report["unmatched"] - ), - ) - - -def _diagnostic(*, row: dict[str, Any], owners: tuple[_PythonSqlOwner, ...]) -> CompilerDiagnostic: - owner: int | None = row["owner"] - resource: list[str] | None = row["resource"] - return CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=row["code"], - message=row["message"], - resource_type=None if resource is None else resource[0], - resource_name=None if resource is None else resource[1], - path=None if row["path"] is None else Path(row["path"]), - location=( - None - if owner is None - else SourceLocation(path=owners[owner].relative_path, line=row["line"], column=1) - ), - help=row["help"], - ) - - -def _owner_row( - *, owner: _PythonSqlOwner, literals: tuple[_LiteralSql, ...], path_order: int -) -> dict[str, object]: - return { - "label": owner.label, - "kind": str(owner.kind), - "path_order": path_order, - "own_refs": _ref_rows(tuple(owner.own_refs)), - "upstream_loaders": list(owner.upstream_loader_by_source.items()), - "literals": [ - {"method": literal.method, "sql": literal.sql, "line": literal.line} - for literal in literals - ], - } - - -def _resource_rows(project: CompiledProjectView) -> list[dict[str, object]]: - return [ - *(_graph_row(key=model.key_row, deps=model.dep_rows) for model in project.models), - *( - _graph_row(key=_key_row(resource.key), deps=_key_rows(resource.deps)) - for resource in (*project.sources, *project.seeds, *project.functions) - ), - ] - - -def _graph_row(*, key: tuple[str, str], deps: list[tuple[str, str]]) -> dict[str, object]: - return {"key": key, "deps": deps} - - -def _key_row(key: CompiledObjectKey) -> tuple[str, str]: - return (str(key.resource_type), key.name) - - -def _key_rows(keys: tuple[CompiledObjectKey, ...]) -> list[tuple[str, str]]: - return [_key_row(key) for key in keys] - - -def _model_row(*, model: CompiledModelView, old_name_retention: str | None) -> dict[str, object]: - raw: object = model.config.values.get(MIGRATE_FROM_CONFIG_KEY) - retained: bool = isinstance(raw, str) and ( - resolve_old_name_view_retention( - config_values=model.config.values, project_retention=old_name_retention - ) - is not None - ) - return { - "name": model.name, - "path": str(model.relative_path), - "destination": _destination_row(model.destination_row), - "python_hooks": list(model_python_hook_names(model=model)), - "sql_hooks": list( - chain.from_iterable( - _sql_hook_rows(hook_key=hook_key, entries=model.config.values.get(hook_key)) - for hook_key in _MODEL_HOOK_KEYS - ) - ), - "migrate_from": raw if retained else None, - } - - -def _sql_hook_rows(*, hook_key: str, entries: object) -> list[dict[str, object]]: - """SQL hook entries with declared reads, at their position among the key's entries.""" - - items: tuple[object, ...] = tuple(entries) if isinstance(entries, list | tuple) else (entries,) - return [ + return _native.explicit_reference_report( + project, { - "hook_key": hook_key, - "index": index, - "name": entry.name, - "path": None if entry.relative_path is None else str(entry.relative_path), - "reads": _ref_rows(entry.reads), - } - for index, entry in enumerate(items) - if isinstance(entry, SqlHookEntry) and entry.reads - ] - - -def _source_row(source: CompiledSourceView) -> dict[str, object]: - entry: SourceEntry = source.source_entry - return { - "name": source.name, - "table": entry.table, - "schema": entry.schema, - "database": entry.database, - "expression": entry.expression is not None, - } - - -def _audit_row(audit: CompiledAuditView) -> dict[str, object]: - target: tuple[str, str] | None = ( - ( - str(AttachedAuditTargetKind(audit.attached_target_kind).resource_type), - audit.attached_target_name, - ) - if audit.attached_target_kind is not None and audit.attached_target_name is not None - else None + "retains_old_names": discovered_inputs.project_config.migrations.old_name_views + is not None, + "enforce_explicit": enforce_explicit, + "explicit_help": ( + explicit_references_help(allowed="hard-coded relation names") + if enforce_explicit + else "" + ), + "placeholder": _VALUE_PLACEHOLDER, + }, + [ + (owner, path_order[owner.relative_path], _literal_sql_calls(owner.function)) + for owner in owners + ], ) - return { - "name": audit.name, - "path": str(audit.audit_file.relative_path), - "target": target, - "scope_deps": _key_rows(audit.scope_deps), - } - - -def _location_row(location: CompiledRelationLocation) -> dict[str, str | None]: - return _destination_row((location.database, location.schema, location.name)) - - -def _destination_row(row: tuple[str | None, str | None, str]) -> dict[str, str | None]: - database, schema, name = row - return {"name": name, "schema": schema, "database": database} - - -def _ref_rows(refs: tuple[SqlResourceRef, ...]) -> list[tuple[str, str]]: - return [(str(ref.kind), ref.name) for ref in refs] def _owners( @@ -369,16 +139,13 @@ def _loader_owners( return tuple(owners) -def _literal_sql_calls(function: Callable[..., object]) -> tuple[_LiteralSql, ...]: +def _literal_sql_calls(function: Callable[..., object]) -> list[tuple[str, str, int]]: try: lines: list[str] start_line: int lines, start_line = inspect.getsourcelines(function) except (OSError, TypeError): - return () - return tuple( - _LiteralSql(method=method, sql=sql, line=line) - for method, sql, line in _native.python_literal_sql_calls( - textwrap.dedent("".join(lines)), start_line, _VALUE_PLACEHOLDER - ) + return [] + return _native.python_literal_sql_calls( + textwrap.dedent("".join(lines)), start_line, _VALUE_PLACEHOLDER ) diff --git a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py index 1901cba38e..48af17b0f7 100644 --- a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py +++ b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py @@ -4,7 +4,9 @@ from pathlib import Path -from sqlbuild.compiler.compile._helpers.attachment.target import build_compile_target_context +from sqlbuild._native import ( + build_compile_target_context, +) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge from sqlbuild.compiler.macro_bridge.main._claim_first_compile import claim_first_compile diff --git a/src/sqlbuild/compiler/compile/_helpers/named_declarations/model_schemas.py b/src/sqlbuild/compiler/compile/_helpers/named_declarations/model_schemas.py deleted file mode 100644 index ad4f50f1d8..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/named_declarations/model_schemas.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Scope facts for reusable model schemas: parent schemas and enum column types.""" - -from __future__ import annotations - -from itertools import chain -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic -from sqlbuild.compiler.compile.models import CompilerDiagnostic, DeclarationScopeResolver -from sqlbuild.compiler.compile.types import DiagnosticPhase, DiagnosticSeverity -from sqlbuild.compiler.discovery.models import ( - DiscoveredModelSchemaFile, - EnumDeclaration, - ModelSchemaDeclaration, -) -from sqlbuild.compiler.scopes.exceptions import InvalidScopePathError -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationRecord, - ResourceIdentity, - UsageRecord, -) -from sqlbuild.compiler.scopes.types import DeclarationKind, ScopeDiagnosticCode - -_SCHEMA_USAGE_KINDS: frozenset[DeclarationKind] = frozenset( - {DeclarationKind.SCHEMA, DeclarationKind.ENUM} -) - - -def model_schema_scope_usages( - *, - schema_files: tuple[DiscoveredModelSchemaFile, ...], - resolver: DeclarationScopeResolver | None, -) -> tuple[UsageRecord, ...]: - """Require parent schemas and enum column types to be visible from each schema's folder.""" - - if resolver is None: - return () - records: list[DeclarationRecord] = [ - found[0] - for identity, found in resolver.lookup.declarations.items() - if found and identity.owner is None and identity.kind in _SCHEMA_USAGE_KINDS - ] - usages: list[tuple[int, int]] - diagnostics: list[tuple[list[str], str, str, str]] - error: str | None - usages, diagnostics, error = _native.scope_schema_usages( - list(map(_schema_row, chain.from_iterable(item.declarations for item in schema_files))), - list(map(_record_row, records)), - ) - if error is not None: - raise InvalidScopePathError(error) - for key, message, path, help_text in diagnostics: - report_compile_diagnostic( - key=tuple(key), - diagnostic=CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=ScopeDiagnosticCode.INACCESSIBLE_DECLARATION.value, - message=message, - path=Path(path), - help=help_text, - ), - ) - return tuple( - UsageRecord(consumer=records[consumer].identity, declaration=records[used].identity) - for consumer, used in usages - ) - - -def _schema_row( - declaration: ModelSchemaDeclaration, -) -> tuple[str, str | None, list[tuple[str, str | None]]]: - return ( - declaration.name, - declaration.extends, - [(column.name, column.type) for column in declaration.columns], - ) - - -def _record_row( - record: DeclarationRecord, -) -> tuple[tuple[str, str], tuple[str, str, tuple[str, str] | None, str | None, str]]: - owner: ResourceIdentity | None = record.identity.owner - return ( - (record.identity.kind.value, record.identity.name), - ( - record.path, - record.scope.value, - None if owner is None else (owner.kind.value, owner.name), - record.owning_path, - record.ownership_root.path, - ), - ) - - -def model_schema_enum_declarations( - *, schema: ModelSchemaDeclaration, resolver: DeclarationScopeResolver | None -) -> dict[str, EnumDeclaration]: - """Return enum column types resolved from the schema's own location.""" - - if resolver is None: - return {} - enums: dict[str, EnumDeclaration] = {} - for column in schema.columns: - if column.type is None: - continue - value: object = resolver.projection.declarations.get( - DeclarationIdentity(DeclarationKind.ENUM, column.type) - ) - if isinstance(value, EnumDeclaration): - enums[column.type] = value - return enums diff --git a/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py b/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py index 9121966adb..444018a284 100644 --- a/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py +++ b/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py @@ -4,16 +4,12 @@ from typing import Any -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.main._analyze_native_model_sql import ( +from sqlbuild._native import ( analyze_native_model_sql, -) -from sqlbuild.compiler.analysis_session.main._infer_native_expression_source_shapes import ( infer_native_expression_source_shapes, -) -from sqlbuild.compiler.analysis_session.main._prove_native_dynamic_contracts import ( prove_native_dynamic_contracts, ) +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.analysis_session.models import ( NativeModelAnalyses, NativeModelAnalysisRequest, diff --git a/src/sqlbuild/compiler/compile/_helpers/refs/native.py b/src/sqlbuild/compiler/compile/_helpers/refs/native.py deleted file mode 100644 index bcb6cb5038..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/refs/native.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Logical SQL references extracted by the native scanner.""" - -from __future__ import annotations - -from functools import cache -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import ( - CompileSqlReference, - InvalidSqlReferenceCall, - SqlReferenceScan, -) -from sqlbuild.compiler.compile.types import SqlReferenceScanFailure -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - -type _NativeExtraction = ( - tuple[ - tuple[ - list[tuple[str, str, str | None, int | None]], - list[tuple[str, str, int, str, str, str]], - ], - None, - ] - | tuple[None, tuple[str, int]] -) - - -def extract_native_sql_references( - *, sql: str, syntax: SqlLexicalSyntax -) -> SqlReferenceScan | SqlReferenceScanFailure: - """Return the references and rejected calls, or the scan error and its start.""" - - extraction: _NativeExtraction = _scanner(syntax).extract(sql) - scanned, failure = extraction - if scanned is None: - return cast(SqlReferenceScanFailure, failure) - references, invalid_calls = scanned - return SqlReferenceScan( - references=tuple( - CompileSqlReference( - ref_kind=SqlReferenceKind(kind), - ref_name=name, - ref_package=package, - call_argument_count=call_argument_count, - ) - for kind, name, package, call_argument_count in references - ), - invalid_calls=tuple( - InvalidSqlReferenceCall( - ref_kind=SqlReferenceKind(kind), - call=call, - start=start, - message=call_message, - help=help_text, - corrected_call=corrected_call, - ) - for kind, call, start, call_message, help_text, corrected_call in invalid_calls - ), - ) - - -@cache -def _scanner(syntax: SqlLexicalSyntax) -> _native.SqlReferenceScanner: - return _native.SqlReferenceScanner(syntax.native_mapping) diff --git a/src/sqlbuild/compiler/compile/_helpers/refs/references.py b/src/sqlbuild/compiler/compile/_helpers/refs/references.py index ce170e1ba8..e241cd04f6 100644 --- a/src/sqlbuild/compiler/compile/_helpers/refs/references.py +++ b/src/sqlbuild/compiler/compile/_helpers/refs/references.py @@ -2,238 +2,79 @@ from __future__ import annotations -from pathlib import Path +from functools import cache +from typing import cast +import sqlbuild._native as _native from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic -from sqlbuild.compiler.compile._helpers.refs.native import extract_native_sql_references -from sqlbuild.compiler.compile._helpers.render.spans import map_through_passes -from sqlbuild.compiler.compile.constants import ( - REFERENCE_CALL_SYNTAX_CODE, -) +from sqlbuild.compiler.compile.constants import REFERENCE_CALL_SYNTAX_CODE from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( CompilerDiagnostic, CompileSqlReference, - ExpansionSpan, InvalidSqlReferenceCall, - MappedOffset, - SqlReferenceOrigin, SqlReferenceScan, - SqlReferenceSourceMap, ) from sqlbuild.compiler.compile.types import ( DiagnosticPhase, DiagnosticSeverity, SqlReferenceScanFailure, ) +from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import SourceLocation def extract_sql_references( - *, sql: str, syntax: SqlLexicalSyntax, origin: SqlReferenceOrigin | None = None + *, sql: str, syntax: SqlLexicalSyntax ) -> tuple[CompileSqlReference, ...]: """Return logical SQL refs outside comments and quotes, reporting rejected calls as P012.""" - scan: SqlReferenceScan = scan_sql_reference_calls(sql=sql, syntax=syntax, origin=origin) - report_invalid_reference_calls(invalid_calls=scan.invalid_calls, origin=origin, syntax=syntax) - return scan.references - - -def scan_sql_reference_calls( - *, sql: str, syntax: SqlLexicalSyntax, origin: SqlReferenceOrigin | None = None -) -> SqlReferenceScan: - """Return valid references and rejected calls; a scan error is located in `origin`.""" - - outcome: SqlReferenceScan | SqlReferenceScanFailure = _reference_scan_outcome( - sql=sql, syntax=syntax - ) - if isinstance(outcome, SqlReferenceScan): - return outcome - raise _located_scan_error(failure=outcome, sql=sql, origin=origin, syntax=syntax) - - -def _reference_scan_outcome( - *, sql: str, syntax: SqlLexicalSyntax -) -> SqlReferenceScan | SqlReferenceScanFailure: - outcome: SqlReferenceScan | SqlReferenceScanFailure = extract_native_sql_references( - sql=sql, syntax=syntax - ) - return outcome - - -def _located_scan_error( - *, - failure: SqlReferenceScanFailure, - sql: str, - origin: SqlReferenceOrigin | None, - syntax: SqlLexicalSyntax, -) -> CompileInputError: - """Locate a scan error in authored text; unmapped or expansion-shaped faults stay unlocated.""" - - message, start = failure - if origin is None: - return CompileInputError(message) - authored_offset: int | None = _authored_fault_offset( - failure=failure, sql=sql, origin=origin, syntax=syntax - ) - path: str = origin.relative_path.as_posix() - if authored_offset is None: - return CompileInputError(f"{path}: {message}") - contents: str = origin.contents - line: int = contents.count("\n", 0, authored_offset) + 1 - column: int = authored_offset - (contents.rfind("\n", 0, authored_offset) + 1) + 1 - return CompileInputError(f"{path}:{line}:{column}: {message}") - - -def _authored_fault_offset( - *, - failure: SqlReferenceScanFailure, - sql: str, - origin: SqlReferenceOrigin, - syntax: SqlLexicalSyntax, -) -> int | None: - """Map the fault to authored text unless expansion output produced or shaped the failure.""" - - source_map: SqlReferenceSourceMap | None = origin.source_map - start: int = failure[1] - body_start: int | None = source_map.body_start() if source_map is not None else None - if source_map is None or body_start is None or start >= len(sql): - return None - mapped: MappedOffset = map_through_passes(offset=start, passes=source_map.passes) - authored_offset: int = body_start + mapped.offset - if ( - mapped.generated - or authored_offset >= len(origin.contents) - or origin.contents[authored_offset] != sql[start] - ): - return None - blanked: list[str] = list(sql) - for output_start, output_end in _expansion_output_ranges(source_map.passes): - blanked[output_start:output_end] = " " * (output_end - output_start) - if _reference_scan_outcome(sql="".join(blanked), syntax=syntax) != failure: - return None - return authored_offset - - -def _expansion_output_ranges( - passes: tuple[tuple[ExpansionSpan, ...], ...], -) -> list[tuple[int, int]]: - """Return every expansion's output range in the coordinates of the final pass's output.""" - - ranges: list[tuple[int, int]] = [] - for index, spans in enumerate(passes): - for span in spans: - output_start: int = span.output_start - output_end: int = span.output_end - for later_spans in passes[index + 1 :]: - output_start = _later_pass_offset(offset=output_start, spans=later_spans, end=False) - output_end = _later_pass_offset(offset=output_end, spans=later_spans, end=True) - ranges.append((output_start, output_end)) - return ranges - - -def _later_pass_offset(*, offset: int, spans: tuple[ExpansionSpan, ...], end: bool) -> int: - """Carry a range boundary through one later pass, widening it over a span that covers it.""" - - shift: int = 0 - for span in spans: - if offset < span.source_start or (end and offset == span.source_start): - break - if offset < span.source_end or (end and offset == span.source_end): - return span.output_end if end else span.output_start - shift += (span.output_end - span.output_start) - (span.source_end - span.source_start) - return offset + shift - - -def report_invalid_reference_calls( - *, - invalid_calls: tuple[InvalidSqlReferenceCall, ...], - origin: SqlReferenceOrigin | None, - syntax: SqlLexicalSyntax, -) -> None: - """Report each rejected reference call at its authored location when one is known.""" - - if not invalid_calls: - return - authored_starts: dict[str, list[int]] = _authored_invalid_call_starts( - origin=origin, syntax=syntax - ) - seen: dict[str, int] = {} - for invalid_call in invalid_calls: - occurrence: int = seen.get(invalid_call.call, 0) - seen[invalid_call.call] = occurrence + 1 - starts: list[int] = authored_starts.get(invalid_call.call, []) - location: SourceLocation | None = ( - reference_call_location( - path=origin.relative_path, - text=origin.contents, - start=starts[occurrence], - call=invalid_call.call, - ) - if origin is not None and occurrence < len(starts) - else None - ) + scan: SqlReferenceScan = scan_sql_reference_calls(sql=sql, syntax=syntax) + for invalid_call in scan.invalid_calls: report_compile_diagnostic( - key=reference_call_syntax_key( - file_path=origin.file_path if origin is not None else None, - call=invalid_call.call, - location=location, - ), + key=(REFERENCE_CALL_SYNTAX_CODE, "", invalid_call.call), diagnostic=CompilerDiagnostic( phase=DiagnosticPhase.COMPILE, severity=DiagnosticSeverity.ERROR, code=REFERENCE_CALL_SYNTAX_CODE, message=invalid_call.message, - resource_type=origin.resource_type if origin is not None else None, - resource_name=origin.resource_name if origin is not None else None, - path=origin.relative_path if origin is not None and location is None else None, - location=location, help=invalid_call.help, ), ) + return scan.references -def reference_call_syntax_key( - *, file_path: Path | None, call: str, location: SourceLocation | None -) -> tuple[str, ...]: - """Return the collector key shared by every report of one rejected reference call.""" - - owner: str = file_path.as_posix() if file_path is not None else "" - if location is None: - return (REFERENCE_CALL_SYNTAX_CODE, owner, call) - return ( - REFERENCE_CALL_SYNTAX_CODE, - owner, - f"{location.line:09d}:{location.column:09d}", - call, - ) - - -def _authored_invalid_call_starts( - *, origin: SqlReferenceOrigin | None, syntax: SqlLexicalSyntax -) -> dict[str, list[int]]: - if origin is None: - return {} - authored: SqlReferenceScan | SqlReferenceScanFailure = _reference_scan_outcome( - sql=origin.contents, syntax=syntax +def scan_sql_reference_calls(*, sql: str, syntax: SqlLexicalSyntax) -> SqlReferenceScan: + """Return valid references and rejected calls; a scan error raises unlocated.""" + + scanned, failure = _scanner(syntax).extract(sql) + if scanned is None: + raise CompileInputError(cast(SqlReferenceScanFailure, failure)[0]) + references, invalid_calls = scanned + return SqlReferenceScan( + references=tuple( + CompileSqlReference( + ref_kind=SqlReferenceKind(kind), + ref_name=name, + ref_package=package, + call_argument_count=call_argument_count, + ) + for kind, name, package, call_argument_count in references + ), + invalid_calls=tuple( + InvalidSqlReferenceCall( + ref_kind=SqlReferenceKind(kind), + call=call, + start=start, + message=call_message, + help=help_text, + corrected_call=corrected_call, + ) + for kind, call, start, call_message, help_text, corrected_call in invalid_calls + ), ) - if not isinstance(authored, SqlReferenceScan): - return {} - starts: dict[str, list[int]] = {} - for invalid_call in authored.invalid_calls: - starts.setdefault(invalid_call.call, []).append(invalid_call.start) - return starts -def reference_call_location(*, path: Path, text: str, start: int, call: str) -> SourceLocation: - """Return the span of ``call`` starting at ``start`` in ``text`` as a source location.""" - - end: int = start + len(call) - return SourceLocation( - path=path, - line=text.count("\n", 0, start) + 1, - column=start - (text.rfind("\n", 0, start) + 1) + 1, - end_line=text.count("\n", 0, end) + 1, - end_column=end - (text.rfind("\n", 0, end) + 1) + 1, - ) +@cache +def _scanner(syntax: SqlLexicalSyntax) -> _native.SqlReferenceScanner: + return _native.SqlReferenceScanner(syntax.native_mapping) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py b/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py index 5160e57304..4f6baacade 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py @@ -21,13 +21,13 @@ def resolve_early_model_templates( return cast( dict[str, object], - expand_config_templates( + expand_template_data( value=values, variables=effective_vars, - context_values=build_run_context_values( - effective_target_name=effective_target_name, - run_id=run_id, - ), + context_values={ + CompileContextKey.RUN_ID: run_id, + CompileContextKey.RUN_TARGET: effective_target_name, + }, context_label="model config", allow_context=True, preserve_context_tokens=False, @@ -36,17 +36,6 @@ def resolve_early_model_templates( ) -def build_run_context_values( - *, effective_target_name: str | None, run_id: str -) -> dict[str, str | None]: - """Build the compile-time CTX values known before resource-specific resolution.""" - - return { - CompileContextKey.RUN_ID: run_id, - CompileContextKey.RUN_TARGET: effective_target_name, - } - - def resolve_run_id(*, selected_run_id: str | None) -> str: """Resolve a stable compile invocation id.""" @@ -55,26 +44,3 @@ def resolve_run_id(*, selected_run_id: str | None) -> str: timestamp_prefix: str = datetime.now(tz=UTC).strftime("%Y%m%dT%H%M%SZ") unique_suffix: str = uuid4().hex[:12] return f"{timestamp_prefix}_{unique_suffix}" - - -def expand_config_templates( - *, - value: object, - variables: dict[str, object], - context_values: dict[str, str | None], - context_label: str, - allow_context: bool, - preserve_context_tokens: bool, - preserve_unknown_context: bool, -) -> object: - """Expand `${...}` templates in a config value natively, recording what it reads.""" - - return expand_template_data( - value=value, - variables=variables, - context_values=context_values, - context_label=context_label, - allow_context=allow_context, - preserve_context_tokens=preserve_context_tokens, - preserve_unknown_context=preserve_unknown_context, - ) diff --git a/src/sqlbuild/compiler/compile/main/_scope_index_with_compile_usages.py b/src/sqlbuild/compiler/compile/main/_scope_index_with_compile_usages.py index 60e92e5e21..8ccdcc776a 100644 --- a/src/sqlbuild/compiler/compile/main/_scope_index_with_compile_usages.py +++ b/src/sqlbuild/compiler/compile/main/_scope_index_with_compile_usages.py @@ -2,9 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.compile._helpers.named_declarations.model_schemas import ( - model_schema_scope_usages, -) +import sqlbuild._native as _native from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.scopes.main._get_placement_validated_scope_index import ( get_placement_validated_scope_index, @@ -16,11 +14,9 @@ def scope_index_with_compile_usages(*, inputs: CompileInputsView) -> ScopeIndex: """Return compile inputs' canonical index with all usage and placement facts.""" collected_usages: list[UsageRecord] = list( - model_schema_scope_usages( - schema_files=inputs.discovered_inputs.model_schema_files, - resolver=( - inputs.declaration_scope.resolver if inputs.declaration_scope is not None else None - ), + _native.model_schema_scope_usages( + inputs.discovered_inputs.model_schema_files, + inputs.declaration_scope.resolver if inputs.declaration_scope is not None else None, ) ) for model_input in inputs.model_inputs: diff --git a/src/sqlbuild/compiler/compile/main/effective_config.py b/src/sqlbuild/compiler/compile/main/effective_config.py index 5e48d0741d..5bb306fe43 100644 --- a/src/sqlbuild/compiler/compile/main/effective_config.py +++ b/src/sqlbuild/compiler/compile/main/effective_config.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.compile._helpers.attachment.core import ( +from sqlbuild._native import ( build_effective_connection, build_effective_vars, ) diff --git a/src/sqlbuild/compiler/compile/main/effective_runtime.py b/src/sqlbuild/compiler/compile/main/effective_runtime.py index f0a4a4e6db..719c6713cc 100644 --- a/src/sqlbuild/compiler/compile/main/effective_runtime.py +++ b/src/sqlbuild/compiler/compile/main/effective_runtime.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.compile._helpers.attachment.core import ( +from sqlbuild._native import ( build_effective_vars, ) from sqlbuild.compiler.compile._helpers.render.context_templates import resolve_run_id diff --git a/src/sqlbuild/compiler/compile/main/effective_settings.py b/src/sqlbuild/compiler/compile/main/effective_settings.py index e33f7860a1..dbe8e63a9b 100644 --- a/src/sqlbuild/compiler/compile/main/effective_settings.py +++ b/src/sqlbuild/compiler/compile/main/effective_settings.py @@ -2,7 +2,9 @@ from __future__ import annotations -from sqlbuild.compiler.compile._helpers.attachment.core import build_effective_settings +from sqlbuild._native import ( + build_effective_settings, +) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.spec.contracts.models import SettingsConfig diff --git a/src/sqlbuild/compiler/compile/main/effective_target_namespace.py b/src/sqlbuild/compiler/compile/main/effective_target_namespace.py index 2e4f8937fb..87184dda4a 100644 --- a/src/sqlbuild/compiler/compile/main/effective_target_namespace.py +++ b/src/sqlbuild/compiler/compile/main/effective_target_namespace.py @@ -1,6 +1,6 @@ """Public effective target namespace operation.""" -from sqlbuild.compiler.compile._helpers.attachment.target import ( +from sqlbuild._native import ( build_effective_target_namespace as _build_effective_target_namespace, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs diff --git a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py b/src/sqlbuild/compiler/compile/main/sql_expansion_context.py index ebc498b355..d3b65cd047 100644 --- a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py +++ b/src/sqlbuild/compiler/compile/main/sql_expansion_context.py @@ -5,10 +5,10 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild._native import discover_project_inputs -from sqlbuild.compiler.compile._helpers.attachment.core import build_effective_vars -from sqlbuild.compiler.compile._helpers.attachment.declaration_scope import ( +from sqlbuild._native import ( build_declaration_scope, + build_effective_vars, + discover_project_inputs, rebind_declaration_scope, ) from sqlbuild.compiler.compile._helpers.render.declarations import ( diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index e8218a86c1..59439b520e 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -384,6 +384,23 @@ class DeclarationExpansionResult: usages: tuple[UsageRecord, ...] = field(default_factory=tuple) +@dataclass(frozen=True) +class VisibleModelDeclarations: + """One model's visible declarations and macros, as the native model loop resolves them.""" + + local_enums: dict[str, EnumDeclaration] + local_constants: dict[str, ConstantDeclaration] + enums: dict[str, EnumDeclaration] + constants: dict[str, ConstantDeclaration] + inaccessible_enums: dict[str, DeclarationRecord] + inaccessible_constants: dict[str, DeclarationRecord] + enum_visibility: dict[str, tuple[VisibilityRecord, ...]] + constant_visibility: dict[str, tuple[VisibilityRecord, ...]] + macros: dict[str, LoadedMacro] + macro_records: dict[str, DeclarationRecord] + inaccessible_macros: dict[str, DeclarationRecord] + + @dataclass(frozen=True) class DeclarationResolutionContext: """Declarations visible during one authored SQL expansion.""" @@ -498,26 +515,6 @@ class SqlReferenceScan: invalid_calls: tuple[InvalidSqlReferenceCall, ...] = () -@dataclass(frozen=True) -class SqlReferenceSourceMap: - """Where the authored body starts in its file and the expansion passes from it to the SQL.""" - - body_start: Callable[[], int | None] - passes: tuple[tuple[ExpansionSpan, ...], ...] - - -@dataclass(frozen=True) -class SqlReferenceOrigin: - """The authored file that SQL handed to reference extraction came from.""" - - file_path: Path - relative_path: Path - contents: str - resource_type: CompiledResourceType | None = None - resource_name: str | None = None - source_map: SqlReferenceSourceMap | None = None - - @dataclass(frozen=True) class CompiledLineageSourceFact: """Compact upstream column fact extracted during SQL analysis.""" diff --git a/src/sqlbuild/compiler/compile/types.py b/src/sqlbuild/compiler/compile/types.py index 44f5090117..01c2391b5d 100644 --- a/src/sqlbuild/compiler/compile/types.py +++ b/src/sqlbuild/compiler/compile/types.py @@ -8,7 +8,7 @@ from sqlbuild.sql_values.models import SqlValue if TYPE_CHECKING: - from collections.abc import Sequence + from collections.abc import Callable, Sequence from pathlib import Path import sqlbuild._native as _native @@ -673,3 +673,4 @@ class SqlTestMode(StrEnum): type SqlReferenceScanFailure = tuple[str, int] +type PythonResourceRow = tuple[str, str, str | None, Callable[[], SourceLocation], str] diff --git a/src/sqlbuild/compiler/compiled_project/__init__.py b/src/sqlbuild/compiler/compiled_project/__init__.py deleted file mode 100644 index 31c048c1f1..0000000000 --- a/src/sqlbuild/compiler/compiled_project/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""The compile's natively owned project and the recorders producers retain facts with.""" diff --git a/src/sqlbuild/compiler/compiled_project/_helpers/__init__.py b/src/sqlbuild/compiler/compiled_project/_helpers/__init__.py deleted file mode 100644 index d6aacb385a..0000000000 --- a/src/sqlbuild/compiler/compiled_project/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Compiled-project recording helpers.""" diff --git a/src/sqlbuild/compiler/compiled_project/_helpers/assurance_attachment.py b/src/sqlbuild/compiler/compiled_project/_helpers/assurance_attachment.py deleted file mode 100644 index c652443256..0000000000 --- a/src/sqlbuild/compiler/compiled_project/_helpers/assurance_attachment.py +++ /dev/null @@ -1,212 +0,0 @@ -"""SQL tests, scenarios and audits attached natively by the compile's model loop.""" - -from __future__ import annotations - -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.compiler.auditing.main._builtins import builtin_audit_definitions -from sqlbuild.compiler.auditing.types import AuditEvaluationMode -from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache -from sqlbuild.compiler.compile.constants import SQL_TEST_CTE_SCAN_ALGORITHM -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompileDirectLogicSqlTestInputPayload, - CompileModelSqlTestInputPayload, - CompilerDiagnostic, - CompileSqlScenarioCte, - CompileSqlScenarioInput, - CompileSqlTestCte, - CompileSqlTestInput, -) -from sqlbuild.compiler.compile.types import ( - AuditInput, - SourceInput, - SqlTestMode, - TypedSqlValueRenderer, -) -from sqlbuild.compiler.compiled_project._helpers.resource_rows import ( - loader_rows, - python_resource_rows, -) -from sqlbuild.compiler.discovery.models import ( - DiscoveredAuditFile, - DiscoveredProjectInputs, - DiscoveredSqlScenarioFile, - DiscoveredSqlTestBlock, - DiscoveredSqlTestFile, -) -from sqlbuild.compiler.profiling.main.record import record_compile_timing -from sqlbuild.compiler.scopes.models import DeclarationIdentity -from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind, UsageKind -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import SettingsConfig, SourceLocation -from sqlbuild.sql_values.exceptions import SqlValueRenderingError, SqlValueValidationError -from sqlbuild.sql_values.models import SqlValue - - -def attach_sql_tests_impl( - *, - discovered_inputs: DiscoveredProjectInputs, - native_project: _native.NativeCompiledProject, - value_renderer: TypedSqlValueRenderer, - syntax: SqlLexicalSyntax, - scan_cache: SqlTestScanCache, -) -> tuple[tuple[CompileSqlTestInput, ...], tuple[CompileSqlScenarioInput, ...]]: - """Attach every SQL test case, then every scenario, saving the scan store afterwards.""" - - with record_compile_timing("test_input_compile_ms"): - tests, scenarios = native_project.attach_sql_tests( - [_test_file_row(test_file) for test_file in discovered_inputs.test_files], - [_scenario_row(scenario) for scenario in discovered_inputs.scenario_files], - { - "test_input": CompileSqlTestInput, - "model_payload": CompileModelSqlTestInputPayload, - "direct_payload": CompileDirectLogicSqlTestInputPayload, - "test_cte": CompileSqlTestCte, - "scenario_input": CompileSqlScenarioInput, - "scenario_cte": CompileSqlScenarioCte, - "modes": {mode.value: mode for mode in SqlTestMode}, - "test_kind": ResourceKind.TEST, - "scenario_kind": ResourceKind.SCENARIO, - "runtime_usage": UsageKind.RUNTIME, - "render_parameter": _parameter_renderer(value_renderer), - "adapter_name": value_renderer.adapter_name, - "source_location": SourceLocation, - "scan_store": scan_cache.opened(), - "scan_key": (SQL_TEST_CTE_SCAN_ALGORITHM, syntax.cache_key), - }, - ) - scan_cache.save() - return tuple(tests), tuple(scenarios) - - -def attach_audits_impl( - *, - discovered_inputs: DiscoveredProjectInputs, - native_project: _native.NativeCompiledProject, - effective_settings: SettingsConfig, -) -> tuple[tuple[AuditInput, ...], tuple[CompilerDiagnostic, ...]]: - """Attach every singular audit, then each model's, source's and seed's generic audits.""" - - default_severity: object | None = effective_settings.default_audit_severity - audits, diagnostics = native_project.attach_audits( - [_audit_file_row(audit_file) for audit_file in discovered_inputs.audit_files], - { - "default_severity": None if default_severity is None else str(default_severity), - "default_run_scope": effective_settings.default_audit_run_scope, - "builtins": [_audit_file_row(audit_file) for audit_file in builtin_audit_definitions()], - "declaration_identity": DeclarationIdentity, - "singular_kind": DeclarationKind.SINGULAR_AUDIT, - "audit_kind": DeclarationKind.AUDIT, - "resource_kinds": {kind.value: kind for kind in ResourceKind}, - }, - ) - return tuple(audits), tuple(diagnostics) - - -def _parameter_renderer(value_renderer: TypedSqlValueRenderer) -> object: - def render(value: SqlValue) -> tuple[str | None, str | None]: - try: - return value_renderer.render_typed_scalar(value=value), None - except (SqlValueRenderingError, SqlValueValidationError) as error: - return None, str(error) - - return render - - -def _test_file_row(test_file: DiscoveredSqlTestFile) -> tuple[object, ...]: - return ( - str(test_file.file_path), - str(test_file.relative_path), - test_file.relative_path.stem, - test_file, - test_file.contents, - [_test_block_row(block) for block in test_file.blocks], - ) - - -def _test_block_row(block: DiscoveredSqlTestBlock) -> tuple[object, ...]: - return ( - block, - block.name, - block.sql_body, - block.mode.value, - [parameter.name for parameter in block.parameters], - [(case, case.name, list(case.values)) for case in block.cases], - block.sql_body_span, - ) - - -def _scenario_row(scenario: DiscoveredSqlScenarioFile) -> tuple[object, ...]: - return ( - str(scenario.file_path), - str(scenario.relative_path), - scenario, - scenario.contents, - scenario.sql_body, - scenario.name, - ) - - -def _audit_file_row(audit_file: DiscoveredAuditFile) -> tuple[object, ...]: - return ( - str(audit_file.file_path), - str(audit_file.relative_path), - audit_file.file_path.stem, - audit_file, - audit_file.declaration_kind is DeclarationKind.AUDIT, - audit_file.contents, - [ - ( - block, - block.name, - block.sql_body, - block.evidence_sql, - block.evaluation_mode == AuditEvaluationMode.MEASUREMENT, - block.header_values, - ) - for block in audit_file.blocks - ], - ) - - -def resource_diagnostics_impl( - *, - discovered_inputs: DiscoveredProjectInputs, - native_project: _native.NativeCompiledProject, - source_inputs: tuple[SourceInput, ...], -) -> tuple[CompilerDiagnostic, ...]: - """Orphaned audit factory warnings, then missing descriptions, then cursor-input errors.""" - - orphans, descriptions, cursors = native_project.resource_diagnostics( - { - "compiler_diagnostic": CompilerDiagnostic, - "compile_error": CompileInputError, - "source_location": SourceLocation, - "path_class": Path, - "model_files": list(discovered_inputs.model_files), - "audit_factories": [ - (factory.name, str(factory.relative_path), factory.line) - for factory in discovered_inputs.audit_factories - ], - "scenarios": [ - ( - scenario.name, - str(scenario.relative_path), - scenario.contents, - scenario.header_values.get("description"), - ) - for scenario in discovered_inputs.scenario_files - ], - "sql_hooks": [ - (hook.name, str(hook.relative_path), hook.contents, hook.description) - for hook in discovered_inputs.sql_hook_files - ], - "loaders": loader_rows(discovered_inputs), - "python_resources": python_resource_rows( - discovered_inputs=discovered_inputs, source_inputs=source_inputs - ), - } - ) - return (*orphans, *descriptions, *cursors) diff --git a/src/sqlbuild/compiler/compiled_project/_helpers/loop_inputs.py b/src/sqlbuild/compiler/compiled_project/_helpers/loop_inputs.py deleted file mode 100644 index c107089052..0000000000 --- a/src/sqlbuild/compiler/compiled_project/_helpers/loop_inputs.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Project declarations and scope classes the native model loop resolves models with.""" - -import inspect -from collections.abc import Callable -from inspect import Parameter - -from sqlbuild.compiler.compile.models import DeclarationScopeResolver, ModelInputBuildContext -from sqlbuild.compiler.discovery.models import EnumDeclaration -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationRecord, - ResourceIdentity, - VisibilityRecord, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - ResourceKind, - ScopeKind, - VisibilityReason, -) - -_NAMED_KINDS: frozenset[DeclarationKind] = frozenset( - { - DeclarationKind.AUDIT, - DeclarationKind.SCHEMA, - DeclarationKind.SQL_HOOK, - DeclarationKind.PYTHON_HOOK, - } -) - - -def named_records( - resolver: DeclarationScopeResolver | None, -) -> dict[tuple[str, str], DeclarationRecord]: - """Each project-owned generic audit's, schema's and hook's first record, by `(kind, name)`.""" - - if resolver is None: - return {} - records: dict[tuple[str, str], DeclarationRecord] = {} - for identity, declarations in resolver.lookup.declarations.items(): - if identity.kind in _NAMED_KINDS and identity.owner is None and declarations: - records[identity.kind.value, identity.name] = declarations[0] - return records - - -def hook_signature(function: Callable[..., object]) -> list[tuple[str, int, bool]]: - """A hook function's parameters as `inspect.signature` reads them: name, kind, default.""" - - return [ - (parameter.name, int(parameter.kind), parameter.default is not Parameter.empty) - for parameter in inspect.signature(function).parameters.values() - ] - - -def schema_enums(resolver: DeclarationScopeResolver | None) -> dict[str, EnumDeclaration]: - """Project-owned enums by name, as `model_schema_enum_declarations` resolves column types.""" - - if resolver is None: - return {} - enums: dict[str, EnumDeclaration] = {} - for identity, value in resolver.projection.declarations.items(): - if ( - identity.kind is DeclarationKind.ENUM - and identity.owner is None - and isinstance(value, EnumDeclaration) - ): - enums[identity.name] = value - return enums - - -def visibility_classes( - *, context: ModelInputBuildContext, visible_declarations: object -) -> dict[str, object]: - """What the loop resolves each model's visible declarations from.""" - - resolver: DeclarationScopeResolver | None = context.declaration_resolver - return { - "resolver": resolver, - "projects": resolver is not None and bool(resolver.lookup.index.declarations), - "resource_specific": ( - frozenset( - declaration.identity.owner - for declaration in resolver.lookup.index.declarations - if declaration.scope is ScopeKind.PRIVATE and declaration.identity.owner is not None - ) - | frozenset(resolver.lookup.grants_by_resource) - if resolver is not None - else frozenset() - ), - "public_enums": context.public_enums, - "public_constants": context.public_constants, - "visible_declarations": visible_declarations, - "resource_identity": ResourceIdentity, - "declaration_identity": DeclarationIdentity, - "model_kind": ResourceKind.MODEL, - "enum_kind": DeclarationKind.ENUM, - "constant_kind": DeclarationKind.CONSTANT, - "private_owner": VisibilityReason.PRIVATE_OWNER, - "visibility_record": VisibilityRecord, - } diff --git a/src/sqlbuild/compiler/compiled_project/_helpers/model_updates.py b/src/sqlbuild/compiler/compiled_project/_helpers/model_updates.py deleted file mode 100644 index b346300292..0000000000 --- a/src/sqlbuild/compiler/compiled_project/_helpers/model_updates.py +++ /dev/null @@ -1,220 +0,0 @@ -"""Record what later compile stages finish for a model, natively when its owner holds it.""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import fields, replace -from typing import Any, cast - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import ( - CompiledAudit, - CompiledFunction, - CompiledModel, - CompiledProject, - CompiledSeed, - CompiledSource, - CompileProjectInputs, - CompilerDiagnostic, - InferredColumn, -) -from sqlbuild.compiler.compile.types import ( - CompiledAuditView, - CompiledFunctionView, - CompiledModelView, - CompiledProjectView, - CompiledSeedView, - CompiledSourceView, - CompileInputsView, -) - - -def compiled_model_record_impl(model: CompiledModelView) -> CompiledModel: - """The model as a `CompiledModel`, projected from its owner when it is a native view.""" - - if isinstance(model, CompiledModel): - return model - return CompiledModel( - **{field.name: getattr(model, field.name) for field in fields(CompiledModel)} - ) - - -def with_binding_diagnostics_impl( - *, model: CompiledModelView, diagnostics: tuple[CompilerDiagnostic, ...] -) -> CompiledModelView: - """The model with these final binding diagnostics.""" - - if isinstance(model, CompiledModel): - return replace(model, binding_diagnostics=diagnostics) - return cast(_native.NativeCompiledModel, model).with_binding_diagnostics(diagnostics) - - -def with_type_recovery_impl( - *, - model: CompiledModelView, - diagnostics: tuple[CompilerDiagnostic, ...], - poisoned: frozenset[tuple[str, str]], -) -> CompiledModelView: - """The model with its recovered bindings and its poisoned outputs' types forgotten.""" - - if not isinstance(model, CompiledModel): - native: _native.NativeCompiledModel = cast(_native.NativeCompiledModel, model) - return native.with_type_recovery( - diagnostics, [column for name, column in poisoned if name == native.name] - ) - columns: tuple[InferredColumn, ...] | None = None - if model.inferred_columns is not None: - columns = tuple( - replace(column, type=None) if (model.name, column.name) in poisoned else column - for column in model.inferred_columns - ) - unchecked: frozenset[str] = frozenset(column for name, column in poisoned if name == model.name) - return replace( - model, - binding_diagnostics=diagnostics, - inferred_columns=columns, - unchecked_output_columns=unchecked, - ) - - -def with_target_defaults_impl( - *, - model: _native.NativeCompiledModel, - defaults: tuple[str | None, str | None], - render_qualified_name: Callable[..., str | None], -) -> CompiledModelView: - """The model with the adapter's `(schema, database)` defaults filled into its destination.""" - - return model.with_target_defaults(defaults, render_qualified_name) - - -def compiled_project_record_impl( - *, project: CompiledProjectView, include_sql_expansions: bool = True -) -> CompiledProject: - """The project as a `CompiledProject`, projected from its owner when it is a native view.""" - - if isinstance(project, CompiledProject): - return project if include_sql_expansions else replace(project, sql_expansions={}) - fields: dict[str, Any] = cast(_native.NativeProjectView, project).fields(include_sql_expansions) - fields["seeds"] = tuple(map(compiled_seed_record_impl, fields["seeds"])) - fields["sources"] = tuple(map(compiled_source_record_impl, fields["sources"])) - fields["functions"] = tuple(map(compiled_function_record_impl, fields["functions"])) - fields["audits"] = tuple(map(compiled_audit_record_impl, fields["audits"])) - return CompiledProject(**fields) - - -def compiled_audit_record_impl(audit: CompiledAuditView) -> CompiledAudit: - """The audit as a `CompiledAudit`, projected from its owner when it is a native view.""" - - if isinstance(audit, CompiledAudit): - return audit - return CompiledAudit( - key=audit.key, - scope_deps=audit.scope_deps, - name=audit.name, - definition_name=audit.definition_name, - audit_file=audit.audit_file, - audit_block=audit.audit_block, - sql_body=audit.sql_body, - evaluation_mode=audit.evaluation_mode, - measurement_contract=audit.measurement_contract, - thresholds=audit.thresholds, - minimum_samples=audit.minimum_samples, - measure_sql=audit.measure_sql, - evidence_sql=audit.evidence_sql, - evidence_limit=audit.evidence_limit, - references=audit.references, - attached_target_kind=audit.attached_target_kind, - attached_target_name=audit.attached_target_name, - attached_column_name=audit.attached_column_name, - severity=audit.severity, - run_scope=audit.run_scope, - always_run=audit.always_run, - description=audit.description, - ) - - -def compiled_function_record_impl(function: CompiledFunctionView) -> CompiledFunction: - """The function as a `CompiledFunction`, projected from its owner when it is a native view.""" - - if isinstance(function, CompiledFunction): - return function - return CompiledFunction( - key=function.key, - deps=function.deps, - name=function.name, - relative_path=function.relative_path, - arguments=function.arguments, - returns=function.returns, - body_sql=function.body_sql, - destination=function.destination, - fingerprint_destination=function.fingerprint_destination, - return_columns=function.return_columns, - references=function.references, - language=function.language, - source_file_path=function.source_file_path, - runtime_version=function.runtime_version, - entry_point=function.entry_point, - packages=function.packages, - tags=function.tags, - description=function.description, - ) - - -def compiled_source_record_impl(source: CompiledSourceView) -> CompiledSource: - """The source as a `CompiledSource`, projected from its owner when it is a native view.""" - - if isinstance(source, CompiledSource): - return source - return CompiledSource( - key=source.key, - deps=source.deps, - name=source.name, - source_entry=source.source_entry, - source_file=source.source_file, - ) - - -def compiled_seed_record_impl(seed: CompiledSeedView) -> CompiledSeed: - """The seed as a `CompiledSeed`, projected from its owner when it is a native view.""" - - if isinstance(seed, CompiledSeed): - return seed - return CompiledSeed( - key=seed.key, - deps=seed.deps, - name=seed.name, - seed_file=seed.seed_file, - schema_entry=seed.schema_entry, - schema_file=seed.schema_file, - destination=seed.destination, - external=seed.external, - ) - - -def project_with_impl(*, project: CompiledProjectView, **changes: Any) -> CompiledProjectView: - """The project with `changes`; a native view shares its owner's models.""" - - if isinstance(project, CompiledProject): - return replace(project, **changes) - return cast(_native.NativeProjectView, project).with_changes(**changes) - - -def inputs_with_impl(*, inputs: CompileInputsView, **changes: Any) -> CompileInputsView: - """The compile inputs with `changes`; a native view shares its owner's model inputs.""" - - if isinstance(inputs, CompileProjectInputs): - return replace(inputs, **changes) - return cast(_native.NativeInputsView, inputs).with_changes(**changes) - - -def compile_inputs_view_impl( - *, fields: dict[str, Any], native_project: _native.NativeCompiledProject | None -) -> CompileInputsView: - """The native owner's inputs view when it holds every model input; else a record.""" - - if native_project is not None and all( - isinstance(model, _native.NativeModelInput) for model in fields["model_inputs"] - ): - return native_project.inputs_view(fields) - return CompileProjectInputs(**fields, native_project=native_project) diff --git a/src/sqlbuild/compiler/compiled_project/_helpers/resource_attachment.py b/src/sqlbuild/compiler/compiled_project/_helpers/resource_attachment.py deleted file mode 100644 index 0b27e7e4b1..0000000000 --- a/src/sqlbuild/compiler/compiled_project/_helpers/resource_attachment.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Seeds, functions and sources attached natively and held by the compile's native project.""" - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.types import FunctionInput, SeedInput, SourceInput -from sqlbuild.compiler.discovery.models import ( - DiscoveredProjectInputs, - DiscoveredSchemaFile, - DiscoveredSourceFile, -) -from sqlbuild.compiler.scopes.types import ResourceKind -from sqlbuild.spec.contracts.models import ( - DefaultsConfig, - SchemaSeedEntry, - SettingsConfig, - SourceEntry, - TargetConfig, -) - - -def attach_seeds_impl( - *, - discovered_inputs: DiscoveredProjectInputs, - native_project: _native.NativeCompiledProject, -) -> tuple[SeedInput, ...]: - """Pair every declared seed with its CSV file; raise for a declaration without one.""" - - declarations: list[tuple[SchemaSeedEntry, DiscoveredSchemaFile]] = [] - for schema_file in discovered_inputs.schema_files: - declarations.extend((seed_entry, schema_file) for seed_entry in schema_file.seed_entries) - return tuple( - native_project.attach_seeds( - declarations, list(discovered_inputs.seed_files), CompileInputError - ) - ) - - -def attach_functions_impl( - *, - discovered_inputs: DiscoveredProjectInputs, - native_project: _native.NativeCompiledProject, - settings: tuple[SettingsConfig, TargetConfig | None, str], - no_sql_validation: bool, - python_functions_inherit_default_namespace: bool, -) -> tuple[FunctionInput, ...]: - """Attach and validate every SQL, then Python, function in discovery order.""" - - effective_settings, target_config, adapter_name = settings - defaults: DefaultsConfig = discovered_inputs.project_config.defaults - sql_rows: list[tuple[object, ...]] = [ - ( - str(function_file.file_path), - str(function_file.relative_path), - function_file.file_path.stem, - function_file, - function_file.header_values, - function_file.body_sql, - function_file.contents, - ) - for function_file in discovered_inputs.sql_function_files - ] - python_rows: list[tuple[object, ...]] = [ - ( - str(function_file.file_path), - str(function_file.relative_path), - function_file.file_path.stem, - function_file, - function_file.header_values, - function_file.body_python, - None, - ) - for function_file in discovered_inputs.python_function_files - ] - return tuple( - native_project.attach_functions( - (sql_rows, python_rows), - ( - adapter_name, - effective_settings.sql_analysis and not no_sql_validation, - ( - defaults.function_database - if defaults.function_database is not None - else defaults.database, - defaults.function_schema - if defaults.function_schema is not None - else defaults.schema, - ), - None if target_config is None else (target_config.database, target_config.schema), - python_functions_inherit_default_namespace, - ResourceKind.FUNCTION, - ), - ) - ) - - -def attach_sources_impl( - *, - discovered_inputs: DiscoveredProjectInputs, - native_project: _native.NativeCompiledProject, - effective_settings: SettingsConfig, - no_sql_validation: bool, -) -> tuple[SourceInput, ...]: - """Expand and validate every discovered source in declaration order.""" - - rows: list[tuple[str, DiscoveredSourceFile, SourceEntry]] = [] - for source_file in discovered_inputs.source_files: - rows.extend( - (str(source_file.file_path), source_file, entry) for entry in source_file.source_entries - ) - return tuple( - native_project.attach_sources( - rows, - ( - discovered_inputs.project_config.defaults.contract, - effective_settings.sql_analysis and not no_sql_validation, - ResourceKind.SOURCE, - ), - ) - ) diff --git a/src/sqlbuild/compiler/compiled_project/_helpers/rows.py b/src/sqlbuild/compiler/compiled_project/_helpers/rows.py deleted file mode 100644 index 8e069f74f7..0000000000 --- a/src/sqlbuild/compiler/compiled_project/_helpers/rows.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Rows producers hand the native compiled project when they retain a resource's facts.""" - -from __future__ import annotations - -from dataclasses import dataclass -from decimal import Decimal -from pathlib import Path -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import ( - CompileModelConfig, - CompileSqlReference, - DynamicColumnContractProof, -) -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration -from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.models import SchemaModelEntry -from sqlbuild.sql_values.models import SqlValue -from sqlbuild.sql_values.types import SqlValueKind - - -@dataclass(frozen=True) -class _ModelDeclarations: - enum_columns: tuple[str, ...] - enums: tuple[EnumDeclaration, ...] - constants: tuple[ConstantDeclaration, ...] - - -def record_model_analyses_impl( - *, project: _native.NativeCompiledProject, models: tuple[CompiledModelView, ...] -) -> int: - """Transfer each assembled model's typed analysis to its retained facts; return how many.""" - - transferred: int = project.record_view_analyses( - [model for model in models if isinstance(model, _native.NativeCompiledModel)] - ) - for model in models: - if isinstance(model, _native.NativeCompiledModel): - continue - proof: DynamicColumnContractProof | None = model.dynamic_column_contract - inferred: list[tuple[str, str | None, str]] | None = model.inferred_rows - columns: list[tuple[str, str | None, str]] = inferred or [] - transferred += project.record_model_analysis( - model.name, - ( - inferred is not None, - [(name, column_type) for name, column_type, _ in columns], - [nullability == InferredNullability.NULLABLE for _, _, nullability in columns], - bool(model.fast_lineage_has_star), - list(model.unchecked_output_columns), - ), - None - if proof is None - else ( - proof.output_proven, - proof.bare_dynamic_pivot, - [(family.name.casefold(), family.inferred_type) for family in proof.families], - proof.failure_reason, - [(family.name, family.inferred_type) for family in proof.families], - ), - ) - return transferred - - -def hand_built_project_impl(project: CompiledProjectView) -> _native.NativeCompiledProject: - """A native project for a `CompiledProject` built without the compile-input stage.""" - - native: _native.NativeCompiledProject = _native.NativeCompiledProject() - for model in project.models: - native.record_model( - _model_row( - identity=( - model.name, - model.relative_path.as_posix(), - model.query_sql, - model.authored_query_sql, - model.authored_sql, - ), - config=model.config, - references=model.references, - schema_entry=model.schema_entry, - relative_path=model.relative_path, - declarations=_ModelDeclarations( - enum_columns=tuple(model.enum_columns), - enums=model.enum_declarations, - constants=model.constant_declarations, - ), - ) - ) - _ = record_model_analyses_impl(project=native, models=project.models) - return native - - -def enum_row(declaration: EnumDeclaration) -> tuple[object, ...]: - """An enum declaration as the native project row.""" - - return ( - declaration.name, - declaration.relative_path.as_posix(), - [(member.name, member.value) for member in declaration.members], - None, - None, - None, - ) - - -def constant_row(declaration: ConstantDeclaration) -> tuple[object, ...]: - """A constant declaration as the native project row, its value made JSON-safe.""" - - return ( - declaration.name, - declaration.relative_path.as_posix(), - [], - typed_value_payload(declaration.value), - declaration.logical_type.display_name, - declaration.render_as.value if declaration.render_as is not None else None, - ) - - -def typed_value_payload(value: SqlValue) -> object: - """A typed SQL value as the plain value native consumers read.""" - - if value.kind == SqlValueKind.DECIMAL: - return str(cast(Decimal, value.value)) - if value.kind in { - SqlValueKind.STRING, - SqlValueKind.INTEGER, - SqlValueKind.BOOLEAN, - SqlValueKind.FLOAT, - SqlValueKind.NULL, - }: - return value.value - if value.kind in {SqlValueKind.LIST, SqlValueKind.SET}: - return [typed_value_payload(item) for item in cast(tuple[SqlValue, ...], value.value)] - return { - key: typed_value_payload(item) - for key, item in cast(tuple[tuple[str, SqlValue], ...], value.value) - } - - -def _model_row( - *, - identity: tuple[str, str, str, str, str], - config: CompileModelConfig, - references: tuple[CompileSqlReference, ...], - schema_entry: SchemaModelEntry | None, - relative_path: Path, - declarations: _ModelDeclarations, -) -> tuple[object, ...]: - return ( - identity, - (config.values, list(config.model_header_keys), config.layer_schema), - [ - (str(reference.ref_kind), reference.ref_name, reference.ref_package) - for reference in references - ], - _schema_row(entry=schema_entry, relative_path=relative_path), - list(declarations.enum_columns), - ( - [enum_row(declaration) for declaration in declarations.enums], - [constant_row(declaration) for declaration in declarations.constants], - ), - ) - - -def _schema_row(*, entry: SchemaModelEntry | None, relative_path: Path) -> object: - if entry is None: - return None - return ( - len(entry.audits), - [ - ( - column.name, - column.type, - column.nullable, - len(column.audits), - column.location is not None and column.location.path != relative_path, - ) - for column in entry.columns - ], - [ - ( - family.name.casefold(), - family.name, - family.pivot_column, - family.value_column, - family.aggregate, - family.type, - family.name_pattern, - ) - for family in entry.dynamic_columns - ], - (bool(entry.type_enforcement), entry.model_schema is not None), - ) diff --git a/src/sqlbuild/compiler/compiled_project/constants.py b/src/sqlbuild/compiler/compiled_project/constants.py deleted file mode 100644 index 5208508531..0000000000 --- a/src/sqlbuild/compiler/compiled_project/constants.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Constants of the native compiled project and its model loop.""" - -from sqlbuild.compiler.planner.types import MaterializationType - -TABLE_BACKED_MATERIALIZATIONS: tuple[MaterializationType, ...] = ( - MaterializationType.TABLE, - MaterializationType.INCREMENTAL, - MaterializationType.SNAPSHOT, -) diff --git a/src/sqlbuild/compiler/compiled_project/main/__init__.py b/src/sqlbuild/compiler/compiled_project/main/__init__.py deleted file mode 100644 index 5de9ede1eb..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Compiled-project entrypoints.""" diff --git a/src/sqlbuild/compiler/compiled_project/main/_attach_checks.py b/src/sqlbuild/compiler/compiled_project/main/_attach_checks.py deleted file mode 100644 index 80c703c218..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/_attach_checks.py +++ /dev/null @@ -1,61 +0,0 @@ -"""SQL tests, scenarios and audits attached natively and held by the compile's native project.""" - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache -from sqlbuild.compiler.compile.models import ( - CompilerDiagnostic, - CompileSqlScenarioInput, - CompileSqlTestInput, -) -from sqlbuild.compiler.compile.types import AuditInput, SourceInput, TypedSqlValueRenderer -from sqlbuild.compiler.compiled_project._helpers.assurance_attachment import ( - attach_audits_impl, - attach_sql_tests_impl, - resource_diagnostics_impl, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import SettingsConfig - - -def attach_checks( - *, - discovered_inputs: DiscoveredProjectInputs, - native_project: _native.NativeCompiledProject, - settings: tuple[SettingsConfig, TypedSqlValueRenderer, SqlLexicalSyntax], - scan_cache: SqlTestScanCache, - source_inputs: tuple[SourceInput, ...], -) -> tuple[ - tuple[CompileSqlTestInput, ...], - tuple[CompileSqlScenarioInput, ...], - tuple[AuditInput, ...], - tuple[CompilerDiagnostic, ...], -]: - """Attach every SQL test case, scenario and audit, then report every resource diagnostic.""" - - effective_settings, value_renderer, syntax = settings - tests, scenarios = attach_sql_tests_impl( - discovered_inputs=discovered_inputs, - native_project=native_project, - value_renderer=value_renderer, - syntax=syntax, - scan_cache=scan_cache, - ) - audits, diagnostics = attach_audits_impl( - discovered_inputs=discovered_inputs, - native_project=native_project, - effective_settings=effective_settings, - ) - return ( - tests, - scenarios, - audits, - ( - *diagnostics, - *resource_diagnostics_impl( - discovered_inputs=discovered_inputs, - native_project=native_project, - source_inputs=source_inputs, - ), - ), - ) diff --git a/src/sqlbuild/compiler/compiled_project/main/_attach_resources.py b/src/sqlbuild/compiler/compiled_project/main/_attach_resources.py deleted file mode 100644 index 02e9d66513..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/_attach_resources.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Seeds, functions and sources attached natively and held by the compile's native project.""" - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.types import FunctionInput, SeedInput, SourceInput -from sqlbuild.compiler.compiled_project._helpers.resource_attachment import ( - attach_functions_impl, - attach_seeds_impl, - attach_sources_impl, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import SettingsConfig, TargetConfig - - -def attach_resources( - *, - discovered_inputs: DiscoveredProjectInputs, - native_project: _native.NativeCompiledProject, - settings: tuple[SettingsConfig, TargetConfig | None, str], - no_sql_validation: bool, - python_functions_inherit_default_namespace: bool, -) -> tuple[tuple[SeedInput, ...], tuple[FunctionInput, ...], tuple[SourceInput, ...]]: - """Pair every seed, then attach every function and source, in that order.""" - - seeds: tuple[SeedInput, ...] = attach_seeds_impl( - discovered_inputs=discovered_inputs, native_project=native_project - ) - functions: tuple[FunctionInput, ...] = attach_functions_impl( - discovered_inputs=discovered_inputs, - native_project=native_project, - settings=settings, - no_sql_validation=no_sql_validation, - python_functions_inherit_default_namespace=python_functions_inherit_default_namespace, - ) - sources: tuple[SourceInput, ...] = attach_sources_impl( - discovered_inputs=discovered_inputs, - native_project=native_project, - effective_settings=settings[0], - no_sql_validation=no_sql_validation, - ) - return seeds, functions, sources diff --git a/src/sqlbuild/compiler/compiled_project/main/_compile_inputs_view.py b/src/sqlbuild/compiler/compiled_project/main/_compile_inputs_view.py deleted file mode 100644 index 1b3cba941f..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/_compile_inputs_view.py +++ /dev/null @@ -1,15 +0,0 @@ -"""A compile's attached inputs, held by its native owner when it finished every model.""" - -from typing import Any - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.compiled_project._helpers.model_updates import compile_inputs_view_impl - - -def compile_inputs_view( - *, fields: dict[str, Any], native_project: _native.NativeCompiledProject | None -) -> CompileInputsView: - """The native owner's inputs view when it holds every model input; else a record.""" - - return compile_inputs_view_impl(fields=fields, native_project=native_project) diff --git a/src/sqlbuild/compiler/compiled_project/main/_model_loop_classes.py b/src/sqlbuild/compiler/compiled_project/main/_model_loop_classes.py deleted file mode 100644 index 48c83f1684..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/_model_loop_classes.py +++ /dev/null @@ -1,135 +0,0 @@ -"""The Python classes and project inputs the native model loop builds models with.""" - -from dataclasses import replace -from functools import partial -from pathlib import Path -from typing import cast - -from sqlbuild.compiler.auditing.types import AuditSeverity -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompactLineageFacts, - CompiledLineageColumnFact, - CompiledLineageSourceFact, - CompiledObjectKey, - CompiledRelationLocation, - CompiledSqlExpansion, - CompilerDiagnostic, - CompileSqlReference, - DeclarationScopeResolver, - DynamicColumnContractProof, - DynamicColumnFamilyProof, - ExpansionSpan, - FunctionArgument, - FunctionReturnColumn, - InferredColumn, - ModelInputBuildContext, - NativeModelConfigSession, -) -from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, FunctionLanguage -from sqlbuild.compiler.compiled_project._helpers.loop_inputs import ( - hook_signature, - named_records, - schema_enums, - visibility_classes, -) -from sqlbuild.compiler.compiled_project.main._native_config_classes import native_config_classes -from sqlbuild.compiler.compiled_project.main.declaration_rows import declaration_rows -from sqlbuild.compiler.discovery.models import ( - DiscoveredAuditFactory, - DiscoveredHookFunction, - PythonHookEntry, - SqlHookEntry, -) -from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.model_config.main._native_config_error import native_config_error -from sqlbuild.compiler.planner.types import ContractPolicy -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver, SqlReferenceKind -from sqlbuild.compiler.scopes.exceptions import InvalidScopePathError -from sqlbuild.compiler.scopes.models import UsageRecord -from sqlbuild.compiler.scopes.types import UsageKind -from sqlbuild.spec.contracts.models import SchemaModelEntry, SourceLocation - - -def model_loop_classes( - *, - audit_factories: tuple[DiscoveredAuditFactory, ...], - context: ModelInputBuildContext, - external_resolver: ExternalSqlReferenceResolver | None, - host_inputs: dict[str, object], -) -> dict[str, object]: - """Return the classes, error factories and project declarations the model loop reads.""" - - resolver: DeclarationScopeResolver | None = context.declaration_resolver - hook_functions: tuple[DiscoveredHookFunction, ...] = cast( - tuple[DiscoveredHookFunction, ...], host_inputs["hook_functions"] - ) - provider_names: frozenset[str] = cast(frozenset[str], host_inputs["provider_names"]) - session: NativeModelConfigSession = cast( - NativeModelConfigSession, host_inputs["config_session"] - ) - return { - "usage_record": UsageRecord, - "compile_error": CompileInputError, - "syntax_help": host_inputs["syntax_help"], - "source_syntax_help": host_inputs["source_syntax_help"], - "macros": host_inputs["macros"], - "sql_hooks": host_inputs["sql_hooks"], - "scope_path_error": InvalidScopePathError, - "sql_hook_entry": SqlHookEntry, - "config_error": native_config_error, - "audit_factories": {factory.name: factory for factory in audit_factories}, - "generated_usage": UsageKind.GENERATED, - "contract_enforced": ContractPolicy.ENFORCED, - "model_schemas": context.public_model_schemas, - "named_records": named_records(resolver), - "schema_enums": schema_enums(resolver), - "compiler_diagnostic": CompilerDiagnostic, - "report_diagnostic": host_inputs["report_diagnostic"], - "path_class": Path, - "external_resolver": external_resolver, - "reference_kind": SqlReferenceKind, - "partial": partial, - "config": native_config_classes( - session=session, record_template_reads=host_inputs["record_template_reads"] - ), - "hooks": { - "functions": {function.name: function for function in hook_functions}, - "provider_names": sorted(provider_names), - "python_hook_entry": PythonHookEntry, - "signature": hook_signature, - }, - "visibility": visibility_classes( - context=context, visible_declarations=host_inputs["visible_declarations"] - ), - "record": {"replace": replace, "declaration_rows": declaration_rows}, - "input_view": { - "sql_reference": CompileSqlReference, - "reference_kind": SqlReferenceKind, - "schema_model_entry": SchemaModelEntry, - "sql_expansion": CompiledSqlExpansion, - "expansion_span": ExpansionSpan, - "audits": { - "audit_severity": AuditSeverity, - "attached_target_kind": AttachedAuditTargetKind, - }, - "functions": { - "function_argument": FunctionArgument, - "function_return_column": FunctionReturnColumn, - "function_language": FunctionLanguage, - }, - "compiled": { - "object_key": CompiledObjectKey, - "relation_location": CompiledRelationLocation, - "inferred_column": InferredColumn, - "nullability": InferredNullability, - "compiler_diagnostic": CompilerDiagnostic, - "source_location": SourceLocation, - "compact_lineage_facts": CompactLineageFacts, - "lineage_column_fact": CompiledLineageColumnFact, - "lineage_source_fact": CompiledLineageSourceFact, - "contract_proof": DynamicColumnContractProof, - "family_proof": DynamicColumnFamilyProof, - }, - }, - } diff --git a/src/sqlbuild/compiler/compiled_project/main/_native_config_classes.py b/src/sqlbuild/compiler/compiled_project/main/_native_config_classes.py deleted file mode 100644 index 630cfcc029..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/_native_config_classes.py +++ /dev/null @@ -1,27 +0,0 @@ -"""The builder, storage defaults and classes a model's `CompileModelConfig` is made of.""" - -from sqlbuild.compiler.compile.models import CompileModelConfig, NativeModelConfigSession -from sqlbuild.compiler.compiled_project.constants import TABLE_BACKED_MATERIALIZATIONS -from sqlbuild.spec.contracts.models import ResolvedTableType, ResolvedTimeTravelRetention -from sqlbuild.spec.contracts.types import TableType, TableTypeSource, TimeTravelRetentionSource - - -def native_config_classes( - *, session: NativeModelConfigSession, record_template_reads: object -) -> dict[str, object]: - """The builder, storage defaults and classes a model's `CompileModelConfig` is made of.""" - - return { - "builder": session.builder, - "inherited_storage": session.inherited_storage, - "table_backed": tuple( - str(materialization) for materialization in TABLE_BACKED_MATERIALIZATIONS - ), - "compile_model_config": CompileModelConfig, - "resolved_retention": ResolvedTimeTravelRetention, - "retention_source": TimeTravelRetentionSource.MODEL, - "resolved_table_type": ResolvedTableType, - "table_type": TableType, - "table_type_source": TableTypeSource.MODEL, - "record_template_reads": record_template_reads, - } diff --git a/src/sqlbuild/compiler/compiled_project/main/_record_model_analyses.py b/src/sqlbuild/compiler/compiled_project/main/_record_model_analyses.py deleted file mode 100644 index 437f3562ab..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/_record_model_analyses.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Transfer assembled models' typed analysis results to the native compiled project.""" - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.compiled_project._helpers.rows import record_model_analyses_impl - - -def record_model_analyses( - *, project: _native.NativeCompiledProject, models: tuple[CompiledModelView, ...] -) -> int: - """Attach each model's inferred columns and dynamic pivot proof to its retained facts.""" - - return record_model_analyses_impl(project=project, models=models) diff --git a/src/sqlbuild/compiler/compiled_project/main/_with_binding_diagnostics.py b/src/sqlbuild/compiler/compiled_project/main/_with_binding_diagnostics.py deleted file mode 100644 index 481393f0a7..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/_with_binding_diagnostics.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Record a compiled model's final binding diagnostics.""" - -from sqlbuild.compiler.compile.models import CompilerDiagnostic -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.compiled_project._helpers.model_updates import with_binding_diagnostics_impl - - -def with_binding_diagnostics( - *, model: CompiledModelView, diagnostics: tuple[CompilerDiagnostic, ...] -) -> CompiledModelView: - """The model with these final binding diagnostics.""" - - return with_binding_diagnostics_impl(model=model, diagnostics=diagnostics) diff --git a/src/sqlbuild/compiler/compiled_project/main/_with_target_defaults.py b/src/sqlbuild/compiler/compiled_project/main/_with_target_defaults.py deleted file mode 100644 index b449833d34..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/_with_target_defaults.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Fill an adapter's default schema and database into a native compiled model's destination.""" - -from collections.abc import Callable - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.compiled_project._helpers.model_updates import with_target_defaults_impl - - -def with_target_defaults( - *, - model: _native.NativeCompiledModel, - defaults: tuple[str | None, str | None], - render_qualified_name: Callable[..., str | None], -) -> CompiledModelView: - """The model with the adapter's `(schema, database)` defaults filled into its destination.""" - - return with_target_defaults_impl( - model=model, defaults=defaults, render_qualified_name=render_qualified_name - ) diff --git a/src/sqlbuild/compiler/compiled_project/main/_with_type_recovery.py b/src/sqlbuild/compiler/compiled_project/main/_with_type_recovery.py deleted file mode 100644 index 860c502cc7..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/_with_type_recovery.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Record a compiled model's semantic type recovery.""" - -from sqlbuild.compiler.compile.models import CompilerDiagnostic -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.compiled_project._helpers.model_updates import with_type_recovery_impl - - -def with_type_recovery( - *, - model: CompiledModelView, - diagnostics: tuple[CompilerDiagnostic, ...], - poisoned: frozenset[tuple[str, str]], -) -> CompiledModelView: - """The model with its recovered bindings and its poisoned outputs' types forgotten.""" - - return with_type_recovery_impl(model=model, diagnostics=diagnostics, poisoned=poisoned) diff --git a/src/sqlbuild/compiler/compiled_project/main/compiled_audit_record.py b/src/sqlbuild/compiler/compiled_project/main/compiled_audit_record.py deleted file mode 100644 index bd551710db..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/compiled_audit_record.py +++ /dev/null @@ -1,11 +0,0 @@ -"""A compiled audit as a `CompiledAudit` record, for callers that copy or expose it.""" - -from sqlbuild.compiler.compile.models import CompiledAudit -from sqlbuild.compiler.compile.types import CompiledAuditView -from sqlbuild.compiler.compiled_project._helpers.model_updates import compiled_audit_record_impl - - -def compiled_audit_record(audit: CompiledAuditView) -> CompiledAudit: - """The audit as a `CompiledAudit`, projected from its owner when it is a native view.""" - - return compiled_audit_record_impl(audit) diff --git a/src/sqlbuild/compiler/compiled_project/main/compiled_function_record.py b/src/sqlbuild/compiler/compiled_project/main/compiled_function_record.py deleted file mode 100644 index c988babf6e..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/compiled_function_record.py +++ /dev/null @@ -1,12 +0,0 @@ -"""A compiled function as a `CompiledFunction` record, for callers that copy or expose it.""" - -from sqlbuild.compiler.compile.types import CompiledFunctionView -from sqlbuild.compiler.compiled_project._helpers.model_updates import ( - compiled_function_record_impl, -) - - -def compiled_function_record(function: CompiledFunctionView) -> CompiledFunctionView: - """The function as a `CompiledFunction`, projected from its owner when it is a native view.""" - - return compiled_function_record_impl(function) diff --git a/src/sqlbuild/compiler/compiled_project/main/compiled_model_record.py b/src/sqlbuild/compiler/compiled_project/main/compiled_model_record.py deleted file mode 100644 index bcdc2613bb..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/compiled_model_record.py +++ /dev/null @@ -1,11 +0,0 @@ -"""A compiled model as a `CompiledModel` record, for callers that copy or ship it.""" - -from sqlbuild.compiler.compile.models import CompiledModel -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.compiled_project._helpers.model_updates import compiled_model_record_impl - - -def compiled_model_record(model: CompiledModelView) -> CompiledModel: - """The model as a `CompiledModel`, projected from its owner when it is a native view.""" - - return compiled_model_record_impl(model) diff --git a/src/sqlbuild/compiler/compiled_project/main/compiled_project_facts.py b/src/sqlbuild/compiler/compiled_project/main/compiled_project_facts.py deleted file mode 100644 index 1c1b65fcca..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/compiled_project_facts.py +++ /dev/null @@ -1,13 +0,0 @@ -"""The native compiled project a consumer reads for one `CompiledProject`.""" - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project._helpers.rows import hand_built_project_impl - - -def compiled_project_facts(project: CompiledProjectView) -> _native.NativeCompiledProject: - """The compile's retained project, or one recorded from a hand-built project.""" - - if project.native_project is not None: - return project.native_project - return hand_built_project_impl(project) diff --git a/src/sqlbuild/compiler/compiled_project/main/compiled_project_record.py b/src/sqlbuild/compiler/compiled_project/main/compiled_project_record.py deleted file mode 100644 index d698d9c082..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/compiled_project_record.py +++ /dev/null @@ -1,15 +0,0 @@ -"""A compiled project as a `CompiledProject` record, for callers that copy or ship it.""" - -from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project._helpers.model_updates import compiled_project_record_impl - - -def compiled_project_record( - project: CompiledProjectView, *, include_sql_expansions: bool = True -) -> CompiledProject: - """The project as a `CompiledProject`, projected from its owner when it is a native view.""" - - return compiled_project_record_impl( - project=project, include_sql_expansions=include_sql_expansions - ) diff --git a/src/sqlbuild/compiler/compiled_project/main/compiled_seed_record.py b/src/sqlbuild/compiler/compiled_project/main/compiled_seed_record.py deleted file mode 100644 index 33f6724975..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/compiled_seed_record.py +++ /dev/null @@ -1,11 +0,0 @@ -"""A compiled seed as a `CompiledSeed` record, for callers that copy or expose it.""" - -from sqlbuild.compiler.compile.models import CompiledSeed -from sqlbuild.compiler.compile.types import CompiledSeedView -from sqlbuild.compiler.compiled_project._helpers.model_updates import compiled_seed_record_impl - - -def compiled_seed_record(seed: CompiledSeedView) -> CompiledSeed: - """The seed as a `CompiledSeed`, projected from its owner when it is a native view.""" - - return compiled_seed_record_impl(seed) diff --git a/src/sqlbuild/compiler/compiled_project/main/compiled_source_record.py b/src/sqlbuild/compiler/compiled_project/main/compiled_source_record.py deleted file mode 100644 index 51c0c910e3..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/compiled_source_record.py +++ /dev/null @@ -1,11 +0,0 @@ -"""A compiled source as a `CompiledSource` record, for callers that copy or expose it.""" - -from sqlbuild.compiler.compile.models import CompiledSource -from sqlbuild.compiler.compile.types import CompiledSourceView -from sqlbuild.compiler.compiled_project._helpers.model_updates import compiled_source_record_impl - - -def compiled_source_record(source: CompiledSourceView) -> CompiledSource: - """The source as a `CompiledSource`, projected from its owner when it is a native view.""" - - return compiled_source_record_impl(source) diff --git a/src/sqlbuild/compiler/compiled_project/main/declaration_rows.py b/src/sqlbuild/compiler/compiled_project/main/declaration_rows.py deleted file mode 100644 index d1c346b4a1..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/declaration_rows.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Declaration rows native consumers read.""" - -from sqlbuild.compiler.compiled_project._helpers.rows import constant_row, enum_row -from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration - - -def declaration_rows( - *, enums: tuple[EnumDeclaration, ...], constants: tuple[ConstantDeclaration, ...] -) -> tuple[list[tuple[object, ...]], list[tuple[object, ...]]]: - """Return enum and constant declarations as native rows, in the given order.""" - - return [enum_row(declaration) for declaration in enums], [ - constant_row(declaration) for declaration in constants - ] diff --git a/src/sqlbuild/compiler/compiled_project/main/inputs_with.py b/src/sqlbuild/compiler/compiled_project/main/inputs_with.py deleted file mode 100644 index f05e7123af..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/inputs_with.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Compile inputs with some fields changed, natively when their owner holds them.""" - -from typing import Any - -from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.compiled_project._helpers.model_updates import inputs_with_impl - - -def inputs_with(*, inputs: CompileInputsView, **changes: Any) -> CompileInputsView: - """The compile inputs with `changes`; a native view shares its owner's model inputs.""" - - return inputs_with_impl(inputs=inputs, **changes) diff --git a/src/sqlbuild/compiler/compiled_project/main/plain_typed_value.py b/src/sqlbuild/compiler/compiled_project/main/plain_typed_value.py deleted file mode 100644 index 2f3fe093dc..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/plain_typed_value.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Typed SQL values as the plain values native consumers read.""" - -from sqlbuild.compiler.compiled_project._helpers.rows import typed_value_payload -from sqlbuild.sql_values.models import SqlValue - - -def plain_typed_value(value: SqlValue) -> object: - """Return the value JSON-safe: decimals as text, collections and mappings recursively.""" - - return typed_value_payload(value) diff --git a/src/sqlbuild/compiler/compiled_project/main/project_with.py b/src/sqlbuild/compiler/compiled_project/main/project_with.py deleted file mode 100644 index bb011f16cc..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/project_with.py +++ /dev/null @@ -1,12 +0,0 @@ -"""A compiled project with some fields changed, natively when its owner holds it.""" - -from typing import Any - -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project._helpers.model_updates import project_with_impl - - -def project_with(*, project: CompiledProjectView, **changes: Any) -> CompiledProjectView: - """The project with `changes`; a native view shares its owner's models.""" - - return project_with_impl(project=project, **changes) diff --git a/src/sqlbuild/compiler/compiled_project/types.py b/src/sqlbuild/compiler/compiled_project/types.py deleted file mode 100644 index 68f523dc8a..0000000000 --- a/src/sqlbuild/compiler/compiled_project/types.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Row shapes the native model loop hands its Python host.""" - -from collections.abc import Callable - -from sqlbuild.spec.contracts.models import SourceLocation - -type ReferenceRow = tuple[str, str, str | None, int | None] -type SpanRow = tuple[int, int, int, int] -type PythonResourceRow = tuple[str, str, str | None, Callable[[], SourceLocation], str] diff --git a/src/sqlbuild/compiler/contracts/_helpers/native_contracts.py b/src/sqlbuild/compiler/contracts/_helpers/native_contracts.py index 46aa691589..f157e9fd84 100644 --- a/src/sqlbuild/compiler/contracts/_helpers/native_contracts.py +++ b/src/sqlbuild/compiler/contracts/_helpers/native_contracts.py @@ -3,6 +3,9 @@ from __future__ import annotations import sqlbuild._native as _native +from sqlbuild._native import ( + compiled_project_facts, +) from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.compiler.compile.models import ( CompilerDiagnostic, @@ -15,7 +18,6 @@ DiagnosticPhase, DiagnosticSeverity, ) -from sqlbuild.compiler.compiled_project.main.compiled_project_facts import compiled_project_facts from sqlbuild.compiler.contracts._helpers.output_locations import ( output_column_location, output_related_locations, diff --git a/src/sqlbuild/compiler/contracts/_helpers/native_promotion.py b/src/sqlbuild/compiler/contracts/_helpers/native_promotion.py index 45cbab049f..c077905736 100644 --- a/src/sqlbuild/compiler/contracts/_helpers/native_promotion.py +++ b/src/sqlbuild/compiler/contracts/_helpers/native_promotion.py @@ -5,6 +5,9 @@ from pathlib import Path import sqlbuild._native as _native +from sqlbuild._native import ( + compiled_project_facts, +) from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.compile.types import ( @@ -13,7 +16,6 @@ DiagnosticPhase, DiagnosticSeverity, ) -from sqlbuild.compiler.compiled_project.main.compiled_project_facts import compiled_project_facts def native_promotion_conflicts( diff --git a/src/sqlbuild/compiler/graph/_helpers/execution_gates.py b/src/sqlbuild/compiler/graph/_helpers/execution_gates.py index d2180b5c69..8df4b01988 100644 --- a/src/sqlbuild/compiler/graph/_helpers/execution_gates.py +++ b/src/sqlbuild/compiler/graph/_helpers/execution_gates.py @@ -2,24 +2,18 @@ from __future__ import annotations -from itertools import chain from pathlib import Path -import orjson - import sqlbuild._native as _native from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import ( - AttachedAuditTargetKind, - CompiledAuditView, CompiledModelView, CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.discovery.models import PythonHookEntry, SqlHookEntry +from sqlbuild.compiler.discovery.models import PythonHookEntry from sqlbuild.compiler.graph.models import AttachedAuditGateEdge, HookReadEdge from sqlbuild.compiler.graph.types import HookReadType, NativeKey -from sqlbuild.python_nodes.models import SqlResourceRef _MODEL_HOOK_KEYS: tuple[str, ...] = ("pre_hooks", "post_hooks") @@ -42,7 +36,7 @@ def attached_audit_gate_edges_impl( ) -> tuple[AttachedAuditGateEdge, ...]: """Return edges from each extra audit read to its model or seed target, or source triggers.""" - gates, _ = _native.native_execution_gate_edges(_request(project)) + gates, _ = _native.native_execution_gate_edges(project) return tuple( AttachedAuditGateEdge( audit_name=audit_name, target=_key(target), gated=_key(gated), read=_key(read) @@ -54,7 +48,7 @@ def attached_audit_gate_edges_impl( def hook_read_edges_impl(*, project: CompiledProjectView) -> tuple[HookReadEdge, ...]: """Return edges from each hook read to the model that runs the hook.""" - _, hooks = _native.native_execution_gate_edges(_request(project)) + _, hooks = _native.native_execution_gate_edges(project) return tuple( HookReadEdge( hook_name=hook_name, @@ -69,73 +63,3 @@ def hook_read_edges_impl(*, project: CompiledProjectView) -> tuple[HookReadEdge, def _key(pair: NativeKey) -> CompiledObjectKey: return CompiledObjectKey(resource_type=CompiledResourceType(pair[0]), name=pair[1]) - - -def _request(project: CompiledProjectView) -> str: - return orjson.dumps( - { - "resources": [ - {"key": model.key_row, "deps": model.dep_rows} for model in project.models - ], - "models": [_model_row(model) for model in project.models], - "audits": [_audit_row(audit) for audit in project.audits], - "hooks": [ - { - "name": hook.name, - "path": hook.relative_path.as_posix(), - "reads": _ref_rows(hook.reads), - } - for hook in project.hook_functions - ], - } - ).decode() - - -def _model_row(model: CompiledModelView) -> dict[str, object]: - return { - "name": model.name, - "path": str(model.relative_path), - "python_hooks": list(model_python_hook_names_impl(model)), - "sql_hooks": list( - chain.from_iterable( - _sql_hook_rows(hook_key=hook_key, entries=model.config.values.get(hook_key)) - for hook_key in _MODEL_HOOK_KEYS - ) - ), - } - - -def _sql_hook_rows(*, hook_key: str, entries: object) -> list[dict[str, object]]: - items: tuple[object, ...] = tuple(entries) if isinstance(entries, list | tuple) else (entries,) - return [ - { - "hook_key": hook_key, - "index": index, - "name": entry.name, - "path": None if entry.relative_path is None else str(entry.relative_path), - "reads": _ref_rows(entry.reads), - } - for index, entry in enumerate(items) - if isinstance(entry, SqlHookEntry) and entry.reads - ] - - -def _audit_row(audit: CompiledAuditView) -> dict[str, object]: - target: tuple[str, str] | None = ( - ( - str(AttachedAuditTargetKind(audit.attached_target_kind).resource_type), - audit.attached_target_name, - ) - if audit.attached_target_kind is not None and audit.attached_target_name is not None - else None - ) - return { - "name": audit.name, - "path": "", - "target": target, - "scope_deps": [(str(key.resource_type), key.name) for key in audit.scope_deps], - } - - -def _ref_rows(refs: tuple[SqlResourceRef, ...]) -> list[tuple[str, str]]: - return [(str(ref.kind), ref.name) for ref in refs] diff --git a/src/sqlbuild/compiler/pipeline/_helpers/target_defaults.py b/src/sqlbuild/compiler/pipeline/_helpers/target_defaults.py index a4d9953c01..1c3f4505a5 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/target_defaults.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/target_defaults.py @@ -7,6 +7,9 @@ from typing import cast import sqlbuild._native as _native +from sqlbuild._native import ( + project_with, +) from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledRelationLocation, @@ -18,8 +21,6 @@ CompiledProjectView, CompiledSeedView, ) -from sqlbuild.compiler.compiled_project.main._with_target_defaults import with_target_defaults -from sqlbuild.compiler.compiled_project.main.project_with import project_with def apply_target_defaults( @@ -76,10 +77,8 @@ def _model_with_defaults( render_qualified_name: Callable[..., str | None], ) -> CompiledModelView: if not isinstance(model, CompiledModel): - return with_target_defaults( - model=cast(_native.NativeCompiledModel, model), - defaults=(default_schema, default_database), - render_qualified_name=render_qualified_name, + return cast(_native.NativeCompiledModel, model).with_target_defaults( + (default_schema, default_database), render_qualified_name ) return replace( model, diff --git a/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py b/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py index b5d053ff31..4f7b8e4e5a 100644 --- a/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py +++ b/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py @@ -6,6 +6,9 @@ from collections.abc import Callable from dataclasses import replace +from sqlbuild._native import ( + compiled_model_record, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.constants import ( @@ -28,7 +31,6 @@ CompiledResourceType, CompiledSeedView, ) -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record from sqlbuild.compiler.planner._helpers.fixtures.completion import ( build_relation_fixture_completion, ) diff --git a/src/sqlbuild/compiler/references/_helpers/remedies.py b/src/sqlbuild/compiler/references/_helpers/remedies.py deleted file mode 100644 index 3e5944d18e..0000000000 --- a/src/sqlbuild/compiler/references/_helpers/remedies.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Remedies for project relation names hard-coded in Python SQL.""" - -from __future__ import annotations - -from collections.abc import Mapping - -from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind -from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.python_nodes.types import SqlResourceRefKind - - -def hard_coded_relation_remedy_impl( - *, - owner_kind: HardCodedRelationOwnerKind, - ref: SqlResourceRef, - upstream_loader_by_source: Mapping[str, str], -) -> str: - """Return how the owner should read ``ref`` instead of naming it in SQL.""" - - typed: str = f'{ref.kind.value}("{ref.name}")' - if owner_kind is HardCodedRelationOwnerKind.LOADER: - if ref.kind is not SqlResourceRefKind.SOURCE: - return ( - "loaders run before models and seeds and cannot read them; read " - f"{ref.kind.value}:{ref.name} from a task or asset instead" - ) - loader_name: str | None = upstream_loader_by_source.get(ref.name) - if loader_name is not None: - return f"use ctx.loader({loader_name}) instead of the relation name" - return f'use ctx.source("{ref.name}") instead of the relation name' - declaration: str = ( - f"@hook(reads={typed})" - if owner_kind is HardCodedRelationOwnerKind.HOOK - else f"depends_on={typed}" - ) - return ( - f"declare it with {declaration} and use ctx.relation({typed}) instead of the relation name" - ) diff --git a/src/sqlbuild/compiler/references/main/_old_name_reference_message.py b/src/sqlbuild/compiler/references/main/_old_name_reference_message.py index ea7d3df620..5fde835797 100644 --- a/src/sqlbuild/compiler/references/main/_old_name_reference_message.py +++ b/src/sqlbuild/compiler/references/main/_old_name_reference_message.py @@ -2,14 +2,12 @@ from __future__ import annotations +import sqlbuild._native as _native + def old_name_reference_message( *, owner_label: str, written: str, method: str, model_name: str ) -> str: """Return the P008 message for SQL naming a relation kept only as a compatibility view.""" - return ( - f"{owner_label} names '{written}' in SQL passed to ctx.{method}(); it is the old name " - f"of model:{model_name}, kept only as a compatibility view for consumers outside the " - "project" - ) + return _native.old_name_reference_message(owner_label, written, method, model_name) diff --git a/src/sqlbuild/compiler/references/main/hard_coded_relation_remedy.py b/src/sqlbuild/compiler/references/main/hard_coded_relation_remedy.py index 94ee894819..9a72167c23 100644 --- a/src/sqlbuild/compiler/references/main/hard_coded_relation_remedy.py +++ b/src/sqlbuild/compiler/references/main/hard_coded_relation_remedy.py @@ -4,7 +4,7 @@ from collections.abc import Mapping -from sqlbuild.compiler.references._helpers.remedies import hard_coded_relation_remedy_impl +import sqlbuild._native as _native from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind from sqlbuild.python_nodes.models import SqlResourceRef @@ -17,10 +17,8 @@ def hard_coded_relation_remedy( ) -> str: """Return how Python code should read ``ref`` instead of naming it in SQL.""" - return hard_coded_relation_remedy_impl( - owner_kind=owner_kind, - ref=ref, - upstream_loader_by_source={} - if upstream_loader_by_source is None - else upstream_loader_by_source, + return _native.hard_coded_relation_remedy( + owner_kind.value, + (ref.kind.value, ref.name), + list((upstream_loader_by_source or {}).items()), ) diff --git a/src/sqlbuild/compiler/semantic_checks/__init__.py b/src/sqlbuild/compiler/semantic_checks/__init__.py deleted file mode 100644 index 6796d94ab5..0000000000 --- a/src/sqlbuild/compiler/semantic_checks/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Native semantic completion.""" diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/__init__.py b/src/sqlbuild/compiler/semantic_checks/_helpers/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/completion.py b/src/sqlbuild/compiler/semantic_checks/_helpers/completion.py deleted file mode 100644 index f623cf26ea..0000000000 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/completion.py +++ /dev/null @@ -1,175 +0,0 @@ -"""Native diagnostic recovery, explanations and opt-out rejection after the metadata checks.""" - -from __future__ import annotations - -from dataclasses import replace -from typing import Any - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.classes.semantic_completion_inputs import SemanticCompletionInputs -from sqlbuild.compiler.compile.constants import UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE -from sqlbuild.compiler.compile.models import CompilerDiagnostic -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledResourceType, - DiagnosticPhase, - DiagnosticSeverity, -) -from sqlbuild.compiler.compiled_project.main._with_binding_diagnostics import ( - with_binding_diagnostics, -) -from sqlbuild.compiler.compiled_project.main.project_with import project_with -from sqlbuild.compiler.semantic_checks._helpers.payloads import diagnostic_ids, lineage_payload -from sqlbuild.spec.contracts.models import SourceLocation - -_RECOVERED_CODE: str = "B002" - -type _Location = tuple[int, int, int | None, int | None] -type _Completed = tuple[ - int, str, str | None, list[str], _Location | None, int | None, int | None, bool -] - - -def complete_native_diagnostics( - *, - project: CompiledProjectView, - catalog: object, - session: Any | None, - session_models: frozenset[str], -) -> CompiledProjectView: - """The recovered, explained and opt-out-checked project; a native internal failure raises.""" - - ids: dict[CompilerDiagnostic, int] = diagnostic_ids(project) - recovers: bool = any(item.code == _RECOVERED_CODE for item in project.diagnostics) - shapes: dict[str, dict[str, str]] = ( - SemanticCompletionInputs.shapes(project) if project.diagnostics else {} - ) - request: Any = ( - project.sql_analysis_dialect, - [_diagnostic_payload(item=item, ids=ids) for item in project.diagnostics], - [ - _model_payload(model=model, lineage=recovers, from_session=model.name in session_models) - for model in project.models - ], - [(name, list(columns.items())) for name, columns in shapes.items()], - ) - completed, model_bindings, order = _native.complete_semantic_checks(catalog, request, session) - diagnostics: list[CompilerDiagnostic] = [ - _completed(original=project.diagnostics[row[0]], row=row) for row in completed - ] - models: tuple[CompiledModelView, ...] = project.models - if model_bindings is not None: - models = tuple( - with_binding_diagnostics( - model=model, diagnostics=_positions(diagnostics=diagnostics, positions=positions) - ) - for model, positions in zip(project.models, model_bindings, strict=True) - ) - final: list[CompilerDiagnostic] = [] - for position, opt_out in order: - if position is not None: - final.append(diagnostics[position]) - elif opt_out is not None: - index, message, note, help_text = opt_out - final.append( - _opt_out( - model=project.models[index], message=message, note=note, help_text=help_text - ) - ) - return project_with(project=project, models=models, diagnostics=tuple(final)) - - -def _positions( - *, diagnostics: list[CompilerDiagnostic], positions: list[int] -) -> tuple[CompilerDiagnostic, ...]: - return tuple(diagnostics[position] for position in positions) - - -def _diagnostic_payload( - *, item: CompilerDiagnostic, ids: dict[CompilerDiagnostic, int] -) -> tuple[Any, ...]: - location: SourceLocation | None = item.location - return ( - ids[item], - item.code, - item.message, - None if item.resource_type is None else str(item.resource_type), - item.resource_name, - item.line, - item.column, - None - if location is None - else (location.line, location.column, location.end_line, location.end_column), - item.help, - list(item.notes), - ) - - -def _model_payload( - *, model: CompiledModelView, lineage: bool, from_session: bool -) -> tuple[Any, ...]: - opt_out: SourceLocation | None = model.rejected_sql_analysis_opt_out - return ( - model.name, - model.query_sql, - model.authored_sql, - None if from_session else [column.name for column in model.inferred_columns or ()], - _lineage(model=model, lineage=lineage, from_session=from_session), - [item.code for item in model.binding_diagnostics], - None if opt_out is None else opt_out.path.name, - ) - - -def _completed(*, original: CompilerDiagnostic, row: _Completed) -> CompilerDiagnostic: - _, message, help_text, notes, location, line, column, changed = row - if not changed: - return original - current: SourceLocation | None = original.location - if current is not None and location is not None: - line_number, column_number, end_line, end_column = location - if (current.line, current.column, current.end_line, current.end_column) != location: - current = replace( - current, - line=line_number, - column=column_number, - end_line=end_line, - end_column=end_column, - ) - return replace( - original, - message=message, - help=help_text, - notes=tuple(notes), - location=current, - line=line, - column=column, - ) - - -def _opt_out( - *, model: CompiledModelView, message: str, note: str, help_text: str -) -> CompilerDiagnostic: - return CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE, - message=message, - resource_type=CompiledResourceType.MODEL, - resource_name=model.name, - location=model.rejected_sql_analysis_opt_out, - notes=(note,), - help=help_text, - ) - - -def _lineage( - *, model: CompiledModelView, lineage: bool, from_session: bool -) -> list[tuple[str, list[tuple[str, str]]]] | None: - """The model's lineage rows; None asks for the session's, and only recovery reads them.""" - - if not lineage: - return [] - if from_session: - return None - return lineage_payload(model) diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/metadata.py b/src/sqlbuild/compiler/semantic_checks/_helpers/metadata.py deleted file mode 100644 index 5c96d1a84f..0000000000 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/metadata.py +++ /dev/null @@ -1,301 +0,0 @@ -"""Native semantic metadata checks, with Python's audit and resource SQL checks in order.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any, cast - -import sqlbuild._native as _native -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.adapter.type_system.main.normalize_type import normalize_type -from sqlbuild.compiler.compile.classes.semantic_completion_inputs import SemanticCompletionInputs -from sqlbuild.compiler.compile.models import ( - CompiledModelSqlTestPayload, - CompiledSqlTest, - CompilerDiagnostic, -) -from sqlbuild.compiler.compile.types import ( - CompiledFunctionView, - CompiledModelView, - CompiledProjectView, - CompiledResourceType, - CompiledSourceView, - DiagnosticPhase, - DiagnosticSeverity, -) -from sqlbuild.compiler.discovery.constants import SQL_ANALYSIS_CONFIG_KEY -from sqlbuild.compiler.semantic_checks.constants import ( - METADATA_CONFIG_KEYS, - METADATA_FUNCTION_REFERENCE_KINDS, -) -from sqlbuild.spec.contracts.models import SchemaAuditInstance, SourceLocation - -type _ErrorRow = tuple[str, str, int, int] -type _NamesRow = tuple[str, list[str]] - - -def native_metadata_diagnostics( - *, - project: CompiledProjectView, - profile: ExpressionInferenceProfile, - resource_sql_analysis: bool, - catalog: object, -) -> tuple[CompilerDiagnostic, ...]: - """The project's semantic metadata diagnostics; a native internal failure raises.""" - - if not project.settings.sql_analysis: - return () - shapes: dict[str, dict[str, str]] = SemanticCompletionInputs.metadata_shapes( - project=project, profile=profile - ) - tests: list[CompiledSqlTest] = [ - test for test in project.sql_tests if isinstance(test.payload, CompiledModelSqlTestPayload) - ] - columns_by_sql: dict[str, tuple[str, ...]] = SemanticCompletionInputs.sql_test_columns( - project=project, profile=profile - ) - functions: dict[str, CompiledFunctionView] = { - function.name.casefold(): function for function in project.functions - } - file_texts: dict[Path, str] = {} - for source in project.sources: - _ = file_texts.setdefault(source.source_file.file_path, source.source_file.contents) - for test in tests: - _ = file_texts.setdefault(test.test_file.file_path, test.test_file.contents) - files: dict[Path, int] = {path: index for index, path in enumerate(file_texts)} - request: Any = ( - profile.sql_analysis_dialect, - profile.quoted_identifiers_ignore_case, - list(profile.function_return_types.items()), - [_function_payload(key=key, function=function) for key, function in functions.items()], - [(name, list(columns.items())) for name, columns in shapes.items()], - [_model_payload(model) for model in project.models], - list(file_texts.values()), - [_source_payload(source=source, files=files) for source in project.sources], - [ - _sql_test_payload(test=test, columns_by_sql=columns_by_sql, files=files) - for test in tests - ], - ) - model_rows, source_rows, test_rows, fallback_types = _native.check_semantic_metadata_rows( - catalog, request - ) - for type_sql in fallback_types: - _ = normalize_type(type_sql=type_sql, dialect=profile.sql_analysis_dialect) - diagnostics: list[CompilerDiagnostic] = [] - for model, (function_rows, audit_rows, reference_rows) in zip( - project.models, model_rows, strict=True - ): - if not _checked(model): - continue - diagnostics.extend(_model_error(model=model, row=row) for row in function_rows) - diagnostics.extend(_audit_error(model=model, row=row) for row in audit_rows) - diagnostics.extend(_model_error(model=model, row=row) for row in reference_rows) - diagnostics.extend( - _source_error(source=project.sources[index], row=row) for index, row in source_rows - ) - diagnostics.extend( - _sql_test_error(test=tests[index], row=row, end_column=end_column) - for index, row, end_column in test_rows - ) - if resource_sql_analysis: - diagnostics.extend( - SemanticCompletionInputs.resource_sql_diagnostics( - project=project, shapes=shapes, profile=profile - ) - ) - return tuple(diagnostics) - - -def _function_payload( - *, key: str, function: CompiledFunctionView -) -> tuple[str, list[tuple[str, str]]]: - return (key, [(argument.name, argument.type) for argument in function.arguments]) - - -def _checked(model: CompiledModelView) -> bool: - return model.config.values.get(SQL_ANALYSIS_CONFIG_KEY) is not False and model.binding_validated - - -def _string(value: object) -> str | None: - return value if isinstance(value, str) else None - - -def _model_payload(model: CompiledModelView) -> tuple[Any, ...]: - values: dict[str, object] = model.config.values - references: list[_NamesRow] = [ - (key, list(SemanticCompletionInputs.column_names(values.get(key)))) - for key in METADATA_CONFIG_KEYS - ] - custom: object = values.get("config") - if isinstance(custom, dict): - partition: object = cast(dict[str, object], custom).get("partition_column") - references.append( - ("partition_column", list(SemanticCompletionInputs.column_names(partition))) - ) - tolerances: object = values.get("row_diff_tolerances") - by_column: object = ( - cast(dict[str, object], tolerances).get("by_column") - if isinstance(tolerances, dict) - else None - ) - if isinstance(by_column, dict): - references.append(("row_diff_tolerances", [str(name) for name in by_column])) - cursor_inputs: object = values.get("cursor_inputs") - upstreams: list[_NamesRow] = [] - if isinstance(cursor_inputs, dict): - upstreams = [ - (str(upstream), list(SemanticCompletionInputs.column_names(columns))) - for upstream, columns in cursor_inputs.items() - ] - return ( - model.name, - model.query_sql, - model.authored_sql, - _checked(model), - any( - reference.ref_kind in METADATA_FUNCTION_REFERENCE_KINDS - for reference in model.references - ), - references, - upstreams, - _string(values.get("cursor")), - _string(values.get("cursor_type")), - _audit_payloads(model), - ) - - -def _audit_payloads(model: CompiledModelView) -> list[tuple[object, ...]]: - if model.schema_entry is None: - return [] - attached: list[tuple[str | None, SchemaAuditInstance]] = [ - (None, audit) for audit in model.schema_entry.audits - ] - for schema_column in model.schema_entry.columns: - attached.extend((schema_column.name, audit) for audit in schema_column.audits) - payloads: list[tuple[object, ...]] = [] - for column, audit in attached: - arguments: dict[str, object] = audit.arguments - expression: object = arguments.get("expression") - values: object = arguments.get("values") - payloads.append( - ( - column, - audit.definition_name, - expression if isinstance(expression, str) else None, - str(arguments.get("to", "")), - str(arguments.get("field", "")), - [_audit_value(value) for value in values] - if isinstance(values, (list, tuple)) - else None, - ) - ) - return payloads - - -def _audit_value(value: object) -> tuple[str, str]: - """An accepted value by Python type: null, bool, number, text or other.""" - - if value is None: - return ("null", "") - if isinstance(value, bool): - return ("bool", "true" if value else "") - if isinstance(value, int | float): - return ("number", str(value)) - if isinstance(value, str): - return ("text", value) - return ("other", "") - - -def _source_payload( - *, source: CompiledSourceView, files: dict[Path, int] -) -> tuple[str, str | None, int]: - return ( - source.name, - source.source_entry.cursor_column or None, - files[source.source_file.file_path], - ) - - -def _sql_test_payload( - *, test: CompiledSqlTest, columns_by_sql: dict[str, tuple[str, ...]], files: dict[Path, int] -) -> tuple[int, list[_NamesRow]]: - payload: CompiledModelSqlTestPayload = cast(CompiledModelSqlTestPayload, test.payload) - return ( - files[test.test_file.file_path], - [ - (cte.name, list(columns_by_sql[cte.sql_body])) - for cte in (*payload.authored_ctes, *payload.expected_ctes) - ], - ) - - -def _model_error(*, model: CompiledModelView, row: _ErrorRow) -> CompilerDiagnostic: - code, message, line, column = row - return CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=code, - message=message, - resource_type=CompiledResourceType.MODEL, - resource_name=model.name, - path=model.relative_path, - line=line, - column=column, - ) - - -def _audit_error( - *, model: CompiledModelView, row: tuple[str, str, int, int, bool] -) -> CompilerDiagnostic: - code, message, line, column, is_error = row - return CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR if is_error else DiagnosticSeverity.WARNING, - code=code, - message=message, - resource_type=CompiledResourceType.MODEL, - resource_name=model.name, - path=model.relative_path, - line=line, - column=column, - ) - - -def _source_error(*, source: CompiledSourceView, row: _ErrorRow) -> CompilerDiagnostic: - code, message, line, column = row - return CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=code, - message=message, - resource_type=CompiledResourceType.SOURCE, - resource_name=source.name, - path=source.source_file.relative_path, - line=line, - column=column, - ) - - -def _sql_test_error( - *, test: CompiledSqlTest, row: _ErrorRow, end_column: int -) -> CompilerDiagnostic: - code, message, line, column = row - return CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=code, - message=message, - resource_type=CompiledResourceType.SQL_TEST, - resource_name=test.name, - path=test.test_file.relative_path, - line=line, - column=column, - location=SourceLocation( - path=test.test_file.relative_path, - line=line, - column=column, - end_line=line, - end_column=end_column, - ), - ) diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/payloads.py b/src/sqlbuild/compiler/semantic_checks/_helpers/payloads.py deleted file mode 100644 index 38ba30d904..0000000000 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/payloads.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Plain-data payloads of compiled facts for the native semantic completion requests.""" - -from __future__ import annotations - -from typing import Any, cast - -from sqlbuild.compiler.analysis_session.types import DiagnosticRow -from sqlbuild.compiler.compile.models import ( - CompiledLineageColumnFact, - CompilerDiagnostic, - DynamicColumnContractProof, -) -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView - - -def lineage_payload(model: CompiledModelView) -> list[tuple[str, list[tuple[str, str]]]]: - """Each output column with the `(resource, column)` pairs it reads.""" - - return [_output_payload(output) for output in model.fast_lineage_columns or ()] - - -def session_fact_models(*, project: CompiledProjectView, session: Any | None) -> frozenset[str]: - """Models whose output names and lineage are the finished session's, so need no payload.""" - - if session is None: - return frozenset() - kept: frozenset[str] = frozenset(cast(list[str], session.fact_models)) - return frozenset( - model.name - for model in project.models - if model.name in kept and not _output_proven(model.dynamic_column_contract) - ) - - -def _output_proven(proof: DynamicColumnContractProof | None) -> bool: - """Whether a dynamic proof replaced the analysed output names.""" - - return proof is not None and proof.output_proven - - -def _output_payload(output: CompiledLineageColumnFact) -> tuple[str, list[tuple[str, str]]]: - return ( - output.output_column, - [(source.resource_name, source.column_name) for source in output.upstream_columns], - ) - - -def diagnostic_ids(project: CompiledProjectView) -> dict[CompilerDiagnostic, int]: - """One id per distinct diagnostic value, as Python's value-keyed dicts and sets see them.""" - - ids: dict[CompilerDiagnostic, int] = {} - for item in project.diagnostics: - _ = ids.setdefault(item, len(ids)) - for model in project.models: - for item in model.binding_diagnostics: - _ = ids.setdefault(item, len(ids)) - return ids - - -def raw_binding_ids( - binding_results: dict[str, list[DiagnosticRow]], -) -> dict[DiagnosticRow, int]: - """One id per distinct raw binding row value.""" - - ids: dict[DiagnosticRow, int] = {} - for rows in binding_results.values(): - for row in rows: - _ = ids.setdefault(row, len(ids)) - return ids - - -def has_blocking_binding(project: CompiledProjectView) -> bool: - """Whether any model owns a binding error, without which type recovery changes nothing.""" - - for model in project.models: - for item in model.binding_diagnostics: - if item.is_error: - return True - return False diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py b/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py deleted file mode 100644 index d0b498e3c9..0000000000 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py +++ /dev/null @@ -1,52 +0,0 @@ -"""The native semantic completion stage: type recovery, metadata checks, then completion.""" - -from __future__ import annotations - -from typing import Any - -import sqlbuild._native as _native -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.types import DiagnosticRow -from sqlbuild.compiler.compile.models import CompilerDiagnostic -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.project_with import project_with -from sqlbuild.compiler.semantic_checks._helpers.completion import complete_native_diagnostics -from sqlbuild.compiler.semantic_checks._helpers.metadata import native_metadata_diagnostics -from sqlbuild.compiler.semantic_checks._helpers.payloads import session_fact_models -from sqlbuild.compiler.semantic_checks._helpers.type_recovery import recover_native_output_types -from sqlbuild.compiler.semantic_checks.constants import NATIVE_SEMANTIC_NO_CATALOG - - -def completed_semantic_project( - *, - project: CompiledProjectView, - profile: ExpressionInferenceProfile, - binding_results: dict[str, list[DiagnosticRow]], - resource_sql_analysis: bool, - session: Any | None, -) -> CompiledProjectView: - """The completed project; a native internal failure raises `NativeCompilerError`.""" - - if project.binding_catalog is None: - raise _native.NativeCompilerError(NATIVE_SEMANTIC_NO_CATALOG) - catalog: object = project.binding_catalog.native - session_models: frozenset[str] = session_fact_models(project=project, session=session) - recovered: CompiledProjectView = recover_native_output_types( - project=project, - binding_results=binding_results, - catalog=catalog, - session=session, - session_models=session_models, - ) - metadata: tuple[CompilerDiagnostic, ...] = native_metadata_diagnostics( - project=recovered, - profile=profile, - resource_sql_analysis=resource_sql_analysis, - catalog=catalog, - ) - return complete_native_diagnostics( - project=project_with(project=recovered, diagnostics=(*recovered.diagnostics, *metadata)), - catalog=catalog, - session=session, - session_models=session_models, - ) diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/type_recovery.py b/src/sqlbuild/compiler/semantic_checks/_helpers/type_recovery.py deleted file mode 100644 index 582011908b..0000000000 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/type_recovery.py +++ /dev/null @@ -1,162 +0,0 @@ -"""Native output-type recovery, with Python's revalidation on the compile's binding catalog.""" - -from __future__ import annotations - -from dataclasses import replace -from typing import Any - -import sqlbuild._native as _native -from sqlbuild.compiler.analysis_session.types import DiagnosticRow -from sqlbuild.compiler.compile.classes.semantic_completion_inputs import SemanticCompletionInputs -from sqlbuild.compiler.compile.models import ( - CompilerDiagnostic, -) -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledResourceType, -) -from sqlbuild.compiler.compiled_project.main._with_type_recovery import with_type_recovery -from sqlbuild.compiler.compiled_project.main.project_with import project_with -from sqlbuild.compiler.semantic_checks._helpers.payloads import ( - diagnostic_ids, - has_blocking_binding, - lineage_payload, - raw_binding_ids, -) -from sqlbuild.compiler.semantic_checks.constants import ( - NATIVE_TYPE_RECOVERY_UNCHANGED, - UNKNOWN_RECOVERED_TYPE, -) -from sqlbuild.compiler.sql_analysis.main._schema_validation import get_schema_validations -from sqlbuild.compiler.sql_analysis.models import SqlBindingResult - -type _Revised = list[tuple[str, int | None, int | None]] - - -def recover_native_output_types( - *, - project: CompiledProjectView, - binding_results: dict[str, list[DiagnosticRow]], - catalog: object, - session: Any | None, - session_models: frozenset[str], -) -> CompiledProjectView: - """The project with recovered output types; a native internal failure raises.""" - - if not has_blocking_binding(project): - return project - ids: dict[CompilerDiagnostic, int] = diagnostic_ids(project) - raw_ids: dict[DiagnosticRow, int] = raw_binding_ids(binding_results) - request: Any = ( - project.sql_analysis_dialect, - [ - _model_payload( - model=model, - binding_results=binding_results, - ids=ids, - raw_ids=raw_ids, - from_session=model.name in session_models, - ) - for model in project.models - ], - [ - ( - ids[item], - item.code, - item.resource_type == CompiledResourceType.MODEL, - item.resource_name, - ) - for item in project.diagnostics - ], - ) - recovery: _native.SemanticTypeRecovery = _native.plan_semantic_type_recovery( - catalog, request, session - ) - if recovery.status == NATIVE_TYPE_RECOVERY_UNCHANGED: - return project - poisoned: list[tuple[str, str]] = recovery.poisoned - outcome: tuple[list[tuple[int, str | None]], list[list[int]]] = recovery.finish( - _revalidated(project=project, revalidated=recovery.revalidated, poisoned=poisoned) - ) - kept, model_bindings = outcome - diagnostics: tuple[CompilerDiagnostic, ...] = tuple( - _with_note(diagnostic=project.diagnostics[index], note=note) for index, note in kept - ) - poisoned_outputs: frozenset[tuple[str, str]] = frozenset(poisoned) - models: tuple[CompiledModelView, ...] = tuple( - with_type_recovery( - model=model, - diagnostics=_positions(diagnostics=diagnostics, positions=positions), - poisoned=poisoned_outputs, - ) - for model, positions in zip(project.models, model_bindings, strict=True) - ) - return project_with(project=project, models=models, diagnostics=diagnostics) - - -def _revalidated( - *, project: CompiledProjectView, revalidated: list[int], poisoned: list[tuple[str, str]] -) -> list[_Revised]: - """Python's revalidation of each planned model with unknown types for poisoned outputs.""" - - if not revalidated: - return [] - shapes: dict[str, dict[str, str]] = _unknown_shapes( - shapes=SemanticCompletionInputs.shapes(project), poisoned=poisoned - ) - revised: list[_Revised] = [] - for index in revalidated: - result: SqlBindingResult = get_schema_validations( - requests=( - SemanticCompletionInputs.revalidation_request( - project=project, model=project.models[index], shapes=shapes - ), - ) - )[0] - revised.append([(item.code, item.start, item.end) for item in result.diagnostics]) - return revised - - -def _model_payload( - *, - model: CompiledModelView, - binding_results: dict[str, list[DiagnosticRow]], - ids: dict[CompilerDiagnostic, int], - raw_ids: dict[DiagnosticRow, int], - from_session: bool, -) -> tuple[Any, ...]: - raws: list[DiagnosticRow] = ( - binding_results.get(model.name, []) if model.binding_diagnostics else [] - ) - return ( - model.name, - model.query_sql, - None - if from_session or model.inferred_columns is None - else [column.name for column in model.inferred_columns], - [reference.ref_name for reference in model.references], - None if from_session else lineage_payload(model), - [(ids[item], item.code, item.message, item.is_error) for item in model.binding_diagnostics], - [(raw_ids[raw], raw[0], raw[1], raw[4], raw[5]) for raw in raws], - ) - - -def _with_note(*, diagnostic: CompilerDiagnostic, note: str | None) -> CompilerDiagnostic: - return replace(diagnostic, notes=(*diagnostic.notes, *((note,) if note is not None else ()))) - - -def _positions( - *, diagnostics: tuple[CompilerDiagnostic, ...], positions: list[int] -) -> tuple[CompilerDiagnostic, ...]: - return tuple(diagnostics[position] for position in positions) - - -def _unknown_shapes( - *, shapes: dict[str, dict[str, str]], poisoned: list[tuple[str, str]] -) -> dict[str, dict[str, str]]: - unknown: dict[str, dict[str, str]] = {name: dict(columns) for name, columns in shapes.items()} - for name, column in poisoned: - if name in unknown and column in unknown[name]: - unknown[name][column] = UNKNOWN_RECOVERED_TYPE - return unknown diff --git a/src/sqlbuild/compiler/semantic_checks/constants.py b/src/sqlbuild/compiler/semantic_checks/constants.py deleted file mode 100644 index 853738663f..0000000000 --- a/src/sqlbuild/compiler/semantic_checks/constants.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Native semantic completion statuses and inputs.""" - -from sqlbuild.compiler.references.types import SqlReferenceKind - -NATIVE_TYPE_RECOVERY_UNCHANGED: str = "unchanged" -NATIVE_SEMANTIC_NO_CATALOG: str = ( - "NativeCompilerError: native semantic completion: the project has no analysis catalog" -) -UNKNOWN_RECOVERED_TYPE: str = "UNKNOWN" -METADATA_CONFIG_KEYS: tuple[str, ...] = ( - "unique_key", - "cursor", - "partition_column", - "row_diff_exclude_columns", -) -METADATA_FUNCTION_REFERENCE_KINDS: frozenset[SqlReferenceKind] = frozenset( - {SqlReferenceKind.UDF, SqlReferenceKind.TABLE_FUNCTION} -) diff --git a/src/sqlbuild/compiler/semantic_checks/main/__init__.py b/src/sqlbuild/compiler/semantic_checks/main/__init__.py deleted file mode 100644 index b7664e28d3..0000000000 --- a/src/sqlbuild/compiler/semantic_checks/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Native semantic completion entry points.""" diff --git a/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py b/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py deleted file mode 100644 index dc14400120..0000000000 --- a/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Complete semantic diagnostics natively.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.types import DiagnosticRow -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.semantic_checks._helpers.stage import completed_semantic_project - - -def complete_native_semantic_diagnostics( - *, - project: CompiledProjectView, - profile: ExpressionInferenceProfile, - binding_results: dict[str, list[DiagnosticRow]], - resource_sql_analysis: bool, - session: Any | None = None, -) -> CompiledProjectView: - """The completed project; a native internal failure raises `NativeCompilerError`.""" - - return completed_semantic_project( - project=project, - profile=profile, - binding_results=binding_results, - resource_sql_analysis=resource_sql_analysis, - session=session, - ) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py index 2299101a90..e6805a87d2 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py @@ -13,9 +13,11 @@ import orjson +from sqlbuild._native import ( + compiled_project_record, +) from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.compiled_project_record import compiled_project_record from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity from sqlbuild.compiler.scopes.models import ScopeIndex from sqlbuild.rule_engine._helpers.engine.custom_rule_evidence import ( diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_host_pool.py b/src/sqlbuild/rule_engine/_helpers/host/custom_host_pool.py index 654e0851cc..3dbf3ea5d2 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_host_pool.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_host_pool.py @@ -17,9 +17,11 @@ import orjson import sqlbuild._native as _native +from sqlbuild._native import ( + compiled_project_record, +) from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.compiled_project.main.compiled_project_record import compiled_project_record from sqlbuild.rule_engine._helpers.host.host_capacity import available_cores from sqlbuild.rule_engine.constants import ( CUSTOM_HOST_ABANDONED_SUFFIX, diff --git a/src/sqlbuild/rule_engine/_helpers/run/native_rows.py b/src/sqlbuild/rule_engine/_helpers/run/native_rows.py index ae4eaef3ea..5f67c8da7e 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/native_rows.py +++ b/src/sqlbuild/rule_engine/_helpers/run/native_rows.py @@ -9,6 +9,11 @@ import orjson import sqlbuild._native as _native +from sqlbuild._native import ( + compiled_project_facts, + declaration_rows, + plain_typed_value, +) from sqlbuild.adapter.type_system.main.types_equal import types_equal from sqlbuild.compiler.compile.models import ( CompiledModelSqlTestPayload, @@ -17,9 +22,6 @@ CompileSqlTestCte, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.compiled_project_facts import compiled_project_facts -from sqlbuild.compiler.compiled_project.main.declaration_rows import declaration_rows -from sqlbuild.compiler.compiled_project.main.plain_typed_value import plain_typed_value from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity from sqlbuild.compiler.scopes.main.scope_metadata import scope_metadata_projection from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index 8d046b6218..9f3710fcd6 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -14,6 +14,9 @@ from typing import Any import sqlbuild._native as _native +from sqlbuild._native import ( + project_with, +) from sqlbuild.compiler.compile.constants import MACRO_TOKEN from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -28,7 +31,6 @@ CompileInputsView, ModelInput, ) -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.lint.classes.stop_context import LintStopContext diff --git a/src/sqlbuild/rule_engine/classes/audit_facts.py b/src/sqlbuild/rule_engine/classes/audit_facts.py index 8a9c0ba1cc..361b2dce77 100644 --- a/src/sqlbuild/rule_engine/classes/audit_facts.py +++ b/src/sqlbuild/rule_engine/classes/audit_facts.py @@ -2,9 +2,11 @@ from functools import cached_property +from sqlbuild._native import ( + compiled_audit_record, +) from sqlbuild.compiler.compile.models import CompiledAudit from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.compiled_audit_record import compiled_audit_record from sqlbuild.rule_engine.models import Model diff --git a/src/sqlbuild/rule_engine/classes/project_facts.py b/src/sqlbuild/rule_engine/classes/project_facts.py index 595de795c9..b07de8a219 100644 --- a/src/sqlbuild/rule_engine/classes/project_facts.py +++ b/src/sqlbuild/rule_engine/classes/project_facts.py @@ -2,13 +2,13 @@ from pathlib import Path -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.compiled_audit_record import compiled_audit_record -from sqlbuild.compiler.compiled_project.main.compiled_function_record import ( +from sqlbuild._native import ( + compiled_audit_record, compiled_function_record, + compiled_seed_record, + compiled_source_record, ) -from sqlbuild.compiler.compiled_project.main.compiled_seed_record import compiled_seed_record -from sqlbuild.compiler.compiled_project.main.compiled_source_record import compiled_source_record +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine.classes.project_tree import ProjectTree, public_model from sqlbuild.rule_engine.models import Model diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index da417a095e..dfa12101fa 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -248,15 +248,6 @@ class DenseCompileGuardTestCase: expected_fingerprint: str -@dataclass(frozen=True) -class DenseMetadataTextGuardTestCase: - """A dense project whose semantic metadata request may carry each file's text only once.""" - - description: str - model_count: int - expected_requests: int = 1 - - @dataclass(frozen=True) class DenseWarmEditCompileGuardTestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 49f6b58631..81336c5963 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -3510,31 +3510,6 @@ def compare_incremental_compile(*, project_dir: Path) -> IncrementalEditComparis return IncrementalEditComparison(incremental=incremental, reference=reference) -def record_metadata_text_characters(monkeypatch: pytest.MonkeyPatch) -> list[int]: - """Record the file-text characters each native semantic metadata request carries.""" - - sizes: list[int] = [] - check: Callable[..., object] = cast( - Callable[..., object], native_module.check_semantic_metadata_rows - ) - - def recorded(catalog: object, request: tuple[object, ...]) -> object: - sizes.append(sum(len(text) for text in cast(list[str], request[5]))) - return check(catalog, request) - - monkeypatch.setattr(native_module, "check_semantic_metadata_rows", recorded) - return sizes - - -def project_text_characters(project_dir: Path) -> int: - """Characters in every file of a project that has not compiled yet.""" - - return sum( - len(path.read_text(encoding="utf-8")) - for path in filter(Path.is_file, project_dir.rglob("*")) - ) - - def in_process_reuse_run( *, project_dir: Path, capsys: pytest.CaptureFixture[str], args: tuple[str, ...] = () ) -> CompileReuseRun: @@ -4840,7 +4815,7 @@ def full_macro_call_runs( (native_stages, "infer_native_expression_source_shapes"), (native_stages, "analyze_native_model_sql"), (native_sql_test_stage, "assemble_native_sql_tests"), - (diagnostic_recovery, "complete_native_semantic_diagnostics"), + (diagnostic_recovery, "complete_semantic_project"), (contract_validation, "evaluate_native_model_contracts"), (promotion_conflicts, "native_promotion_conflict_diagnostics"), (compile_lineage, "build_compile_lineage"), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index a3c7bff6fe..dec0bf7ea8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -19,18 +19,13 @@ from sqlbuild.rule_engine.models import Rule, RulesConfig from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( DenseCompileGuardTestCase, - DenseMetadataTextGuardTestCase, DenseWarmEditCompileGuardTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( - CompileReuseRun, FreshProcessCompileBenchmarkResult, _run_fresh_process_compile_benchmark, fresh_process_compile_cache_metrics, - in_process_reuse_run, measure_model_sql_bytes, - project_text_characters, - record_metadata_text_characters, run_dense_warm_edit_benchmark, ) @@ -136,36 +131,6 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ assert result.peak_rss_bytes < test_case.expected_max_rss_bytes -@pytest.mark.performance -@pytest.mark.cold_compile_performance -@pytest.mark.parametrize( - "test_case", - [ - DenseMetadataTextGuardTestCase(f"dense_models_{models}_metadata_file_texts", models) - for models in (1000, 3000, 5000, 10000) - ], - ids=lambda case: case.description, -) -def test_given_dense_project_when_checking_semantic_metadata_then_each_file_text_crosses_once( - test_case: DenseMetadataTextGuardTestCase, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - project_dir: Path = tmp_path / "dense_orders" - write_dense_compile_project(project_dir=project_dir, model_count=test_case.model_count) - budget: int = project_text_characters(project_dir) - sizes: list[int] = record_metadata_text_characters(monkeypatch) - - run: CompileReuseRun = in_process_reuse_run( - project_dir=project_dir, capsys=capsys, args=("--no-cache",) - ) - - assert run.returncode == 0, run.stderr - assert len(sizes) == test_case.expected_requests - assert 0 < sizes[0] <= budget, (sizes, budget) - - @pytest.mark.performance @pytest.mark.cold_compile_performance @pytest.mark.parametrize( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index e8f977cece..b710e6cb6c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -33,7 +33,7 @@ "infer_native_expression_source_shapes": [ANY], "analyze_native_model_sql": [ANY], "assemble_native_sql_tests": [ANY], - "complete_native_semantic_diagnostics": [ANY], + "complete_semantic_project": [ANY], "evaluate_native_model_contracts": [ANY], "native_promotion_conflict_diagnostics": [()], "build_compile_lineage": [ANY], @@ -76,9 +76,7 @@ def test_given_project_when_compiling_then_native_seams_answer( assert planned_artifacts assert all(isinstance(artifact, NativeSqlTestArtifact) for artifact in planned_artifacts) planned_counts.append(len(planned_artifacts)) - assert isinstance( - seams["complete_native_semantic_diagnostics"][0], native_module.NativeProjectView - ) + assert isinstance(seams["complete_semantic_project"][0], native_module.NativeProjectView) assert isinstance(seams["evaluate_native_model_contracts"][0], ContractValidationResult) assert [ len(call.args[0].tests) for call in native_sql_test_planning.call_args_list diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 2bfb3c12c1..3d2284cab8 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -12,7 +12,6 @@ from _pytest.capture import CaptureResult import sqlbuild._native as native_module -import sqlbuild.compiler.compile._helpers.attachment.declaration_scope as declaration_scope_module import sqlbuild.compiler.compile._helpers.render.macros as macros_module import sqlbuild.compiler.compile.main.sql_expansion_context as expansion_context_module import sqlbuild.rule_engine._helpers.host.custom_host_pool as custom_host_pool_module @@ -791,9 +790,7 @@ def test_given_warm_scoped_macro_sql_test_when_compiling_then_rules_reuse_compil macro_finding: str = f"tests/unit/test_macro_orders.sql:{test_case.expected_code}" cold: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) scope_builds: list[Path] = [] - original_build: Callable[..., DeclarationScopeBuild] = ( - declaration_scope_module.build_declaration_scope - ) + original_build: Callable[..., DeclarationScopeBuild] = native_module.build_declaration_scope def counting_build( *, @@ -810,7 +807,7 @@ def counting_build( compile_cache_dir=compile_cache_dir, ) - monkeypatch.setattr(declaration_scope_module, "build_declaration_scope", counting_build) + monkeypatch.setattr(native_module, "build_declaration_scope", counting_build) monkeypatch.setattr(expansion_context_module, "build_declaration_scope", counting_build) warm: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) @@ -850,9 +847,7 @@ def test_given_cold_scoped_macro_sql_test_when_compiling_then_early_lint_uses_pr ) scope_builds: list[Path] = [] macro_loads: list[dict[str, LoadedMacro]] = [] - original_build: Callable[..., DeclarationScopeBuild] = ( - declaration_scope_module.build_declaration_scope - ) + original_build: Callable[..., DeclarationScopeBuild] = native_module.build_declaration_scope original_load: Callable[..., dict[str, LoadedMacro]] = ( expansion_context_module.load_project_macros ) @@ -877,7 +872,7 @@ def recording_load(macro_files: tuple[DiscoveredMacroFile, ...]) -> dict[str, Lo macro_loads.append(loaded) return loaded - monkeypatch.setattr(declaration_scope_module, "build_declaration_scope", counting_build) + monkeypatch.setattr(native_module, "build_declaration_scope", counting_build) monkeypatch.setattr(expansion_context_module, "build_declaration_scope", counting_build) monkeypatch.setattr(macros_module, "load_project_macros", recording_load) monkeypatch.setattr(expansion_context_module, "load_project_macros", recording_load) diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/_test_helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/_test_helpers.py index 6db0951552..70165aa640 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/_test_helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/_test_helpers.py @@ -14,13 +14,29 @@ conditional_result_nullability, ) from sqlbuild.adapter.type_system.main.first_arg_nullability import first_arg_nullability +from sqlbuild.adapter.type_system.main.safe_cast_nullability import safe_cast_nullability from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.analysis_session._helpers.profile_rows import nullability_rule_rows + +_NULLABILITY_RULE_IDS: dict[object, str] = { + first_arg_nullability: "first_arg", + conditional_result_nullability: "conditional_result", + safe_cast_nullability: "safe_cast", +} + + +def nullability_rule_rows(profile: ExpressionInferenceProfile) -> list[tuple[str, str]]: + """The profile's nullability rules as the native `(name, rule id)` rows.""" + + return [ + (name, _NULLABILITY_RULE_IDS.get(rule, "python")) + for name, rule in profile.function_nullability_rules.items() + ] + type Generate = Callable[[random.Random, list[str], int], str] type CteFactView = ( diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index b919d9b2d3..10ecbee4ec 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -16,26 +16,19 @@ import sqlbuild._native as native_module import sqlbuild.compiler.compile._helpers.diagnostics.recovery as semantic_recovery import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stage_assembly -from sqlbuild._native import build_compile_inputs, discover_project_inputs -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.cli.entry.main.entry import main -from sqlbuild.compiler.analysis_session.constants import LINEAGE_NATIVE -from sqlbuild.compiler.analysis_session.main._analyze_native_model_sql import ( +from sqlbuild._native import ( analyze_native_model_sql, -) -from sqlbuild.compiler.analysis_session.main._dynamic_contract_proof import ( - dynamic_contract_proof as contract_proof, -) -from sqlbuild.compiler.analysis_session.main._infer_native_expression_source_shapes import ( + build_compile_inputs, + discover_project_inputs, infer_native_expression_source_shapes, -) -from sqlbuild.compiler.analysis_session.main._prove_native_dynamic_contracts import ( prove_native_dynamic_contracts, -) -from sqlbuild.compiler.analysis_session.main._session_outcome_analysis import ( session_outcome_analysis, ) +from sqlbuild._native import dynamic_contract_proof as contract_proof +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.cli.entry.main.entry import main +from sqlbuild.compiler.analysis_session.constants import LINEAGE_NATIVE from sqlbuild.compiler.analysis_session.models import ( NativeModelAnalyses, NativeModelAnalysisRequest, diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py index 5590867d80..77a6e7efc4 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py @@ -11,13 +11,10 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import assemble_project +from sqlbuild._native import analyze_native_model_sql, assemble_project from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.analysis_session.main._analyze_native_model_sql import ( - analyze_native_model_sql, -) from sqlbuild.compiler.analysis_session.models import NativeModelAnalysisRequest from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py index 3e507ee1c6..82852ea94d 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py @@ -13,10 +13,8 @@ import sqlbuild._native as native_module import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stage_assembly +from sqlbuild._native import infer_native_query_columns from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.main._infer_native_query_columns import ( - infer_native_query_columns, -) from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeModelAnalyses from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView diff --git a/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/_test_types.py b/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/_test_types.py index 208352ff9c..c6fc409c30 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/_test_types.py @@ -2,8 +2,6 @@ from dataclasses import dataclass -from sqlbuild.compiler.compile.models import SqlReferenceSourceMap - @dataclass(frozen=True) class CraftedReferenceTestCase: @@ -23,18 +21,6 @@ class ReferenceExtractionTestCase: sql: str expected_references: tuple[tuple[str, str, str | None, int | None], ...] = () expected_error: str = "" - expected_native_calls: int = 0 - - -@dataclass(frozen=True) -class NativeReferenceErrorTestCase: - """Expanded SQL whose native error is located through its source map without Python.""" - - description: str - sql: str - contents: str - source_map: SqlReferenceSourceMap | None - expected_message: str @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/helpers.py deleted file mode 100644 index 7169985f1c..0000000000 --- a/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/helpers.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Source maps for located reference extraction errors.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import ExpansionSpan, SqlReferenceSourceMap - - -def source_map_at( - *, body_start: int, passes: tuple[tuple[ExpansionSpan, ...], ...] = () -) -> SqlReferenceSourceMap: - """Return a source map whose authored body starts at `body_start` in its file.""" - - return SqlReferenceSourceMap(body_start=lambda: body_start, passes=passes) diff --git a/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/test_native_references.py b/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/test_native_references.py index 63dd69079f..9eb38d107e 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/test_native_references.py +++ b/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/test_native_references.py @@ -1,29 +1,22 @@ -"""Native reference extraction: references, rejected calls, located errors and scan bounds.""" +"""Native reference extraction: references, rejected calls and scan bounds.""" from __future__ import annotations import time from collections.abc import Callable -from operator import itemgetter -from pathlib import Path from typing import Any import pytest -from sqlbuild.compiler.compile._helpers.refs import references -from sqlbuild.compiler.compile._helpers.refs.native import extract_native_sql_references +import sqlbuild._native as _native from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ExpansionSpan, SqlReferenceOrigin, SqlReferenceScan -from sqlbuild.compiler.compile.types import SqlReferenceScanFailure from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.integration.src.sqlbuild.compiler.compile._helpers.refs._test_types import ( CraftedReferenceTestCase, - NativeReferenceErrorTestCase, ReferenceExtractionTestCase, ReferenceScanBoundTestCase, ) -from tests.integration.src.sqlbuild.compiler.compile._helpers.refs.helpers import source_map_at _GENERIC_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() _REFERENCE_SQLS: tuple[ @@ -57,13 +50,14 @@ '__table_fn("orders_for")(' + ", ".join(f"(c{index} /* , */)" for index in range(100_000)) + ")" ) _NESTED_CALLS: str = '__table_fn("orders_for")(' * 1_000 + "1" + ")" * 1_000 +_SCANNER: _native.SqlReferenceScanner = _native.SqlReferenceScanner(_GENERIC_SYNTAX.native_mapping) _OUTCOME_SUMMARIES: dict[type, Callable[[Any], int | str]] = { - SqlReferenceScan: lambda scan: len(scan.references) + len(scan.invalid_calls), - tuple: itemgetter(0), + tuple: lambda outcome: len(outcome[0][0]) + len(outcome[0][1]), + type(None): lambda outcome: outcome[1][0], } _REJECTED_AND_FAILED: dict[type, Callable[[Any], tuple[int, int]]] = { - SqlReferenceScan: lambda scan: (len(scan.invalid_calls), 0), - tuple: lambda _failure: (0, 1), + tuple: lambda outcome: (len(outcome[0][1]), 0), + type(None): lambda _outcome: (0, 1), } _MANY_REFERENCES: str = "SELECT 1 FROM " + " JOIN ".join( f'__ref("orders_{index}")' for index in range(50_000) @@ -188,11 +182,9 @@ def test_given_crafted_reference_sql_when_extracting_then_rejected_and_failed_counts_match( test_case: CraftedReferenceTestCase, ) -> None: - outcome: SqlReferenceScan | SqlReferenceScanFailure = extract_native_sql_references( - sql=test_case.sql, syntax=_GENERIC_SYNTAX - ) + outcome: Any = _SCANNER.extract(test_case.sql) - assert _REJECTED_AND_FAILED[type(outcome)](outcome) == ( + assert _REJECTED_AND_FAILED[type(outcome[0])](outcome) == ( test_case.expected_rejected, test_case.expected_failed, ) @@ -206,28 +198,14 @@ def test_given_crafted_reference_sql_when_extracting_then_rejected_and_failed_co sql=sql, expected_references=expected_references, expected_error=expected_error, - expected_native_calls=1, ) for name, sql, expected_references, expected_error in _REFERENCE_SQLS ], ids=lambda case: case.description, ) def test_given_reference_sql_when_extracting_references_then_native_scanner_runs( - test_case: ReferenceExtractionTestCase, monkeypatch: pytest.MonkeyPatch + test_case: ReferenceExtractionTestCase, ) -> None: - native_calls: list[str] = [] - native_extract: Callable[..., SqlReferenceScan | SqlReferenceScanFailure] = ( - references.extract_native_sql_references - ) - - def counting_extract( - *, sql: str, syntax: SqlLexicalSyntax - ) -> SqlReferenceScan | SqlReferenceScanFailure: - native_calls.append(sql) - return native_extract(sql=sql, syntax=syntax) - - monkeypatch.setattr(references, "extract_native_sql_references", counting_extract) - error: str = "" extracted: tuple[tuple[str, str, str | None, int | None], ...] = () try: @@ -238,119 +216,7 @@ def counting_extract( except CompileInputError as raised: error = str(raised) - assert (extracted, error, len(native_calls)) == ( - test_case.expected_references, - test_case.expected_error, - test_case.expected_native_calls, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - NativeReferenceErrorTestCase( - description="unclosed_reference_call_points_at_the_call_after_the_header", - sql='SELECT *\nFROM __ref("orders"', - contents='MODEL (description "Orders");\n\nSELECT *\nFROM __ref("orders"', - source_map=source_map_at(body_start=31), - expected_message="models/orders.sql:4:6: SQL reference contains an unclosed parenthesis", - ), - NativeReferenceErrorTestCase( - description="empty_table_function_argument_after_a_rejected_call", - sql='SELECT * FROM __ref(orders)\nJOIN __table_fn("orders_for")(1,,2)', - contents='SELECT * FROM __ref(orders)\nJOIN __table_fn("orders_for")(1,,2)', - source_map=source_map_at(body_start=0), - expected_message="models/orders.sql:2:6: SQL reference contains an empty argument", - ), - NativeReferenceErrorTestCase( - description="unclosed_quote_points_at_the_quote_in_code_points", - sql="SELECT 'é', __ref(\"orders\") WHERE note = 'open", - contents="-- é\nSELECT 'é', __ref(\"orders\") WHERE note = 'open", - source_map=source_map_at(body_start=5), - expected_message="models/orders.sql:2:42: SQL reference contains an unclosed quoted string", - ), - NativeReferenceErrorTestCase( - description="fault_after_neutral_expansions_in_two_passes_maps_through_both", - sql="SELECT 'open', 1 + 2\nFROM __ref(\"orders\"", - contents='SELECT @enum("s").OPEN, @m()\nFROM __ref("orders"', - source_map=source_map_at( - body_start=0, - passes=( - (ExpansionSpan(source_start=7, source_end=22, output_start=7, output_end=13),), - ( - ExpansionSpan( - source_start=15, source_end=19, output_start=15, output_end=20 - ), - ), - ), - ), - expected_message="models/orders.sql:2:6: SQL reference contains an unclosed parenthesis", - ), - NativeReferenceErrorTestCase( - description="fault_inside_macro_output_is_unlocated", - sql='SELECT * FROM __ref("orders") /* expanded', - contents='SELECT * FROM __ref("orders") @expand()', - source_map=source_map_at( - body_start=0, - passes=( - ( - ExpansionSpan( - source_start=30, source_end=39, output_start=30, output_end=41 - ), - ), - ), - ), - expected_message="models/orders.sql: SQL reference contains an unclosed block comment", - ), - NativeReferenceErrorTestCase( - description="authored_fault_shaped_by_an_unclosed_macro_quote_is_unlocated", - sql="SELECT 'x || ' || note || 'open", - contents="SELECT @q() || ' || note || 'open", - source_map=source_map_at( - body_start=0, - passes=( - (ExpansionSpan(source_start=7, source_end=11, output_start=7, output_end=9),), - ), - ), - expected_message="models/orders.sql: SQL reference contains an unclosed quoted string", - ), - NativeReferenceErrorTestCase( - description="without_a_source_map_the_file_alone_is_named", - sql='SELECT *\nFROM __ref("orders"', - contents='SELECT *\nFROM __ref("orders"', - source_map=None, - expected_message="models/orders.sql: SQL reference contains an unclosed parenthesis", - ), - ], - ids=lambda case: case.description, -) -def test_given_native_reference_error_when_extracting_then_raised_located( - test_case: NativeReferenceErrorTestCase, monkeypatch: pytest.MonkeyPatch -) -> None: - - with pytest.raises(CompileInputError) as raised: - _ = extract_sql_references( - sql=test_case.sql, - syntax=_GENERIC_SYNTAX, - origin=SqlReferenceOrigin( - file_path=Path("/project/models/orders.sql"), - relative_path=Path("models/orders.sql"), - contents=test_case.contents, - source_map=test_case.source_map, - ), - ) - - assert ( - type(raised.value), - str(raised.value), - raised.value.code, - raised.value.help, - ) == ( - CompileInputError, - test_case.expected_message, - "P001", - None, - ) + assert (extracted, error) == (test_case.expected_references, test_case.expected_error) @pytest.mark.parametrize( @@ -393,12 +259,10 @@ def test_given_worst_case_reference_sql_when_extracting_natively_then_finishes_q test_case: ReferenceScanBoundTestCase, ) -> None: started: float = time.thread_time() - outcome: SqlReferenceScan | SqlReferenceScanFailure = extract_native_sql_references( - sql=test_case.sql, syntax=_GENERIC_SYNTAX - ) + outcome: Any = _SCANNER.extract(test_case.sql) elapsed: float = time.thread_time() - started - summary: int | str = _OUTCOME_SUMMARIES[type(outcome)](outcome) + summary: int | str = _OUTCOME_SUMMARIES[type(outcome[0])](outcome) assert (summary, elapsed < test_case.expected_maximum_seconds) == ( test_case.expected_outcome, diff --git a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py index cab46a6b74..144f892d98 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py @@ -6,11 +6,7 @@ from typing import cast import sqlbuild._native as _native -from sqlbuild.compiler.compile._helpers.attachment.model_config import ( - native_model_config_session, - native_model_validation, - native_validation_error, -) +from sqlbuild._native import native_model_config_session from sqlbuild.compiler.compile.models import ( CompileModelConfig, CompileSqlReference, @@ -19,6 +15,7 @@ NativeModelConfigSession, ) from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from sqlbuild.compiler.model_config.main._native_config_error import native_config_error from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.spec.contracts.models import ProjectConfig @@ -66,12 +63,28 @@ def validation_outcome(*, values: dict[str, object]) -> object: ) outcome: _native.NativeConfigError | None = cast( _native.NativeConfigError | None, - native_model_validation(session=session, request=request, rejected=[False]), + session.validator.validate( + request.config.values, + ( + request.model_file.file_path.stem, + request.query_sql, + str(request.model_file.relative_path), + ), + ( + [ + (str(reference.ref_kind), reference.ref_name, False) + for reference in request.references + ], + None, + request.config.time_travel_retention.unmanaged, + request.config.table_type.declared, + ), + ), ) return [ _ACCEPTED, *( - raised_error_shape(native_validation_error(error=error, values=values)) + raised_error_shape(native_config_error(error=error, values=values)) for error in filter(None, (outcome,)) ), ][-1] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 7b2dfaa4e3..21fe8a2f83 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -8,7 +8,8 @@ from collections.abc import Callable, Iterator from dataclasses import replace from pathlib import Path -from typing import Any +from types import ModuleType +from typing import Any, cast import pytest @@ -23,7 +24,6 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.models import MeasurementThresholdBound, MeasurementThresholds from sqlbuild.compiler.compile import models as compile_models -from sqlbuild.compiler.compile._helpers.attachment import core as attachment_core from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references from sqlbuild.compiler.compile.models import ( CompileAdapterContext, @@ -38,10 +38,13 @@ CompiledModelView, CompiledProjectView, CompileInputsView, + ModelInput, ) -from sqlbuild.compiler.compiled_project.main import compiled_project_facts as facts_module +from sqlbuild.compiler.contracts._helpers import native_contracts as native_contracts_module +from sqlbuild.compiler.contracts._helpers import native_promotion as native_promotion_module from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from sqlbuild.rule_engine._helpers.run import native_rows as native_rows_module from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, SourceColumnEntry from sqlbuild.sql_values.types import CollectionRendering from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -66,13 +69,23 @@ } +_FACTS_CONSUMERS: tuple[ModuleType, ...] = ( + native_rows_module, + native_contracts_module, + native_promotion_module, +) + + def forbid_hand_built_projects(*, monkeypatch: pytest.MonkeyPatch) -> None: """Fail if a consumer records a project from `CompiledProject` instead of the compile's.""" - def refuse(project: object) -> object: - raise AssertionError(f"rules rebuilt project facts from {type(project).__name__}") + def retained_only(project: CompiledProjectView) -> _native.NativeCompiledProject: + retained: _native.NativeCompiledProject | None = project.native_project + assert retained is not None, f"rules rebuilt project facts from {type(project).__name__}" + return retained - monkeypatch.setattr(facts_module, "hand_built_project_impl", refuse) + for module in _FACTS_CONSUMERS: + monkeypatch.setattr(module, "compiled_project_facts", retained_only) def skip_model_retention(*, monkeypatch: pytest.MonkeyPatch) -> None: @@ -82,22 +95,21 @@ def skip_model_retention(*, monkeypatch: pytest.MonkeyPatch) -> None: attaches its other resources. """ - native_loop: Callable[..., object] = attachment_core._run_native_model_loop - - def discarded(**kwargs: Any) -> object: - loop: Any = kwargs["loop"] - _ = native_loop( - **{ - **kwargs, - "loop": replace( - loop, discovered_inputs=replace(loop.discovered_inputs, model_files=()) - ), - } - ) - kwargs["native_project"] = _native.NativeCompiledProject() - return native_loop(**kwargs) + build_inputs: Callable[..., tuple[ModelInput, ...]] = cast( + Callable[..., tuple[ModelInput, ...]], _native.build_model_inputs + ) - monkeypatch.setattr(attachment_core, "_run_native_model_loop", discarded) + def discarded(**kwargs: Any) -> tuple[ModelInput, ...]: + discovered: DiscoveredProjectInputs = kwargs["discovered_inputs"] + without_models: dict[str, Any] = { + **kwargs, + "discovered_inputs": replace(discovered, model_files=()), + } + _ = build_inputs(**without_models) + throwaway: dict[str, Any] = {**kwargs, "native_project": _native.NativeCompiledProject()} + return build_inputs(**throwaway) + + monkeypatch.setattr(_native, "build_model_inputs", discarded) LEGACY_PROJECT_TYPES: tuple[str, ...] = ( diff --git a/tests/integration/src/sqlbuild/compiler/contracts/helpers.py b/tests/integration/src/sqlbuild/compiler/contracts/helpers.py index 56de61f101..189adaf725 100644 --- a/tests/integration/src/sqlbuild/compiler/contracts/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/contracts/helpers.py @@ -12,7 +12,12 @@ import pytest import sqlbuild.compiler.contracts.main._evaluate_native_model_contracts as contract_entry -from sqlbuild._native import build_project_graph, discover_project_inputs +from sqlbuild._native import ( + build_project_graph, + compiled_model_record, + compiled_project_record, + discover_project_inputs, +) from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( @@ -22,8 +27,6 @@ InferredColumn, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.compiled_project_record import compiled_project_record from sqlbuild.compiler.contracts.main.validate import evaluate_model_contracts from sqlbuild.compiler.contracts.models import ContractValidationResult from sqlbuild.compiler.lineage.types import InferredNullability diff --git a/tests/integration/src/sqlbuild/compiler/lineage/helpers.py b/tests/integration/src/sqlbuild/compiler/lineage/helpers.py index df42624aac..924b2d468b 100644 --- a/tests/integration/src/sqlbuild/compiler/lineage/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/lineage/helpers.py @@ -10,11 +10,15 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import ProjectColumnLineage, build_project_graph, discover_project_inputs +from sqlbuild._native import ( + ProjectColumnLineage, + build_project_graph, + compiled_model_record, + discover_project_inputs, + project_with, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.lineage.types import ColumnLineageMode _PROJECT_TOML: str = ( diff --git a/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py b/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py index 44fddb43c8..e9c5cec76c 100644 --- a/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py +++ b/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py @@ -12,10 +12,12 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import ProjectColumnLineage +from sqlbuild._native import ( + ProjectColumnLineage, + compiled_model_record, + project_with, +) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with from tests.integration.src.sqlbuild.compiler.helpers import ( GoldenEntry, golden_differences, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index afbeadcaeb..90d2179bbd 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -1,5 +1,6 @@ from __future__ import annotations +import functools import json from collections.abc import Callable from pathlib import Path @@ -7,7 +8,8 @@ import pytest -import sqlbuild._native as native_module +import sqlbuild.compiler.compile._helpers.assembly.semantic_shapes as semantic_shapes_module +import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stage_assembly from sqlbuild._native import ( build_compile_inputs, build_compiled_project, @@ -15,6 +17,7 @@ run_compile_pipeline, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import semantic_shapes @@ -212,17 +215,32 @@ def apply_warehouse_setup_sql(*, project_dir: Path, statements: tuple[str, ...]) def traced_shape_batches(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, ...]]: - """Record the expressions of every native expression-source shape batch.""" + """Record each shape request's expressions the binding catalog's shape cache misses.""" - original: Callable[..., Any] = native_module.infer_expression_source_shapes batches: list[tuple[str, ...]] = [] + for module in (native_stage_assembly, semantic_shapes_module): + original: Callable[..., Any] = module.infer_native_expression_source_shapes + monkeypatch.setattr( + module, + "infer_native_expression_source_shapes", + functools.partial(_traced_shapes, original=original, batches=batches), + ) + return batches - def traced(catalog: object, request: tuple[Any, ...], *rules: Any) -> Any: - batches.append(tuple(request[4])) - return original(catalog, request, *rules) - monkeypatch.setattr(native_module, "infer_expression_source_shapes", traced) - return batches +def _traced_shapes( + *, + original: Callable[..., Any], + batches: list[tuple[str, ...]], + expressions: tuple[str, ...], + profile: ExpressionInferenceProfile, +) -> tuple[dict[str, str] | None, ...]: + cache: dict[str, object] = getattr(profile.binding_catalog, "expression_shapes", {}) + pending: tuple[str, ...] = tuple( + dict.fromkeys(filter(lambda item: item not in cache, expressions)) + ) + batches.extend((pending,) * bool(pending)) + return original(expressions=expressions, profile=profile) _OFFLINE_POSTGRES_CONNECTION: dict[str, object] = { diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py index b14f5b45d7..9775f7f77e 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py @@ -6,10 +6,12 @@ import pytest +from sqlbuild._native import ( + project_with, +) from sqlbuild.cli.commands.main.entrypoint.entry import main from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import semantic_shapes from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.project_with import project_with from tests.integration.src.sqlbuild.compiler.lineage.helpers import compiled_project from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( ExpressionBatchCase, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_diagnostic_ux.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_diagnostic_ux.py index 4b1ebb1190..236ab18434 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_diagnostic_ux.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_diagnostic_ux.py @@ -9,8 +9,8 @@ import pytest from _pytest.capture import CaptureResult +import sqlbuild._native as _native from sqlbuild.cli.commands.main.entrypoint.entry import main -from sqlbuild.compiler.compile._helpers.diagnostics.help import semantic_help_catalogue from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( DiagnosticUxCase, ) @@ -200,7 +200,10 @@ def test_given_semantic_failure_when_compiling_cold_and_warm_then_reports_root_e def test_given_semantic_codes_when_explaining_then_each_code_owns_distinct_help( test_case: DiagnosticUxCase, ) -> None: - help_by_code: dict[str, str] = {**semantic_help_catalogue(), "B002": "did you mean 'quantity'?"} + help_by_code: dict[str, str] = { + **dict(_native.semantic_help_catalogue()), + "B002": "did you mean 'quantity'?", + } assert len(help_by_code) - len(set(help_by_code.values())) == test_case.expected_errors assert all( "review the SQL expression, input types, and authoritative schema" not in value diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py index 14ced6d8fc..f48e3a7d86 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -6,6 +6,10 @@ from pathlib import Path from typing import Any +from sqlbuild._native import ( + compiled_model_record, + project_with, +) from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( @@ -25,8 +29,6 @@ CompiledSeedView, CompiledSourceView, ) -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import ( DiscoveredSchemaFile, DiscoveredSeedFile, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py index c658ac2fc1..592eafe31a 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py @@ -6,6 +6,10 @@ import pytest +from sqlbuild._native import ( + compiled_model_record, + project_with, +) from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( @@ -17,8 +21,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.fingerprints.main.write import write_fingerprint from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.warehouse.snapshot import gather_warehouse_snapshot diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 7c839aea91..617a777127 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -12,7 +12,12 @@ from types import MappingProxyType from typing import Any, cast -from sqlbuild._native import discover_project_inputs, run_compile_pipeline +from sqlbuild._native import ( + compiled_model_record, + discover_project_inputs, + project_with, + run_compile_pipeline, +) from sqlbuild.adapter.contract.models import RelationInfo, RelationReadProbe from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( @@ -36,8 +41,6 @@ CompiledSourceView, FunctionLanguage, ) -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import ( DiscoveredProjectInputs, DiscoveredSchemaFile, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py index c59d3bccce..893d4b43bb 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py @@ -7,12 +7,14 @@ import pytest +from sqlbuild._native import ( + compiled_model_record, + project_with, +) from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.output.main.plan import format_plan from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.planner._helpers.changes import detect as change_detection from sqlbuild.compiler.planner._helpers.planning import identities as identity_planning from sqlbuild.compiler.planner.exceptions import PlannerInputError diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py index 5b7ea297e8..2b3f192183 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py @@ -13,14 +13,18 @@ import pytest import sqlbuild._native as _native -from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs +from sqlbuild._native import ( + assemble_project, + build_compile_inputs, + discover_project_inputs, + inputs_with, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS from sqlbuild.compiler.compile.models import ( CompileAdapterContext, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView -from sqlbuild.compiler.compiled_project.main.inputs_with import inputs_with from sqlbuild.compiler.project_assembly.types import ProjectRequestRow from sqlbuild.sql_values.types import CollectionRendering diff --git a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py index fd1c905a1e..ebaa9d828e 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py @@ -13,10 +13,11 @@ import pytest import sqlbuild._native as _native -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import ( + build_declaration_scope, + discover_project_inputs, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile._helpers.attachment import declaration_scope -from sqlbuild.compiler.compile._helpers.attachment.declaration_scope import build_declaration_scope from sqlbuild.compiler.compile._helpers.render.declarations import resolve_declaration_context from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros from sqlbuild.compiler.compile.exceptions import CompileInputError @@ -32,6 +33,7 @@ DiscoveredProjectInputs, EnumDeclaration, ) +from sqlbuild.compiler.scopes.main import _open_native_scope_index as open_index_module from sqlbuild.compiler.scopes.main._declaration_lexical_path import declaration_lexical_path from sqlbuild.compiler.scopes.main._open_native_scope_index import open_native_scope_index from sqlbuild.compiler.scopes.main._resolve_scope_declaration_visibility import ( @@ -328,7 +330,7 @@ def counted( discovered_inputs=discovered_inputs, loaded_macros=loaded_macros ) - monkeypatch.setattr(declaration_scope, "open_native_scope_index", counted) + monkeypatch.setattr(open_index_module, "open_native_scope_index", counted) outcome: ScopeOutcome = _scope_outcome( discovered=discovered, macros=macros, diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py index 578c6a74b6..c17d00de94 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py @@ -14,8 +14,7 @@ import sqlbuild._native as native_module import sqlbuild.compiler.compile._helpers.diagnostics.recovery as semantic_recovery -import sqlbuild.compiler.semantic_checks._helpers.stage as semantic_stage -from sqlbuild._native import build_project_graph, discover_project_inputs +from sqlbuild._native import build_project_graph, discover_project_inputs, project_with from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.analysis_session.types import DiagnosticRow @@ -24,12 +23,6 @@ ) from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.compiled_project.main.project_with import project_with -from sqlbuild.compiler.semantic_checks._helpers.metadata import native_metadata_diagnostics -from sqlbuild.compiler.semantic_checks._helpers.payloads import session_fact_models -from sqlbuild.compiler.semantic_checks.main._complete_native_semantic_diagnostics import ( - complete_native_semantic_diagnostics, -) from tests.integration.src.sqlbuild.compiler.analysis_session.helpers import ( generated_analysis_files, ) @@ -245,7 +238,7 @@ def note_kinds(project: CompiledProjectView, kinds: tuple[str, ...]) -> Counter[ def native_completion(inputs: SemanticInputs) -> CompiledProjectView: """The native stage's completed project.""" - return complete_native_semantic_diagnostics( + return native_module.complete_semantic_project( project=inputs.project, profile=inputs.profile, binding_results=inputs.binding_results, @@ -281,18 +274,27 @@ def without_session(inputs: SemanticInputs) -> SemanticInputs: return replace(inputs, native_session=None) -def record_session_models(*, monkeypatch: pytest.MonkeyPatch) -> list[frozenset[str]]: - """Keep each completion's models whose output names and lineage the session supplies.""" +def session_fact_models(inputs: SemanticInputs) -> frozenset[str]: + """Models whose output names and lineage the finished session supplies to the stage.""" + + kept: frozenset[str] = frozenset(getattr(inputs.native_session, "fact_models", ())) + proven: frozenset[str] = frozenset( + model.name for model in filter(_output_proven, inputs.project.models) + ) + return frozenset(model.name for model in inputs.project.models) & kept - proven + + +def _output_proven(model: CompiledModelView) -> bool: + return model.dynamic_column_contract is not None and model.dynamic_column_contract.output_proven - selections: list[frozenset[str]] = [] - def recorded(**arguments: Any) -> frozenset[str]: - selected: frozenset[str] = session_fact_models(**arguments) - selections.append(selected) - return selected +def has_blocking_binding(project: CompiledProjectView) -> bool: + """Whether any model owns a binding error, so the stage plans a type recovery.""" - monkeypatch.setattr(semantic_stage, "session_fact_models", recorded) - return selections + bindings: list[tuple[CompilerDiagnostic, ...]] = [ + model.binding_diagnostics for model in project.models + ] + return any(item.is_error for item in chain.from_iterable(bindings)) def completion_view(project: CompiledProjectView) -> tuple[object, ...]: @@ -313,29 +315,6 @@ def _model_view(model: CompiledModelView) -> tuple[object, ...]: ) -def record_native_statuses(*, monkeypatch: pytest.MonkeyPatch) -> Counter[str]: - """Count native type recovery plans, completions and completed diagnostics.""" - - statuses: Counter[str] = Counter() - plan: Callable[..., Any] = native_module.plan_semantic_type_recovery - complete: Callable[..., Any] = native_module.complete_semantic_checks - - def counted_plan(*arguments: Any) -> Any: - recovery: Any = plan(*arguments) - statuses[f"type_recovery_{recovery.status}"] += 1 - return recovery - - def counted_complete(*arguments: Any) -> Any: - outcome: Any = complete(*arguments) - statuses["completion_native"] += 1 - statuses["diagnostics_native"] += len(outcome[0]) - return outcome - - monkeypatch.setattr(native_module, "plan_semantic_type_recovery", counted_plan) - monkeypatch.setattr(native_module, "complete_semantic_checks", counted_complete) - return statuses - - def non_ascii_variant(*, files: dict[str, str], rng: random.Random) -> dict[str, str]: """The project with some column names spelt beyond ASCII and a comment after each model header holding non-ASCII text, digits and Python-only line separators.""" @@ -545,37 +524,26 @@ def generated_metadata_files(*, rng: random.Random, model_count: int) -> dict[st def native_metadata(inputs: SemanticInputs) -> tuple[CompilerDiagnostic, ...]: """The native stage's metadata diagnostics.""" - catalog: Any = cast(Any, inputs.project.binding_catalog) - return native_metadata_diagnostics( + return native_module.semantic_metadata_diagnostics( project=inputs.project, profile=inputs.profile, resource_sql_analysis=inputs.resource_sql_analysis, - catalog=catalog.native, ) -def _family_counts(outcome: Any) -> Counter[str]: - counts: Counter[str] = Counter() - for function_rows, reference_rows, schema_audit_rows in outcome[0]: - counts.update(f"function {row[0]}" for row in function_rows) - counts.update(f"reference {row[0]}" for row in reference_rows) - counts.update(f"schema_audit {row[0]}" for row in schema_audit_rows) - counts.update(f"source {row[0]}" for _, row in outcome[1]) - counts.update(f"sql_test {row[0]}" for _, row, _ in outcome[2]) - return counts - - -def record_metadata_families(*, monkeypatch: pytest.MonkeyPatch) -> Counter[str]: - """Count native metadata runs and errors by family and code.""" +def _completed(outcome: tuple[str, object]) -> bool: + return outcome[0] == "completed" - counts: Counter[str] = Counter() - check: Callable[..., Any] = native_module.check_semantic_metadata_rows - def counted(*arguments: Any) -> Any: - outcome: Any = check(*arguments) - counts["native"] += 1 - counts.update(_family_counts(outcome)) - return outcome +def metadata_families(views: dict[str, object]) -> Counter[str]: + """Completed metadata runs and their diagnostics counted by resource type and code.""" - monkeypatch.setattr(native_module, "check_semantic_metadata_rows", counted) - return counts + completed: list[tuple[CompilerDiagnostic, ...]] = [ + cast(tuple[CompilerDiagnostic, ...], value) + for _, value in filter(_completed, cast(list[tuple[str, object]], list(views.values()))) + ] + families: Counter[str] = Counter( + f"{item.resource_type} {item.code}" for item in chain.from_iterable(completed) + ) + families["native"] = len(completed) + return families diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_metadata_checks.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_metadata_checks.py index cd3675054c..dd452c37c5 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_metadata_checks.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_metadata_checks.py @@ -24,8 +24,8 @@ captured_semantic_inputs, files_with_non_ascii, generated_metadata_files, + metadata_families, native_metadata, - record_metadata_families, semantic_outcome, with_dialect, ) @@ -44,10 +44,9 @@ golden_prefix="metadata_generated", expected_minimum_native=32, expected_minimum_families={ - "function B102": 100, - "function B301": 100, - "reference B300": 50, - "reference B301": 16, + "model B102": 100, + "model B300": 50, + "model B301": 116, "source B300": 4, "sql_test B302": 16, }, @@ -61,7 +60,7 @@ non_ascii=True, golden_prefix="metadata_formerly_deferred", expected_minimum_native=12, - expected_minimum_families={"function B301": 10, "reference B300": 4}, + expected_minimum_families={"model B300": 4, "model B301": 10}, ), ], ids=lambda case: case.description, @@ -72,7 +71,7 @@ def test_given_generated_projects_when_checking_metadata_natively_then_matches_r monkeypatch: pytest.MonkeyPatch, ) -> None: rng: random.Random = random.Random(test_case.seed) - families: Counter[str] = record_metadata_families(monkeypatch=monkeypatch) + families: Counter[str] = Counter() golden: list[GoldenEntry] = [] for index in range(test_case.count): project_dir: Path = tmp_path / f"project_{index}" @@ -91,6 +90,7 @@ def test_given_generated_projects_when_checking_metadata_natively_then_matches_r ) for dialect in test_case.dialects } + families.update(metadata_families(views)) golden.append(golden_entry("metadata", views, masked=(str(project_dir),))) assert ( diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py index e23e6e4e18..1ec117546b 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py @@ -10,8 +10,10 @@ import pytest import sqlbuild._native as native_module +from sqlbuild._native import ( + project_with, +) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.project_with import project_with from tests.integration.src.sqlbuild.compiler.helpers import ( GoldenEntry, golden_differences, @@ -32,14 +34,14 @@ completion_view, files_with_non_ascii, generated_semantic_files, + has_blocking_binding, native_completion, note_kinds, outcome_label, proven_output_count, - record_native_statuses, - record_session_models, semantic_outcome, session_corpus_files, + session_fact_models, with_dialect, without_session, ) @@ -78,7 +80,7 @@ def test_given_generated_failing_projects_when_completing_natively_then_matches_ monkeypatch: pytest.MonkeyPatch, ) -> None: rng: random.Random = random.Random(test_case.seed) - statuses: Counter[str] = record_native_statuses(monkeypatch=monkeypatch) + statuses: Counter[str] = Counter() codes: Counter[str] = Counter() golden: list[GoldenEntry] = [] for index in range(test_case.count): @@ -93,6 +95,9 @@ def test_given_generated_failing_projects_when_completing_natively_then_matches_ views: dict[str, object] = {} for dialect in test_case.dialects: native: CompiledProjectView = completed_natively(with_dialect(captured, dialect)) + statuses["completion_native"] += 1 + statuses["diagnostics_native"] += len(native.diagnostics) + statuses["type_recovery_planned"] += has_blocking_binding(captured.project) codes.update(item.code for item in native.diagnostics) codes.update(note_kinds(native, _NOTE_KINDS)) views[dialect] = completion_view(native) @@ -261,7 +266,7 @@ def test_given_compiles_when_completing_from_the_session_then_matches_payload_an ) for index in range(test_case.count) ] - selections: list[frozenset[str]] = record_session_models(monkeypatch=monkeypatch) + selections: list[frozenset[str]] = [session_fact_models(inputs) for inputs in captured] codes: Counter[str] = Counter() golden: list[GoldenEntry] = [] payload_views: list[tuple[object, ...]] = [] diff --git a/tests/integration/src/sqlbuild/lint/main/helpers.py b/tests/integration/src/sqlbuild/lint/main/helpers.py index 7bbe835f06..78e74f1825 100644 --- a/tests/integration/src/sqlbuild/lint/main/helpers.py +++ b/tests/integration/src/sqlbuild/lint/main/helpers.py @@ -5,11 +5,14 @@ from dataclasses import replace from pathlib import Path -from sqlbuild._native import build_compiled_project, discover_project_inputs +from sqlbuild._native import ( + build_compiled_project, + compiled_model_record, + discover_project_inputs, + project_with, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with _PROJECT_TOML: str = 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' diff --git a/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py b/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py index ebec9b591a..b6ad426c3f 100644 --- a/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py +++ b/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py @@ -4,11 +4,11 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.compiled_project_record import ( +from sqlbuild._native import ( compiled_project_record, ) +from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.engine.custom_rules import host_project from sqlbuild.rule_engine._helpers.host.custom_host_pool import host_payload_project from sqlbuild.rule_engine.classes.project_tree import public_model diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py index 19c92acac4..c12849f906 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -6,6 +6,9 @@ from pathlib import Path from typing import Any +from sqlbuild._native import ( + project_with, +) from sqlbuild.adapter.contract.models import LifeCycleEvent, QueryResult from sqlbuild.adapter.contract.types import LifeCycleEventKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -29,7 +32,6 @@ CompiledResourceType, FunctionLanguage, ) -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.compiler.planner.models import ( AuditPlanEntry, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py index bfc8b5a41f..6751d0e54a 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py @@ -6,7 +6,11 @@ import pytest -from sqlbuild._native import ProjectColumnLineage +from sqlbuild._native import ( + ProjectColumnLineage, + compiled_model_record, + project_with, +) from sqlbuild.cli.commands._helpers.compile.output import ( format_compile_json, format_compile_text, @@ -17,8 +21,6 @@ ) from sqlbuild.compiler.compile.models import CompiledObjectKey, CompileModelConfig from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import SqlHookEntry from sqlbuild.compiler.pipeline.models import ProjectGraph from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py index 8382d5bb4b..0911791559 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py @@ -8,6 +8,10 @@ import pytest import sqlbuild._native as _native +from sqlbuild._native import ( + compiled_model_record, + project_with, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.commands._helpers.compile import target_writer as target_writer_module from sqlbuild.cli.commands._helpers.compile.target_writer import ( @@ -24,8 +28,6 @@ CompiledResourceType, DiagnosticPhase, ) -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.manifest.main.build import build_manifest from sqlbuild.compiler.planner.models import ChainStep, PlanOutput, SqlTestPlanEntry from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index 354ec57e68..972ab91505 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -849,9 +849,8 @@ class DescriptionInputClassificationCase: @dataclass(frozen=True) class CollectedReferenceCallSyntaxTestCase: description: str - contents: str sql: str - expected_diagnostics: tuple[tuple[str, str, Path, SourceLocation | None], ...] + expected_diagnostics: tuple[tuple[str, str, Path | None, SourceLocation | None], ...] @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py index 2eccb09ea6..06c6fd07b7 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py @@ -2,12 +2,8 @@ import pytest -from sqlbuild.compiler.analysis_session._helpers.dynamic_contracts import ( - native_dynamic_contracts, -) -from sqlbuild.compiler.analysis_session.main._dynamic_contract_proof import ( - dynamic_contract_proof as contract_proof, -) +from sqlbuild._native import dynamic_contract_proof as contract_proof +from sqlbuild._native import prove_native_dynamic_contracts from sqlbuild.compiler.analysis_session.models import NativePivotTables from sqlbuild.compiler.compile.models import DynamicColumnContractProof from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily @@ -151,7 +147,7 @@ def test_given_dynamic_family_when_analyzing_output_then_requires_supported_closed_shape( test_case: DynamicPivotAnalysisTestCase, ) -> None: - (row,) = native_dynamic_contracts( + (row,) = prove_native_dynamic_contracts( session=None, tables=NativePivotTables( dialect=test_case.dialect, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py index 5abc2b166c..0fbec2669a 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py @@ -1,28 +1,32 @@ """Helpers for model configuration reuse and native validation tests.""" from pathlib import Path +from typing import cast import sqlbuild._native as _native -from sqlbuild._native import build_compile_inputs, discover_project_inputs -from sqlbuild.compiler.compile._helpers.attachment.model_config import ( - build_native_model_config, +from sqlbuild._native import ( + build_compile_inputs, + discover_project_inputs, native_model_config_session, - native_validation_error, ) +from sqlbuild.compiler.compile._helpers.render.templating import record_template_reads from sqlbuild.compiler.compile.models import ( CompileModelConfig, NativeModelConfigInputs, NativeModelConfigSession, ) from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from sqlbuild.compiler.model_config.main._native_config_error import native_config_error from sqlbuild.spec.contracts.models import ( DefaultsConfig, MaterializationDefaultsConfig, ProjectConfig, + ResolvedTableType, + ResolvedTimeTravelRetention, SchemaColumn, TargetConfig, ) +from sqlbuild.spec.contracts.types import TableType, TableTypeSource, TimeTravelRetentionSource from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, ) @@ -83,7 +87,7 @@ def validate_natively( # noqa: PLR0913 ) assert not isinstance(outcome, int) for error in filter(None, (outcome,)): - raise native_validation_error(error=error, values=config.values) + raise native_config_error(error=error, values=config.values) def build_config_natively( # noqa: PLR0913 @@ -117,16 +121,32 @@ def build_config_natively( # noqa: PLR0913 ({"orders"}, set(), set(), set(), set()), set(), False ), ) - return build_native_model_config( - session=session, - model_file=DiscoveredSqlModelFile( - file_path=Path("models/marts/orders.sql"), - relative_path=Path("models/marts/orders.sql"), - contents="SELECT 1", - header_values=model_header_values, - header_column_locations={}, - output_column_locations={}, - query_sql="SELECT 1", - ), - matched_path_default=matched_path_default, + built: object = session.builder.model_config( + model_header_values, + (matched_path_default, "orders"), + native_config_classes(session=session), ) + for error in filter(_is_config_error, (built,)): + raise native_config_error(error=cast(_native.NativeConfigError, error)) + return cast(CompileModelConfig, built) + + +def native_config_classes(*, session: NativeModelConfigSession) -> dict[str, object]: + """The builder, storage defaults and classes a model's `CompileModelConfig` is made of.""" + + return { + "builder": session.builder, + "inherited_storage": session.inherited_storage, + "table_backed": ("table", "incremental", "snapshot"), + "compile_model_config": CompileModelConfig, + "resolved_retention": ResolvedTimeTravelRetention, + "retention_source": TimeTravelRetentionSource.MODEL, + "resolved_table_type": ResolvedTableType, + "table_type": TableType, + "table_type_source": TableTypeSource.MODEL, + "record_template_reads": record_template_reads, + } + + +def _is_config_error(value: object) -> bool: + return isinstance(value, _native.NativeConfigError) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index e63daf8241..3c9327d4be 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -5,7 +5,12 @@ from types import MappingProxyType from typing import cast -from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs +from sqlbuild._native import ( + assemble_project, + build_compile_inputs, + discover_project_inputs, + infer_native_query_columns, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter @@ -13,9 +18,6 @@ from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.analysis_session.main._infer_native_query_columns import ( - infer_native_query_columns, -) from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( collect_compile_diagnostics, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py index d0eb22c4da..8520ec0c33 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py @@ -2,10 +2,8 @@ import pytest +from sqlbuild._native import infer_native_query_columns from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.main._infer_native_query_columns import ( - infer_native_query_columns, -) from sqlbuild.compiler.analysis_session.models import NativeColumnQuery from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( cursor_intrinsics_analysis_sql, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py index 2df8110bb7..935e5a6369 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py @@ -7,9 +7,9 @@ import pytest -from sqlbuild._native import discover_project_inputs -from sqlbuild.compiler.compile._helpers.attachment.declaration_scope import ( +from sqlbuild._native import ( build_declaration_scope, + discover_project_inputs, rebind_declaration_scope, ) from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py index 9b54407c9e..d729ea72fb 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py @@ -2,10 +2,8 @@ import pytest +from sqlbuild._native import infer_native_query_columns from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.main._infer_native_query_columns import ( - infer_native_query_columns, -) from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns from sqlbuild.compiler.compile._helpers.analysis.reference_names import ( substitute_placeholder_defaults, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py index 8dd6bb50a1..ea9d99f453 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py @@ -3,7 +3,6 @@ from __future__ import annotations import re -from pathlib import Path import pytest @@ -15,11 +14,9 @@ from sqlbuild.compiler.compile._helpers.diagnostics.collector import collect_compile_diagnostics from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompileSqlReference, SqlReferenceOrigin -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.models import CompileSqlReference from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import SourceLocation from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( CollectedReferenceCallSyntaxTestCase, DialectSqlScanTestCase, @@ -28,7 +25,6 @@ ) _GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() -_ORIGIN_PATH: Path = Path("models/marts/order_totals.sql") @pytest.mark.parametrize( @@ -213,69 +209,26 @@ def test_given_reference_call_compile_cannot_replace_when_extracting_then_raises "test_case", [ CollectedReferenceCallSyntaxTestCase( - description="every rejected call is reported at its authored location", - contents=( - 'MODEL (description "Order totals.");\n\n' - "SELECT * FROM __ref(stg_orders) o\n" - "JOIN __source('raw_payments') p USING (order_id)\n" - ), + description="every rejected call is reported once without a location", sql=( "SELECT * FROM __ref(stg_orders) o\n" "JOIN __source('raw_payments') p USING (order_id)\n" + "JOIN __ref(stg_orders) again USING (order_id)\n" ), expected_diagnostics=( - ( - "P012", - "__ref(stg_orders) is not a valid __ref() call", - _ORIGIN_PATH, - SourceLocation(path=_ORIGIN_PATH, line=3, column=15, end_line=3, end_column=32), - ), - ( - "P012", - "__source('raw_payments') is not a valid __source() call", - _ORIGIN_PATH, - SourceLocation(path=_ORIGIN_PATH, line=4, column=6, end_line=4, end_column=30), - ), - ), - ), - CollectedReferenceCallSyntaxTestCase( - description="a commented-out rejected call does not take the authored location", - contents=("-- was __ref(stg_orders)\nSELECT * FROM __ref(stg_orders)\n"), - sql="SELECT * FROM __ref(stg_orders)\n", - expected_diagnostics=( - ( - "P012", - "__ref(stg_orders) is not a valid __ref() call", - _ORIGIN_PATH, - SourceLocation(path=_ORIGIN_PATH, line=2, column=15, end_line=2, end_column=32), - ), - ), - ), - CollectedReferenceCallSyntaxTestCase( - description="a call missing from the authored text keeps the file path", - contents="SELECT * FROM @staged_orders()\n", - sql="SELECT * FROM __ref(stg_orders)\n", - expected_diagnostics=( - ("P012", "__ref(stg_orders) is not a valid __ref() call", _ORIGIN_PATH, None), + ("P012", "__ref(stg_orders) is not a valid __ref() call", None, None), + ("P012", "__source('raw_payments') is not a valid __source() call", None, None), ), ), ], ids=lambda case: case.description, ) -def test_given_rejected_calls_when_collecting_diagnostics_then_reports_each_with_location( +def test_given_rejected_calls_when_collecting_diagnostics_then_reports_each_unlocated( test_case: CollectedReferenceCallSyntaxTestCase, ) -> None: - origin: SqlReferenceOrigin = SqlReferenceOrigin( - file_path=Path("/project/models/marts/order_totals.sql"), - relative_path=_ORIGIN_PATH, - contents=test_case.contents, - resource_type=CompiledResourceType.MODEL, - resource_name="order_totals", - ) - with collect_compile_diagnostics() as collected: references: tuple[CompileSqlReference, ...] = extract_sql_references( - sql=test_case.sql, syntax=_GENERIC_SQL_SYNTAX, origin=origin + sql=test_case.sql, syntax=_GENERIC_SQL_SYNTAX ) assert references == () @@ -286,7 +239,6 @@ def test_given_rejected_calls_when_collecting_diagnostics_then_reports_each_with ) == test_case.expected_diagnostics ) - assert {diagnostic.resource_name for diagnostic in collected.diagnostics} == {"order_totals"} @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py index 621b130840..5daf7e3da1 100644 --- a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py +++ b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py @@ -13,12 +13,12 @@ build_compile_inputs, build_dag_json, discover_project_inputs, + project_with, ) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.dag.types import NodeKind from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.graph.main.project_lineage_views import project_lineage_views diff --git a/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py b/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py index 76def5dffc..9349c6c40f 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py @@ -4,7 +4,10 @@ import pytest -from sqlbuild._native import ProjectColumnLineage +from sqlbuild._native import ( + ProjectColumnLineage, + compiled_model_record, +) from sqlbuild.compiler.compile.models import ( CompiledLineageColumnFact, CompiledLineageSourceFact, @@ -14,7 +17,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record from sqlbuild.compiler.lineage.models import ColumnLineage, ModelColumnLineage from sqlbuild.compiler.lineage.types import ( ColumnLineageConfidence, diff --git a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py index 7a549c4bce..bbacfc9788 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py @@ -8,6 +8,9 @@ import pytest import sqlbuild._native as _native +from sqlbuild._native import ( + project_with, +) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, LoadedMacro, @@ -18,7 +21,6 @@ CompiledSeedView, CompiledSourceView, ) -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import DiscoveredSqlHookFile, SqlHookEntry from sqlbuild.compiler.manifest.main.build import build_manifest from sqlbuild.compiler.planner.models import AuditPlanEntry, ChainStep, PlanOutput, SqlTestPlanEntry diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py index 9453017c0c..094cc05f91 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -12,6 +12,10 @@ from typing import ClassVar import sqlbuild._native as _native +from sqlbuild._native import ( + compiled_model_record, + project_with, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.compiler.auditing.types import AuditSeverity @@ -46,8 +50,6 @@ CompiledSourceView, SqlTestMode, ) -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import ( DiscoveredAuditBlock, DiscoveredAuditFile, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py index 769f17f68d..318f7f9605 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py @@ -5,6 +5,9 @@ import pytest +from sqlbuild._native import ( + project_with, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter @@ -20,7 +23,6 @@ CompileSqlTestCte, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile from sqlbuild.compiler.planner._helpers.sql_tests.native_planning import ( plan_compiled_sql_test_artifacts, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py index 9e44554ac2..a789428dd2 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py @@ -5,6 +5,10 @@ import pytest +from sqlbuild._native import ( + compiled_model_record, + project_with, +) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledSqlTest, @@ -15,8 +19,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.planner._helpers.fixtures import completion as fixture_completion from sqlbuild.compiler.planner._helpers.output.plan_output import build_selected_test_entries from sqlbuild.compiler.planner._helpers.sql_tests import assembly as sql_test_assembly diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py index 208e7ebed7..587ed8102f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py @@ -6,6 +6,9 @@ import pytest +from sqlbuild._native import ( + project_with, +) from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -17,7 +20,6 @@ CompileSqlScenarioCte, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import DiscoveredHookFunction from sqlbuild.compiler.planner._helpers.scenario.relations import ( build_scenario_execution_plan, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py index 69fef260e4..a6dbdd897b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py @@ -6,13 +6,15 @@ import pytest +from sqlbuild._native import ( + compiled_model_record, +) from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_SEED from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.planning.scopes import resolve_planner_scopes diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py index 0773889e39..b8f8830034 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py @@ -7,6 +7,9 @@ import pytest +from sqlbuild._native import ( + project_with, +) from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, @@ -19,7 +22,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.planner._helpers.graph.loader_dag import ( build_intermediate_source_map, expand_selected_loader_dependencies, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py index 57ca0ac495..cd00e48e5d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py @@ -1,6 +1,9 @@ from dataclasses import replace from pathlib import Path +from sqlbuild._native import ( + compiled_model_record, +) from sqlbuild.compiler.compile.models import ( CompiledFunction, CompiledModel, @@ -10,7 +13,6 @@ FunctionReturnColumn, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record from sqlbuild.compiler.planner._helpers.graph.core import build_downstream_deps from sqlbuild.compiler.planner._helpers.identity.direct import ( build_direct_model_version_identities, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py index 3224a08207..890873ae51 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py @@ -5,10 +5,12 @@ import pytest +from sqlbuild._native import ( + project_with, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import DiscoveredHookFunction from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py index 21a78e3899..b9b34cd0d0 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py @@ -9,10 +9,12 @@ import pytest +from sqlbuild._native import ( + compiled_model_record, + project_with, +) from sqlbuild.compiler.compile.models import CompileModelConfig from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledSeedView -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import DiscoveredSeedFile from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.main.commands._relations import resolve_static_relation_context diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py index e4378bf8b7..20fb1f390c 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py @@ -8,6 +8,9 @@ import pytest +from sqlbuild._native import ( + project_with, +) from sqlbuild.compiler.compile.models import ( CompiledDirectLogicSqlTestPayload, CompiledModelSqlTestPayload, @@ -17,7 +20,6 @@ CompileSqlTestCte, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile from sqlbuild.compiler.scopes.models import ScopeIndex from sqlbuild.rule_engine._helpers.engine import native diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py index 9c013f18a7..43d9d99bfa 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py @@ -5,10 +5,12 @@ import pytest +from sqlbuild._native import ( + compiled_model_record, + project_with, +) from sqlbuild.compiler.compile.models import CompactLineageFacts from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.main._evaluate import evaluate from sqlbuild.rule_engine.models import ( diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py index 80fe76fc77..51c428b117 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py @@ -3,13 +3,15 @@ import pytest +from sqlbuild._native import ( + project_with, +) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledSqlScenario, CompiledSqlTest, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import ( DiscoveredSqlScenarioFile, ) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py index fb8d88879b..81b9d91e6d 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py @@ -3,7 +3,9 @@ import pytest -from sqlbuild.compiler.compiled_project.main.declaration_rows import declaration_rows +from sqlbuild._native import ( + declaration_rows, +) from sqlbuild.compiler.discovery.models import ConstantDeclaration from sqlbuild.sql_values.main.normalize import normalize_sql_value from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py index e805376121..daa5f077eb 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py @@ -13,8 +13,10 @@ import pytest import sqlbuild._native as native_module +from sqlbuild._native import ( + project_with, +) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached from sqlbuild.rule_engine._helpers.host import custom_host_pool diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/test_audit_facts.py b/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/test_audit_facts.py index 04ba55c983..f48e681c8e 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/test_audit_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/test_audit_facts.py @@ -4,8 +4,10 @@ import pytest +from sqlbuild._native import ( + project_with, +) from sqlbuild.compiler.compile.types import CompiledAuditView, CompiledProjectView -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.rule_engine.classes.audit_facts import AuditFacts from sqlbuild.rule_engine.models import Model from tests.unit.src.sqlbuild.rule_engine.classes.audit_facts._test_types import ( diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/model_test_facts/test_model_test_facts.py b/tests/unit/src/sqlbuild/rule_engine/classes/model_test_facts/test_model_test_facts.py index d26e3ba301..2db8ccba43 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/model_test_facts/test_model_test_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/model_test_facts/test_model_test_facts.py @@ -5,9 +5,11 @@ import pytest +from sqlbuild._native import ( + project_with, +) from sqlbuild.compiler.compile.models import CompiledSqlTest from sqlbuild.compiler.compile.types import CompiledProjectView, SqlTestMode -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.rule_engine.classes.model_test_facts import TestFacts as ModelTestFacts from sqlbuild.rule_engine.models import Model from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import ( diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py b/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py index 4b6084dc9b..1fe8015782 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py @@ -1,12 +1,14 @@ from pathlib import Path +from sqlbuild._native import ( + project_with, +) from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledSource from sqlbuild.compiler.compile.types import ( CompiledProjectView, CompiledResourceType, CompiledSourceView, ) -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import DiscoveredSourceFile from sqlbuild.rule_engine.classes.rule_context import RuleFactViews, build_rule_fact_views from sqlbuild.spec.contracts.models import SourceEntry diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py index df0e6a0334..e94a7aa128 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py @@ -5,9 +5,11 @@ import pytest +from sqlbuild._native import ( + compiled_model_record, + project_with, +) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.main._evaluate import evaluate from sqlbuild.rule_engine.models import RuleExemption, RuleIgnore, RulesConfig, RulesResult