diff --git a/Cargo.lock b/Cargo.lock index 6cb9facbb2..bd18c4472d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1318,6 +1318,15 @@ dependencies = [ "sqlbuild-core", ] +[[package]] +name = "sqlbuild-project" +version = "0.164.3" +dependencies = [ + "sqlbuild-analysis", + "sqlbuild-core", + "sqlbuild-sqltext", +] + [[package]] name = "sqlbuild-pysource" version = "0.164.3" @@ -1350,6 +1359,7 @@ dependencies = [ "sqlbuild-discovery", "sqlbuild-host", "sqlbuild-model-config", + "sqlbuild-project", "sqlbuild-pysource", "sqlbuild-recall", "sqlbuild-refactor", diff --git a/Cargo.toml b/Cargo.toml index 3b5f1fd0ea..0d8d709785 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ members = [ "crates/sqlbuild-analysis", "crates/sqlbuild-rules", "crates/sqlbuild-refactor", + "crates/sqlbuild-project", "crates/sqlbuild-cli", "crates/sqlbuild-python", ] @@ -46,6 +47,7 @@ order = [ "sqlbuild-analysis", "sqlbuild-rules", "sqlbuild-refactor", + "sqlbuild-project", "sqlbuild-cli", "sqlbuild-python", ] @@ -126,6 +128,7 @@ sqlbuild-render = { path = "crates/sqlbuild-render" } sqlbuild-refactor = { path = "crates/sqlbuild-refactor" } sqlbuild-rules = { path = "crates/sqlbuild-rules" } sqlbuild-model-config = { path = "crates/sqlbuild-model-config" } +sqlbuild-project = { path = "crates/sqlbuild-project" } sqlbuild-pysource = { path = "crates/sqlbuild-pysource" } sqlbuild-recall = { path = "crates/sqlbuild-recall" } sqlbuild-scopes = { path = "crates/sqlbuild-scopes" } diff --git a/crates/README.md b/crates/README.md index be42de8f89..37c7b39166 100644 --- a/crates/README.md +++ b/crates/README.md @@ -23,6 +23,7 @@ that can be tested and benchmarked with `cargo test` alone. | `sqlbuild-analysis` | SQL analysis over polyglot: query analysis, binding, semantic validation and usage, column references, type normalization, FAST/RICH lineage, SQL-test/scenario extraction and compiler-side assembly, and Snowflake cost statement classification and query-tag parsing. | | `sqlbuild-rules` | Built-in rules and the rules engine, the custom-rule host, SQL lint, quality checks and formatting, rules configuration and the request models. It also owns the build identity script. | | `sqlbuild-refactor` | Native rename/move plans, contract-adoption edits, MODEL/SCHEMA/YAML text edits and atomic staging/commit with rollback. Shared scope placement delegates to the native scope implementation; user Python source inspection remains at the Python AST boundary. | +| `sqlbuild-project` | The compiled project as plain Rust: `CompiledProject` with the producers' facts and each resource's records (model inputs and schemas, compiled models and their analysis, sources, functions, audits and seeds), and what those records answer without Python, such as lineage rows, resolved namespaces and dependency keys. Values the host owns, such as user config and discovered files, and the views it builds lazily live in one slot per record, whose types `sqlbuild-python` supplies through the `ProjectHost` trait; reads that need Python stay in the bindings as extension traits. | | `sqlbuild-cli` | The `sqb` and `sqlbuild` executables: `--version`, unchanged compiles replayed from the shared reuse store with Python's exact streams, exit code and compute log capture, and a handoff of every other invocation to the installed `sqlbuild-python` script. Built as its own wheel (`crates/sqlbuild-cli/pyproject.toml`). | | `sqlbuild-python` | The only PyO3 crate: the `_native` module, its Python classes and functions, conversions from Python objects, and the process allocator. | @@ -32,7 +33,7 @@ Dependencies point one way, from the top of this graph to the bottom: ```text sqlbuild-python - -> sqlbuild-refactor -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-attachments + -> sqlbuild-project -> sqlbuild-refactor -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-attachments -> sqlbuild-model-config -> sqlbuild-scopes -> sqlbuild-render -> sqlbuild-discovery -> sqlbuild-host -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-pysource -> sqlbuild-state -> sqlbuild-cursor -> sqlbuild-core sqlbuild-cli -> sqlbuild-cache, sqlbuild-core, sqlbuild-build-features diff --git a/crates/sqlbuild-project/Cargo.toml b/crates/sqlbuild-project/Cargo.toml new file mode 100644 index 0000000000..112044bb2c --- /dev/null +++ b/crates/sqlbuild-project/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "sqlbuild-project" +version.workspace = true +description = "The compiled project as plain Rust: model, source, function, audit and seed records, with the host's values carried as opaque handles." +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +sqlbuild-analysis.workspace = true +sqlbuild-core.workspace = true +sqlbuild-sqltext.workspace = true diff --git a/crates/sqlbuild-project/src/_helpers/analysis_records.rs b/crates/sqlbuild-project/src/_helpers/analysis_records.rs new file mode 100644 index 0000000000..aa0e8d15eb --- /dev/null +++ b/crates/sqlbuild-project/src/_helpers/analysis_records.rs @@ -0,0 +1,22 @@ +//! What a model analysis record answers without the host. + +use sqlbuild_analysis::assembly::analysis_session::models::{LineageFacts, LineageRow}; + +use crate::models::AnalysisRecord; +use crate::types::ProjectHost; + +impl AnalysisRecord { + /// The session's lineage rows, or `None` when the model kept no lineage. + #[inline] + pub fn lineage_rows(&self) -> Option<&[LineageRow]> { + self.lineage.as_ref().map(|lineage| match lineage { + LineageFacts::Native(rows) | LineageFacts::NativeFacts(rows) => rows.as_slice(), + }) + } + + /// The number of upstream sources across the model's lineage rows, or `None` without lineage. + pub fn lineage_edge_count(&self) -> Option { + let rows: &[LineageRow] = self.lineage_rows()?; + Some(rows.iter().map(|row| row.sources.len()).sum()) + } +} diff --git a/crates/sqlbuild-project/src/_helpers/mod.rs b/crates/sqlbuild-project/src/_helpers/mod.rs new file mode 100644 index 0000000000..51293fc21a --- /dev/null +++ b/crates/sqlbuild-project/src/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod analysis_records; +pub(crate) mod project_records; +pub(crate) mod resource_records; diff --git a/crates/sqlbuild-project/src/_helpers/project_records.rs b/crates/sqlbuild-project/src/_helpers/project_records.rs new file mode 100644 index 0000000000..c1891cb420 --- /dev/null +++ b/crates/sqlbuild-project/src/_helpers/project_records.rs @@ -0,0 +1,79 @@ +//! What the compiled project answers and records without the host. + +use sqlbuild_analysis::compiled_project::main::model_facts::model_facts; +use sqlbuild_analysis::compiled_project::main::record_model::record_model; +use sqlbuild_analysis::compiled_project::models::{ + CompiledModelFacts, CompiledProjectFacts, ModelSchemaFact, +}; + +use crate::models::{ + AuditRecord, CompiledProject, FunctionRecord, ModelInputRecord, SeedRecord, SourceRecord, +}; +use crate::types::ProjectHost; + +impl Default for CompiledProject { + fn default() -> Self { + Self::with_facts(CompiledProjectFacts::default()) + } +} + +impl CompiledProject { + /// A project holding only `facts`, which analyses can be recorded into apart from the compile's. + pub fn with_facts(facts: CompiledProjectFacts) -> Self { + Self { + facts, + inputs: Vec::new(), + graph_builds: 0, + seeds: Vec::new(), + sources: Vec::new(), + functions: Vec::new(), + audits: Vec::new(), + } + } + + /// Retain facts a producer finished natively. + pub fn retain(&mut self, facts: CompiledModelFacts) { + record_model(&mut self.facts, facts); + } + + /// Retain a finished model's input record; return its index for the model's view. + pub fn retain_input(&mut self, record: ModelInputRecord) -> usize { + self.inputs.push(record); + self.inputs.len() - 1 + } + + /// The schema fact recorded for an input record's model. + pub fn input_schema_fact(&self, record: &ModelInputRecord) -> Option<&ModelSchemaFact> { + model_facts(&self.facts, &record.name).and_then(|facts| facts.schema.as_ref()) + } + + /// The input record at `index`, or `None` past the last model. + #[inline] + pub fn input(&self, index: usize) -> Option<&ModelInputRecord> { + self.inputs.get(index) + } + + /// Every attached audit, in attachment order. + #[inline] + pub fn audits(&self) -> &[AuditRecord] { + &self.audits + } + + /// Every attached function, in attachment order. + #[inline] + pub fn functions(&self) -> &[FunctionRecord] { + &self.functions + } + + /// Every attached source, in attachment order. + #[inline] + pub fn sources(&self) -> &[SourceRecord] { + &self.sources + } + + /// Every attached seed, in attachment order. + #[inline] + pub fn seeds(&self) -> &[SeedRecord] { + &self.seeds + } +} diff --git a/crates/sqlbuild-project/src/_helpers/resource_records.rs b/crates/sqlbuild-project/src/_helpers/resource_records.rs new file mode 100644 index 0000000000..e16c95548f --- /dev/null +++ b/crates/sqlbuild-project/src/_helpers/resource_records.rs @@ -0,0 +1,56 @@ +//! What source, function and audit records answer without the host. + +use crate::models::{ + AttachedAudit, AttachedFunction, AttachedSource, AuditRecord, FunctionRecord, SourceRecord, +}; +use crate::types::{ObjectKeyRow, ProjectHost, SourceNamespace}; + +const TABLE_FUNCTION_TYPE: &str = "table_fn"; +const SCALAR_FUNCTION_TYPE: &str = "udf"; + +impl SourceRecord { + /// The attached source. + #[inline] + pub fn attached(&self) -> &AttachedSource { + &self.attached + } + + /// Keep the namespace native project assembly resolved for the source. + pub fn set_namespace(&mut self, namespace: SourceNamespace) { + self.namespace = Some(namespace); + } +} + +impl AttachedFunction { + /// `function_node_type`: a table function when it declares return columns. + #[inline] + pub fn node_type(&self) -> &'static str { + if self.return_columns.is_empty() { + SCALAR_FUNCTION_TYPE + } else { + TABLE_FUNCTION_TYPE + } + } +} + +impl FunctionRecord { + /// Keep the dependency keys native project assembly resolved for the function. + pub fn set_deps(&mut self, deps: Vec) { + self.deps = Some(deps); + } +} + +impl AttachedAudit { + /// `_resolve_audit_name`: the instance name, else the definition name. + #[inline] + pub fn compiled_name(&self) -> &str { + self.name.as_deref().unwrap_or(&self.definition_name) + } +} + +impl AuditRecord { + /// Keep the scope dependency keys native project assembly resolved for the audit. + pub fn set_scope_deps(&mut self, deps: Vec) { + self.scope_deps = Some(deps); + } +} diff --git a/crates/sqlbuild-project/src/lib.rs b/crates/sqlbuild-project/src/lib.rs new file mode 100644 index 0000000000..6fce054ebb --- /dev/null +++ b/crates/sqlbuild-project/src/lib.rs @@ -0,0 +1,11 @@ +//! The compiled project as plain Rust: each resource's records, with host values held as opaque handles. + +#![forbid(unsafe_code)] + +mod _helpers; + +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-project/src/models.rs b/crates/sqlbuild-project/src/models.rs new file mode 100644 index 0000000000..705efc3138 --- /dev/null +++ b/crates/sqlbuild-project/src/models.rs @@ -0,0 +1,230 @@ +//! The plain records the compiled project keeps for its resources. + +use std::sync::Arc; + +use sqlbuild_sqltext::compiler::models::CharSpan; + +use sqlbuild_analysis::assembly::analysis_session::models::{ContractProof, LineageFacts}; +use sqlbuild_analysis::binding_locations::models::LocatedDiagnostic; +use sqlbuild_analysis::compiled_project::models::CompiledProjectFacts; + +use sqlbuild_core::json::models::JsonValue; + +use crate::types::{InferredRow, NamedTypeRow, ObjectKeyRow, ProjectHost, SourceNamespace}; + +/// One reference a resource's SQL makes: kind, name, package and call argument count. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ReferenceRow { + pub kind: String, + pub name: String, + pub package: Option, + pub call_argument_count: Option, +} + +/// A compiled relation location's parts. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DestinationFact { + pub database: Option, + pub schema: Option, + pub name: String, + pub qualified_name: Option, + pub logical_schema: Option, + pub logical_database: Option, +} + +/// A model's name, compiled query, authored file, and whether it calls a function. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ModelMetadataTexts { + pub name: String, + pub query_sql: String, + pub authored_sql: String, + pub calls_functions: bool, +} + +/// The expansion passes of a model's query, from authored text to expanded SQL. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ExpansionRecord { + /// The authored query below the header. + pub authored_sql: Arc, + /// The declaration- and macro-expanded SQL before canonicalisation. + pub expanded_sql: Arc, + pub interpolation_spans: Vec, + pub declaration_spans: Vec, + pub macro_spans: Vec, + /// Whether interpolation left the authored query unchanged. + pub authored: bool, +} + +/// A model's attached header schema; the host keeps its columns, audits and dynamic columns. +#[derive(Debug)] +pub struct SchemaRecord { + pub description: Option, + pub type_enforcement: bool, + pub model_schema: Option, + pub host: H::Schema, +} + +/// What the model loop finished for one model beyond its facts; the host keeps its user values. +#[derive(Debug)] +pub struct ModelInputRecord { + /// The model's name, its file stem. + pub name: String, + /// Immutable text shared with discovery and expansions. + pub query_sql: Arc, + pub macro_source_sql: Arc, + pub references: Vec, + pub schema: Option>, + pub sql_validation_enabled: bool, + pub macro_deps: Vec, + pub expansion: ExpansionRecord, + /// What assembly and the semantic checks finished, once the model is assembled. + pub compiled: Option>, + pub host: H::ModelInput, +} + +/// What one model's analysis left for its compiled model. +#[derive(Debug)] +pub struct AnalysisRecord { + pub inferred: Option>, + pub lineage: Option, + pub has_star: bool, + pub star_resolved: bool, + pub binding: Vec, + pub binding_validated: bool, + pub proof: Option, + pub host: H::Analysis, +} + +/// What assembly and the semantic checks finished for one compiled model. +#[derive(Debug)] +pub struct CompiledRecord { + /// `(resource type, name)` of each dependency. + pub deps: Vec<(String, String)>, + pub destination: DestinationFact, + /// Columns, lineage, binding diagnostics and dynamic proof kept from the native session. + pub analysis: AnalysisRecord, + pub macro_deps: Vec, + pub host: H::CompiledModel, +} + +/// What the source loop attached for one source: its expanded metadata. +#[derive(Debug)] +pub struct AttachedSource { + pub name: String, + pub contract: Option, + pub database: Option, + pub schema: Option, + pub table: Option, + pub expression: Option, + pub description: Option, + pub host: H::AttachedSource, +} + +/// One attached source held by the compiled project. +#[derive(Debug)] +pub struct SourceRecord { + pub attached: AttachedSource, + /// The namespace native project assembly resolved, once it ran. + pub namespace: Option, + pub host: H::Source, +} + +/// A Python function's runtime, entry point and packages. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PythonRuntime { + pub runtime_version: Option, + pub entry_point: Option, + pub packages: Vec, +} + +/// What the function loop attached for one SQL or Python function. +#[derive(Debug)] +pub struct AttachedFunction { + /// The file's relative path, as messages name it. + pub relative_path: String, + pub name: String, + pub arguments: Vec, + pub returns: String, + pub return_columns: Vec, + pub body: String, + pub references: Vec, + pub destination: DestinationFact, + pub fingerprint_destination: DestinationFact, + pub python: Option, + pub tags: Vec, + pub description: Option, + pub host: H::AttachedFunction, +} + +/// One attached function held by the compiled project. +#[derive(Debug)] +pub struct FunctionRecord { + pub attached: AttachedFunction, + /// The dependency keys native project assembly resolved, once it ran. + pub deps: Option>, + pub host: H::Function, +} + +/// The target an attached audit checks: its kind, name and column. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AttachedTarget { + pub kind: String, + pub name: String, + pub column: Option, +} + +/// What the audit loop attached for one singular audit block or attached generic audit. +#[derive(Debug)] +pub struct AttachedAudit { + /// The block's name, else the file's stem. + pub definition_name: String, + pub sql_body: String, + pub measurement: bool, + pub evidence_sql: Option, + pub references: Vec, + pub target: Option, + pub severity: &'static str, + pub run_scope: String, + pub always_run: bool, + pub name: Option, + pub description: Option, + /// The authored mapping of JSON values reported with results, or `None`. + pub meta: Option, + pub host: H::AttachedAudit, +} + +/// One attached audit held by the compiled project. +#[derive(Debug)] +pub struct AuditRecord { + pub attached: AttachedAudit, + /// The scope dependency keys native project assembly resolved, once it ran. + pub scope_deps: Option>, + pub host: H::Audit, +} + +/// One seed CSV paired with its declaration; the host keeps the discovered objects. +#[derive(Debug)] +pub struct SeedRecord { + pub name: String, + /// The destination native project assembly resolved, once it ran. + pub destination: Option, + pub host: H::Seed, +} + +/// One compile's project: the facts producers record and each resource's records. +#[derive(Debug)] +pub struct CompiledProject { + pub facts: CompiledProjectFacts, + /// What the model loop finished for each model beyond its facts, in loop order. + pub inputs: Vec>, + /// Lineage graphs its project views built. + pub graph_builds: usize, + /// Each attached seed, in attachment order. + pub seeds: Vec>, + /// Each attached source, in attachment order. + pub sources: Vec>, + /// Each attached SQL, then Python, function in attachment order. + pub functions: Vec>, + /// Each singular, then model-, source- and seed-attached, audit in attachment order. + pub audits: Vec>, +} diff --git a/crates/sqlbuild-project/src/tests.rs b/crates/sqlbuild-project/src/tests.rs new file mode 100644 index 0000000000..3f722391e0 --- /dev/null +++ b/crates/sqlbuild-project/src/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_analysis_records.rs"] +mod test_analysis_records; +#[path = "tests/test_project_records.rs"] +mod test_project_records; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-project/src/tests/helpers.rs b/crates/sqlbuild-project/src/tests/helpers.rs new file mode 100644 index 0000000000..884fc4073c --- /dev/null +++ b/crates/sqlbuild-project/src/tests/helpers.rs @@ -0,0 +1,75 @@ +use sqlbuild_analysis::assembly::analysis_session::models::{LineageFacts, LineageRow}; + +use sqlbuild_analysis::compiled_project::models::{ + CompiledModelFacts, ModelSchemaFact, SchemaColumnFact, +}; + +use crate::models::{AnalysisRecord, ExpansionRecord, ModelInputRecord}; +use crate::tests::test_types::TestHost; + +pub(super) fn lineage_row(output: &str, sources: &[&str]) -> LineageRow { + LineageRow { + output_column: output.to_owned(), + transform_code: 0, + confidence_code: 0, + sources: sources + .iter() + .map(|source| { + ( + "orders".to_owned(), + (*source).to_owned(), + "model".to_owned(), + ) + }) + .collect(), + } +} + +pub(super) fn analysis(lineage: Option) -> AnalysisRecord { + AnalysisRecord { + inferred: None, + lineage, + has_star: false, + star_resolved: false, + binding: Vec::new(), + binding_validated: false, + proof: None, + host: (), + } +} + +pub(super) fn model_input(name: &str) -> ModelInputRecord { + ModelInputRecord { + name: name.to_owned(), + query_sql: "SELECT 1".into(), + macro_source_sql: "SELECT 1".into(), + references: Vec::new(), + schema: None, + sql_validation_enabled: true, + macro_deps: Vec::new(), + expansion: ExpansionRecord { + authored_sql: "SELECT 1".into(), + expanded_sql: "SELECT 1".into(), + interpolation_spans: Vec::new(), + declaration_spans: Vec::new(), + macro_spans: Vec::new(), + authored: true, + }, + compiled: None, + host: (), + } +} + +pub(super) fn schema_facts(name: &str, column: &str) -> CompiledModelFacts { + CompiledModelFacts { + name: name.to_owned(), + schema: Some(ModelSchemaFact { + columns: vec![SchemaColumnFact { + name: column.to_owned(), + ..SchemaColumnFact::default() + }], + ..ModelSchemaFact::default() + }), + ..CompiledModelFacts::default() + } +} diff --git a/crates/sqlbuild-project/src/tests/test_analysis_records.rs b/crates/sqlbuild-project/src/tests/test_analysis_records.rs new file mode 100644 index 0000000000..2eb411099c --- /dev/null +++ b/crates/sqlbuild-project/src/tests/test_analysis_records.rs @@ -0,0 +1,51 @@ +use sqlbuild_analysis::assembly::analysis_session::models::{LineageFacts, LineageRow}; + +use crate::models::AnalysisRecord; +use crate::tests::helpers::{analysis, lineage_row}; +use crate::tests::test_types::{LineageTestCase, TestHost}; + +#[test] +fn given_analysis_lineage_when_reading_then_rows_and_edges_follow_the_kept_facts() { + let test_cases = [ + LineageTestCase { + description: "a model without lineage has no rows or edges", + lineage: None, + expected_outputs: None, + expected_edge_count: None, + }, + LineageTestCase { + description: "native rows count every upstream source", + lineage: Some(LineageFacts::Native(vec![ + lineage_row("order_id", &["order_id"]), + lineage_row("total", &["amount", "tax"]), + ])), + expected_outputs: Some(vec!["order_id", "total"]), + expected_edge_count: Some(3), + }, + LineageTestCase { + description: "plain facts read the same as native rows", + lineage: Some(LineageFacts::NativeFacts(vec![lineage_row("status", &[])])), + expected_outputs: Some(vec!["status"]), + expected_edge_count: Some(0), + }, + ]; + + for test_case in test_cases { + let record: AnalysisRecord = analysis(test_case.lineage); + let outputs: Option> = record + .lineage_rows() + .map(|rows: &[LineageRow]| rows.iter().map(|row| row.output_column.as_str()).collect()); + + assert_eq!( + outputs, test_case.expected_outputs, + "{}", + test_case.description + ); + assert_eq!( + record.lineage_edge_count(), + test_case.expected_edge_count, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-project/src/tests/test_project_records.rs b/crates/sqlbuild-project/src/tests/test_project_records.rs new file mode 100644 index 0000000000..cc7b8fb2fe --- /dev/null +++ b/crates/sqlbuild-project/src/tests/test_project_records.rs @@ -0,0 +1,76 @@ +use sqlbuild_analysis::compiled_project::models::ModelSchemaFact; + +use crate::models::{CompiledProject, ModelInputRecord}; +use crate::tests::helpers::{model_input, schema_facts}; +use crate::tests::test_types::{ProjectInputTestCase, TestHost}; + +#[test] +fn given_retained_inputs_when_reading_then_indexes_and_schema_facts_follow_loop_order() { + let test_cases = [ + ProjectInputTestCase { + description: "inputs keep loop order and their recorded schema", + retained: &["orders", "customers"], + recorded_schemas: &["customers"], + read_index: 1, + expected_indexes: vec![0, 1], + expected_name: Some("customers"), + expected_schema_column: Some("customer_id"), + }, + ProjectInputTestCase { + description: "an input without recorded facts has no schema", + retained: &["orders"], + recorded_schemas: &[], + read_index: 0, + expected_indexes: vec![0], + expected_name: Some("orders"), + expected_schema_column: None, + }, + ProjectInputTestCase { + description: "an index past the last input reads nothing", + retained: &["orders"], + recorded_schemas: &["orders"], + read_index: 1, + expected_indexes: vec![0], + expected_name: None, + expected_schema_column: None, + }, + ]; + + for test_case in test_cases { + let mut project: CompiledProject = CompiledProject::default(); + for name in test_case.recorded_schemas { + project.retain(schema_facts( + name, + &format!("{}_id", name.trim_end_matches('s')), + )); + } + let indexes: Vec = test_case + .retained + .iter() + .map(|name| project.retain_input(model_input(name))) + .collect(); + let input: Option<&ModelInputRecord> = project.input(test_case.read_index); + let schema: Option<&ModelSchemaFact> = + input.and_then(|record| project.input_schema_fact(record)); + + assert_eq!( + indexes, test_case.expected_indexes, + "{}", + test_case.description + ); + assert_eq!( + input.map(|record| record.name.as_str()), + test_case.expected_name, + "{}", + test_case.description + ); + assert_eq!( + schema + .and_then(|schema| schema.columns.first()) + .map(|column| column.name.as_str()), + test_case.expected_schema_column, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-project/src/tests/test_types.rs b/crates/sqlbuild-project/src/tests/test_types.rs new file mode 100644 index 0000000000..1d256a10d9 --- /dev/null +++ b/crates/sqlbuild-project/src/tests/test_types.rs @@ -0,0 +1,38 @@ +use sqlbuild_analysis::assembly::analysis_session::models::LineageFacts; + +use crate::types::ProjectHost; + +/// A host that keeps nothing beside the records. +#[derive(Debug)] +pub(super) struct TestHost; + +impl ProjectHost for TestHost { + type ModelInput = (); + type Schema = (); + type CompiledModel = (); + type Analysis = (); + type AttachedSource = (); + type Source = (); + type AttachedFunction = (); + type Function = (); + type AttachedAudit = (); + type Audit = (); + type Seed = (); +} + +pub(super) struct LineageTestCase { + pub(super) description: &'static str, + pub(super) lineage: Option, + pub(super) expected_outputs: Option>, + pub(super) expected_edge_count: Option, +} + +pub(super) struct ProjectInputTestCase { + pub(super) description: &'static str, + pub(super) retained: &'static [&'static str], + pub(super) recorded_schemas: &'static [&'static str], + pub(super) read_index: usize, + pub(super) expected_indexes: Vec, + pub(super) expected_name: Option<&'static str>, + pub(super) expected_schema_column: Option<&'static str>, +} diff --git a/crates/sqlbuild-project/src/types.rs b/crates/sqlbuild-project/src/types.rs new file mode 100644 index 0000000000..9b969d313a --- /dev/null +++ b/crates/sqlbuild-project/src/types.rs @@ -0,0 +1,41 @@ +//! What the host supplies beside each compiled record. + +use std::fmt::Debug; + +/// The host's own values and lazily built views, kept in one slot beside each compiled record. +pub trait ProjectHost { + /// A model input's host values: its model file, config, declarations, usages and views. + type ModelInput: Debug; + /// A model schema's host values: its header columns, audits and dynamic columns. + type Schema: Debug; + /// A compiled model's lazily built views. + type CompiledModel: Debug; + /// A model analysis's lazily built views. + type Analysis: Debug; + /// An attached source's host values: its file, entry, metadata, specs and usages. + type AttachedSource: Debug; + /// A source record's lazily built facts and views. + type Source: Debug; + /// An attached function's host values: its file and usages. + type AttachedFunction: Debug; + /// A function record's lazily built views. + type Function: Debug; + /// An attached audit's host values: its file, block, measurement policy specs and usages. + type AttachedAudit: Debug; + /// An audit record's lazily built views. + type Audit: Debug; + /// A seed's host values: its CSV file, declaration, schema file and views. + type Seed: Debug; +} + +/// `(name, type, nullability)` of one inferred output column. +pub type InferredRow = (String, Option, String); + +/// `(database, schema)` native project assembly resolved for a source, or `None`. +pub type SourceNamespace = Option<(Option, Option)>; + +/// `(name, type)` of a function argument or return column, its types template-expanded. +pub type NamedTypeRow = (String, String); + +/// `(kind, name)` of a compiled object key. +pub type ObjectKeyRow = (String, String); diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 854e25fc40..a0d4b49fb4 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -33,6 +33,7 @@ sqlbuild-cursor.workspace = true sqlbuild-discovery.workspace = true sqlbuild-host.workspace = true sqlbuild-model-config.workspace = true +sqlbuild-project.workspace = true sqlbuild-pysource.workspace = true sqlbuild-recall.workspace = true sqlbuild-refactor.workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index 583038d3f1..8956bf6161 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -23,7 +23,6 @@ use crate::bindings::_helpers::cache::native_store::{content_digest, current_cod use crate::bindings::_helpers::compile_driver::analysis_cache::{ AnalysisCacheCounts, record_analysis_cache_metrics, }; -use crate::bindings::_helpers::compiled_project::rows::ReferenceRow; use crate::bindings::_helpers::compiled_views::model_input_view::{ AttachedSchema, NativeModelInput, attached_schema, input_model_name, input_query_sql, input_reference_rows, @@ -35,6 +34,7 @@ use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::fields::mapping_items; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::semantic_shapes::shapes::{VALID_FROM_DEFAULT, VALID_TO_DEFAULT}; +use sqlbuild_project::models::ReferenceRow; const LOGGING_MODULE: &str = "logging"; const DEBUG_LOGGER: &str = "sqlbuild.compile"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs index 6a5e490b73..9ea5b32496 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs @@ -4,12 +4,12 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; -use crate::bindings::_helpers::compiled_project::rows::ReferenceRow; use crate::bindings::_helpers::compiled_views::model_input_view::input_reference_rows; use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::ids::RecordId; +use sqlbuild_project::models::ReferenceRow; const CURSOR_INPUTS_KEY: &str = "cursor_inputs"; const FILTER_ROLE: &str = "filter"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs index ac37b569e5..5ef5b8dec2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs @@ -10,15 +10,15 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyDictMethods, PyString}; use pyo3::{FromPyObject, pyclass, pymethods}; use serde_json::Value; -use sqlbuild_analysis::compiled_project::main::model_facts::model_facts as project_model; use sqlbuild_analysis::compiled_project::main::project_models::project_models; -use sqlbuild_analysis::compiled_project::main::record_model::record_model; use sqlbuild_analysis::compiled_project::main::record_model_analysis::record_model_analysis; use sqlbuild_analysis::compiled_project::models::{ CompiledModelFacts, CompiledProjectFacts, DeclarationFact, DynamicProofFact, ModelAnalysisFact, ModelConfigFacts, ModelReferenceFact, ModelSchemaFact, }; +use sqlbuild_project::models::CompiledProject; + use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::compiled_project::attachment_inputs::AttachedSqlTests; use crate::bindings::_helpers::compiled_project::attachment_inputs::AuditFileRow; @@ -35,26 +35,30 @@ use crate::bindings::_helpers::compiled_project::attachment_inputs::TestFileRow; use crate::bindings::_helpers::compiled_project::attachment_inputs::resource_attachment; use crate::bindings::_helpers::compiled_project::values::{object_value, plain_value}; use crate::bindings::_helpers::compiled_views::audit_views::{ - AttachedAudit, AuditRecord, NativeAuditInput, NativeCompiledAudit, + AttachedAudit, AuditRecord, NativeAuditInput, NativeCompiledAudit, audit_record, }; use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - CompiledViewClasses, InferredRow, NativeCompiledModel, + CompiledViewClasses, NativeCompiledModel, }; use crate::bindings::_helpers::compiled_views::function_views::{ - AttachedFunction, FunctionRecord, NativeCompiledFunction, NativeFunctionInput, ObjectKeyRow, + AttachedFunction, FunctionRecord, NativeCompiledFunction, NativeFunctionInput, function_record, }; use crate::bindings::_helpers::compiled_views::inputs_view::NativeInputsView; use crate::bindings::_helpers::compiled_views::model_input_view::{ ModelInputRecord, NativeModelInput, }; +use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; use crate::bindings::_helpers::compiled_views::seed_views::{ - NativeCompiledSeed, NativeSeedInput, SeedNamespace, SeedRecord, paired_seeds, + NativeCompiledSeed, NativeSeedInput, SeedNamespace, SeedRecord, SeedViews, paired_seeds, }; use crate::bindings::_helpers::compiled_views::source_views::{ - NativeCompiledSource, NativeSourceInput, SourceNamespace, SourceRecord, + NativeCompiledSource, NativeSourceInput, SourceRecord, }; use crate::bindings::_helpers::render::declaration_values::ConstantRendering; +use sqlbuild_project::types::InferredRow; +use sqlbuild_project::types::ObjectKeyRow; +use sqlbuild_project::types::SourceNamespace; pub(crate) const CONTRACT_KEY: &str = "contract"; const INPUT_CLASSES_UNSET: &str = "no model loop recorded inputs into this project"; @@ -125,27 +129,15 @@ pub(crate) struct DeclarationPy<'py>( Option, ); -/// One compile's project facts, recorded by the stages that produce them. +/// One compile's project, recorded by the stages that produce it, with its Python projections. #[pyclass(module = "sqlbuild._native")] #[derive(Default)] pub(crate) struct NativeCompiledProject { - facts: CompiledProjectFacts, - /// What the model loop finished for each model beyond its facts, in loop order. - inputs: Vec, + project: CompiledProject, /// The Python classes model input views project their attributes with. input_classes: Option>, /// Shared immutable column projections, allocated only when Python consumers request them. inferred_column_projections: Mutex>>, - /// Lineage graphs its project views built. - graph_builds: usize, - /// Each attached seed, in attachment order. - seeds: Vec, - /// Each attached source, in attachment order. - sources: Vec, - /// Each attached SQL, then Python, function in attachment order. - functions: Vec, - /// Each singular, then model-, source- and seed-attached, audit in attachment order. - audits: Vec, /// The model loop and its constant rendering, which attach the project's other resources. attachment: Option<(Py, Py)>, } @@ -153,26 +145,25 @@ pub(crate) struct NativeCompiledProject { impl NativeCompiledProject { /// The facts recorded so far, for native consumers such as the rules request. pub(crate) fn facts(&self) -> &CompiledProjectFacts { - &self.facts + &self.project.facts } /// A project holding only `facts`, which analyses can be recorded into apart from the compile's. pub(crate) fn with_facts(facts: CompiledProjectFacts) -> Self { Self { - facts, + project: CompiledProject::with_facts(facts), ..Self::default() } } /// Retain facts a producer finished natively. pub(crate) fn retain(&mut self, facts: CompiledModelFacts) { - record_model(&mut self.facts, facts); + self.project.retain(facts); } /// Retain a finished model's input record; return its index for the model's view. pub(crate) fn retain_input(&mut self, record: ModelInputRecord) -> usize { - self.inputs.push(record); - self.inputs.len() - 1 + self.project.retain_input(record) } /// Keep the classes model input views project their attributes with. @@ -182,7 +173,7 @@ impl NativeCompiledProject { /// Count one lineage graph a project view built. pub(crate) fn record_graph_build(&mut self) { - self.graph_builds += 1; + self.project.graph_builds += 1; } /// The kept model loop and its constant rendering. @@ -207,65 +198,68 @@ impl NativeCompiledProject { /// Retain attached functions; return the index of the first. pub(crate) fn extend_functions(&mut self, attached: Vec) -> usize { - let first: usize = self.functions.len(); - self.functions - .extend(attached.into_iter().map(FunctionRecord::new)); + let first: usize = self.project.functions.len(); + self.project + .functions + .extend(attached.into_iter().map(function_record)); first } /// Retain attached audits; return the index of the first. pub(crate) fn extend_audits(&mut self, attached: Vec) -> usize { - let first: usize = self.audits.len(); - self.audits - .extend(attached.into_iter().map(AuditRecord::new)); + let first: usize = self.project.audits.len(); + self.project + .audits + .extend(attached.into_iter().map(audit_record)); first } /// Every attached audit, in attachment order. pub(crate) fn audits(&self) -> &[AuditRecord] { - &self.audits + self.project.audits() } /// Every attached function, in attachment order. pub(crate) fn functions(&self) -> &[FunctionRecord] { - &self.functions + self.project.functions() } /// Every attached source, in attachment order. pub(crate) fn sources(&self) -> &[SourceRecord] { - &self.sources + self.project.sources() } /// Keep one attached source. pub(crate) fn retain_source(&mut self, source: SourceRecord) { - self.sources.push(source); + self.project.sources.push(source); } /// Every attached seed, in attachment order. pub(crate) fn seeds(&self) -> &[SeedRecord] { - &self.seeds + self.project.seeds() } /// The schema fact recorded for an input record's model. pub(crate) fn input_schema_fact(&self, record: &ModelInputRecord) -> Option<&ModelSchemaFact> { - project_model(&self.facts, &record.name).and_then(|facts| facts.schema.as_ref()) + self.project.input_schema_fact(record) } /// Every model's input record, in loop order. pub(crate) fn inputs(&self) -> &[ModelInputRecord] { - &self.inputs + &self.project.inputs } /// The input record at `index`. pub(crate) fn input(&self, index: usize) -> PyResult<&ModelInputRecord> { - self.inputs - .get(index) + self.project + .input(index) .ok_or_else(|| PyIndexError::new_err(index)) } /// The input record at `index`, for recording what later stages finish. pub(crate) fn input_mut(&mut self, index: usize) -> PyResult<&mut ModelInputRecord> { - self.inputs + self.project + .inputs .get_mut(index) .ok_or_else(|| PyIndexError::new_err(index)) } @@ -349,7 +343,7 @@ impl NativeCompiledProject { fast_lineage_has_star, unchecked_output_columns: unchecked, }; - record_model_analysis(&mut self.facts, name, analysis) + record_model_analysis(&mut self.project.facts, name, analysis) } /// Record each assembled native model view's final typed analysis; return how many. @@ -365,7 +359,11 @@ impl NativeCompiledProject { let mut owner = slf.borrow_mut(); let mut recorded: usize = 0; for (name, analysis) in facts { - recorded += usize::from(record_model_analysis(&mut owner.facts, &name, analysis)); + recorded += usize::from(record_model_analysis( + &mut owner.project.facts, + &name, + analysis, + )); } Ok(recorded) } @@ -395,7 +393,7 @@ impl NativeCompiledProject { ) -> PyResult> { let seeds: Vec = paired_seeds(declarations, &seed_files, compile_error)?; let count: usize = seeds.len(); - slf.borrow_mut().seeds = seeds; + slf.borrow_mut().project.seeds = seeds; Ok((0..count) .map(|index| NativeSeedInput::new(slf.clone().unbind(), index)) .collect()) @@ -404,7 +402,8 @@ impl NativeCompiledProject { /// Keep the destinations native project assembly resolved, in seed order. fn record_seed_namespaces(&mut self, namespaces: Vec) -> PyResult<()> { for (index, namespace) in namespaces.into_iter().enumerate() { - self.seeds + self.project + .seeds .get_mut(index) .ok_or_else(|| PyIndexError::new_err(index))? .set_namespace(namespace); @@ -460,7 +459,8 @@ impl NativeCompiledProject { /// Keep the scope dependency keys native project assembly resolved, in audit order. fn record_audit_scope_deps(&mut self, deps: Vec>) -> PyResult<()> { for (index, keys) in deps.into_iter().enumerate() { - self.audits + self.project + .audits .get_mut(index) .ok_or_else(|| PyIndexError::new_err(index))? .set_scope_deps(keys); @@ -470,7 +470,7 @@ impl NativeCompiledProject { /// The assembled audit at `index`. fn compiled_audit(slf: &Bound<'_, Self>, index: usize) -> PyResult { - if index >= slf.borrow().audits.len() { + if index >= slf.borrow().project.audits.len() { return Err(PyIndexError::new_err(index)); } Ok(NativeCompiledAudit::new(slf.clone().unbind(), index)) @@ -479,7 +479,8 @@ impl NativeCompiledProject { /// Keep the dependency keys native project assembly resolved, in function order. fn record_function_deps(&mut self, deps: Vec>) -> PyResult<()> { for (index, keys) in deps.into_iter().enumerate() { - self.functions + self.project + .functions .get_mut(index) .ok_or_else(|| PyIndexError::new_err(index))? .set_deps(keys); @@ -489,7 +490,7 @@ impl NativeCompiledProject { /// The assembled function at `index`. fn compiled_function(slf: &Bound<'_, Self>, index: usize) -> PyResult { - if index >= slf.borrow().functions.len() { + if index >= slf.borrow().project.functions.len() { return Err(PyIndexError::new_err(index)); } Ok(NativeCompiledFunction::new(slf.clone().unbind(), index)) @@ -498,7 +499,8 @@ impl NativeCompiledProject { /// Keep the namespaces native project assembly resolved, in source order. fn record_source_namespaces(&mut self, namespaces: Vec) -> PyResult<()> { for (index, namespace) in namespaces.into_iter().enumerate() { - self.sources + self.project + .sources .get_mut(index) .ok_or_else(|| PyIndexError::new_err(index))? .set_namespace(namespace); @@ -508,7 +510,7 @@ impl NativeCompiledProject { /// The assembled source at `index`. fn compiled_source(slf: &Bound<'_, Self>, index: usize) -> PyResult { - if index >= slf.borrow().sources.len() { + if index >= slf.borrow().project.sources.len() { return Err(PyIndexError::new_err(index)); } Ok(NativeCompiledSource::new(slf.clone().unbind(), index)) @@ -516,7 +518,7 @@ impl NativeCompiledProject { /// The assembled seed at `index`. fn compiled_seed(slf: &Bound<'_, Self>, index: usize) -> PyResult { - if index >= slf.borrow().seeds.len() { + if index >= slf.borrow().project.seeds.len() { return Err(PyIndexError::new_err(index)); } Ok(NativeCompiledSeed::new(slf.clone().unbind(), index)) @@ -524,17 +526,17 @@ impl NativeCompiledProject { /// How many lineage graphs this owner's project views built. fn graph_builds(&self) -> usize { - self.graph_builds + self.project.graph_builds } /// Whether both projects retained the same facts. fn __eq__(&self, other: &Self) -> bool { - self.facts == other.facts + self.project.facts == other.project.facts } /// The recorded models' names, in production order. fn model_names(&self) -> Vec { - project_models(&self.facts) + project_models(&self.project.facts) .iter() .map(|model| model.name.clone()) .collect() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/rows.rs index ed1a1bbe64..a898fdc251 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/rows.rs @@ -1,13 +1,30 @@ //! Positional rows shared by the compiled views and the input loops that fill them. +use sqlbuild_project::models::ReferenceRow; + positional_record! { - extract, into; - /// `(kind, name, package, call argument count)`. - #[derive(Clone, Debug, PartialEq, Eq)] - pub(crate) struct ReferenceRow { + into; + /// A reference as the tuple Python reads: kind, name, package and call argument count. + pub(crate) struct ReferenceTuple { pub(crate) kind: String, pub(crate) name: String, - pub(crate) package: Option, - pub(crate) call_argument_count: Option, + package: Option, + call_argument_count: Option, + } +} + +/// A reference as the tuple Python reads. +pub(crate) fn reference_row_tuple(row: ReferenceRow) -> ReferenceTuple { + let ReferenceRow { + kind, + name, + package, + call_argument_count, + } = row; + ReferenceTuple { + kind, + name, + package, + call_argument_count, } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs index f620ecffef..f8d81ee417 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs @@ -6,16 +6,18 @@ use pyo3::sync::PyOnceLock; use pyo3::types::PyTuple; use pyo3::{FromPyObject, IntoPyObjectExt, pyclass, pymethods}; +use sqlbuild_project::models as project_models; +use sqlbuild_project::types::ObjectKeyRow; + use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_project::rows::ReferenceRow; use crate::bindings::_helpers::compiled_views::compiled_model_view::object_key; -use crate::bindings::_helpers::compiled_views::function_views::ObjectKeyRow; use crate::bindings::_helpers::compiled_views::model_input_view::{ InputViewClasses, reference_tuple, view_classes, }; +use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::scopes::scope_report_objects::json_to_python; -use sqlbuild_core::json::models::JsonValue; +use sqlbuild_project::models::ReferenceRow; const AUDIT_RESOURCE: &str = "audit"; const AUDIT_CLASSES: &str = "audits"; @@ -35,69 +37,40 @@ pub(crate) type AuditAssemblyRow = ( Option<(String, String)>, ); -/// The target an attached audit checks: its kind, name and column. -pub(crate) struct AttachedTarget { - pub(crate) kind: String, - pub(crate) name: String, - pub(crate) column: Option, -} - /// What the audit loop attached for one singular audit block or attached generic audit. -pub(crate) struct AttachedAudit { +pub(crate) type AttachedAudit = project_models::AttachedAudit; + +/// One attached audit held by the compiled project. +pub(crate) type AuditRecord = project_models::AuditRecord; + +/// An attached audit's user values. +pub(crate) struct AuditHandles { /// The discovered (or built-in) audit file and block the SQL comes from. pub(crate) audit_file: Py, pub(crate) audit_block: Py, - /// The block's name, else the file's stem. - pub(crate) definition_name: String, - pub(crate) sql_body: String, - pub(crate) measurement: bool, - pub(crate) evidence_sql: Option, /// The authored measurement policy specs, carrying user values. pub(crate) thresholds: Py, pub(crate) minimum_samples: Py, pub(crate) evidence_limit: Py, - pub(crate) references: Vec, - pub(crate) target: Option, - pub(crate) severity: &'static str, - pub(crate) run_scope: String, - pub(crate) always_run: bool, /// Usage records, the identities the scope index is built from. pub(crate) usages: Py, - pub(crate) name: Option, - pub(crate) description: Option, - /// The authored mapping of JSON values reported with results, or `None`. - pub(crate) meta: Option, } -impl AttachedAudit { - /// `_resolve_audit_name`: the instance name, else the definition name. - fn compiled_name(&self) -> &str { - self.name.as_deref().unwrap_or(&self.definition_name) - } +/// An audit record's Python projections, each built once on first access. +pub(crate) struct AuditProjections { + pub(crate) references: PyOnceLock>, + pub(crate) key: PyOnceLock>, } -/// One attached audit held by the compiled project. -pub(crate) struct AuditRecord { - pub(crate) attached: AttachedAudit, - /// The scope dependency keys native project assembly resolved, once it ran. - scope_deps: Option>, - references: PyOnceLock>, - key: PyOnceLock>, -} - -impl AuditRecord { - pub(crate) fn new(attached: AttachedAudit) -> Self { - Self { - attached, - scope_deps: None, +/// The audit record of an attached audit, before assembly resolves its scope dependencies. +pub(crate) fn audit_record(attached: AttachedAudit) -> AuditRecord { + AuditRecord { + attached, + scope_deps: None, + host: AuditProjections { references: PyOnceLock::new(), key: PyOnceLock::new(), - } - } - - /// Keep the scope dependency keys native project assembly resolved for the audit. - pub(crate) fn set_scope_deps(&mut self, deps: Vec) { - self.scope_deps = Some(deps); + }, } } @@ -137,14 +110,14 @@ macro_rules! audit_getters { #[getter] pub(crate) fn audit_file(&self, py: Python<'_>) -> PyResult> { read_audit(&self.owner, self.index, py, |audit, py| { - Ok(audit.attached.audit_file.clone_ref(py)) + Ok(audit.attached.host.audit_file.clone_ref(py)) }) } #[getter] pub(crate) fn audit_block(&self, py: Python<'_>) -> PyResult> { read_audit(&self.owner, self.index, py, |audit, py| { - Ok(audit.attached.audit_block.clone_ref(py)) + Ok(audit.attached.host.audit_block.clone_ref(py)) }) } @@ -160,7 +133,7 @@ macro_rules! audit_getters { read_audit(&self.owner, self.index, py, |audit, py| { Ok(native_field(&audit .attached - .audit_block + .host.audit_block .bind(py), "evaluation_mode")? .unbind()) }) @@ -171,7 +144,7 @@ macro_rules! audit_getters { read_audit(&self.owner, self.index, py, |audit, py| { Ok(native_field(&audit .attached - .audit_block + .host.audit_block .bind(py), "measurement_contract")? .unbind()) }) @@ -180,14 +153,14 @@ macro_rules! audit_getters { #[getter] pub(crate) fn thresholds(&self, py: Python<'_>) -> PyResult> { read_audit(&self.owner, self.index, py, |audit, py| { - Ok(audit.attached.thresholds.clone_ref(py)) + Ok(audit.attached.host.thresholds.clone_ref(py)) }) } #[getter] pub(crate) fn minimum_samples(&self, py: Python<'_>) -> PyResult> { read_audit(&self.owner, self.index, py, |audit, py| { - Ok(audit.attached.minimum_samples.clone_ref(py)) + Ok(audit.attached.host.minimum_samples.clone_ref(py)) }) } @@ -211,7 +184,7 @@ macro_rules! audit_getters { #[getter] pub(crate) fn evidence_limit(&self, py: Python<'_>) -> PyResult> { read_audit(&self.owner, self.index, py, |audit, py| { - Ok(audit.attached.evidence_limit.clone_ref(py)) + Ok(audit.attached.host.evidence_limit.clone_ref(py)) }) } @@ -219,7 +192,7 @@ macro_rules! audit_getters { pub(crate) fn references(&self, py: Python<'_>) -> PyResult> { let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; read_audit(&self.owner, self.index, py, |audit, _| { - reference_tuple(&classes, &audit.attached.references, &audit.references) + reference_tuple(&classes, &audit.attached.references, &audit.host.references) }) } @@ -311,7 +284,7 @@ audit_getters!(NativeAuditInput, { #[getter] pub(crate) fn declaration_usages(&self, py: Python<'_>) -> PyResult> { read_audit(&self.owner, self.index, py, |audit, py| { - Ok(audit.attached.usages.clone_ref(py)) + Ok(audit.attached.host.usages.clone_ref(py)) }) } @@ -369,6 +342,7 @@ audit_getters!(NativeCompiledAudit, { let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; read_audit(&self.owner, self.index, py, |audit, py| { audit + .host .key .get_or_try_init(py, || { object_key( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs index e1b1151385..7ecd4a4759 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs @@ -16,14 +16,17 @@ use sqlbuild_analysis::binding_locations::main::locate_binding_diagnostics::loca use sqlbuild_analysis::binding_locations::models::{BindingSource, LocatedDiagnostic}; use sqlbuild_analysis::semantic_validation::models::{DiagnosticRow, Expansion}; +use sqlbuild_project::models as project_models; +use sqlbuild_project::types::InferredRow; + use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; use crate::bindings::_helpers::boundary::panics::compiler_error; -use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - CompiledViewClasses, InferredRow, -}; +use crate::bindings::_helpers::compiled_views::compiled_model_view::CompiledViewClasses; use crate::bindings::_helpers::compiled_views::model_input_view::ModelInputRecord; +use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::lineage::lineage_facts::{confidence_member, transform_kind_member}; +use crate::bindings::_helpers::render::macro_call_rows::span_expansion; const COMPILE_PHASE: &str = "compile"; const MODEL_RESOURCE: &str = "model"; @@ -45,19 +48,10 @@ positional_record! { pub(crate) type SessionArgument = (Py, usize); /// What one model's analysis left for its compiled model. -pub(crate) struct AnalysisRecord { - pub(crate) inferred: Option>, - lineage: Option, - pub(crate) has_star: bool, - pub(crate) star_resolved: bool, - binding: Vec, - pub(crate) binding_validated: bool, - pub(crate) proof: Option, - projections: AnalysisProjections, -} +pub(crate) type AnalysisRecord = project_models::AnalysisRecord; /// Python projections of the analysis, each built once on first access. -struct AnalysisProjections { +pub(crate) struct AnalysisProjections { lineage: PyOnceLock>, binding: PyOnceLock>, proof: PyOnceLock>, @@ -122,7 +116,7 @@ pub(crate) fn analysis_record( binding: Vec::new(), binding_validated: false, proof: standalone_proof.map(proof_fact), - projections: AnalysisProjections::default(), + host: AnalysisProjections::default(), }; if let Some((session, index)) = analysis { let session = session.borrow(py); @@ -159,7 +153,7 @@ fn located(input: &ModelInputRecord, outcome: &ModelOutcome) -> PyResult = resolved_cleaned_sql(outcome).map_err(compiler_error)?; - let model_file: &NativeModelFile = input.model_file.get(); + let model_file: &NativeModelFile = input.host.model_file.get(); let authored: &str = model_file.contents_text(); let query: &str = model_file.query_text(); let expansion = &input.expansion; @@ -168,13 +162,13 @@ fn located(input: &ModelInputRecord, outcome: &ModelOutcome) -> PyResult = BindingSource { @@ -188,21 +182,39 @@ fn located(input: &ModelInputRecord, outcome: &ModelOutcome) -> PyResult Option<&[LineageRow]> { - self.lineage.as_ref().map(|lineage| match lineage { - LineageFacts::Native(rows) | LineageFacts::NativeFacts(rows) => rows.as_slice(), - }) - } +/// The Python objects a model analysis record projects, each built once. +pub(crate) trait AnalysisViews { + /// The session's lineage as `CompactLineageFacts` or plain facts, or `None`. + fn lineage_object( + &self, + py: Python<'_>, + classes: &CompiledViewClasses<'_>, + ) -> PyResult>; + /// The model's `CompilerDiagnostic` tuple, built once. + fn binding_tuple( + &self, + py: Python<'_>, + classes: &CompiledViewClasses<'_>, + model: (&str, &Bound<'_, PyAny>), + ) -> PyResult>; + + /// The model's `DynamicColumnContractProof`, or `None`. + fn proof_object( + &self, + py: Python<'_>, + classes: &CompiledViewClasses<'_>, + ) -> PyResult>; +} + +impl AnalysisViews for AnalysisRecord { /// The session's lineage as `CompactLineageFacts` or plain facts, or `None`. - pub(crate) fn lineage_object( + fn lineage_object( &self, py: Python<'_>, classes: &CompiledViewClasses<'_>, ) -> PyResult> { - self.projections + self.host .lineage .get_or_try_init(py, || match &self.lineage { None => Ok(py.None()), @@ -212,24 +224,14 @@ impl AnalysisRecord { .map(|lineage| lineage.clone_ref(py)) } - /// The number of upstream sources across the model's lineage rows, or `None` without lineage. - pub(crate) fn lineage_edge_count(&self) -> Option { - match &self.lineage { - None => None, - Some(LineageFacts::Native(rows) | LineageFacts::NativeFacts(rows)) => { - Some(rows.iter().map(|row| row.sources.len()).sum()) - } - } - } - /// The model's `CompilerDiagnostic` tuple, built once. - pub(crate) fn binding_tuple( + fn binding_tuple( &self, py: Python<'_>, classes: &CompiledViewClasses<'_>, model: (&str, &Bound<'_, PyAny>), ) -> PyResult> { - self.projections + self.host .binding .get_or_try_init(py, || { let mut diagnostics: Vec> = Vec::with_capacity(self.binding.len()); @@ -242,12 +244,12 @@ impl AnalysisRecord { } /// The model's `DynamicColumnContractProof`, or `None`. - pub(crate) fn proof_object( + fn proof_object( &self, py: Python<'_>, classes: &CompiledViewClasses<'_>, ) -> PyResult> { - self.projections + self.host .proof .get_or_try_init(py, || match &self.proof { None => Ok(py.None()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs index 0123c3f6e1..2bbd812f1a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs @@ -1,5 +1,8 @@ //! A compiled model, read from the native compiled project that owns it. +use sqlbuild_project::models as project_models; +use sqlbuild_project::models::{DestinationFact, ModelMetadataTexts}; +use sqlbuild_project::types::InferredRow; use std::sync::Arc; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; @@ -15,14 +18,15 @@ use sqlbuild_analysis::compiled_project::models::{ }; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_project::rows::ReferenceRow; +use crate::bindings::_helpers::compiled_project::rows::{ReferenceTuple, reference_row_tuple}; use crate::bindings::_helpers::compiled_views::compiled_model_analysis::{ - AnalysisRecord, ProofArgument, SessionArgument, analysis_record, + AnalysisRecord, AnalysisViews, ProofArgument, SessionArgument, analysis_record, }; use crate::bindings::_helpers::compiled_views::model_input_view::{ InputViewClasses, ModelInputRecord, model_relation_key_row, query_sql_text, read_input, read_input_with_owner, references, rejected_opt_out, schema_entry, view_classes, }; +use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; use crate::bindings::_helpers::compiled_views::relation_keys::RelationKeyRow; use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::graph::project_graph::ResourceRow; @@ -58,9 +62,6 @@ pub(crate) struct SqlRuleKey { /// A schema's entry audits, column specs and each audited column's index and name. pub(crate) type AuditedSchema = (Py, Arc, Vec<(usize, String)>); -/// `(name, type, nullability)` of one inferred output column. -pub(crate) type InferredRow = (String, Option, String); - /// `(name, type)` of a declared column. pub(crate) type DeclaredRow = (String, Option); @@ -87,30 +88,11 @@ pub(crate) struct CompiledViewClasses<'py> { pub(crate) family_proof: Bound<'py, PyAny>, } -/// A compiled relation location's parts. -#[derive(Clone)] -pub(crate) struct DestinationFact { - pub(crate) database: Option, - pub(crate) schema: Option, - pub(crate) name: String, - pub(crate) qualified_name: Option, - pub(crate) logical_schema: Option, - pub(crate) logical_database: Option, -} - /// What assembly and the semantic checks finished for one compiled model. -pub(crate) struct CompiledRecord { - /// `(resource type, name)` of each dependency. - deps: Vec<(String, String)>, - destination: DestinationFact, - /// Columns, lineage, binding diagnostics and dynamic proof kept from the native session. - analysis: AnalysisRecord, - macro_deps: Vec, - projections: CompiledProjections, -} +pub(crate) type CompiledRecord = project_models::CompiledRecord; /// A compiled model's Python attribute projections, each built once on first access. -struct CompiledProjections { +pub(crate) struct CompiledProjections { key: PyOnceLock>, deps: PyOnceLock>, } @@ -161,14 +143,6 @@ impl Default for ViewProjections { } } -/// A model's name, compiled query, authored file, and whether it calls a function. -pub(crate) struct ModelMetadataTexts { - pub(crate) name: String, - pub(crate) query_sql: String, - pub(crate) authored_sql: String, - pub(crate) calls_functions: bool, -} - /// Deps, session analysis, failed analysis's columns, standalone proof, and macro deps. #[derive(FromPyObject)] pub(crate) struct CompiledArguments( @@ -189,7 +163,7 @@ pub(crate) fn compiled_record( let CompiledArguments(deps, session, fallback, standalone_proof, macro_deps) = arguments; let analysis: AnalysisRecord = analysis_record(py, input, session, (fallback, standalone_proof))?; - let config: &Bound<'_, PyAny> = input.config.bind(py); + let config: &Bound<'_, PyAny> = input.host.config.bind(py); let values: Bound<'_, PyDict> = native_field(config, VALUES_ATTRIBUTE)?.downcast_into()?; Ok(CompiledRecord { deps, @@ -203,7 +177,7 @@ pub(crate) fn compiled_record( }, analysis, macro_deps, - projections: CompiledProjections::default(), + host: CompiledProjections::default(), }) } @@ -370,7 +344,7 @@ impl NativeCompiledModel { record .schema .as_ref() - .map(|schema| schema.columns.project(py).map(Bound::unbind)) + .map(|schema| schema.host.columns.project(py).map(Bound::unbind)) .transpose() }) } @@ -393,8 +367,8 @@ impl NativeCompiledModel { .map(|(index, column)| (index, column.name.clone())) .collect(); Ok(Some(( - schema.audits.clone_ref(py), - schema.columns.clone(), + schema.host.audits.clone_ref(py), + schema.host.columns.clone(), audited, ))) }) @@ -438,7 +412,7 @@ impl NativeCompiledModel { /// The model's name, `str(relative_path)` and authored file text. pub(crate) fn file_texts(&self, py: Python<'_>) -> PyResult<(String, String, Arc)> { self.read(py, |record, py| { - let model_file: &NativeModelFile = record.model_file.get(); + let model_file: &NativeModelFile = record.host.model_file.get(); Ok(( model_name(record), model_file.relative_text(py)?, @@ -457,7 +431,7 @@ impl NativeCompiledModel { Ok(ModelMetadataTexts { name: model_name(record), query_sql: record.query_sql.to_string(), - authored_sql: record.model_file.get().contents_text().to_string(), + authored_sql: record.host.model_file.get().contents_text().to_string(), calls_functions: record .references .iter() @@ -470,7 +444,7 @@ impl NativeCompiledModel { pub(crate) fn output_texts(&self, py: Python<'_>) -> PyResult<(String, Arc)> { self.read(py, |record, py| { Ok(( - record.model_file.get().posix_path(py)?, + record.host.model_file.get().posix_path(py)?, Arc::clone(&record.query_sql), )) }) @@ -491,7 +465,7 @@ impl NativeCompiledModel { py: Python<'_>, read: impl FnOnce(&NativeModelFile) -> PyResult, ) -> PyResult { - self.read(py, |record, _| read(record.model_file.get())) + self.read(py, |record, _| read(record.host.model_file.get())) } } @@ -527,7 +501,7 @@ impl NativeCompiledModel { self.read(py, |record, py| { let compiled: &CompiledRecord = compiled_of(record)?; compiled - .projections + .host .key .get_or_try_init(py, || { let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; @@ -541,7 +515,7 @@ impl NativeCompiledModel { pub(crate) fn deps(&self, py: Python<'_>) -> PyResult> { self.compiled(py, |compiled, py| { compiled - .projections + .host .deps .get_or_try_init(py, || { let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; @@ -572,7 +546,7 @@ impl NativeCompiledModel { #[getter] pub(crate) fn config(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(record.config.clone_ref(py))) + self.read(py, |record, py| Ok(record.host.config.clone_ref(py))) } #[getter] @@ -601,8 +575,15 @@ impl NativeCompiledModel { /// Each reference as `(kind, name, package, call argument count)`. #[getter] - pub(crate) fn reference_rows(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, _| Ok(record.references.clone())) + pub(crate) fn reference_rows(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, _| { + Ok(record + .references + .iter() + .cloned() + .map(reference_row_tuple) + .collect()) + }) } /// The model's declared key facts, for deriving its unique keys. @@ -688,7 +669,7 @@ impl NativeCompiledModel { Ok(record .schema .as_ref() - .is_some_and(|schema| !schema.dynamic_columns.bind(py).is_empty())) + .is_some_and(|schema| !schema.host.dynamic_columns.bind(py).is_empty())) }) } @@ -702,7 +683,7 @@ impl NativeCompiledModel { Ok(record .schema .as_ref() - .map_or(0, |schema| schema.columns.specs.len())) + .map_or(0, |schema| schema.host.columns.specs.len())) }) } @@ -799,19 +780,21 @@ impl NativeCompiledModel { #[getter] pub(crate) fn enum_declarations(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(record.enum_declarations.clone_ref(py))) + self.read(py, |record, py| { + Ok(record.host.enum_declarations.clone_ref(py)) + }) } #[getter] pub(crate) fn constant_declarations(&self, py: Python<'_>) -> PyResult> { self.read(py, |record, py| { - Ok(record.constant_declarations.clone_ref(py)) + Ok(record.host.constant_declarations.clone_ref(py)) }) } #[getter] pub(crate) fn enum_columns(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(record.enum_columns.clone_ref(py))) + self.read(py, |record, py| Ok(record.host.enum_columns.clone_ref(py))) } #[getter] @@ -822,7 +805,7 @@ impl NativeCompiledModel { let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; self.read(py, |record, py| { let name: String = model_name(record); - let path: Bound<'_, PyAny> = record.model_file.get().path_objects(py)?.1; + let path: Bound<'_, PyAny> = record.host.model_file.get().path_objects(py)?.1; compiled_of(record)? .analysis .binding_tuple(py, &classes.compiled, (&name, &path)) @@ -901,7 +884,7 @@ pub(crate) fn model_graph_row( ) -> PyResult { model.read(py, |record, py| { let compiled: &CompiledRecord = compiled_of(record)?; - let values: Bound<'_, PyAny> = native_field(record.config.bind(py), VALUES_ATTRIBUTE)?; + let values: Bound<'_, PyAny> = native_field(record.host.config.bind(py), VALUES_ATTRIBUTE)?; let mut tags: Vec = Vec::new(); if let Some(value) = values.downcast::()?.get_item(TAGS_KEY)? && (value.is_instance_of::() @@ -911,7 +894,7 @@ pub(crate) fn model_graph_row( tags.push(tag?.str()?.extract()?); } } - let folder: String = record.model_file.get().folder(py)?; + let folder: String = record.host.model_file.get().folder(py)?; Ok(ResourceRow { key: (MODEL_RESOURCE.to_owned(), model_name(record)), deps: compiled.deps.clone(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs index e7c8175d35..7be4f6bee8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs @@ -6,19 +6,22 @@ use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{FromPyObject, IntoPyObjectExt, pyclass, pymethods}; +use sqlbuild_project::models as project_models; +use sqlbuild_project::types::{NamedTypeRow, ObjectKeyRow}; + use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_project::rows::ReferenceRow; use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - DestinationFact, defaulted_destination, object_key, relation_location, + defaulted_destination, object_key, relation_location, }; use crate::bindings::_helpers::compiled_views::model_input_view::{ InputViewClasses, reference_tuple, view_classes, }; +use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; use crate::bindings::_helpers::graph::project_graph::ResourceRow; use crate::bindings::_helpers::records::fields::field as native_field; +use sqlbuild_project::models::DestinationFact; +use sqlbuild_project::models::ReferenceRow; -const TABLE_FUNCTION_TYPE: &str = "table_fn"; -const SCALAR_FUNCTION_TYPE: &str = "udf"; const SQL_LANGUAGE: &str = "sql"; const PYTHON_LANGUAGE: &str = "python"; const FUNCTION_FILE_ATTRIBUTE: &str = "relative_path"; @@ -35,9 +38,6 @@ pub(crate) struct FunctionClasses<'py> { function_language: Bound<'py, PyAny>, } -/// `(name, type)` of an argument or return column, its types template-expanded. -pub(crate) type NamedTypeRow = (String, String); - positional_record! { into; /// `(node type, name, tags, references as (kind, name, package))`. @@ -49,108 +49,74 @@ positional_record! { } } -/// `(kind, name)` of a compiled object key. -pub(crate) type ObjectKeyRow = (String, String); +/// What the function loop attached for one SQL or Python function. +pub(crate) type AttachedFunction = project_models::AttachedFunction; -/// A Python function's runtime, entry point and packages. -pub(crate) struct PythonRuntime { - pub(crate) runtime_version: Option, - pub(crate) entry_point: Option, - pub(crate) packages: Vec, -} +/// One attached function held by the compiled project. +pub(crate) type FunctionRecord = project_models::FunctionRecord; -/// What the function loop attached for one SQL or Python function. -pub(crate) struct AttachedFunction { +/// An attached function's user values. +pub(crate) struct FunctionHandles { /// The discovered function file. pub(crate) function_file: Py, - /// The file's relative path, as messages name it. - pub(crate) relative_path: String, - pub(crate) name: String, - pub(crate) arguments: Vec, - pub(crate) returns: String, - pub(crate) return_columns: Vec, - /// The expanded SQL body, or the Python function's source. - pub(crate) body: String, - pub(crate) references: Vec, - /// The physical and logical destination. - pub(crate) destination: DestinationFact, - /// The destination the function's fingerprint reads. - pub(crate) fingerprint_destination: DestinationFact, - /// `None` for a SQL function. - pub(crate) python: Option, - pub(crate) tags: Vec, - pub(crate) description: Option, /// Usage records, the identities the scope index is built from. pub(crate) usages: Py, } -impl AttachedFunction { - /// `function_node_type`: a table function when it declares return columns. - pub(crate) fn node_type(&self) -> &'static str { - if self.return_columns.is_empty() { - SCALAR_FUNCTION_TYPE - } else { - TABLE_FUNCTION_TYPE - } - } -} - -/// One attached function held by the compiled project. -pub(crate) struct FunctionRecord { - pub(crate) attached: AttachedFunction, - /// The dependency keys native project assembly resolved, once it ran. - deps: Option>, +/// A function record's Python projections, each built once on first access. +pub(crate) struct FunctionProjections { arguments: PyOnceLock>, return_columns: PyOnceLock>, - references: PyOnceLock>, - key: PyOnceLock>, + pub(crate) references: PyOnceLock>, + pub(crate) key: PyOnceLock>, } -impl FunctionRecord { - pub(crate) fn new(attached: AttachedFunction) -> Self { - Self { - attached, - deps: None, +/// The function record of an attached function, before assembly resolves its dependencies. +pub(crate) fn function_record(attached: AttachedFunction) -> FunctionRecord { + FunctionRecord { + attached, + deps: None, + host: FunctionProjections { arguments: PyOnceLock::new(), return_columns: PyOnceLock::new(), references: PyOnceLock::new(), key: PyOnceLock::new(), - } - } - - /// Keep the dependency keys native project assembly resolved for the function. - pub(crate) fn set_deps(&mut self, deps: Vec) { - self.deps = Some(deps); + }, } +} - fn named_types( - rows: &[NamedTypeRow], - class: &Bound<'_, PyAny>, - cache: &PyOnceLock>, - ) -> PyResult> { - let py: Python<'_> = class.py(); - cache - .get_or_try_init(py, || { - let mut items: Vec> = Vec::with_capacity(rows.len()); - for (name, type_text) in rows { - let fields: Bound<'_, PyDict> = PyDict::new(py); - fields.set_item("name", name)?; - fields.set_item("type", type_text)?; - items.push(class.call((), Some(&fields))?); - } - Ok::<_, pyo3::PyErr>(PyTuple::new(py, items)?.unbind()) - }) - .map(|items| items.clone_ref(py)) - } +/// `rows` as `class` records, built once into `cache`. +fn named_types( + rows: &[NamedTypeRow], + class: &Bound<'_, PyAny>, + cache: &PyOnceLock>, +) -> PyResult> { + let py: Python<'_> = class.py(); + cache + .get_or_try_init(py, || { + let mut items: Vec> = Vec::with_capacity(rows.len()); + for (name, type_text) in rows { + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("name", name)?; + fields.set_item("type", type_text)?; + items.push(class.call((), Some(&fields))?); + } + Ok::<_, pyo3::PyErr>(PyTuple::new(py, items)?.unbind()) + }) + .map(|items| items.clone_ref(py)) +} - fn language<'py>(&self, classes: &FunctionClasses<'py>) -> PyResult> { - let value: &str = if self.attached.python.is_some() { - PYTHON_LANGUAGE - } else { - SQL_LANGUAGE - }; - classes.function_language.call1((value,)) - } +/// The function's `FunctionLanguage` member. +fn language<'py>( + function: &FunctionRecord, + classes: &FunctionClasses<'py>, +) -> PyResult> { + let value: &str = if function.attached.python.is_some() { + PYTHON_LANGUAGE + } else { + SQL_LANGUAGE + }; + classes.function_language.call1((value,)) } fn read_function<'py, T>( @@ -197,10 +163,10 @@ macro_rules! function_getters { pub(crate) fn arguments(&self, py: Python<'_>) -> PyResult> { let classes: FunctionClasses<'_> = function_classes(&self.owner, py)?; read_function(&self.owner, self.index, py, |function, _| { - FunctionRecord::named_types( + named_types( &function.attached.arguments, &classes.function_argument, - &function.arguments, + &function.host.arguments, ) }) } @@ -223,10 +189,10 @@ macro_rules! function_getters { pub(crate) fn return_columns(&self, py: Python<'_>) -> PyResult> { let classes: FunctionClasses<'_> = function_classes(&self.owner, py)?; read_function(&self.owner, self.index, py, |function, _| { - FunctionRecord::named_types( + named_types( &function.attached.return_columns, &classes.function_return_column, - &function.return_columns, + &function.host.return_columns, ) }) } @@ -238,7 +204,7 @@ macro_rules! function_getters { reference_tuple( &classes, &function.attached.references, - &function.references, + &function.host.references, ) }) } @@ -247,7 +213,7 @@ macro_rules! function_getters { pub(crate) fn language<'py>(&self, py: Python<'py>) -> PyResult> { let classes: FunctionClasses<'py> = function_classes(&self.owner, py)?; read_function(&self.owner, self.index, py, |function, _| { - function.language(&classes) + language(function, &classes) }) } @@ -319,7 +285,7 @@ function_getters!(NativeFunctionInput, { #[getter] pub(crate) fn function_file(&self, py: Python<'_>) -> PyResult> { read_function(&self.owner, self.index, py, |function, py| { - Ok(function.attached.function_file.clone_ref(py)) + Ok(function.attached.host.function_file.clone_ref(py)) }) } @@ -390,7 +356,7 @@ function_getters!(NativeFunctionInput, { #[getter] pub(crate) fn declaration_usages(&self, py: Python<'_>) -> PyResult> { read_function(&self.owner, self.index, py, |function, py| { - Ok(function.attached.usages.clone_ref(py)) + Ok(function.attached.host.usages.clone_ref(py)) }) } @@ -490,6 +456,7 @@ function_getters!(NativeCompiledFunction, { let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; read_function(&self.owner, self.index, py, |function, py| { function + .host .key .get_or_try_init(py, || { object_key( @@ -523,7 +490,7 @@ function_getters!(NativeCompiledFunction, { pub(crate) fn relative_path(&self, py: Python<'_>) -> PyResult> { read_function(&self.owner, self.index, py, |function, py| { Ok(native_field( - function.attached.function_file.bind(py), + function.attached.host.function_file.bind(py), FUNCTION_FILE_ATTRIBUTE, )? .unbind()) @@ -534,7 +501,7 @@ function_getters!(NativeCompiledFunction, { pub(crate) fn source_file_path(&self, py: Python<'_>) -> PyResult> { read_function(&self.owner, self.index, py, |function, py| { Ok(native_field( - function.attached.function_file.bind(py), + function.attached.host.function_file.bind(py), FILE_PATH_ATTRIBUTE, )? .unbind()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/mod.rs index 41cfd50614..a8c2429d4e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/mod.rs @@ -7,6 +7,7 @@ pub(crate) mod getters; pub(crate) mod inputs_view; pub(crate) mod model_input_view; pub(crate) mod model_reads; +pub(crate) mod project_host; pub(crate) mod project_view; pub(crate) mod records; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs index d0b6281702..32134df859 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs @@ -1,5 +1,7 @@ //! A model's compile input, read from the native compiled project that owns it. +use sqlbuild_project::models as project_models; +use sqlbuild_project::models::ExpansionRecord; use std::sync::Arc; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; @@ -9,16 +11,18 @@ use pyo3::{FromPyObject, IntoPyObject, pyclass, pymethods}; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_project::rows::ReferenceRow; +use crate::bindings::_helpers::compiled_project::rows::{ReferenceTuple, reference_row_tuple}; use crate::bindings::_helpers::compiled_project::schema_facts::python_column_facts; use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - CompiledArguments, CompiledRecord, CompiledViewClasses, NativeCompiledModel, compiled_record, + CompiledArguments, CompiledViewClasses, NativeCompiledModel, compiled_record, }; +use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; use crate::bindings::_helpers::compiled_views::relation_keys::{RelationKeyRow, relation_key_row}; use crate::bindings::_helpers::discovery::model_file_record::{NativeModelFile, projected_text}; use crate::bindings::_helpers::model_config::header_metadata::HeaderColumns; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::render::macro_call_rows::SpanRow; +use crate::bindings::_helpers::render::macro_call_rows::{SpanRow, span_tuple}; +use sqlbuild_project::models::ReferenceRow; const MODEL_RELATION_KIND: &str = "ref"; @@ -34,57 +38,35 @@ pub(crate) struct InputViewClasses<'py> { expansion_span: Bound<'py, PyAny>, } -/// A model's attached header schema; its specs carry user-authored audit arguments. -pub(crate) struct SchemaRecord { - pub(crate) description: Option, +/// A model's attached header schema; the host keeps its columns, audits and dynamic columns. +pub(crate) type SchemaRecord = project_models::SchemaRecord; + +/// What the model loop finished for one model beyond its facts. +pub(crate) type ModelInputRecord = project_models::ModelInputRecord; + +/// A model schema's header columns and specs, which carry user-authored audit arguments. +pub(crate) struct SchemaHandles { pub(crate) columns: Arc, pub(crate) audits: Py, pub(crate) dynamic_columns: Py, - pub(crate) type_enforcement: bool, - pub(crate) model_schema: Option, } -/// The expansion passes of a model's query, from authored text to expanded SQL. -pub(crate) struct ExpansionRecord { - /// The authored query below the header. - pub(crate) authored_sql: Arc, - /// The declaration- and macro-expanded SQL before canonicalisation. - pub(crate) expanded_sql: Arc, - pub(crate) interpolation_spans: Vec, - pub(crate) declaration_spans: Vec, - pub(crate) macro_spans: Vec, - /// Whether interpolation left the authored query unchanged. - pub(crate) authored: bool, -} - -/// What the model loop finished for one model beyond its facts; handles carry user values. -pub(crate) struct ModelInputRecord { - /// The model's name, its file stem. - pub(crate) name: String, +/// A model input's user values and the attribute projections its views build. +pub(crate) struct ModelInputHandles { /// The discovered model file, owned by the discovery inputs. pub(crate) model_file: Py, /// The resolved config, whose values hold user values. pub(crate) config: Py, - /// Immutable text shared with discovery and expansions; Python reads build strings on demand. - pub(crate) query_sql: Arc, - pub(crate) macro_source_sql: Arc, - pub(crate) references: Vec, - pub(crate) schema: Option, - pub(crate) sql_validation_enabled: bool, pub(crate) rejected_opt_out: Option>, /// The model's own enums and constants, whose values are user values. pub(crate) enum_declarations: Py, pub(crate) constant_declarations: Py, pub(crate) enum_columns: Py, - pub(crate) macro_deps: Vec, /// Usage records, the identities the scope index is built from. pub(crate) macro_usages: Py, pub(crate) declaration_usages: Py, - pub(crate) expansion: ExpansionRecord, /// Attribute projections built on first access, shared by every view of the model. pub(crate) projections: InputProjections, - /// What assembly and the semantic checks finished, once the model is assembled. - pub(crate) compiled: Option, } /// A model input's Python attribute projections, each built once on first access. @@ -173,13 +155,13 @@ impl NativeModelInput { #[getter] pub(crate) fn model_file(&self, py: Python<'_>) -> PyResult> { self.read(py, |record, py| { - Ok(record.model_file.clone_ref(py).into_any()) + Ok(record.host.model_file.clone_ref(py).into_any()) }) } #[getter] pub(crate) fn config(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(record.config.clone_ref(py))) + self.read(py, |record, py| Ok(record.host.config.clone_ref(py))) } #[getter] @@ -208,8 +190,15 @@ impl NativeModelInput { /// Each reference as `(kind, name, package, call argument count)`. #[getter] - pub(crate) fn reference_rows(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, _| Ok(record.references.clone())) + pub(crate) fn reference_rows(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, _| { + Ok(record + .references + .iter() + .cloned() + .map(reference_row_tuple) + .collect()) + }) } /// The model's declared key facts, for deriving its unique keys. @@ -243,19 +232,21 @@ impl NativeModelInput { #[getter] pub(crate) fn enum_declarations(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(record.enum_declarations.clone_ref(py))) + self.read(py, |record, py| { + Ok(record.host.enum_declarations.clone_ref(py)) + }) } #[getter] pub(crate) fn constant_declarations(&self, py: Python<'_>) -> PyResult> { self.read(py, |record, py| { - Ok(record.constant_declarations.clone_ref(py)) + Ok(record.host.constant_declarations.clone_ref(py)) }) } #[getter] pub(crate) fn enum_columns(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(record.enum_columns.clone_ref(py))) + self.read(py, |record, py| Ok(record.host.enum_columns.clone_ref(py))) } #[getter] @@ -265,12 +256,14 @@ impl NativeModelInput { #[getter] pub(crate) fn macro_usages(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(record.macro_usages.clone_ref(py))) + self.read(py, |record, py| Ok(record.host.macro_usages.clone_ref(py))) } #[getter] pub(crate) fn declaration_usages(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(record.declaration_usages.clone_ref(py))) + self.read(py, |record, py| { + Ok(record.host.declaration_usages.clone_ref(py)) + }) } /// Record what assembly finished for the model; return its compiled model view. @@ -302,6 +295,7 @@ impl NativeModelInput { let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; self.read(py, |record, py| { record + .host .projections .lint_expansion .get_or_try_init(py, || { @@ -326,11 +320,11 @@ pub(crate) fn query_sql_text<'py>( py: Python<'py>, record: &ModelInputRecord, ) -> Bound<'py, PyString> { - let model_file: &NativeModelFile = record.model_file.get(); + let model_file: &NativeModelFile = record.host.model_file.get(); if Arc::ptr_eq(&record.query_sql, model_file.query_text()) { return model_file.query_sql(py); } - projected_text(py, &record.projections.query_sql, &record.query_sql) + projected_text(py, &record.host.projections.query_sql, &record.query_sql) } /// The authored-to-expanded passes, when interpolation left the authored query unchanged. @@ -340,6 +334,7 @@ pub(crate) fn authored_expansion( ) -> PyResult> { let py: Python<'_> = classes.sql_expansion.py(); record + .host .projections .sql_expansion .get_or_try_init(py, || { @@ -359,6 +354,7 @@ pub(crate) fn authored_expansion( /// Where a rejected `sql_analysis false` was authored, or `None`. pub(crate) fn rejected_opt_out(record: &ModelInputRecord, py: Python<'_>) -> Py { record + .host .rejected_opt_out .as_ref() .map_or_else(|| py.None(), |location| location.clone_ref(py)) @@ -369,7 +365,11 @@ pub(crate) fn references( classes: &InputViewClasses<'_>, record: &ModelInputRecord, ) -> PyResult> { - reference_tuple(classes, &record.references, &record.projections.references) + reference_tuple( + classes, + &record.references, + &record.host.projections.references, + ) } /// `CompileSqlReference` records of `rows`, built once into `cache`. @@ -407,11 +407,12 @@ pub(crate) fn model_relation_key_row( record: &ModelInputRecord, py: Python<'_>, ) -> PyResult { - let values: Bound<'_, PyAny> = native_field(record.config.bind(py), "values")?; + let values: Bound<'_, PyAny> = native_field(record.host.config.bind(py), "values")?; let declared = match (&record.schema, owner.input_schema_fact(record)) { - (Some(schema), Some(fact)) => { - Some((fact.columns.as_slice(), schema.audits.bind(py).as_any())) - } + (Some(schema), Some(fact)) => Some(( + fact.columns.as_slice(), + schema.host.audits.bind(py).as_any(), + )), _ => None, }; relation_key_row( @@ -434,6 +435,7 @@ pub(crate) fn schema_entry( ) -> PyResult> { let py: Python<'_> = classes.schema_model_entry.py(); record + .host .projections .schema_entry .get_or_try_init(py, || { @@ -445,9 +447,9 @@ pub(crate) fn schema_entry( fields.set_item("model_schema", &schema.model_schema)?; fields.set_item("description", &schema.description)?; fields.set_item("type_enforcement", schema.type_enforcement.then_some(true))?; - fields.set_item("columns", schema.columns.project(py)?)?; - fields.set_item("dynamic_columns", schema.dynamic_columns.bind(py))?; - fields.set_item("audits", schema.audits.bind(py))?; + fields.set_item("columns", schema.host.columns.project(py)?)?; + fields.set_item("dynamic_columns", schema.host.dynamic_columns.bind(py))?; + fields.set_item("audits", schema.host.audits.bind(py))?; Ok(classes.schema_model_entry.call((), Some(&fields))?.unbind()) }) .map(|entry| entry.clone_ref(py)) @@ -464,7 +466,11 @@ fn expansion_object( for spans in passes { let mut span_objects: Vec> = Vec::with_capacity(spans.len()); for span in spans.iter() { - span_objects.push(classes.expansion_span.call1(span.into_pyobject(py)?)?); + span_objects.push( + classes + .expansion_span + .call1(span_tuple(span).into_pyobject(py)?)?, + ); } pass_objects.push(PyTuple::new(py, span_objects)?); } @@ -494,7 +500,7 @@ pub(crate) fn attached_schema<'py>( }; Ok(Some(AttachedSchema { name: record.name.clone(), - dynamic_columns: schema.dynamic_columns.bind(py).clone().into_any(), + dynamic_columns: schema.host.dynamic_columns.bind(py).clone().into_any(), type_enforcement: schema.type_enforcement, })) }); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs index e0eebf6b54..752cddb036 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs @@ -3,9 +3,10 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - DeclaredRow, InferredRow, LineageInput, NativeCompiledModel, + DeclaredRow, LineageInput, NativeCompiledModel, }; use crate::bindings::_helpers::records::fields::field as native_field; +use sqlbuild_project::types::InferredRow; /// One lineage output with its transform kind and confidence values, and its sources. pub(crate) struct LineageFactRow { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_host.rs new file mode 100644 index 0000000000..046ad1d0f2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_host.rs @@ -0,0 +1,122 @@ +//! The Python host's slots beside the compiled project's records. + +use std::fmt::{Debug, Formatter, Result as FmtResult}; + +use sqlbuild_project::types::ProjectHost; + +use crate::bindings::_helpers::compiled_views::audit_views::{AuditHandles, AuditProjections}; +use crate::bindings::_helpers::compiled_views::compiled_model_analysis::AnalysisProjections; +use crate::bindings::_helpers::compiled_views::compiled_model_view::CompiledProjections; +use crate::bindings::_helpers::compiled_views::function_views::{ + FunctionHandles, FunctionProjections, +}; +use crate::bindings::_helpers::compiled_views::model_input_view::{ + ModelInputHandles, SchemaHandles, +}; +use crate::bindings::_helpers::compiled_views::seed_views::SeedHandles; +use crate::bindings::_helpers::compiled_views::source_views::{SourceHandles, SourceProjections}; + +/// The extension's host: Python handles and the views built from them. +pub(crate) struct PyProjectHost; + +impl ProjectHost for PyProjectHost { + type ModelInput = ModelInputHandles; + type Schema = SchemaHandles; + type CompiledModel = CompiledProjections; + type Analysis = AnalysisProjections; + type AttachedSource = SourceHandles; + type Source = SourceProjections; + type AttachedFunction = FunctionHandles; + type Function = FunctionProjections; + type AttachedAudit = AuditHandles; + type Audit = AuditProjections; + type Seed = SeedHandles; +} + +impl Debug for ModelInputHandles { + fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter + .debug_struct("ModelInputHandles") + .finish_non_exhaustive() + } +} + +impl Debug for SchemaHandles { + fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter + .debug_struct("SchemaHandles") + .finish_non_exhaustive() + } +} + +impl Debug for CompiledProjections { + fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter + .debug_struct("CompiledProjections") + .finish_non_exhaustive() + } +} + +impl Debug for AnalysisProjections { + fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter + .debug_struct("AnalysisProjections") + .finish_non_exhaustive() + } +} + +impl Debug for SourceHandles { + fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter + .debug_struct("SourceHandles") + .finish_non_exhaustive() + } +} + +impl Debug for SourceProjections { + fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter + .debug_struct("SourceProjections") + .finish_non_exhaustive() + } +} + +impl Debug for FunctionHandles { + fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter + .debug_struct("FunctionHandles") + .finish_non_exhaustive() + } +} + +impl Debug for FunctionProjections { + fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter + .debug_struct("FunctionProjections") + .finish_non_exhaustive() + } +} + +impl Debug for AuditHandles { + fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter + .debug_struct("AuditHandles") + .finish_non_exhaustive() + } +} + +impl Debug for AuditProjections { + fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter + .debug_struct("AuditProjections") + .finish_non_exhaustive() + } +} + +impl Debug for SeedHandles { + fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter + .debug_struct("SeedHandles") + .finish_non_exhaustive() + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs index ee166d5fb3..90cdf45297 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs @@ -503,7 +503,8 @@ impl NativeProjectView { for record in owner.inputs() { let expansion: Py = authored_expansion(&classes, record)?; if !expansion.is_none(py) { - let path: Bound<'_, PyAny> = record.model_file.get().path_objects(py)?.0; + let path: Bound<'_, PyAny> = + record.host.model_file.get().path_objects(py)?.0; expansions.set_item(path, expansion)?; } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/relation_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/relation_keys.rs index b388aee974..3330e9d5df 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/relation_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/relation_keys.rs @@ -17,7 +17,9 @@ use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompil use crate::bindings::_helpers::compiled_views::model_input_view::{ NativeModelInput, model_relation_key_row, }; -use crate::bindings::_helpers::compiled_views::source_views::{SourceKeyFacts, read_source_view}; +use crate::bindings::_helpers::compiled_views::source_views::{ + SourceKeyFacts, SourceViews, read_source_view, +}; use crate::bindings::_helpers::discovery::project_facts::seed_csv_settings; use crate::bindings::_helpers::records::fields::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs index 9fe4958451..b6f857e5f5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs @@ -7,13 +7,17 @@ use pyo3::types::PyTuple; use pyo3::{IntoPyObjectExt, pyclass, pymethods}; use sqlbuild_attachments::seeds::main::pair_seed_declarations::pair_seed_declarations; +use sqlbuild_project::models as project_models; + use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - DestinationFact, defaulted_destination, object_key, relation_location, + defaulted_destination, object_key, relation_location, }; use crate::bindings::_helpers::compiled_views::model_input_view::{InputViewClasses, view_classes}; +use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; use crate::bindings::_helpers::graph::project_graph::ResourceRow; use crate::bindings::_helpers::records::fields::field as native_field; +use sqlbuild_project::models::DestinationFact; const SEED_RESOURCE: &str = "seed"; const TAGS_ATTRIBUTE: &str = "tags"; @@ -42,19 +46,28 @@ pub(crate) struct SeedParts<'a> { } /// One seed CSV paired with its declaration; the discovered objects are discovery's. -pub(crate) struct SeedRecord { +pub(crate) type SeedRecord = project_models::SeedRecord; + +/// A seed's discovered objects and the key its views build. +pub(crate) struct SeedHandles { /// The discovered CSV file. seed_file: Py, /// The discovered declaration, whose spec carries user-authored column metadata. schema_entry: Py, /// The schema file declaring the seed. schema_file: Py, - name: String, - /// The destination native project assembly resolved, once it ran. - destination: Option, key: PyOnceLock>, } +/// What a seed record reads and keeps through its Python values. +pub(crate) trait SeedViews { + /// The seed's name, CSV file, declaration and declaring schema file. + fn parts(&self) -> SeedParts<'_>; + + /// Keep the destination native project assembly resolved for the seed. + fn set_namespace(&mut self, namespace: SeedNamespace); +} + /// Pair each declaration with the CSV file of its name, raising for the first without one. pub(crate) fn paired_seeds( declarations: Vec<(Py, Py)>, @@ -96,30 +109,30 @@ pub(crate) fn paired_seeds( declarations.into_iter().zip(names).zip(pairs) { seeds.push(SeedRecord { - seed_file: seed_files[file_index].clone_ref(py), - schema_entry, - schema_file, name, destination: None, - key: PyOnceLock::new(), + host: SeedHandles { + seed_file: seed_files[file_index].clone_ref(py), + schema_entry, + schema_file, + key: PyOnceLock::new(), + }, }); } Ok(seeds) } -impl SeedRecord { - /// The seed's name, CSV file, declaration and declaring schema file. - pub(crate) fn parts(&self) -> SeedParts<'_> { +impl SeedViews for SeedRecord { + fn parts(&self) -> SeedParts<'_> { SeedParts { name: &self.name, - seed_file: &self.seed_file, - schema_entry: &self.schema_entry, - schema_file: &self.schema_file, + seed_file: &self.host.seed_file, + schema_entry: &self.host.schema_entry, + schema_file: &self.host.schema_file, } } - /// Keep the destination native project assembly resolved for the seed. - pub(crate) fn set_namespace(&mut self, namespace: SeedNamespace) { + fn set_namespace(&mut self, namespace: SeedNamespace) { let SeedNamespace { database, schema, @@ -169,21 +182,21 @@ impl NativeSeedInput { #[getter] pub(crate) fn seed_file(&self, py: Python<'_>) -> PyResult> { read_seed(&self.owner, self.index, py, |seed, py| { - Ok(seed.seed_file.clone_ref(py)) + Ok(seed.host.seed_file.clone_ref(py)) }) } #[getter] pub(crate) fn schema_entry(&self, py: Python<'_>) -> PyResult> { read_seed(&self.owner, self.index, py, |seed, py| { - Ok(seed.schema_entry.clone_ref(py)) + Ok(seed.host.schema_entry.clone_ref(py)) }) } #[getter] pub(crate) fn schema_file(&self, py: Python<'_>) -> PyResult> { read_seed(&self.owner, self.index, py, |seed, py| { - Ok(seed.schema_file.clone_ref(py)) + Ok(seed.host.schema_file.clone_ref(py)) }) } } @@ -212,7 +225,7 @@ impl NativeCompiledSeed { pub(crate) fn graph_row(&self, py: Python<'_>) -> PyResult { read_seed(&self.owner, self.index, py, |seed, py| { let mut tags: Vec = Vec::new(); - for tag in native_field(seed.schema_entry.bind(py), TAGS_ATTRIBUTE)?.try_iter()? { + for tag in native_field(seed.host.schema_entry.bind(py), TAGS_ATTRIBUTE)?.try_iter()? { tags.push(tag?.str()?.extract()?); } Ok(ResourceRow { @@ -243,7 +256,8 @@ impl NativeCompiledSeed { pub(crate) fn key(&self, py: Python<'_>) -> PyResult> { let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; read_seed(&self.owner, self.index, py, |seed, py| { - seed.key + seed.host + .key .get_or_try_init(py, || { object_key(&classes.compiled, SEED_RESOURCE, &seed.name) }) @@ -264,21 +278,21 @@ impl NativeCompiledSeed { #[getter] pub(crate) fn seed_file(&self, py: Python<'_>) -> PyResult> { read_seed(&self.owner, self.index, py, |seed, py| { - Ok(seed.seed_file.clone_ref(py)) + Ok(seed.host.seed_file.clone_ref(py)) }) } #[getter] pub(crate) fn schema_entry(&self, py: Python<'_>) -> PyResult> { read_seed(&self.owner, self.index, py, |seed, py| { - Ok(seed.schema_entry.clone_ref(py)) + Ok(seed.host.schema_entry.clone_ref(py)) }) } #[getter] pub(crate) fn schema_file(&self, py: Python<'_>) -> PyResult> { read_seed(&self.owner, self.index, py, |seed, py| { - Ok(seed.schema_file.clone_ref(py)) + Ok(seed.host.schema_file.clone_ref(py)) }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_views.rs index 92b648c4cf..36592d98f3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_views.rs @@ -7,10 +7,13 @@ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{IntoPyObjectExt, pyclass, pymethods}; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; +use sqlbuild_project::models as project_models; + use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_project::schema_facts::python_column_facts; use crate::bindings::_helpers::compiled_views::compiled_model_view::object_key; use crate::bindings::_helpers::compiled_views::model_input_view::{InputViewClasses, view_classes}; +use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; use crate::bindings::_helpers::compiled_views::relation_keys::audit_key_rows; use crate::bindings::_helpers::graph::project_graph::ResourceRow; use crate::bindings::_helpers::records::fields::field as native_field; @@ -29,22 +32,18 @@ positional_record! { } } -/// `(database, schema)` native project assembly resolved for a source, or `None`. -pub(crate) type SourceNamespace = Option<(Option, Option)>; - /// What the source loop attached for one source: its expanded metadata and usages. -pub(crate) struct AttachedSource { +pub(crate) type AttachedSource = project_models::AttachedSource; + +/// One attached source held by the compiled project. +pub(crate) type SourceRecord = project_models::SourceRecord; + +/// An attached source's user values. +pub(crate) struct SourceHandles { /// The discovered source file. pub(crate) source_file: Py, /// The discovered entry; its spec carries user-authored metadata. pub(crate) entry: Py, - pub(crate) name: String, - pub(crate) contract: Option, - pub(crate) database: Option, - pub(crate) schema: Option, - pub(crate) table: Option, - pub(crate) expression: Option, - pub(crate) description: Option, /// Template-expanded user metadata. pub(crate) meta: Py, /// Template-expanded column specs, which carry user metadata and audit arguments. @@ -61,52 +60,53 @@ pub(crate) struct SourceKeyFacts { pub(crate) audits: Vec<(String, Option)>, } -/// One attached source held by the compiled project. -pub(crate) struct SourceRecord { - attached: AttachedSource, +/// A source record's key facts and entries, each built once from its Python values. +pub(crate) struct SourceProjections { key_facts: PyOnceLock, - /// The namespace native project assembly resolved, once it ran. - namespace: Option, attached_entry: PyOnceLock>, assembled_entry: PyOnceLock>, - key: PyOnceLock>, + pub(crate) key: PyOnceLock>, } -impl SourceRecord { - pub(crate) fn new(attached: AttachedSource) -> Self { - Self { - attached, +/// The source record of an attached source, before assembly resolves its namespace. +pub(crate) fn source_record(attached: AttachedSource) -> SourceRecord { + SourceRecord { + attached, + namespace: None, + host: SourceProjections { key_facts: PyOnceLock::new(), - namespace: None, attached_entry: PyOnceLock::new(), assembled_entry: PyOnceLock::new(), key: PyOnceLock::new(), - } - } - - /// The attached source. - pub(crate) fn attached(&self) -> &AttachedSource { - &self.attached + }, } +} +/// What a source record projects from its Python values, each built once. +pub(crate) trait SourceViews { /// The source's declared column facts and audit key rows, read once from its expanded specs. - pub(crate) fn key_facts(&self, py: Python<'_>) -> PyResult<&SourceKeyFacts> { - self.key_facts.get_or_try_init(py, || { + fn key_facts(&self, py: Python<'_>) -> PyResult<&SourceKeyFacts>; + + /// The attached `SourceEntry`: the discovered entry with its expanded metadata. + fn attached_entry(&self, py: Python<'_>) -> PyResult>; + + /// The assembled `SourceEntry`: the attached entry in its resolved namespace. + fn assembled_entry(&self, py: Python<'_>) -> PyResult>; +} + +impl SourceViews for SourceRecord { + fn key_facts(&self, py: Python<'_>) -> PyResult<&SourceKeyFacts> { + self.host.key_facts.get_or_try_init(py, || { Ok(SourceKeyFacts { - columns: python_column_facts(self.attached.columns.bind(py).as_any(), None)?, - audits: audit_key_rows(self.attached.audits.bind(py).as_any())?, + columns: python_column_facts(self.attached.host.columns.bind(py).as_any(), None)?, + audits: audit_key_rows(self.attached.host.audits.bind(py).as_any())?, }) }) } - /// Keep the namespace native project assembly resolved for the source. - pub(crate) fn set_namespace(&mut self, namespace: SourceNamespace) { - self.namespace = Some(namespace); - } - - /// The attached `SourceEntry`: the discovered entry with its expanded metadata. fn attached_entry(&self, py: Python<'_>) -> PyResult> { - self.attached_entry + self.host + .attached_entry .get_or_try_init(py, || { let source: &AttachedSource = &self.attached; let changes: Bound<'_, PyDict> = PyDict::new(py); @@ -116,21 +116,21 @@ impl SourceRecord { changes.set_item("table", &source.table)?; changes.set_item("expression", &source.expression)?; changes.set_item("description", &source.description)?; - changes.set_item("meta", source.meta.bind(py))?; - changes.set_item("columns", source.columns.bind(py))?; - changes.set_item("audits", source.audits.bind(py))?; - replaced(py, source.entry.bind(py), &changes) + changes.set_item("meta", source.host.meta.bind(py))?; + changes.set_item("columns", source.host.columns.bind(py))?; + changes.set_item("audits", source.host.audits.bind(py))?; + replaced(py, source.host.entry.bind(py), &changes) }) .map(|entry| entry.clone_ref(py)) } - /// The assembled `SourceEntry`: the attached entry in its resolved namespace. fn assembled_entry(&self, py: Python<'_>) -> PyResult> { let attached: Py = self.attached_entry(py)?; let Some(Some((database, schema))) = &self.namespace else { return Ok(attached); }; - self.assembled_entry + self.host + .assembled_entry .get_or_try_init(py, || { let changes: Bound<'_, PyDict> = PyDict::new(py); changes.set_item("database", database)?; @@ -206,14 +206,14 @@ impl NativeSourceInput { #[getter] pub(crate) fn source_file(&self, py: Python<'_>) -> PyResult> { read_source(&self.owner, self.index, py, |source, py| { - Ok(source.attached.source_file.clone_ref(py)) + Ok(source.attached.host.source_file.clone_ref(py)) }) } #[getter] pub(crate) fn declaration_usages(&self, py: Python<'_>) -> PyResult> { read_source(&self.owner, self.index, py, |source, py| { - Ok(source.attached.usages.clone_ref(py)) + Ok(source.attached.host.usages.clone_ref(py)) }) } @@ -224,7 +224,7 @@ impl NativeSourceInput { let attached: &AttachedSource = &source.attached; Ok(SourceAssemblyRow { name: attached.name.clone(), - managed: !native_field(attached.entry.bind(py), "loader")?.is_none(), + managed: !native_field(attached.host.entry.bind(py), "loader")?.is_none(), database: attached.database.clone(), schema: attached.schema.clone(), }) @@ -265,6 +265,7 @@ impl NativeCompiledSource { let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; read_source(&self.owner, self.index, py, |source, py| { source + .host .key .get_or_try_init(py, || { object_key(&classes.compiled, SOURCE_RESOURCE, &source.attached.name) @@ -295,7 +296,7 @@ impl NativeCompiledSource { #[getter] pub(crate) fn source_file(&self, py: Python<'_>) -> PyResult> { read_source(&self.owner, self.index, py, |source, py| { - Ok(source.attached.source_file.clone_ref(py)) + Ok(source.attached.host.source_file.clone_ref(py)) }) } } @@ -322,7 +323,7 @@ pub(crate) fn retain_sources( let first: usize = project.sources().len(); let count: usize = attached.len(); for source in attached { - project.retain_source(SourceRecord::new(source)); + project.retain_source(source_record(source)); } drop(project); (first..first + count) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs index bee6d46f46..29e2899e36 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, HashMap}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple, PyTupleMethods}; use sqlbuild_analysis::compiled_project::main::merge_call_site_references::merge_call_site_references; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; @@ -27,8 +27,7 @@ use crate::bindings::_helpers::compile_context::delta::record_source_reads; use crate::bindings::_helpers::compiled_project::attachment_inputs::{ AuditBlockRow, AuditFileRow, AuditSettings, }; -use crate::bindings::_helpers::compiled_project::rows::ReferenceRow; -use crate::bindings::_helpers::compiled_views::audit_views::{AttachedAudit, AttachedTarget}; +use crate::bindings::_helpers::compiled_views::audit_views::{AttachedAudit, AuditHandles}; use crate::bindings::_helpers::input_loops::model_loop::{ LoopContext, NativeModelLoop, compile_error, helped_compile_error, }; @@ -46,6 +45,8 @@ use crate::bindings::_helpers::model_config::header_metadata::thresholds; use crate::bindings::_helpers::model_config::model_validator::NativeModelValidator; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::result_output::integration_values::json_value; +use sqlbuild_project::models::AttachedTarget; +use sqlbuild_project::models::ReferenceRow; const THRESHOLDS_KEY: &str = "thresholds"; const MINIMUM_SAMPLES_KEY: &str = "minimum_samples"; @@ -308,24 +309,26 @@ impl<'py> AuditScope<'_, 'py> { usages.push(self.context.classes.usage_record.call((), Some(&fields))?); } Ok(AttachedAudit { - audit_file: audit_file.clone_ref(py), - audit_block: audit_block.clone_ref(py), definition_name: audit_name, sql_body: expanded.sql_body, measurement: *measurement, evidence_sql: expanded.evidence_sql, - thresholds: thresholds.unbind(), - minimum_samples: minimum_samples.unbind(), - evidence_limit: py.None(), references: expanded.references, target: None, severity, run_scope, always_run, - usages: PyTuple::new(py, usages)?.unbind(), name: None, description: None, meta: None, + host: AuditHandles { + audit_file: audit_file.clone_ref(py), + audit_block: audit_block.clone_ref(py), + thresholds: thresholds.unbind(), + minimum_samples: minimum_samples.unbind(), + evidence_limit: py.None(), + usages: PyTuple::new(py, usages)?.unbind(), + }, }) } @@ -451,9 +454,13 @@ impl<'py> AuditScope<'_, 'py> { return Err(compile_error(self.context.classes, &message, None)); } usages.extend(expanded.usages); + let evidence_limit: Py = native_field(instance, "evidence_limit")?.unbind(); + let always_run: bool = native_field(instance, ALWAYS_RUN_KEY)?.is_truthy()?; + let usages: Py = PyTuple::new(py, usages)?.unbind(); + let name: Option = native_field(instance, "name")?.extract()?; + let description: Option = native_field(instance, "description")?.extract()?; + let meta: Option = self.meta(instance, &label)?; Ok(AttachedAudit { - audit_file: definition.file.file.clone_ref(py), - audit_block: definition.block.block.clone_ref(py), definition_name: definition .block .name @@ -462,9 +469,6 @@ impl<'py> AuditScope<'_, 'py> { sql_body: expanded.sql_body, measurement, evidence_sql: expanded.evidence_sql, - thresholds: thresholds.unbind(), - minimum_samples: minimum_samples.unbind(), - evidence_limit: native_field(instance, "evidence_limit")?.unbind(), references: expanded.references, target: Some(AttachedTarget { kind: owner.kind.to_owned(), @@ -473,11 +477,18 @@ impl<'py> AuditScope<'_, 'py> { }), severity, run_scope, - always_run: native_field(instance, ALWAYS_RUN_KEY)?.is_truthy()?, - usages: PyTuple::new(py, usages)?.unbind(), - name: native_field(instance, "name")?.extract()?, - description: native_field(instance, "description")?.extract()?, - meta: self.meta(instance, &label)?, + always_run, + name, + description, + meta, + host: AuditHandles { + audit_file: definition.file.file.clone_ref(py), + audit_block: definition.block.block.clone_ref(py), + thresholds: thresholds.unbind(), + minimum_samples: minimum_samples.unbind(), + evidence_limit, + usages, + }, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs index 74c413a4c3..1798cc6884 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs @@ -34,10 +34,8 @@ use crate::bindings::_helpers::compile_context::delta::record_source_reads; use crate::bindings::_helpers::compiled_project::attachment_inputs::{ FunctionRow, FunctionSettings, }; -use crate::bindings::_helpers::compiled_project::rows::ReferenceRow; -use crate::bindings::_helpers::compiled_views::compiled_model_view::DestinationFact; use crate::bindings::_helpers::compiled_views::function_views::{ - AttachedFunction, NamedTypeRow, PythonRuntime, + AttachedFunction, FunctionHandles, }; use crate::bindings::_helpers::input_loops::model_loop::{ LoopContext, NativeModelLoop, compile_error, @@ -48,6 +46,10 @@ use crate::bindings::_helpers::input_loops::source_loop::{ AuthoredExpansion, expanded_authored_sql, expanded_template, parse_checked, scoped_declarations, }; use crate::bindings::_helpers::model_config::model_validator::NativeModelValidator; +use sqlbuild_project::models::DestinationFact; +use sqlbuild_project::models::PythonRuntime; +use sqlbuild_project::models::ReferenceRow; +use sqlbuild_project::types::NamedTypeRow; const PRESERVE_TARGET_VALUE: &str = "preserve"; const GENERIC_DIALECT: &str = "generic"; @@ -213,7 +215,6 @@ fn sql_function(scope: &FunctionScope<'_, '_>, row: FunctionRow) -> PyResult, row: FunctionRow) -> PyResult, row: FunctionRow) -> PyResult< } raise_stage(context, &header, HeaderStage::Metadata)?; Ok(AttachedFunction { - function_file, relative_path, destination: destination(&name, &namespace, false), fingerprint_destination: destination(&name, &namespace, true), @@ -290,7 +293,10 @@ fn python_function(scope: &FunctionScope<'_, '_>, row: FunctionRow) -> PyResult< }), tags: header.tags, description: header.description, - usages: PyTuple::empty(py).unbind(), + host: FunctionHandles { + function_file, + usages: PyTuple::empty(py).unbind(), + }, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs index f67dd3dbe8..96fe87e2b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs @@ -35,7 +35,6 @@ use crate::bindings::_helpers::compiled_project::attachment_inputs::SourceSettin use crate::bindings::_helpers::compiled_project::attachment_inputs::SqlTestSettings; use crate::bindings::_helpers::compiled_project::attachment_inputs::TestFileRow; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_project::rows::ReferenceRow; use crate::bindings::_helpers::compiled_views::audit_views::AttachedAudit; use crate::bindings::_helpers::compiled_views::function_views::AttachedFunction; use crate::bindings::_helpers::compiled_views::model_input_view::NativeModelInput; @@ -102,6 +101,7 @@ use crate::bindings::_helpers::sqltext::static_sql::{ InterpolationRow as SqlInterpolationRow, InterpolationSources, interpolation_batch, }; use crate::bindings::types::CompilerDetach; +use sqlbuild_project::models::ReferenceRow; const MATERIALIZED_KEY: &str = "materialized"; const CURSOR_KEY: &str = "cursor"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_external.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_external.rs index 5ec1658ae3..fc19a93486 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_external.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_external.rs @@ -3,8 +3,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::PyDict; -use crate::bindings::_helpers::compiled_project::rows::ReferenceRow; use crate::bindings::_helpers::input_loops::model_loop::{LoopContext, compile_error}; +use sqlbuild_project::models::ReferenceRow; const DBT_REF_KIND: &str = "dbt_ref"; const VALIDATE_METHOD: &str = "validate_reference"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_hooks.rs index 74931699bc..4c09d0f900 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_hooks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_hooks.rs @@ -19,7 +19,6 @@ use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_ use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; use crate::bindings::_helpers::attachments::argument_values::argument_pairs; -use crate::bindings::_helpers::compiled_project::rows::ReferenceRow; use crate::bindings::_helpers::input_loops::model_loop::{ LoopContext, NativeModelLoop, VisibleDeclarations, compile_error, record_reads, validation_outcome, @@ -43,6 +42,7 @@ use crate::bindings::_helpers::render::declaration_expansion::{ }; use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan; use crate::bindings::_helpers::sqltext::static_sql::{InterpolationSources, interpolated}; +use sqlbuild_project::models::ReferenceRow; const REFERENCES_METHOD: &str = "references"; const STATEMENT_ATTRIBUTE: &str = "statement"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs index fe1d13a1b3..da752c56ac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs @@ -17,9 +17,8 @@ use crate::bindings::_helpers::compiled_project::project::{ use crate::bindings::_helpers::compiled_project::project::{ DeclarationPy, config_values, declarations, }; -use crate::bindings::_helpers::compiled_project::rows::ReferenceRow; use crate::bindings::_helpers::compiled_views::model_input_view::{ - ExpansionRecord, InputProjections, ModelInputRecord, SchemaRecord, + InputProjections, ModelInputHandles, ModelInputRecord, SchemaHandles, SchemaRecord, }; use crate::bindings::_helpers::discovery::model_file_record::{ModelFileTexts, NativeModelFile}; use crate::bindings::_helpers::input_loops::model_loop_header::AttachedHeader; @@ -29,6 +28,8 @@ use crate::bindings::_helpers::render::declaration_expansion::{ExpandedDeclarati use crate::bindings::_helpers::render::macro_call_rows::{SpanRow, span_row}; use crate::bindings::_helpers::render::macro_expansion::ExpansionOutput; use crate::bindings::_helpers::sqltext::declaration_references::ScannedReference; +use sqlbuild_project::models::ExpansionRecord; +use sqlbuild_project::models::ReferenceRow; const HEADER_KEYS_ATTRIBUTE: &str = "model_header_keys"; const LAYER_SCHEMA_ATTRIBUTE: &str = "layer_schema"; @@ -207,11 +208,13 @@ pub(crate) fn finished_record<'py>( }; let schema: Option = finished.header.schema.as_ref().map(|schema| SchemaRecord { description: finished.header.description.clone(), - columns: schema.columns.clone(), - audits: schema.audits.clone().unbind(), - dynamic_columns: schema.dynamic_columns.clone().unbind(), type_enforcement: schema.type_enforcement, model_schema: schema.model_schema.clone(), + host: SchemaHandles { + columns: schema.columns.clone(), + audits: schema.audits.clone().unbind(), + dynamic_columns: schema.dynamic_columns.clone().unbind(), + }, }); let authored: Arc = authored_query_sql.clone(); let macro_source: Arc = shared_sql(&finished.declarations.spliced.sql, &[&authored]); @@ -219,25 +222,12 @@ pub(crate) fn finished_record<'py>( let query: Arc = shared_sql(&finished.query_sql, &[&authored, ¯o_source, &expanded]); let record: ModelInputRecord = ModelInputRecord { name: name.clone(), - model_file, - config: config.unbind(), query_sql: Arc::clone(&query), macro_source_sql: macro_source, references: finished.references.clone(), schema, sql_validation_enabled: finished.analysis.0, - rejected_opt_out: finished.analysis.1.map(Bound::unbind), - enum_declarations: local_enums.unbind(), - constant_declarations: local_constants.unbind(), - enum_columns: finished - .header - .schema - .as_ref() - .map_or_else(|| PyDict::new(py), |schema| schema.enum_columns.clone()) - .unbind(), macro_deps, - macro_usages: PyTuple::new(py, macro_usages)?.unbind(), - declaration_usages: PyTuple::new(py, unique(py, declaration_usages)?)?.unbind(), expansion: ExpansionRecord { authored: *interpolated_sql == *authored_query_sql, authored_sql: authored, @@ -246,8 +236,23 @@ pub(crate) fn finished_record<'py>( declaration_spans: span_rows(&finished.declarations.spliced), macro_spans, }, - projections: InputProjections::default(), compiled: None, + host: ModelInputHandles { + model_file, + config: config.unbind(), + rejected_opt_out: finished.analysis.1.map(Bound::unbind), + enum_declarations: local_enums.unbind(), + constant_declarations: local_constants.unbind(), + enum_columns: finished + .header + .schema + .as_ref() + .map_or_else(|| PyDict::new(py), |schema| schema.enum_columns.clone()) + .unbind(), + macro_usages: PyTuple::new(py, macro_usages)?.unbind(), + declaration_usages: PyTuple::new(py, unique(py, declaration_usages)?)?.unbind(), + projections: InputProjections::default(), + }, }; let (values, encoding_error) = config_values(&stripped); let facts: CompiledModelFacts = plain_model_facts(PlainModel { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_inputs.rs index bf8ba2d2e2..aaa7af2b28 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_inputs.rs @@ -6,7 +6,7 @@ use sqlbuild_analysis::compiled_project::main::header_metadata_keys::is_header_m use sqlbuild_analysis::compiled_project::models::{AnalysisOverride, ModelReferenceFact}; use sqlbuild_sqltext::sql_references::models::SqlReference; -use crate::bindings::_helpers::compiled_project::rows::ReferenceRow; +use sqlbuild_project::models::ReferenceRow; const SQL_ANALYSIS_KEY: &str = "sql_analysis"; const SQL_VALIDATION_KEY: &str = "sql_validation"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_macros.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_macros.rs index 29dd88c6a0..d0c780d698 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_macros.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_macros.rs @@ -10,7 +10,6 @@ use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_ use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::bindings::_helpers::compiled_project::rows::ReferenceRow; use crate::bindings::_helpers::input_loops::model_loop::{ LoopContext, VisibleDeclarations, compile_error, }; @@ -20,6 +19,7 @@ use crate::bindings::_helpers::render::macro_expansion::{ ExpansionOutput, NativeMacroExpansion, StateInput, expanded, }; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; +use sqlbuild_project::models::ReferenceRow; /// Fields of `_VisibleModelDeclarations` a `DeclarationResolutionContext` is built from. const CONTEXT_FIELDS: [&str; 9] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs index baea73a399..95469078fa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs @@ -9,12 +9,11 @@ use crate::bindings::_helpers::compiled_project::attachment_inputs::{ ResourceAttachment, ScenarioRow, SourceRow, SourceSettings, SqlTestSettings, TestFileRow, }; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_project::rows::ReferenceRow; use crate::bindings::_helpers::compiled_views::audit_views::NativeAuditInput; use crate::bindings::_helpers::compiled_views::function_views::{ AttachedFunction, NativeFunctionInput, }; -use crate::bindings::_helpers::compiled_views::seed_views::SeedParts; +use crate::bindings::_helpers::compiled_views::seed_views::{SeedParts, SeedViews}; use crate::bindings::_helpers::compiled_views::source_views::{AttachedSource, NativeSourceInput}; use crate::bindings::_helpers::input_loops::audit_loop::{ AttachmentOwner, model_owner, resource_owner, @@ -22,6 +21,7 @@ use crate::bindings::_helpers::input_loops::audit_loop::{ use crate::bindings::_helpers::input_loops::model_loop::NativeModelLoop; use crate::bindings::_helpers::input_loops::resource_diagnostics::resource_diagnostics; use crate::bindings::_helpers::records::fields::field as native_field; +use sqlbuild_project::models::ReferenceRow; const RELATIVE_PATH_ATTRIBUTE: &str = "relative_path"; const SOURCE_OWNER: &str = "source"; @@ -68,7 +68,7 @@ pub(crate) fn attach_functions( .iter() .map(|record| { Ok(( - record.model_file.get().relative_text(py)?, + record.host.model_file.get().relative_text(py)?, record.references.clone(), )) }) @@ -147,25 +147,26 @@ fn attachment_owners<'py>( }; owners.push(model_owner( record.name.clone(), - record.model_file.get().relative_text(py)?, + record.host.model_file.get().relative_text(py)?, ( - schema.audits.bind(py), - (schema.columns.as_ref(), &fact.columns), + schema.host.audits.bind(py), + (schema.host.columns.as_ref(), &fact.columns), ), )?); } for source in project.sources() { let attached: &AttachedSource = source.attached(); - let path: String = native_field(attached.source_file.bind(py), RELATIVE_PATH_ATTRIBUTE)? - .str()? - .to_string(); + let path: String = + native_field(attached.host.source_file.bind(py), RELATIVE_PATH_ATTRIBUTE)? + .str()? + .to_string(); owners.push(resource_owner( SOURCE_OWNER, attached.name.clone(), (path.clone(), path), ( - attached.audits.bind(py).as_any(), - attached.columns.bind(py).as_any(), + attached.host.audits.bind(py).as_any(), + attached.host.columns.bind(py).as_any(), ), )?); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs index f4c86683fb..ba96e7ae51 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs @@ -25,10 +25,10 @@ use crate::bindings::_helpers::compiled_project::attachment_inputs::{ ScenarioDescriptionRow, }; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_project::rows::ReferenceRow; -use crate::bindings::_helpers::compiled_views::seed_views::SeedParts; +use crate::bindings::_helpers::compiled_views::seed_views::{SeedParts, SeedViews}; use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::records::fields::field as native_field; +use sqlbuild_project::models::ReferenceRow; const MISSING_DESCRIPTION_CODE: &str = "P010"; const CURSOR_WITHOUT_INPUTS_CODE: &str = "P011"; @@ -222,7 +222,7 @@ fn missing_descriptions<'py>( let py: Python<'py> = diagnostics.py; let mut found: Vec>> = Vec::new(); for record in project.inputs() { - let model_file: &NativeModelFile = record.model_file.get(); + let model_file: &NativeModelFile = record.host.model_file.get(); let name: String = record.name.clone(); let path: String = model_file.relative_text(py)?; let contents: &str = model_file.contents_text(); @@ -360,13 +360,14 @@ fn source_description<'py>( let py: Python<'py> = diagnostics.py; let attached = source.attached(); let name: &str = &attached.name; - let source_file: &Bound<'py, PyAny> = attached.source_file.bind(py); + let source_file: &Bound<'py, PyAny> = attached.host.source_file.bind(py); let path: String = native_field(source_file, "relative_path")? .str()? .extract()?; let contents_value = native_field(source_file, "contents")?; let contents = contents_value.downcast::()?.to_str()?; - let loader_name: Option = native_field(attached.entry.bind(py), "loader")?.extract()?; + let loader_name: Option = + native_field(attached.host.entry.bind(py), "loader")?.extract()?; let loader: Option<(&String, &LoaderRow)> = loader_name .as_ref() .and_then(|loader| diagnostics.settings.loaders.get_key_value(loader)); @@ -406,7 +407,7 @@ fn function_description<'py>( let py: Python<'py> = diagnostics.py; let attached = &function.attached; let name: &str = &attached.name; - let function_file: &Bound<'py, PyAny> = attached.function_file.bind(py); + let function_file: &Bound<'py, PyAny> = attached.host.function_file.bind(py); let path: String = native_field(function_file, "relative_path")? .str()? .extract()?; @@ -472,7 +473,7 @@ fn cursor_without_inputs<'py>( let mut errors: Vec> = Vec::new(); for record in project.inputs() { let values: Bound<'py, PyDict> = - native_field(record.config.bind(py), "values")?.downcast_into()?; + native_field(record.host.config.bind(py), "values")?.downcast_into()?; let kinds: Vec<&str> = record .references .iter() @@ -481,7 +482,7 @@ fn cursor_without_inputs<'py>( let materialized: Option = config_text(&values, "materialized")?; let cursor: Option = config_text(&values, CURSOR_KEY)?; let strategy: Option = config_text(&values, "microbatch_strategy")?; - let model_file: &NativeModelFile = record.model_file.get(); + let model_file: &NativeModelFile = record.host.model_file.get(); let name: String = record.name.clone(); let Some((message, help)) = cursor_lacks_inputs( &name, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs index 06a53815b2..60bf243cb6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs @@ -8,8 +8,9 @@ use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicCheck; use sqlbuild_model_config::templates::models::TemplateOptions; use crate::bindings::_helpers::compiled_project::attachment_inputs::{SourceRow, SourceSettings}; -use crate::bindings::_helpers::compiled_project::rows::ReferenceRow; -use crate::bindings::_helpers::compiled_views::source_views::{AttachedSource, replaced}; +use crate::bindings::_helpers::compiled_views::source_views::{ + AttachedSource, SourceHandles, replaced, +}; use crate::bindings::_helpers::input_loops::model_loop::{ LoopContext, NativeModelLoop, compile_error, record_reads, }; @@ -32,6 +33,7 @@ use crate::bindings::_helpers::render::declaration_expansion::{ use crate::bindings::_helpers::scopes::declaration_contexts::NativeDeclarationContexts; use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan; use crate::bindings::_helpers::sqltext::static_sql::{InterpolationSources, interpolated}; +use sqlbuild_project::models::ReferenceRow; const MACRO_TOKEN: char = '@'; const NATIVE_CONTEXTS_ATTRIBUTE: &str = "native_contexts"; @@ -114,8 +116,6 @@ pub(crate) fn attached_sources<'py>( .extract::>()? .or_else(|| default_contract.clone()); attached.push(AttachedSource { - source_file, - entry: entry.unbind(), name, contract, database, @@ -123,10 +123,14 @@ pub(crate) fn attached_sources<'py>( table, expression, description, - meta: meta.unbind(), - columns: PyTuple::new(py, columns)?.unbind(), - audits: PyTuple::new(py, audits)?.unbind(), - usages: usages.unbind(), + host: SourceHandles { + source_file, + entry: entry.unbind(), + meta: meta.unbind(), + columns: PyTuple::new(py, columns)?.unbind(), + audits: PyTuple::new(py, audits)?.unbind(), + usages: usages.unbind(), + }, }); } Ok(attached) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index e9ca93cb6e..bac3ace9cd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -28,7 +28,6 @@ use sqlbuild_analysis::planner_changes::models::{ }; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with, imported}; -use crate::bindings::_helpers::compiled_views::compiled_model_view::InferredRow; use crate::bindings::_helpers::compiled_views::model_reads::{inferred_rows, star_flags}; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_changes::renames::origin_names; @@ -48,6 +47,7 @@ use crate::bindings::_helpers::records::fields::mapping_items; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::semantic_shapes::shapes::snapshot_validity_columns; use crate::bindings::_helpers::type_system::equality::types_equal as native_types_equal; +use sqlbuild_project::types::InferredRow; const CHANGES_LOGGER: &str = "sqlbuild.planner.changes"; const REPLAY_CONFIG_KEY: &str = "replay_on_change"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs index aed49e1448..edd894a333 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs @@ -5,13 +5,13 @@ use sqlbuild_analysis::planner_fixtures::main::merge_metadata::merge_metadata; use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; use sqlbuild_core::text::main::python_casefold::python_casefold; -use crate::bindings::_helpers::compiled_views::compiled_model_view::InferredRow; use crate::bindings::_helpers::compiled_views::model_reads::{inferred_rows, star_flags}; use crate::bindings::_helpers::model_config::typed_config::{config_value, model_values}; use crate::bindings::_helpers::planner_strategy::constants::ENFORCED_CONTRACT; use crate::bindings::_helpers::planner_strategy::fixture_values::columns; use crate::bindings::_helpers::planner_strategy::models::FixtureRelation; use crate::bindings::_helpers::records::fields::field as native_field; +use sqlbuild_project::types::InferredRow; const CONTRACT_KEY: &str = "contract"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs index e905651bf2..9279669fe2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs @@ -12,7 +12,9 @@ use crate::bindings::_helpers::compiled_project::schema_facts::python_column_fac use crate::bindings::_helpers::compiled_views::model_input_view::{ attached_columns, attached_schema, }; -use crate::bindings::_helpers::compiled_views::source_views::{AttachedSource, read_source_view}; +use crate::bindings::_helpers::compiled_views::source_views::{ + AttachedSource, SourceViews, read_source_view, +}; use crate::bindings::_helpers::records::construct::enum_member; use crate::bindings::_helpers::records::fields::{enum_member_value, field as native_field}; use crate::bindings::_helpers::records::ids::RecordId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs index 3619725387..3042a7bd33 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs @@ -19,7 +19,6 @@ use sqlbuild_model_config::templates::models::Scalar; use crate::bindings::_helpers::authored_values::project_vars::project_var_text; use crate::bindings::_helpers::boundary::panics::{NativeCompilerError, compiler_error}; use crate::bindings::_helpers::boundary::python_calls::{imported, keywords}; -use crate::bindings::_helpers::compiled_project::rows::ReferenceRow; use crate::bindings::_helpers::compiled_views::model_input_view::input_reference_rows; use crate::bindings::_helpers::model_config::template_data::record_reads; use crate::bindings::_helpers::project_assembly::models::{ @@ -31,6 +30,7 @@ use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::fields::mapping_items; use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::types::CompilerDetach; +use sqlbuild_project::models::ReferenceRow; const OS_MODULE: &str = "os"; const PROJECT_ASSEMBLY_CONTEXT: &str = "native project assembly"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs index ad947ea062..13655dfee2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs @@ -24,7 +24,7 @@ use crate::bindings::_helpers::render::declaration_expansion::{ use crate::bindings::_helpers::render::declaration_values::ConstantRendering; use crate::bindings::_helpers::render::declaration_values::resolved_declaration_context; use crate::bindings::_helpers::render::macro_call_rows::SpanRow; -use crate::bindings::_helpers::render::macro_call_rows::span_row; +use crate::bindings::_helpers::render::macro_call_rows::{span_row, span_tuple}; use crate::bindings::_helpers::render::macro_expansion_result::{ExpansionRequest, macro_output}; use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan; use crate::bindings::_helpers::sqltext::static_sql::{ @@ -84,7 +84,7 @@ fn span_objects<'py>(py: Python<'py>, spans: &[SpanRow]) -> PyResult for SpanRow { - fn from(span: CharSpan) -> Self { - Self { - source_start: span.source_start, - source_end: span.source_end, - output_start: span.output_start, - output_end: span.output_end, - } +/// A span as the tuple Python reads. +pub(crate) fn span_tuple(span: &SpanRow) -> SpanTuple { + SpanTuple { + source_start: span.source_start, + source_end: span.source_end, + output_start: span.output_start, + output_end: span.output_end, } } -impl From for Expansion { - fn from(span: SpanRow) -> Self { - Self { - source_start: span.source_start, - source_end: span.source_end, - output_start: span.output_start, - output_end: span.output_end, - } +/// A span as the expansion semantic validation maps offsets through. +pub(crate) fn span_expansion(span: SpanRow) -> Expansion { + Expansion { + source_start: span.source_start, + source_end: span.source_end, + output_start: span.output_start, + output_end: span.output_end, } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs index 356fd854c3..4b051769a3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs @@ -13,7 +13,9 @@ use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::render::macro_call_memo::MacroCallMemo; -use crate::bindings::_helpers::render::macro_call_rows::{SiteRow, SpanRow, site_row, span_row}; +use crate::bindings::_helpers::render::macro_call_rows::{ + SiteRow, SpanTuple, site_row, span_row, span_tuple, +}; use crate::bindings::_helpers::render::macro_expansion::raised; use crate::bindings::types::CompilerDetach; @@ -79,13 +81,19 @@ pub(crate) fn splice_macro_calls( sql: &str, sites: Vec<(usize, usize)>, outputs: Vec, -) -> PyResult<(String, Vec)> { +) -> PyResult<(String, Vec)> { let spliced = py .compiler_detach(|| Ok(splice_calls(sql, &sites, &outputs))) .map_err(compiler_error)?; let (rendered, spans) = spliced.ok_or_else(|| PyValueError::new_err("macro call sites do not fit the SQL"))?; - Ok((rendered, spans.into_iter().map(span_row).collect())) + Ok(( + rendered, + spans + .into_iter() + .map(|span| span_tuple(&span_row(span))) + .collect(), + )) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs index c605e8c5ab..5a61f2a4d4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs @@ -19,7 +19,7 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::render::declaration_expansion::usage_visibility; use crate::bindings::_helpers::render::macro_call_classes::{CallClass, ClassInputs}; use crate::bindings::_helpers::render::macro_call_memo::{MacroCallMemo, poisoned}; -use crate::bindings::_helpers::render::macro_call_rows::{SpanRow, span_row}; +use crate::bindings::_helpers::render::macro_call_rows::{SpanTuple, span_row, span_tuple}; use crate::bindings::_helpers::render::macro_evaluation::{CallText, Evaluation, evaluate_call}; use crate::bindings::_helpers::render::macro_expansion_facts::{ExpansionFacts, ResourceRef}; use crate::bindings::_helpers::render::macro_generated_references::{ @@ -38,7 +38,7 @@ positional_record! { /// `(sql, spans, argument references, dependencies, usages)` of one expansion. struct ExpansionRow { sql: String, - spans: Vec, + spans: Vec, argument_references: Vec, dependencies: Vec>, usages: Vec>, @@ -141,7 +141,11 @@ impl NativeMacroExpansion { let output: ExpansionOutput = expanded(slf, sql.to_str()?, bridge)?; Ok(ExpansionRow { sql: output.sql, - spans: output.spans.into_iter().map(span_row).collect(), + spans: output + .spans + .into_iter() + .map(|span| span_tuple(&span_row(span))) + .collect(), argument_references: output.argument_references, dependencies: output.dependencies, usages: output.usages, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_payload.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_payload.rs index fc3a56479c..7f924a3723 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_payload.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_payload.rs @@ -200,7 +200,7 @@ impl PayloadWriter { let mut model: Map = Map::new(); let (authored, query): (Arc, Arc) = view.read(py, |record, _| { Ok(( - record.model_file.get().contents_text().clone(), + record.host.model_file.get().contents_text().clone(), record.query_sql.clone(), )) })?; @@ -266,8 +266,8 @@ fn native_declared(py: Python<'_>, view: &NativeCompiledModel) -> PyResult = Vec::with_capacity(schema.columns.specs.len()); - for column in &schema.columns.specs { + let mut columns: Vec = Vec::with_capacity(schema.host.columns.specs.len()); + for column in &schema.host.columns.specs { let Some(spec) = column.native_spec() else { return Ok(Declared::Python); }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs index b79144b27a..98bbdb4fec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs @@ -14,9 +14,7 @@ use sqlbuild_rules::project_tree::main::tracked_text_path::tracked_text_path; use sqlbuild_rules::project_tree::models::ProjectTreeEntry; use crate::bindings::_helpers::compiled_views::audit_views::NativeCompiledAudit; -use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - InferredRow, NativeCompiledModel, -}; +use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; use crate::bindings::_helpers::compiled_views::function_views::NativeCompiledFunction; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; use crate::bindings::_helpers::compiled_views::seed_views::NativeCompiledSeed; @@ -32,6 +30,7 @@ use crate::bindings::_helpers::rules::fact_tree::{ Outcome, Raised, TreeQuery, nodes_value, path_string, read_text, relative_parts, selected_nodes, tree_nodes, }; +use sqlbuild_project::types::InferredRow; const MODEL_RECORD: &str = "sqlbuild._native.Model"; const MATERIALIZED: &str = "materialized"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs index 6cc4c01404..3909b55b6b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use crate::bindings::_helpers::compiled_views::model_input_view::{ - ExpansionRecord, ModelInputRecord, NativeModelInput, + ModelInputRecord, NativeModelInput, }; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; use crate::bindings::_helpers::records::fields::field as native_field; @@ -15,12 +15,13 @@ use crate::bindings::_helpers::render::macro_call_rows::SpanRow; use crate::bindings::_helpers::rules::lint_pipeline::{ CompiledExpansion, CompiledSource, span_pass, }; +use sqlbuild_project::models::ExpansionRecord; const FILE_PATH_ATTRIBUTE: &str = "file_path"; /// The model file path of a native input record. fn record_path(record: &ModelInputRecord) -> PathBuf { - record.model_file.get().file_path_buf() + record.host.model_file.get().file_path_buf() } /// The record's shared texts with the given passes, read only when their file is linted. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs index 0f94b34c7b..6251d5cf49 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs @@ -15,9 +15,7 @@ use sqlbuild_analysis::semantic_checks::models::{ use sqlbuild_analysis::semantic_validation::models as validation; use crate::bindings::_helpers::boundary::panics::compiler_error; -use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - ModelMetadataTexts, NativeCompiledModel, -}; +use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; use crate::bindings::_helpers::compiled_views::model_reads::binding_validated; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; @@ -31,6 +29,7 @@ use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; use crate::bindings::_helpers::semantic_checks::type_recovery::items; use crate::bindings::_helpers::semantic_shapes::shapes::project_shapes; use crate::bindings::types::CompilerDetach; +use sqlbuild_project::models::ModelMetadataTexts; /// `SqlReferenceKind.UDF` and `SqlReferenceKind.TABLE_FUNCTION` values. const FUNCTION_REFERENCE_KINDS: [&str; 2] = ["udf", "table_fn"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs index 78d4bec4fb..0c9e594a9d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs @@ -14,7 +14,6 @@ use sqlbuild_analysis::semantic_checks::models::{ }; use crate::bindings::_helpers::boundary::panics::compiler_error; -use crate::bindings::_helpers::compiled_project::rows::ReferenceRow; use crate::bindings::_helpers::compiled_views::audit_views::audit_reference_rows; use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; use crate::bindings::_helpers::records::construct::error_class; @@ -25,6 +24,7 @@ use crate::bindings::_helpers::records::fields::mapping_items; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::models::ProjectCatalog; use crate::bindings::types::CompilerDetach; +use sqlbuild_project::models::ReferenceRow; const SQL_ANALYSIS_CONFIG_KEY: &str = "sql_analysis"; const MODEL_HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs index be48aa10f0..0779a6fd4b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs @@ -7,11 +7,11 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_context::delta::ModelCoverage; -use crate::bindings::_helpers::compiled_views::compiled_model_view::InferredRow; use crate::bindings::_helpers::compiled_views::model_reads::{inferred_rows, star_flags}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::semantic_shapes::delta_shapes::semantic_reasons; use crate::bindings::_helpers::semantic_shapes::shapes::project_shapes; +use sqlbuild_project::types::InferredRow; const TYPE_CHECKED_DIALECTS: [&str; 4] = ["duckdb", "snowflake", "postgresql", "bigquery"]; const SOURCE_KIND: &str = "source"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs index 4f8d7a46ba..2f79dfdbe1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs @@ -11,9 +11,9 @@ use crate::bindings::_helpers::analysis::binding_catalog::{ use crate::bindings::_helpers::analysis::normalization::normalize_analysis_sql; use crate::bindings::_helpers::analysis_session::stage_functions::infer_native_expression_source_shapes; use crate::bindings::_helpers::boundary::keyword_request::Request; -use crate::bindings::_helpers::compiled_project::rows::ReferenceRow; +use crate::bindings::_helpers::compiled_project::rows::ReferenceTuple; use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - DeclaredRow, InferredRow, NativeCompiledModel, + DeclaredRow, NativeCompiledModel, }; use crate::bindings::_helpers::compiled_views::model_reads::{ declared_rows, dynamic_columns_declared, inferred_rows, star_flags, unchecked_columns, @@ -24,6 +24,7 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::fields::replaced_keywords; use crate::bindings::_helpers::records::ids::RecordId; +use sqlbuild_project::types::InferredRow; const UNKNOWN: &str = "UNKNOWN"; const GENERIC_DIALECT: &str = "generic"; @@ -70,11 +71,11 @@ fn relation_names(rows: Vec<(String, String)>) -> Vec { /// The relations a model's reference rows bind; a native model answers from its record. fn model_relation_names(model: &Bound<'_, PyAny>) -> PyResult<(Vec, bool)> { if let Ok(view) = model.downcast::() { - let rows: Vec = view.get().reference_rows(model.py())?; + let rows: Vec = view.get().reference_rows(model.py())?; let referenced: bool = !rows.is_empty(); let names: Vec = relation_names( rows.into_iter() - .map(|ReferenceRow { kind, name, .. }| (kind, name)) + .map(|ReferenceTuple { kind, name, .. }| (kind, name)) .collect(), ); return Ok((names, referenced)); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs index b97b3d0a9b..05d255db81 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs @@ -13,13 +13,13 @@ use sqlbuild_analysis::compiler::models::{ use sqlbuild_core::panics::main::native_failure::native_failure; use crate::bindings::_helpers::boundary::panics::value_error; -use crate::bindings::_helpers::compiled_project::rows::ReferenceRow; use crate::bindings::_helpers::compiled_views::model_input_view::NativeModelInput; use crate::bindings::_helpers::records::fields::{ field as native_field, path_posix_text, path_stem, }; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; use crate::bindings::types::CompilerDetach; +use sqlbuild_project::models::ReferenceRow; const MACRO_TOKEN: char = '@'; const INPUT_FAILURE: &str = "input"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs index 687b20670c..a60ec2f168 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs @@ -159,7 +159,7 @@ pub(crate) fn interpolation_batch( let row = match interpolate_sql(&host, sql, file_path) { Ok(result) => InterpolationRow { sql: result.sql, - spans: result.spans.into_iter().map(SpanRow::from).collect(), + spans: result.spans, reads: read_rows(result.reads), error: None, }, diff --git a/fensu.toml b/fensu.toml index 0b708c5dce..8f39033b71 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1576,6 +1576,7 @@ crate_names = [ "sqlbuild-discovery", "sqlbuild-host", "sqlbuild-model-config", + "sqlbuild-project", "sqlbuild-pysource", "sqlbuild-python", "sqlbuild-recall",