diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs index 3b4eb35339..ce7fe35a60 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs @@ -16,7 +16,7 @@ use crate::assembly::analysis_session::models::{ LineageRow, ModelAnalysis, ModelOutcome, ModelRequest, SessionRequest, }; use crate::assembly::analysis_session::types::{Pairs, Shapes}; -use crate::semantic_validation::types::DiagnosticRow; +use crate::semantic_validation::models::DiagnosticRow; const LINEAGE_NATIVE: u64 = 0; const LINEAGE_FACTS: u64 = 1; @@ -454,7 +454,16 @@ pub(crate) fn encode_outcome( encoder.flag(analysis.has_star); encoder.flag(analysis.star_resolved); encoder.uint(analysis.binding_diagnostics.len() as u64); - for (code, message, line, column, start, end, severity) in &analysis.binding_diagnostics { + for DiagnosticRow { + code, + message, + line, + column, + start, + end, + severity, + } in &analysis.binding_diagnostics + { encoder.text(code); encoder.text(message); for position in [line, column, start, end] { @@ -529,15 +538,15 @@ pub(crate) fn decode_outcome( let diagnostic_count: usize = decoder.index()?; let mut binding_diagnostics: Vec = Vec::new(); for _ in 0..diagnostic_count { - binding_diagnostics.push(( - decoder.text()?, - decoder.text()?, - decoder.optional_index()?, - decoder.optional_index()?, - decoder.optional_index()?, - decoder.optional_index()?, - decoder.text()?, - )); + binding_diagnostics.push(DiagnosticRow { + code: decoder.text()?, + message: decoder.text()?, + line: decoder.optional_index()?, + column: decoder.optional_index()?, + start: decoder.optional_index()?, + end: decoder.optional_index()?, + severity: decoder.text()?, + }); } let binding_validated: bool = decoder.flag()?; decoder.finish()?; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs index 9a274defc7..8679f4480d 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs @@ -23,7 +23,8 @@ use crate::assembly::analysis_session::constants::{ use crate::assembly::analysis_session::models::{ColumnFact, LineageRow}; use crate::assembly::analysis_session::types::{Pairs, Shapes}; use crate::semantic_validation::main::diagnostics::binding_diagnostics; -use crate::semantic_validation::types::{DiagnosticRow, NormalizationRequest, Relations}; +use crate::semantic_validation::models::DiagnosticRow; +use crate::semantic_validation::types::{NormalizationRequest, Relations}; const INVALID_BATCH: &str = "native compact query analysis returned an invalid batch response"; const INVALID_TEMPLATE: &str = "native compact query analysis returned an invalid template"; @@ -1424,15 +1425,15 @@ fn diagnostic_row(error: &Value) -> Result, String> { Some(Value::String(message)) if !message.is_empty() => message.clone(), _ => DEFAULT_BINDING_MESSAGE.to_owned(), }; - Ok(Some(( - code.to_owned(), + Ok(Some(DiagnosticRow { + code: code.to_owned(), message, - position(error, "line")?, - position(error, "column")?, - position(error, "start")?, - position(error, "end")?, - severity.to_owned(), - ))) + line: position(error, "line")?, + column: position(error, "column")?, + start: position(error, "start")?, + end: position(error, "end")?, + severity: severity.to_owned(), + })) } fn position(error: &Value, key: &str) -> Result, String> { diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs index b37e6c4e4b..e49ac7c85c 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs @@ -192,7 +192,12 @@ fn pivot_proof( if truthy(pivot.get("unpivot")) { return failure("UNPIVOT does not produce a dynamic column family".to_owned()); } - let (pivot_columns, aggregates, dynamic, columns_valid) = components(pivot, context.dialect); + let PivotComponents { + pivot_columns, + aggregates, + dynamic, + columns_valid, + } = components(pivot, context.dialect); if !dynamic { return failure( "static pivot values must use ordinary exact column declarations".to_owned(), @@ -309,11 +314,18 @@ fn fixed_names( .collect() } +/// Python's `_pivot_components` for one pivot clause. +struct PivotComponents<'a> { + pivot_columns: Vec<&'a str>, + aggregates: Vec<&'a Value>, + /// Whether the pivot values come from a dynamic source. + dynamic: bool, + /// Whether every pivot column resolved to a name. + columns_valid: bool, +} + /// Python's `_pivot_components`: pivot columns, aggregates, dynamic values, columns valid. -fn components<'a>( - pivot: &'a Map, - dialect: &str, -) -> (Vec<&'a str>, Vec<&'a Value>, bool, bool) { +fn components<'a>(pivot: &'a Map, dialect: &str) -> PivotComponents<'a> { if SIMPLIFIED_PIVOT_DIALECTS.contains(&dialect) && truthy(pivot.get("using")) { let expressions: Vec<&Value> = dict_list(pivot.get("expressions")); let columns: Vec<&str> = expressions @@ -321,12 +333,12 @@ fn components<'a>( .filter_map(|expression| column_name(Some(expression))) .collect(); let valid: bool = columns.len() == expressions.len(); - return ( - columns, - dict_list(pivot.get("using")), - !truthy(pivot.get("fields")), - valid, - ); + return PivotComponents { + pivot_columns: columns, + aggregates: dict_list(pivot.get("using")), + dynamic: !truthy(pivot.get("fields")), + columns_valid: valid, + }; } let fields: Vec<&Value> = dict_list(pivot.get("fields")); let columns: Vec<&str> = fields @@ -335,7 +347,12 @@ fn components<'a>( .collect(); let dynamic: bool = fields.iter().any(|field| has_dynamic_values(field)); let valid: bool = columns.len() == fields.len(); - (columns, dict_list(pivot.get("expressions")), dynamic, valid) + PivotComponents { + pivot_columns: columns, + aggregates: dict_list(pivot.get("expressions")), + dynamic, + columns_valid: valid, + } } /// Python's `_has_dynamic_value_source` for one pivot field. @@ -651,13 +668,24 @@ fn failure(reason: String) -> PivotOutcome { } /// Python's `_families_equal` key for one family. -fn normalized(family: &DynamicFamily) -> (String, String, String, String, String, Option<&str>) { - ( - casefold(&family.name), - casefold(&family.pivot_column), - casefold(&family.value_column), - casefold(&family.aggregate), - casefold(&family.data_type), - family.name_pattern.as_deref(), - ) +#[derive(PartialEq, Eq)] +struct FamilyKey<'a> { + name: String, + pivot_column: String, + value_column: String, + aggregate: String, + data_type: String, + name_pattern: Option<&'a str>, +} + +/// Python's `_families_equal` key for one family. +fn normalized(family: &DynamicFamily) -> FamilyKey<'_> { + FamilyKey { + name: casefold(&family.name), + pivot_column: casefold(&family.pivot_column), + value_column: casefold(&family.value_column), + aggregate: casefold(&family.aggregate), + data_type: casefold(&family.data_type), + name_pattern: family.name_pattern.as_deref(), + } } diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs index 66f3c1196b..2451efca3d 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs @@ -36,8 +36,8 @@ use crate::assembly::analysis_session::models::{ RelationTables, SessionModelFacts, SessionOutcome, SessionRequest, SessionStep, }; use crate::assembly::analysis_session::types::{Pairs, Shapes}; +use crate::semantic_validation::models::DiagnosticRow; use crate::semantic_validation::models::ProjectCatalog; -use crate::semantic_validation::types::DiagnosticRow; /// What an internal native failure of the session names. const SESSION_CONTEXT: &str = "native model analysis"; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs index 3a625a8170..da66105b17 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs @@ -11,7 +11,7 @@ use sqlbuild_cache::store::models::NativeStore; use crate::assembly::analysis_session::_helpers::catalog_state::SessionCatalog; use crate::assembly::analysis_session::_helpers::mappings::ShapeTable; use crate::assembly::analysis_session::types::{NullabilityRuleFn, OutputSources, Pairs, Shapes}; -use crate::semantic_validation::types::DiagnosticRow; +use crate::semantic_validation::models::DiagnosticRow; /// One `ref`, `source`, `seed`, `table_fn` or `udf` call a model's SQL makes. #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs index cef7394585..321905eb3e 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs @@ -230,7 +230,7 @@ pub(crate) fn described(outcome: &ModelOutcome) -> Vec { analysis .binding_diagnostics .iter() - .map(|row| row.0.as_str()) + .map(|row| row.code.as_str()) .collect::>() )); lines diff --git a/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs b/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs index 80221f225b..71738049d7 100644 --- a/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs +++ b/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs @@ -3,9 +3,11 @@ use std::collections::HashSet; use crate::binding_locations::constants::{ERROR_SEVERITY, UNKNOWN_TABLE_INTERNAL_CODE}; -use crate::binding_locations::models::{AuthoredSpan, BindingSource, LocatedDiagnostic}; +use crate::binding_locations::models::{ + AuthoredSpan, BindingSource, LocatedDiagnostic, SeenDiagnostic, +}; +use crate::semantic_validation::models::DiagnosticRow; use crate::semantic_validation::models::{BindingPositions, PositionInput}; -use crate::semantic_validation::types::DiagnosticRow; /// Each reported diagnostic in order; an error mapping a position is the mapper's message. pub fn locate_binding_diagnostics( @@ -17,9 +19,18 @@ pub fn locate_binding_diagnostics( .find(source.query) .map(|start| source.authored[..start].matches('\n').count()); let mut positions: Option = None; - let mut seen: HashSet<(String, String, Option, Option)> = HashSet::new(); + let mut seen: HashSet = HashSet::new(); let mut located: Vec = Vec::new(); - for (code, message, line, column, start, end, severity) in rows { + for DiagnosticRow { + code, + message, + line, + column, + start, + end, + severity, + } in rows + { if code == UNKNOWN_TABLE_INTERNAL_CODE { continue; } @@ -39,7 +50,12 @@ pub fn locate_binding_diagnostics( *column, ), }; - let key = (code.clone(), message.clone(), line, column); + let key = SeenDiagnostic { + code: code.clone(), + message: message.clone(), + line, + column, + }; if seen.contains(&key) && start.is_some() && severity == ERROR_SEVERITY { continue; } diff --git a/crates/sqlbuild-analysis/src/binding_locations/models.rs b/crates/sqlbuild-analysis/src/binding_locations/models.rs index 292ed5690a..efd854bf90 100644 --- a/crates/sqlbuild-analysis/src/binding_locations/models.rs +++ b/crates/sqlbuild-analysis/src/binding_locations/models.rs @@ -1,6 +1,15 @@ //! A model's authored SQL texts and the located diagnostics read from them. -use crate::semantic_validation::types::Expansion; +use crate::semantic_validation::models::Expansion; + +/// A reported diagnostic's identity: an error repeating it at a known start is dropped. +#[derive(Debug, PartialEq, Eq, Hash)] +pub(crate) struct SeenDiagnostic { + pub(crate) code: String, + pub(crate) message: String, + pub(crate) line: Option, + pub(crate) column: Option, +} /// The texts one model's binding diagnostics are mapped back through. #[derive(Clone, Copy, Debug)] diff --git a/crates/sqlbuild-analysis/src/binding_locations/tests/helpers.rs b/crates/sqlbuild-analysis/src/binding_locations/tests/helpers.rs index a09c0b76aa..e7dcf94764 100644 --- a/crates/sqlbuild-analysis/src/binding_locations/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/binding_locations/tests/helpers.rs @@ -1,4 +1,4 @@ -use crate::semantic_validation::types::DiagnosticRow; +use crate::semantic_validation::models::DiagnosticRow; pub(super) const MODEL: &str = "MODEL (\n description \"Orders\"\n);\n\nSELECT o.amount, o.missing\nFROM orders AS o"; @@ -11,15 +11,15 @@ fn row( span: (Option, Option), severity: &str, ) -> DiagnosticRow { - ( - code.to_owned(), - message.to_owned(), - position.0, - position.1, - span.0, - span.1, - severity.to_owned(), - ) + DiagnosticRow { + code: code.to_owned(), + message: message.to_owned(), + line: position.0, + column: position.1, + start: span.0, + end: span.1, + severity: severity.to_owned(), + } } fn missing_column() -> DiagnosticRow { diff --git a/crates/sqlbuild-analysis/src/binding_locations/tests/test_types.rs b/crates/sqlbuild-analysis/src/binding_locations/tests/test_types.rs index 300b0e43f8..336b4dc4d1 100644 --- a/crates/sqlbuild-analysis/src/binding_locations/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/binding_locations/tests/test_types.rs @@ -1,4 +1,4 @@ -use crate::semantic_validation::types::DiagnosticRow; +use crate::semantic_validation::models::DiagnosticRow; /// `(code, line, column, (line, column, end line, end column))` of one reported diagnostic. pub(super) type LocatedCase = ( diff --git a/crates/sqlbuild-analysis/src/lineage/models.rs b/crates/sqlbuild-analysis/src/lineage/models.rs index f660fbad29..d8d4a2657b 100644 --- a/crates/sqlbuild-analysis/src/lineage/models.rs +++ b/crates/sqlbuild-analysis/src/lineage/models.rs @@ -149,15 +149,41 @@ pub enum FastLineageOutcome { UnknownDialect(String), } +/// A lineage outcome flattened into its status, columns, star flag and detail. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LineageOutcomeParts { + pub status: &'static str, + pub columns: Vec, + pub has_star: bool, + pub detail: Option, +} + +impl LineageOutcomeParts { + fn new(status: &'static str, columns: Vec, has_star: bool, detail: Option) -> Self { + Self { + status, + columns, + has_star, + detail, + } + } +} + impl FastLineageOutcome { - /// `(status, columns, has_star, detail)`: detail is the parse error or the unknown dialect. - pub fn into_parts(self) -> (&'static str, Vec, bool, Option) { + /// The status, columns and star flag; detail is the parse error or the unknown dialect. + pub fn into_parts(self) -> LineageOutcomeParts { match self { - Self::StarColumns(columns) => ("star", columns, true, None), - Self::Built { columns, has_star } => ("built", columns, has_star, None), - Self::Omitted => ("omitted", Vec::new(), false, None), - Self::Unparsed(message) => ("unparsed", Vec::new(), false, Some(message)), - Self::UnknownDialect(name) => ("unknown_dialect", Vec::new(), false, Some(name)), + Self::StarColumns(columns) => LineageOutcomeParts::new("star", columns, true, None), + Self::Built { columns, has_star } => { + LineageOutcomeParts::new("built", columns, has_star, None) + } + Self::Omitted => LineageOutcomeParts::new("omitted", Vec::new(), false, None), + Self::Unparsed(message) => { + LineageOutcomeParts::new("unparsed", Vec::new(), false, Some(message)) + } + Self::UnknownDialect(name) => { + LineageOutcomeParts::new("unknown_dialect", Vec::new(), false, Some(name)) + } } } } @@ -217,11 +243,15 @@ pub enum RichLineageOutcome { } impl RichLineageOutcome { - /// `(status, columns, has_star, detail)`: detail is the polyglot error. - pub fn into_parts(self) -> (&'static str, Vec, bool, Option) { + /// The status, columns and star flag; detail is the polyglot error. + pub fn into_parts(self) -> LineageOutcomeParts { match self { - Self::Built { columns, has_star } => ("built", columns, has_star, None), - Self::Skipped(message) => ("skipped", Vec::new(), false, Some(message)), + Self::Built { columns, has_star } => { + LineageOutcomeParts::new("built", columns, has_star, None) + } + Self::Skipped(message) => { + LineageOutcomeParts::new("skipped", Vec::new(), false, Some(message)) + } } } } diff --git a/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs b/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs index 73e206171a..bf332a89da 100644 --- a/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs @@ -4,8 +4,9 @@ use crate::lineage::main::build_fast_lineage::build_fast_lineage; use crate::lineage::main::build_rich_lineage::build_rich_lineage; use crate::lineage::models::RelationFingerprint; use crate::lineage::models::{ - FastLineageModel, FastLineageOutcome, FastLineageRequest, LineageColumn, LineageResourceType, - LineageSchemaResource, RichLineageColumn, RichLineageRequest, RichSchemaResource, + FastLineageModel, FastLineageOutcome, FastLineageRequest, LineageColumn, LineageOutcomeParts, + LineageResourceType, LineageSchemaResource, RichLineageColumn, RichLineageRequest, + RichSchemaResource, }; use crate::semantic_validation::models::ProjectCatalog; @@ -64,7 +65,12 @@ fn catalog() -> ProjectCatalog { fn described_outcome( outcome: FastLineageOutcome, ) -> (&'static str, Vec, bool, Option) { - let (status, columns, has_star, detail) = outcome.into_parts(); + let LineageOutcomeParts { + status, + columns, + has_star, + detail, + } = outcome.into_parts(); (status, described(&columns), has_star, detail) } @@ -139,7 +145,12 @@ pub(crate) fn rich_outcome( }; let mut outcomes = build_rich_lineage(&request).expect("the rich lineage pool builds"); assert_eq!(outcomes.len(), 1); - let (status, columns, has_star, detail) = outcomes.remove(0).into_parts(); + let LineageOutcomeParts { + status, + columns, + has_star, + detail, + } = outcomes.remove(0).into_parts(); ( status, columns.iter().map(described_rich_column).collect(), diff --git a/crates/sqlbuild-analysis/src/lineage/tests/test_rich_lineage_panics.rs b/crates/sqlbuild-analysis/src/lineage/tests/test_rich_lineage_panics.rs index b4a0944961..1f4d748a01 100644 --- a/crates/sqlbuild-analysis/src/lineage/tests/test_rich_lineage_panics.rs +++ b/crates/sqlbuild-analysis/src/lineage/tests/test_rich_lineage_panics.rs @@ -55,7 +55,7 @@ fn given_an_injected_panic_when_building_rich_lineage_then_the_request_fails() { let answered: Result, String> = outcomes.map(|outcomes| { outcomes .into_iter() - .map(|outcome| outcome.into_parts().3.unwrap_or_default()) + .map(|outcome| outcome.into_parts().detail.unwrap_or_default()) .collect() }); assert_eq!( diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/resolved_reads.rs b/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/resolved_reads.rs index 5e70d90bcc..8ab1fbca56 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/resolved_reads.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/resolved_reads.rs @@ -25,9 +25,9 @@ pub(crate) fn resolved_reads( let references: Vec = key .references .iter() - .map(|(kind, name, _, _)| ReferenceInput { - kind: kind.clone(), - name: name.clone(), + .map(|reference| ReferenceInput { + kind: reference.kind.clone(), + name: reference.name.clone(), }) .collect(); cache.misses += 1; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/required_columns.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/required_columns.rs index d2a2c86b91..4134ac1029 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/main/required_columns.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/required_columns.rs @@ -26,14 +26,14 @@ pub fn required_columns( let mut result = FixtureReadRequirements::default(); for model in models { let mut reads = model.high_confidence.clone(); - if model.key.references.iter().any(|(kind, name, _, _)| { - let resource_type = match kind.as_str() { + if model.key.references.iter().any(|reference| { + let resource_type = match reference.kind.as_str() { "ref" => "model", "source" => "source", "seed" => "seed", _ => return false, }; - fixture_keys.contains(&(resource_type.to_owned(), name.clone())) + fixture_keys.contains(&(resource_type.to_owned(), reference.name.clone())) }) { let (updated, resolved) = resolved_reads(cache, &model.key); cache = updated; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/mod.rs b/crates/sqlbuild-analysis/src/planner_fixtures/mod.rs index 42b1da1020..b010d05a00 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/mod.rs @@ -7,4 +7,3 @@ pub mod main; pub mod models; #[cfg(test)] mod tests; -pub mod types; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/models.rs b/crates/sqlbuild-analysis/src/planner_fixtures/models.rs index 703a4defa1..944b845040 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/models.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/models.rs @@ -12,7 +12,14 @@ pub struct FixtureReadSchemas<'a> { use crate::required_columns::models::ColumnRead; -use crate::planner_fixtures::types::FixtureReadReference; +/// The complete reference identity retained by the fixture read cache. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct FixtureReadReference { + pub kind: String, + pub name: String, + pub package: Option, + pub argument_count: Option, +} /// The original SQL, complete references and dialect form the read cache key. #[derive(Clone, Debug, Eq, Hash, PartialEq)] diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_reads.rs b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_reads.rs index 8d19e9e57e..b7832b9526 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_reads.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_reads.rs @@ -1,6 +1,6 @@ use crate::planner_fixtures::_helpers::resolved_reads::resolved_reads; use crate::planner_fixtures::_helpers::stub_markers::stub_markers; -use crate::planner_fixtures::models::{FixtureReadCache, FixtureReadKey}; +use crate::planner_fixtures::models::{FixtureReadCache, FixtureReadKey, FixtureReadReference}; use crate::planner_fixtures::tests::test_types::{FixtureMarkerTestCase, FixtureReadCacheTestCase}; #[test] @@ -50,7 +50,12 @@ fn given_repeated_reads_when_resolving_then_full_reference_identity_controls_reu for test_case in test_cases { let key = FixtureReadKey { sql: "SELECT order_id FROM __source(\"orders\")".to_owned(), - references: vec![("source".into(), "orders".into(), None, None)], + references: vec![FixtureReadReference { + kind: "source".into(), + name: "orders".into(), + package: None, + argument_count: None, + }], dialect: Some("duckdb".into()), }; let (cache, initial) = resolved_reads(FixtureReadCache::default(), &key); @@ -59,12 +64,12 @@ fn given_repeated_reads_when_resolving_then_full_reference_identity_controls_reu assert_eq!(cache.misses, 1, "{}", test_case.description); let changed = FixtureReadKey { sql: test_case.sql.to_owned(), - references: vec![( - "source".into(), - "orders".into(), - test_case.package.map(str::to_owned), - test_case.argument_count, - )], + references: vec![FixtureReadReference { + kind: "source".into(), + name: "orders".into(), + package: test_case.package.map(str::to_owned), + argument_count: test_case.argument_count, + }], dialect: test_case.dialect.map(str::to_owned), }; let (cache, reads) = resolved_reads(cache, &changed); diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/types.rs b/crates/sqlbuild-analysis/src/planner_fixtures/types.rs deleted file mode 100644 index ec569300e0..0000000000 --- a/crates/sqlbuild-analysis/src/planner_fixtures/types.rs +++ /dev/null @@ -1,4 +0,0 @@ -//! Fixture analysis identities. - -/// The complete reference identity retained by the fixture read cache. -pub type FixtureReadReference = (String, String, Option, Option); diff --git a/crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs b/crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs index b5def1c23c..bcd4228bfe 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs @@ -7,7 +7,8 @@ use polyglot_sql::{ ValidationSchema, analyze_query, analyze_query_for_project_projections, }; use rayon::iter::{IntoParallelIterator, ParallelIterator}; -use serde::{Deserialize, Serialize}; +use serde::ser::SerializeTuple; +use serde::{Deserialize, Serialize, Serializer}; use std::collections::{HashMap, HashSet}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Mutex, PoisonError}; @@ -1128,7 +1129,29 @@ fn analyze_project_result(request: ProjectAnalysisRequest) -> Result, u8, u8, u8, Vec<(usize, usize, usize)>); +/// One interned column fact; it serializes as the positional array the batch reader expects. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct CompactProjectColumn { + name: usize, + data_type: Option, + nullability: u8, + transform: u8, + confidence: u8, + upstream: Vec<(usize, usize, usize)>, +} + +impl Serialize for CompactProjectColumn { + fn serialize(&self, serializer: S) -> Result { + let mut fields = serializer.serialize_tuple(6)?; + fields.serialize_element(&self.name)?; + fields.serialize_element(&self.data_type)?; + fields.serialize_element(&self.nullability)?; + fields.serialize_element(&self.transform)?; + fields.serialize_element(&self.confidence)?; + fields.serialize_element(&self.upstream)?; + fields.end() + } +} #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] @@ -1222,14 +1245,14 @@ impl CompactProjectAccumulator { ) }) .collect(); - let fact = ( - self.interner.intern(column.name), - column.data_type.map(|value| self.interner.intern(value)), - nullability_code(column.nullability), - transform_code(lineage.transform_kind), - confidence_code(lineage.confidence), + let fact = CompactProjectColumn { + name: self.interner.intern(column.name), + data_type: column.data_type.map(|value| self.interner.intern(value)), + nullability: nullability_code(column.nullability), + transform: transform_code(lineage.transform_kind), + confidence: confidence_code(lineage.confidence), upstream, - ); + }; let next_index = self.facts.len(); let fact_index = *self.fact_indexes.entry(fact.clone()).or_insert_with(|| { self.facts.push(fact); diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/audits.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/audits.rs index 748ea8a336..c6d39d9a39 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/audits.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/audits.rs @@ -13,8 +13,8 @@ use crate::semantic_checks::models::{ }; use crate::semantic_validation::main::diagnostics::binding_diagnostics; use crate::semantic_validation::main::validations_json; -use crate::semantic_validation::models::{CatalogInput, Columns, ProjectCatalog}; -use crate::semantic_validation::types::{DiagnosticRow, Relations}; +use crate::semantic_validation::models::{CatalogInput, Columns, DiagnosticRow, ProjectCatalog}; +use crate::semantic_validation::types::Relations; const EXPRESSION_AUDIT: &str = "expression_is_true"; const RELATIONSHIPS_AUDIT: &str = "relationships"; @@ -107,15 +107,15 @@ fn response_rows(response: &Value) -> Result, SemanticFailure Some(Value::String(text)) if !text.is_empty() => text.clone(), _ => DEFAULT_BINDING_MESSAGE.to_owned(), }; - rows.push(( - code.to_owned(), + rows.push(DiagnosticRow { + code: code.to_owned(), message, - optional_index(error.get("line")), - optional_index(error.get("column")), - optional_index(error.get("start")), - optional_index(error.get("end")), - severity.to_owned(), - )); + line: optional_index(error.get("line")), + column: optional_index(error.get("column")), + start: optional_index(error.get("start")), + end: optional_index(error.get("end")), + severity: severity.to_owned(), + }); } Ok(rows) } @@ -162,8 +162,13 @@ fn standalone_validation( binding_diagnostics(sql, &dialect, response_rows(response)?).map_err(internal) } -/// One `(code, message, severity)` finding before it is located. -type Unlocated = (String, String, String, String); +/// One audit finding before it is located at the named column or field. +struct Unlocated { + code: String, + message: String, + severity: String, + name: String, +} fn expression_findings( expression: &str, @@ -188,14 +193,12 @@ fn expression_findings( .unwrap_or_default(); Ok(rows .into_iter() - .filter(|row| row.0 != UNKNOWN_TABLE_CODE) - .map(|row| { - ( - row.0, - format!("{EXPRESSION_AUDIT}: {}", row.1), - row.6, - expression.to_owned(), - ) + .filter(|row| row.code != UNKNOWN_TABLE_CODE) + .map(|row| Unlocated { + code: row.code, + message: format!("{EXPRESSION_AUDIT}: {}", row.message), + severity: row.severity, + name: expression.to_owned(), }) .collect()) } @@ -203,17 +206,17 @@ fn expression_findings( fn typed_findings(rows: Vec, label: &str, column: &str) -> Vec { rows.into_iter() .map(|row| { - let code = if row.6 == ERROR_SEVERITY { + let code = if row.severity == ERROR_SEVERITY { AUDIT_TYPE_CODE.to_owned() } else { - row.0 + row.code }; - ( + Unlocated { code, - format!("{label}: {}", row.1), - row.6, - column.to_owned(), - ) + message: format!("{label}: {}", row.message), + severity: row.severity, + name: column.to_owned(), + } }) .collect() } @@ -243,12 +246,12 @@ fn relationship_findings( if target_shape.is_empty() { return Ok(Vec::new()); } - return Ok(vec![( - MISSING_TARGET_COLUMN_CODE.to_owned(), - format!("relationships target has no column '{}'", audit.field), - ERROR_SEVERITY.to_owned(), - audit.field.clone(), - )]); + return Ok(vec![Unlocated { + code: MISSING_TARGET_COLUMN_CODE.to_owned(), + message: format!("relationships target has no column '{}'", audit.field), + severity: ERROR_SEVERITY.to_owned(), + name: audit.field.clone(), + }]); }; let Some((_, left)) = shape.iter().find(|(name, _)| name == column) else { return Ok(Vec::new()); @@ -337,7 +340,13 @@ pub(crate) fn audit_errors( ) -> Result, SemanticFailure> { let mut findings: Vec = Vec::new(); for audit in &model.audits { - for (code, message, severity, name) in audit_findings(audit, shape, context)? { + for Unlocated { + code, + message, + severity, + name, + } in audit_findings(audit, shape, context)? + { let (line, column) = text_position(&model.authored_sql, &name, 0)?; findings.push(AuditFinding { code, diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs index 5df90d2dcc..33ced0fba8 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs @@ -16,7 +16,8 @@ use crate::semantic_checks::_helpers::resource_sql::resources::{ MissingColumnAudit, ResourceSql, audit_label, missing_column_audits, resources, }; use crate::semantic_checks::models::{ - ResourceSqlDiagnostic, ResourceSqlFailure, ResourceSqlKind, ResourceSqlRequest, SemanticFailure, + ResourceSpan, ResourceSqlDiagnostic, ResourceSqlFailure, ResourceSqlKind, ResourceSqlRequest, + SemanticFailure, }; use crate::semantic_validation::main::normalize::normalize_analysis_sql; use crate::semantic_validation::models::{Columns, NormalizationInput, ProjectCatalog}; @@ -32,7 +33,17 @@ const CURSOR_CONTEXT: &str = "Model SQL"; const ANALYSIS_CURSOR_LITERAL: &str = "CAST('2000-01-01 00:00:00' AS TIMESTAMP)"; const GENERIC_DIALECT: &str = "generic"; -type Key = (String, String, String, i64, i64, i64); +/// A finding's identity: its path, code, message, start and native order when unlocated. +#[derive(Clone, PartialEq, Eq, Hash)] +struct Key { + path: String, + code: String, + message: String, + line: i64, + column: i64, + /// The finding's native index when it has no start, else `-1`. + order: i64, +} fn kind_label(kind: ResourceSqlKind) -> &'static str { match kind { @@ -88,26 +99,26 @@ fn binding_request( /// Python's `_reported_diagnostic`: drop open-relation lookups, make parser rejections P001. fn reported(resource: &ResourceSql, finding: &Finding) -> Option { - if finding.0 != UNKNOWN_TABLE_CODE { + if finding.code != UNKNOWN_TABLE_CODE { return Some(finding.clone()); } - if finding.1.starts_with(UNKNOWN_TABLE_PREFIX) { + if finding.message.starts_with(UNKNOWN_TABLE_PREFIX) { return None; } - Some(( - SYNTAX_ERROR_CODE.to_owned(), - format!( + Some(Finding { + code: SYNTAX_ERROR_CODE.to_owned(), + message: format!( "SQL syntax error in {} '{}': {}", kind_label(resource.kind), resource.name, - finding.1 + finding.message ), - finding.2, - finding.3, - finding.4, - finding.5, - ERROR_SEVERITY.to_owned(), - )) + line: finding.line, + column: finding.column, + start: finding.start, + end: finding.end, + severity: ERROR_SEVERITY.to_owned(), + }) } /// What one resource's findings are checked against. @@ -128,31 +139,31 @@ fn resource_diagnostics( continue; }; let span = location(resource, checking.cleaned_sql, &finding)?; - let key: Key = ( - resource.path.clone(), - finding.0.clone(), - finding.1.clone(), - span.0, - span.1, - if finding.4.is_none() { + let key: Key = Key { + path: resource.path.clone(), + code: finding.code.clone(), + message: finding.message.clone(), + line: span.line, + column: span.column, + order: if finding.start.is_none() { index as i64 } else { -1 }, - ); + }; if found.iter().any(|(existing, _)| *existing == key) { continue; } let authored_text = text_from(resource, span); let authored_line = line_at(resource, span); let help = resource_sql_help( - &finding.0, - &finding.1, + &finding.code, + &finding.message, &HelpContext { kind: resource.kind, authored_text: &authored_text, authored_line: &authored_line, - column: span.1, + column: span.column, authored_body: &resource.authored_body, audit_definition: resource.audit_definition.as_deref(), shapes: &checking.request.shapes, @@ -162,16 +173,18 @@ fn resource_diagnostics( ) .map_err(semantic)?; let message = match &resource.label { - Some(label) if finding.0 != SYNTAX_ERROR_CODE => format!("{label}: {}", finding.1), - _ => finding.1.clone(), + Some(label) if finding.code != SYNTAX_ERROR_CODE => { + format!("{label}: {}", finding.message) + } + _ => finding.message.clone(), }; found.push(( key, ResourceSqlDiagnostic { kind: resource.kind, resource_name: resource.name.clone(), - is_error: finding.6 == ERROR_SEVERITY, - code: finding.0.clone(), + is_error: finding.severity == ERROR_SEVERITY, + code: finding.code.clone(), message, path: resource.path.clone(), span, @@ -199,7 +212,7 @@ fn opt_out_diagnostic( code: OPT_OUT_CODE.to_owned(), message, path: resource.path.clone(), - span: (line, column, None, None), + span: ResourceSpan::start(line, column), note: Some(note), help: Some(help), }) @@ -230,7 +243,12 @@ pub(crate) fn missing_column_diagnostic( model.name ), path: model.path.clone(), - span: (line, start, Some(line), Some(end)), + span: ResourceSpan { + line, + column: start, + end_line: Some(line), + end_column: Some(end), + }, note: None, help: Some(format!( "add {column} to the SELECT list of model '{}' or attach the {definition} audit to \ diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/fixtures.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/fixtures.rs index 10693cdcef..bcafbf2275 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/fixtures.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/fixtures.rs @@ -87,14 +87,27 @@ fn empty_fixture_call_body(text: &[char], index: usize) -> Option { literal(text, after, ")", false) } -/// Every empty fixture CTE in `text`: start, prefix end, name and end. -fn empty_fixture_ctes(text: &[char]) -> Vec<(usize, usize, String, usize)> { - let mut found: Vec<(usize, usize, String, usize)> = Vec::new(); +/// One empty fixture CTE: where it starts, where its prefix ends, its name and its end. +struct EmptyFixtureCte { + start: usize, + prefix_end: usize, + name: String, + end: usize, +} + +/// Every empty fixture CTE in `text`. +fn empty_fixture_ctes(text: &[char]) -> Vec { + let mut found: Vec = Vec::new(); let mut index = 0; while index < text.len() { match empty_fixture_cte(text, index) { Some((prefix_end, name, end)) => { - found.push((index, prefix_end, name, end)); + found.push(EmptyFixtureCte { + start: index, + prefix_end, + name, + end, + }); index = end; } None => index += 1, @@ -109,11 +122,11 @@ pub(crate) fn prepared_test_sql(sql_body: &str) -> (String, Vec) { let ctes = empty_fixture_ctes(&body); let mut opened: Vec = Vec::with_capacity(body.len()); let mut copied = 0; - for (start, prefix_end, name, end) in &ctes { - opened.extend_from_slice(&body[copied..*start]); - opened.extend_from_slice(&body[*start..*prefix_end]); - opened.extend(name.chars()); - copied = *end; + for cte in &ctes { + opened.extend_from_slice(&body[copied..cte.start]); + opened.extend_from_slice(&body[cte.start..cte.prefix_end]); + opened.extend(cte.name.chars()); + copied = cte.end; } opened.extend_from_slice(&body[copied..]); let mut sql = String::with_capacity(opened.len()); @@ -128,6 +141,6 @@ pub(crate) fn prepared_test_sql(sql_body: &str) -> (String, Vec) { } } let mut relations = reference_names(&body); - relations.extend(ctes.into_iter().map(|(_, _, name, _)| name)); + relations.extend(ctes.into_iter().map(|cte| cte.name)); (sql, relations) } diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/locations.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/locations.rs index 0cc89868c2..85eadd87bb 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/locations.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/locations.rs @@ -4,25 +4,14 @@ use sqlbuild_core::text::main::ignorecase_key::ignorecase_key; use crate::semantic_checks::_helpers::resource_sql::resources::{ResourceSql, find_code_points}; use crate::semantic_checks::_helpers::resource_sql::scanning::word_at; -use crate::semantic_checks::models::ResourceSqlFailure; +use crate::semantic_checks::models::{ResourceSpan, ResourceSqlFailure}; use crate::semantic_validation::main::truth_test_spans::truth_test_spans; -use crate::semantic_validation::models::{BindingPositions, PositionInput}; - -/// A binding finding: code, message, line, column, start, end and severity. -pub(crate) type Finding = ( - String, - String, - Option, - Option, - Option, - Option, - String, -); +use crate::semantic_validation::models::{BindingPositions, DiagnosticRow, PositionInput}; -const UNSUPPORTED_TRUTH_TEST_CODE: &str = "B103"; +/// A binding finding: code, message, cleaned position and severity. +pub(crate) type Finding = DiagnosticRow; -/// `(line, column, end line, end column)`. -pub(crate) type Span = (i64, i64, Option, Option); +const UNSUPPORTED_TRUTH_TEST_CODE: &str = "B103"; /// Python's `_offset_location`: the 1-based line and column of a code-point offset. pub(crate) fn offset_location(text: &str, offset: usize) -> (i64, i64) { @@ -89,7 +78,7 @@ fn mapped_span( resource: &ResourceSql, cleaned_sql: &str, finding: &Finding, -) -> Result, ResourceSqlFailure> { +) -> Result, ResourceSqlFailure> { let mapper = BindingPositions::new(PositionInput { authored: resource.contents.to_string(), query: resource.authored_body.clone(), @@ -99,27 +88,36 @@ fn mapped_span( }) .map_err(ResourceSqlFailure::Normalization)?; let (line, column) = mapper - .position(finding.4, finding.2, finding.3, &finding.1) + .position( + finding.start, + finding.line, + finding.column, + &finding.message, + ) .map_err(ResourceSqlFailure::Normalization)?; let (Some(line), Some(column)) = (line, column) else { return Ok(None); }; - let (end_line, end_column) = match finding.5 { + let (end_line, end_column) = match finding.end { Some(end) => mapper .position(Some(end), None, None, "") .map_err(ResourceSqlFailure::Normalization)?, None => (None, None), }; - Ok(Some(( - line as i64, - column as i64, - usize_line(end_line), - usize_line(end_column), - ))) + Ok(Some(ResourceSpan { + line: line as i64, + column: column as i64, + end_line: usize_line(end_line), + end_column: usize_line(end_column), + })) } /// The authored span of the truth-value test a B103 finding starts at, matched by its order. -fn truth_test_span(resource: &ResourceSql, cleaned_sql: &str, start: usize) -> Option { +fn truth_test_span( + resource: &ResourceSql, + cleaned_sql: &str, + start: usize, +) -> Option { let cleaned: Vec<(usize, usize)> = truth_test_spans(cleaned_sql); let order: usize = cleaned .iter() @@ -134,7 +132,12 @@ fn truth_test_span(resource: &ResourceSql, cleaned_sql: &str, start: usize) -> O let length: usize = resource.authored_body[test_start..test_end].chars().count(); let (line, column) = offset_location(&resource.contents, offset); let (end_line, end_column) = offset_location(&resource.contents, offset + length); - Some((line, column, Some(end_line), Some(end_column))) + Some(ResourceSpan { + line, + column, + end_line: Some(end_line), + end_column: Some(end_column), + }) } /// Python's `_location`: the offending authored text, falling back to the body start. @@ -142,18 +145,18 @@ pub(crate) fn location( resource: &ResourceSql, cleaned_sql: &str, finding: &Finding, -) -> Result { +) -> Result { let body_offset = find_code_points(&resource.contents, &resource.authored_body).unwrap_or(0); - if finding.4.is_none() { + if finding.start.is_none() { let text: Vec = resource.contents.chars().collect(); - if let Some(offset) = quoted_name_offset(&text, body_offset, &finding.1) { + if let Some(offset) = quoted_name_offset(&text, body_offset, &finding.message) { let (line, column) = offset_location(&resource.contents, offset); - return Ok((line, column, None, None)); + return Ok(ResourceSpan::start(line, column)); } } - if finding.0 == UNSUPPORTED_TRUTH_TEST_CODE + if finding.code == UNSUPPORTED_TRUTH_TEST_CODE && let Some(span) = finding - .4 + .start .and_then(|start| truth_test_span(resource, cleaned_sql, start)) { return Ok(span); @@ -162,7 +165,7 @@ pub(crate) fn location( return Ok(span); } let (line, column) = offset_location(&resource.contents, body_offset); - Ok((line, column, None, None)) + Ok(ResourceSpan::start(line, column)) } /// Code-point starts of each line of `text`. @@ -187,12 +190,12 @@ fn offset(starts: &[usize], line: i64, column: i64) -> Option { } /// Python's `_text_from`: the authored text of a span, or from its start. -pub(crate) fn text_from(resource: &ResourceSql, span: Span) -> String { +pub(crate) fn text_from(resource: &ResourceSql, span: ResourceSpan) -> String { let starts = line_starts(&resource.contents); - let Some(start) = offset(&starts, span.0, span.1) else { + let Some(start) = offset(&starts, span.line, span.column) else { return String::new(); }; - let end = match (span.2, span.3) { + let end = match (span.end_line, span.end_column) { (Some(line), Some(column)) => offset(&starts, line, column), _ => None, }; @@ -204,12 +207,12 @@ pub(crate) fn text_from(resource: &ResourceSql, span: Span) -> String { } /// Python's `_line_at`: the authored line of a span, or empty text. -pub(crate) fn line_at(resource: &ResourceSql, span: Span) -> String { +pub(crate) fn line_at(resource: &ResourceSql, span: ResourceSpan) -> String { let lines: Vec<&str> = resource.contents.lines().collect(); - if span.0 < 1 { + if span.line < 1 { return String::new(); } lines - .get((span.0 - 1) as usize) + .get((span.line - 1) as usize) .map_or_else(String::new, |line| (*line).to_owned()) } diff --git a/crates/sqlbuild-analysis/src/semantic_checks/models.rs b/crates/sqlbuild-analysis/src/semantic_checks/models.rs index d12faf7325..7674a801ce 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/models.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/models.rs @@ -445,6 +445,27 @@ pub struct ResourceSqlRequest { pub hook_files: Vec<(String, Arc)>, } +/// A resource SQL finding's authored start and, when it covers text, its end. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct ResourceSpan { + pub line: i64, + pub column: i64, + pub end_line: Option, + pub end_column: Option, +} + +impl ResourceSpan { + /// A span that marks only its start. + pub fn start(line: i64, column: i64) -> Self { + Self { + line, + column, + end_line: None, + end_column: None, + } + } +} + /// A located resource SQL finding, or a rejected `sql_analysis false` with its note. #[derive(Clone, Debug, PartialEq, Eq)] pub struct ResourceSqlDiagnostic { @@ -454,8 +475,7 @@ pub struct ResourceSqlDiagnostic { pub code: String, pub message: String, pub path: String, - /// `(line, column, end line, end column)`. - pub span: (i64, i64, Option, Option), + pub span: ResourceSpan, pub note: Option, pub help: Option, } diff --git a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_resource_sql.rs b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_resource_sql.rs index 53b07ab184..f91795ba59 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_resource_sql.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_resource_sql.rs @@ -120,9 +120,9 @@ fn given_built_in_audit_on_missing_column_when_checking_then_reports_once_at_its ); ( diagnostic.message, - diagnostic.span.0, - diagnostic.span.1, - diagnostic.span.3.unwrap_or_default(), + diagnostic.span.line, + diagnostic.span.column, + diagnostic.span.end_column.unwrap_or_default(), ) }); assert_eq!( diff --git a/crates/sqlbuild-analysis/src/semantic_usage/main.rs b/crates/sqlbuild-analysis/src/semantic_usage/main.rs index 37a3d7e161..beeedb31e0 100644 --- a/crates/sqlbuild-analysis/src/semantic_usage/main.rs +++ b/crates/sqlbuild-analysis/src/semantic_usage/main.rs @@ -227,16 +227,26 @@ fn physical_source_name(expression: &Expression) -> Option { } } +/// The identity two usage facts share when one repeats the other. +#[derive(PartialEq, Eq, Hash)] +struct UsageKey { + context: String, + expression_sql: String, + source_name: String, + column: String, + start: Option, +} + fn dedupe(mut uses: Vec) -> Vec { - let mut seen: HashSet<(String, String, String, String, Option)> = HashSet::new(); + let mut seen: HashSet = HashSet::new(); uses.retain(|fact| { - seen.insert(( - fact.context.clone(), - fact.expression_sql.clone(), - fact.source_name.clone(), - fact.column.clone(), - fact.start, - )) + seen.insert(UsageKey { + context: fact.context.clone(), + expression_sql: fact.expression_sql.clone(), + source_name: fact.source_name.clone(), + column: fact.column.clone(), + start: fact.start, + }) }); uses } diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs index a771ef8731..7f892c622e 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs @@ -6,10 +6,10 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, PoisonError}; use crate::semantic_validation::models::{ - CatalogInput, Columns, CompactAnalysisJob, FunctionProbes, ProjectCatalog, + CatalogInput, Columns, CompactAnalysisJob, DiagnosticRow, FunctionProbes, ProjectCatalog, }; use crate::semantic_validation::types::{ - BindingRequest, DiagnosticRow, NormalizationRequest, PreparedCompactAnalysis, Relations, + BindingRequest, NormalizationRequest, PreparedCompactAnalysis, Relations, }; use crate::semantic_validation::{ _helpers::{diagnostics, identifiers}, @@ -616,8 +616,8 @@ impl ProjectCatalog { /// The diagnostic row of `error`, naming the failure when the parser gives no message. fn messaged_row(error: ValidationError) -> DiagnosticRow { let mut row: DiagnosticRow = diagnostic_row(error); - if row.1.is_empty() { - DEFAULT_BINDING_MESSAGE.clone_into(&mut row.1); + if row.message.is_empty() { + DEFAULT_BINDING_MESSAGE.clone_into(&mut row.message); } row } @@ -627,15 +627,15 @@ pub(super) fn diagnostic_row(error: ValidationError) -> DiagnosticRow { polyglot_sql::ValidationSeverity::Error => "error", _ => "warning", }; - ( - error.code, - error.message, - error.line, - error.column, - error.start, - error.end, - severity.to_owned(), - ) + DiagnosticRow { + code: error.code, + message: error.message, + line: error.line, + column: error.column, + start: error.start, + end: error.end, + severity: severity.to_owned(), + } } fn has_exact_columns(table: &SchemaTable) -> bool { diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs index 0983e58f70..17175e9868 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs @@ -4,8 +4,8 @@ use crate::semantic_validation::_helpers::catalog::diagnostic_row; use crate::semantic_validation::_helpers::function_names; use crate::semantic_validation::_helpers::truth_tests; use crate::semantic_validation::_helpers::type_locations::locate_type_findings; +use crate::semantic_validation::models::DiagnosticRow; use crate::semantic_validation::models::FunctionProbes; -use crate::semantic_validation::types::DiagnosticRow; use polyglot_sql::expressions::Select; use polyglot_sql::{ Dialect, DialectType, Expression, ExpressionWalk, ValidationError, ValidationResult, @@ -393,21 +393,19 @@ pub(crate) fn binding_diagnostics( ) -> Result, String> { let errors: Vec = rows .into_iter() - .map( - |(code, message, line, column, start, end, severity)| ValidationError { - code, - message, - line, - column, - start, - end, - severity: if severity == ERROR { - ValidationSeverity::Error - } else { - ValidationSeverity::Warning - }, + .map(|row| ValidationError { + code: row.code, + message: row.message, + line: row.line, + column: row.column, + start: row.start, + end: row.end, + severity: if row.severity == ERROR { + ValidationSeverity::Error + } else { + ValidationSeverity::Warning }, - ) + }) .collect(); let result = map_diagnostics( sql, diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/function_names.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/function_names.rs index fba1e4be4c..ca0337945e 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/function_names.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/function_names.rs @@ -12,7 +12,7 @@ use std::mem::Discriminant; use std::sync::{LazyLock, OnceLock, PoisonError}; const UNKNOWN_FUNCTION: &str = "B101"; -const SNOWFLAKE_FUNCTIONS: &str = include_str!("../../../function_names/snowflake.txt"); +const SNOWFLAKE_FUNCTIONS: &str = include_str!("snowflake_functions.txt"); const DUCKDB_CATALOG: &str = "duckdb"; const PROBE_PREFIX: &str = "SELECT "; const TYPE_POSITION_KEYWORD: &str = "AS"; diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/positions.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/positions.rs index d59718854b..0cf001abe2 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/positions.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/positions.rs @@ -2,7 +2,7 @@ use crate::semantic_validation::_helpers::diagnostics::column_pattern; use crate::semantic_validation::_helpers::normalization::normalize; -use crate::semantic_validation::models::{BindingPositions, PositionInput}; +use crate::semantic_validation::models::{BindingPositions, Expansion, PositionInput}; use regex::Regex; use std::collections::HashMap; use std::sync::{LazyLock, OnceLock}; @@ -10,7 +10,6 @@ use std::sync::{LazyLock, OnceLock}; const POPULAR_TOKEN_THRESHOLD: usize = 200; static TOKENS: LazyLock> = LazyLock::new(|| Regex::new(r"\w+|[^\w\s]").map_err(|error| error.to_string())); -type Mapping = (usize, usize, usize, usize); impl BindingPositions { pub fn new(request: PositionInput) -> Result { @@ -75,7 +74,13 @@ impl BindingPositions { .unwrap_or_else(|| *self.offsets.last().unwrap_or(&0)); for pass in self.passes.iter().rev() { let mut adjusted = mapped as isize; - for &(source_start, source_end, output_start, output_end) in pass { + for &Expansion { + source_start, + source_end, + output_start, + output_end, + } in pass + { if mapped < output_start { break; } @@ -227,28 +232,31 @@ fn align_offsets(original: &str, normalized: &str) -> Result, String> blocks.sort(); let ac = byte_char_map(original); let bc = byte_char_map(normalized); - let mut mappings: Vec = Vec::new(); + let mut mappings: Vec = Vec::new(); for (ai, bi, size) in blocks { for i in 0..size { - mappings.push(( - ac[a[ai + i].start()], - ac[a[ai + i].end()], - bc[b[bi + i].start()], - bc[b[bi + i].end()], - )); + mappings.push(Expansion { + source_start: ac[a[ai + i].start()], + source_end: ac[a[ai + i].end()], + output_start: bc[b[bi + i].start()], + output_end: bc[b[bi + i].end()], + }); } } let mut offsets = Vec::with_capacity(normalized.chars().count() + 1); let original_len = original.chars().count(); let mut index = 0; for offset in 0..=normalized.chars().count() { - while index < mappings.len() && offset > mappings[index].3 { + while index < mappings.len() && offset > mappings[index].output_end { index += 1; } offsets.push(match mappings.get(index) { - Some(&(start, end, target, _)) => { - start + offset.saturating_sub(target).min(end - start) - } + Some(&Expansion { + source_start: start, + source_end: end, + output_start: target, + .. + }) => start + offset.saturating_sub(target).min(end - start), None => original_len, }); } diff --git a/crates/sqlbuild-analysis/function_names/snowflake.txt b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/snowflake_functions.txt similarity index 100% rename from crates/sqlbuild-analysis/function_names/snowflake.txt rename to crates/sqlbuild-analysis/src/semantic_validation/_helpers/snowflake_functions.txt diff --git a/crates/sqlbuild-analysis/src/semantic_validation/main/diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_validation/main/diagnostics.rs index 2efcff0313..ae7f192632 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/main/diagnostics.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/main/diagnostics.rs @@ -1,4 +1,4 @@ -use crate::semantic_validation::types::DiagnosticRow; +use crate::semantic_validation::models::DiagnosticRow; pub fn binding_diagnostics( sql: &str, diff --git a/crates/sqlbuild-analysis/src/semantic_validation/models.rs b/crates/sqlbuild-analysis/src/semantic_validation/models.rs index 7f1805efcf..a24345c77a 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/models.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/models.rs @@ -1,6 +1,6 @@ //! Compile-owned native catalog, mapping state and boundary requests. -use crate::semantic_validation::types::{Expansion, PreparedCompactAnalysis, ProbeKey, Relations}; +use crate::semantic_validation::types::{PreparedCompactAnalysis, ProbeKey, Relations}; use polyglot_sql::validation::SchemaTable; use polyglot_sql::{DialectType, SchemaValidationOptions}; use rayon::ThreadPool; @@ -8,6 +8,27 @@ use std::collections::HashMap; use std::sync::atomic::AtomicUsize; use std::sync::{Arc, Mutex}; +/// One schema-validation diagnostic with its cleaned-SQL position. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DiagnosticRow { + pub code: String, + pub message: String, + pub line: Option, + pub column: Option, + pub start: Option, + pub end: Option, + pub severity: String, +} + +/// One macro-expansion edit: the authored char span and the span it expanded to. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Expansion { + pub source_start: usize, + pub source_end: usize, + pub output_start: usize, + pub output_end: usize, +} + #[derive(Default, Debug)] pub struct Columns(pub Vec<(String, Option)>); diff --git a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_type_locations.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_type_locations.rs index 06c47b5f23..7bbe92b98e 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_type_locations.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_type_locations.rs @@ -1,4 +1,5 @@ use crate::semantic_validation::main::diagnostics::binding_diagnostics; +use crate::semantic_validation::models::DiagnosticRow; use crate::semantic_validation::tests::test_types::TypeLocationTestCase; #[test] @@ -68,16 +69,14 @@ fn given_position_less_type_findings_when_mapping_then_each_points_at_its_own_ex let rows = test_case .findings .iter() - .map(|(code, message)| { - ( - (*code).to_owned(), - (*message).to_owned(), - None, - None, - None, - None, - "error".to_owned(), - ) + .map(|(code, message)| DiagnosticRow { + code: (*code).to_owned(), + message: (*message).to_owned(), + line: None, + column: None, + start: None, + end: None, + severity: "error".to_owned(), }) .collect(); let mapped = binding_diagnostics(test_case.sql, test_case.dialect, rows)?; @@ -85,7 +84,7 @@ fn given_position_less_type_findings_when_mapping_then_each_points_at_its_own_ex .iter() .zip(test_case.expected_starts) .map(|(row, expected)| { - row.4.map_or_else(String::new, |start| { + row.start.map_or_else(String::new, |start| { test_case .sql .chars() @@ -107,7 +106,9 @@ fn given_position_less_type_findings_when_mapping_then_each_points_at_its_own_ex test_case.description ); assert!( - mapped.iter().all(|row| row.2.is_some() && row.3.is_some()), + mapped + .iter() + .all(|row| row.line.is_some() && row.column.is_some()), "{}", test_case.description ); diff --git a/crates/sqlbuild-analysis/src/semantic_validation/types.rs b/crates/sqlbuild-analysis/src/semantic_validation/types.rs index a6a8770c55..8ab94dd7a8 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/types.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/types.rs @@ -5,16 +5,6 @@ pub type Relations = HashMap; pub type BindingRequest = (String, Vec<(String, bool)>, Relations); /// One analysis SQL with its relation stubs and placeholder defaults. pub type NormalizationRequest = (String, HashMap, HashMap); -pub type DiagnosticRow = ( - String, - String, - Option, - Option, - Option, - Option, - String, -); -pub type Expansion = (usize, usize, usize, usize); pub(crate) type ProbeKey = (polyglot_sql::DialectType, String, usize); /// Compact analysis whose catalog schemas are resolved; it runs with an options-only catalog. pub(crate) type PreparedCompactAnalysis = diff --git a/crates/sqlbuild-attachments/src/functions/_helpers/namespace.rs b/crates/sqlbuild-attachments/src/functions/_helpers/namespace.rs index 8a302d80e9..958cf3bd53 100644 --- a/crates/sqlbuild-attachments/src/functions/_helpers/namespace.rs +++ b/crates/sqlbuild-attachments/src/functions/_helpers/namespace.rs @@ -1,13 +1,14 @@ //! One of database or schema of a function namespace. -pub(crate) type Part = ( - Option, - Option, - Option, - Option, -); +/// The physical, logical and fingerprint names of one of database or schema. +pub(crate) struct Part { + pub(crate) physical: Option, + pub(crate) logical: Option, + pub(crate) fingerprint: Option, + pub(crate) fingerprint_logical: Option, +} -/// `(physical, logical, fingerprint, fingerprint logical)` for one of database or schema. +/// The physical, logical, fingerprint and fingerprint logical names of one namespace part. pub(crate) fn part( header: Option<&String>, default: Option<&String>, @@ -26,5 +27,10 @@ pub(crate) fn part( }; let fingerprint_logical: Option = header.or(default).cloned(); let fingerprint: Option = target.cloned().or_else(|| fingerprint_logical.clone()); - (physical, logical, fingerprint, fingerprint_logical) + Part { + physical, + logical, + fingerprint, + fingerprint_logical, + } } diff --git a/crates/sqlbuild-attachments/src/functions/main/resolve_function_namespace.rs b/crates/sqlbuild-attachments/src/functions/main/resolve_function_namespace.rs index ae40e9d40b..74bff9d485 100644 --- a/crates/sqlbuild-attachments/src/functions/main/resolve_function_namespace.rs +++ b/crates/sqlbuild-attachments/src/functions/main/resolve_function_namespace.rs @@ -8,26 +8,26 @@ use crate::functions::models::{FunctionLanguage, FunctionNamespace, NamespaceInp pub fn resolve_function_namespace(inputs: &NamespaceInputs) -> FunctionNamespace { let inherit: bool = inputs.language == FunctionLanguage::Sql || inputs.inherit_default_namespace; - let (database, logical_database, fingerprint_database, fingerprint_logical_database) = part( + let database = part( inputs.header_database.as_ref(), inputs.default_database.as_ref(), inputs.target_database.as_ref(), inherit, ); - let (schema, logical_schema, fingerprint_schema, fingerprint_logical_schema) = part( + let schema = part( inputs.header_schema.as_ref(), inputs.default_schema.as_ref(), inputs.target_schema.as_ref(), inherit, ); FunctionNamespace { - database, - schema, - logical_database, - logical_schema, - fingerprint_database, - fingerprint_schema, - fingerprint_logical_database, - fingerprint_logical_schema, + database: database.physical, + schema: schema.physical, + logical_database: database.logical, + logical_schema: schema.logical, + fingerprint_database: database.fingerprint, + fingerprint_schema: schema.fingerprint, + fingerprint_logical_database: database.fingerprint_logical, + fingerprint_logical_schema: schema.fingerprint_logical, } } diff --git a/crates/sqlbuild-core/src/dbt_plan/main/render_dbt_execution_summary.rs b/crates/sqlbuild-core/src/dbt_plan/main/render_dbt_execution_summary.rs index 6c7719b1ad..251dda70cb 100644 --- a/crates/sqlbuild-core/src/dbt_plan/main/render_dbt_execution_summary.rs +++ b/crates/sqlbuild-core/src/dbt_plan/main/render_dbt_execution_summary.rs @@ -14,7 +14,10 @@ pub fn render_dbt_execution_summary( if results.is_empty() { return None; } - let (mut passed, mut warned, mut failed, mut skipped): (i64, i64, i64, i64) = (0, 0, 0, 0); + let mut passed: i64 = 0; + let mut warned: i64 = 0; + let mut failed: i64 = 0; + let mut skipped: i64 = 0; let mut elapsed: f64 = 0.0; for result in results { let status: String = result.status.to_lowercase(); diff --git a/crates/sqlbuild-core/src/lifecycle_events/constants.rs b/crates/sqlbuild-core/src/lifecycle_events/constants.rs index fd5d643514..70e6a04f84 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/constants.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/constants.rs @@ -1,6 +1,6 @@ //! The versioned lifecycle event catalogue and the payload value contracts it validates. -use crate::lifecycle_events::types::CatalogEntry; +use crate::lifecycle_events::models::EventDefinition; /// The lifecycle event schema version SQLBuild produces. pub const CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION: i64 = 2; @@ -277,45 +277,51 @@ pub const OPERATION_ADAPTERS: [&str; 10] = [ ]; /// The schema version 1 catalogue in declaration order. -pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ - ("invocation_started", &[], &[], &["command"], false), - ( - "invocation_completed", - &[], - &[], - &["command", "duration_ms", "exit_code"], - true, - ), - ( - "invocation_failed", - &[], - &[], - &[ +pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [EventDefinition; 20] = [ + EventDefinition { + event_type: "invocation_started", + required_correlations: &[], + required_payload: &[], + allowed_payload: &["command"], + terminal: false, + }, + EventDefinition { + event_type: "invocation_completed", + required_correlations: &[], + required_payload: &[], + allowed_payload: &["command", "duration_ms", "exit_code"], + terminal: true, + }, + EventDefinition { + event_type: "invocation_failed", + required_correlations: &[], + required_payload: &[], + allowed_payload: &[ "command", "duration_ms", "error_code", "error_type", "exit_code", ], - true, - ), - ( - "run_started", - &["run_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "run_started", + required_correlations: &["run_id"], + required_payload: &[], + allowed_payload: &[ "configured_concurrency", "run_kind", "selected_count", "worker_count", ], - false, - ), - ( - "run_completed", - &["run_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "run_completed", + required_correlations: &["run_id"], + required_payload: &[], + allowed_payload: &[ "duration_ms", "fail_count", "failed_count", @@ -325,13 +331,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "succeeded_count", "warn_count", ], - true, - ), - ( - "run_failed", - &["run_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "run_failed", + required_correlations: &["run_id"], + required_payload: &[], + allowed_payload: &[ "duration_ms", "error_code", "error_type", @@ -343,32 +349,32 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "succeeded_count", "warn_count", ], - true, - ), - ( - "resource_attempt_started", - &["resource_attempt_id", "resource_id", "run_id"], - &[], - &["attempt_number", "resource_kind", "resource_name"], - false, - ), - ( - "resource_attempt_completed", - &["resource_attempt_id", "resource_id", "run_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "resource_attempt_started", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[], + allowed_payload: &["attempt_number", "resource_kind", "resource_name"], + terminal: false, + }, + EventDefinition { + event_type: "resource_attempt_completed", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[], + allowed_payload: &[ "attempt_number", "duration_ms", "resource_kind", "resource_name", ], - true, - ), - ( - "resource_attempt_failed", - &["resource_attempt_id", "resource_id", "run_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "resource_attempt_failed", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[], + allowed_payload: &[ "attempt_number", "duration_ms", "error_code", @@ -376,18 +382,18 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "resource_kind", "resource_name", ], - true, - ), - ( - "resource_attempt_skipped", - &["resource_attempt_id", "resource_id", "run_id"], - &[ + terminal: true, + }, + EventDefinition { + event_type: "resource_attempt_skipped", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[ "attempt_number", "resource_kind", "resource_name", "skip_code", ], - &[ + allowed_payload: &[ "attempt_number", "duration_ms", "resource_kind", @@ -395,13 +401,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "skip_code", "skip_mode", ], - true, - ), - ( - "operation_started", - &["operation_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "operation_started", + required_correlations: &["operation_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "hook_index", "hook_name", @@ -415,13 +421,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "strategy", "target_kind", ], - false, - ), - ( - "operation_completed", - &["operation_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "operation_completed", + required_correlations: &["operation_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "duration_ms", "exit_code", @@ -439,13 +445,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "strategy", "target_kind", ], - true, - ), - ( - "operation_failed", - &["operation_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "operation_failed", + required_correlations: &["operation_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "duration_ms", "error_code", @@ -465,30 +471,30 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "strategy", "target_kind", ], - true, - ), - ( - "retry_scheduled", - &["resource_attempt_id", "resource_id", "run_id"], - &[ + terminal: true, + }, + EventDefinition { + event_type: "retry_scheduled", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[ "delay_ms", "error_type", "failed_attempt_number", "next_attempt_number", ], - &[ + allowed_payload: &[ "delay_ms", "error_code", "error_type", "failed_attempt_number", "next_attempt_number", ], - false, - ), - ( - "audit_completed", - &["run_id"], - &[ + terminal: false, + }, + EventDefinition { + event_type: "audit_completed", + required_correlations: &["run_id"], + required_payload: &[ "attachment_kind", "audit_definition_name", "audit_name", @@ -497,7 +503,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "run_scope_phase", "severity", ], - &[ + allowed_payload: &[ "attached_column_name", "attached_target_kind", "attached_target_name", @@ -534,13 +540,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "thresholds", "violation_count", ], - true, - ), - ( - "statement_started", - &["statement_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "statement_started", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -552,13 +558,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "sql_digest", "statement_kind", ], - false, - ), - ( - "statement_submitted", - &["statement_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "statement_submitted", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -569,13 +575,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "sql_digest", "statement_kind", ], - false, - ), - ( - "statement_heartbeat", - &["statement_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "statement_heartbeat", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -588,13 +594,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "sql_digest", "statement_kind", ], - false, - ), - ( - "statement_completed", - &["statement_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "statement_completed", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -607,13 +613,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "sql_digest", "statement_kind", ], - true, - ), - ( - "statement_failed", - &["statement_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "statement_failed", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -628,55 +634,55 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "sql_digest", "statement_kind", ], - true, - ), + terminal: true, + }, ]; /// The schema version 2 catalogue in declaration order. -pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ - ( - "invocation_started", - &[], - &[], - &["adapter", "command", "selection_count", "target"], - false, - ), - ( - "invocation_completed", - &[], - &[], - &["command", "duration_ms", "exit_code"], - true, - ), - ( - "invocation_failed", - &[], - &[], - &[ +pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [EventDefinition; 23] = [ + EventDefinition { + event_type: "invocation_started", + required_correlations: &[], + required_payload: &[], + allowed_payload: &["adapter", "command", "selection_count", "target"], + terminal: false, + }, + EventDefinition { + event_type: "invocation_completed", + required_correlations: &[], + required_payload: &[], + allowed_payload: &["command", "duration_ms", "exit_code"], + terminal: true, + }, + EventDefinition { + event_type: "invocation_failed", + required_correlations: &[], + required_payload: &[], + allowed_payload: &[ "command", "duration_ms", "error_code", "error_type", "exit_code", ], - true, - ), - ( - "run_started", - &["run_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "run_started", + required_correlations: &["run_id"], + required_payload: &[], + allowed_payload: &[ "configured_concurrency", "run_kind", "selected_count", "worker_count", ], - false, - ), - ( - "run_completed", - &["run_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "run_completed", + required_correlations: &["run_id"], + required_payload: &[], + allowed_payload: &[ "duration_ms", "fail_count", "failed_count", @@ -686,13 +692,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "succeeded_count", "warn_count", ], - true, - ), - ( - "run_failed", - &["run_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "run_failed", + required_correlations: &["run_id"], + required_payload: &[], + allowed_payload: &[ "duration_ms", "error_code", "error_type", @@ -704,26 +710,26 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "succeeded_count", "warn_count", ], - true, - ), - ( - "resource_attempt_started", - &["resource_attempt_id", "resource_id", "run_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "resource_attempt_started", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[], + allowed_payload: &[ "attempt_number", "plan_action", "plan_reason_code", "resource_kind", "resource_name", ], - false, - ), - ( - "resource_attempt_completed", - &["resource_attempt_id", "resource_id", "run_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "resource_attempt_completed", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[], + allowed_payload: &[ "attempt_number", "duration_ms", "plan_action", @@ -731,13 +737,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "resource_kind", "resource_name", ], - true, - ), - ( - "resource_attempt_failed", - &["resource_attempt_id", "resource_id", "run_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "resource_attempt_failed", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[], + allowed_payload: &[ "attempt_number", "duration_ms", "error_code", @@ -747,18 +753,18 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "resource_kind", "resource_name", ], - true, - ), - ( - "resource_attempt_skipped", - &["resource_attempt_id", "resource_id", "run_id"], - &[ + terminal: true, + }, + EventDefinition { + event_type: "resource_attempt_skipped", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[ "attempt_number", "resource_kind", "resource_name", "skip_code", ], - &[ + allowed_payload: &[ "attempt_number", "duration_ms", "plan_action", @@ -768,13 +774,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "skip_code", "skip_mode", ], - true, - ), - ( - "operation_started", - &["operation_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "operation_started", + required_correlations: &["operation_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "hook_index", "hook_name", @@ -788,13 +794,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "strategy", "target_kind", ], - false, - ), - ( - "operation_completed", - &["operation_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "operation_completed", + required_correlations: &["operation_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "duration_ms", "exit_code", @@ -812,13 +818,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "strategy", "target_kind", ], - true, - ), - ( - "operation_failed", - &["operation_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "operation_failed", + required_correlations: &["operation_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "duration_ms", "error_code", @@ -838,30 +844,30 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "strategy", "target_kind", ], - true, - ), - ( - "retry_scheduled", - &["resource_attempt_id", "resource_id", "run_id"], - &[ + terminal: true, + }, + EventDefinition { + event_type: "retry_scheduled", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[ "delay_ms", "error_type", "failed_attempt_number", "next_attempt_number", ], - &[ + allowed_payload: &[ "delay_ms", "error_code", "error_type", "failed_attempt_number", "next_attempt_number", ], - false, - ), - ( - "audit_completed", - &["run_id"], - &[ + terminal: false, + }, + EventDefinition { + event_type: "audit_completed", + required_correlations: &["run_id"], + required_payload: &[ "attachment_kind", "audit_definition_name", "audit_name", @@ -870,7 +876,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "run_scope_phase", "severity", ], - &[ + allowed_payload: &[ "attached_column_name", "attached_target_kind", "attached_target_name", @@ -907,13 +913,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "thresholds", "violation_count", ], - true, - ), - ( - "statement_started", - &["statement_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "statement_started", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -926,13 +932,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "sql_digest", "statement_kind", ], - false, - ), - ( - "statement_submitted", - &["statement_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "statement_submitted", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -944,13 +950,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "sql_digest", "statement_kind", ], - false, - ), - ( - "statement_heartbeat", - &["statement_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "statement_heartbeat", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -964,13 +970,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "sql_digest", "statement_kind", ], - false, - ), - ( - "statement_completed", - &["statement_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "statement_completed", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -984,13 +990,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "sql_digest", "statement_kind", ], - true, - ), - ( - "statement_failed", - &["statement_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "statement_failed", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -1006,12 +1012,12 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "sql_digest", "statement_kind", ], - true, - ), - ( - "microbatch_started", - &["resource_attempt_id", "resource_id", "run_id"], - &[ + terminal: true, + }, + EventDefinition { + event_type: "microbatch_started", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[ "batch_count", "batch_index", "batch_kind", @@ -1024,7 +1030,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "planned_cursor_end_exclusive", "planned_cursor_start", ], - &[ + allowed_payload: &[ "batch_count", "batch_index", "batch_kind", @@ -1037,12 +1043,12 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "planned_cursor_end_exclusive", "planned_cursor_start", ], - false, - ), - ( - "microbatch_completed", - &["resource_attempt_id", "resource_id", "run_id"], - &[ + terminal: false, + }, + EventDefinition { + event_type: "microbatch_completed", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[ "batch_count", "batch_index", "batch_kind", @@ -1055,7 +1061,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "planned_cursor_end_exclusive", "planned_cursor_start", ], - &[ + allowed_payload: &[ "affected_rows", "batch_count", "batch_index", @@ -1070,12 +1076,12 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "planned_cursor_end_exclusive", "planned_cursor_start", ], - true, - ), - ( - "microbatch_failed", - &["resource_attempt_id", "resource_id", "run_id"], - &[ + terminal: true, + }, + EventDefinition { + event_type: "microbatch_failed", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[ "batch_count", "batch_index", "batch_kind", @@ -1089,7 +1095,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "planned_cursor_end_exclusive", "planned_cursor_start", ], - &[ + allowed_payload: &[ "batch_count", "batch_index", "batch_kind", @@ -1105,6 +1111,6 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "planned_cursor_end_exclusive", "planned_cursor_start", ], - true, - ), + terminal: true, + }, ]; diff --git a/crates/sqlbuild-core/src/lifecycle_events/main/catalog.rs b/crates/sqlbuild-core/src/lifecycle_events/main/catalog.rs index 74d374d42a..3d8922df03 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/main/catalog.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/main/catalog.rs @@ -2,26 +2,14 @@ use crate::lifecycle_events::constants::{LIFECYCLE_EVENT_CATALOG_V1, LIFECYCLE_EVENT_CATALOG_V2}; use crate::lifecycle_events::models::EventDefinition; -use crate::lifecycle_events::types::CatalogEntry; - -fn definition(entry: &CatalogEntry) -> EventDefinition { - let (event_type, required_correlations, required_payload, allowed_payload, terminal) = *entry; - EventDefinition { - event_type, - required_correlations, - required_payload, - allowed_payload, - terminal, - } -} /// Every definition of `schema_version`, or `None` for an uncatalogued version. #[must_use] pub fn catalog(schema_version: i64) -> Option> { - let entries: &[CatalogEntry] = match schema_version { + let entries: &[EventDefinition] = match schema_version { 1 => &LIFECYCLE_EVENT_CATALOG_V1, 2 => &LIFECYCLE_EVENT_CATALOG_V2, _ => return None, }; - Some(entries.iter().map(definition).collect()) + Some(entries.to_vec()) } diff --git a/crates/sqlbuild-core/src/lifecycle_events/types.rs b/crates/sqlbuild-core/src/lifecycle_events/types.rs index 2af438e15f..7719efd781 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/types.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/types.rs @@ -1,13 +1,4 @@ -//! The shape of one catalogue entry. - -/// One catalogued event: type, required correlations, required and allowed payload, terminal. -pub(crate) type CatalogEntry = ( - &'static str, - &'static [&'static str], - &'static [&'static str], - &'static [&'static str], - bool, -); +//! The shape of one event-family payload rule. /// One event-family payload rule: the first violation in a payload, if any. pub(crate) type PayloadCheck = diff --git a/crates/sqlbuild-discovery/src/_helpers/locations.rs b/crates/sqlbuild-discovery/src/_helpers/locations.rs index e74876e9b0..14e7d4fa22 100644 --- a/crates/sqlbuild-discovery/src/_helpers/locations.rs +++ b/crates/sqlbuild-discovery/src/_helpers/locations.rs @@ -18,7 +18,13 @@ pub(crate) fn header_column_locations( let header: Vec = contents[header_start..header_end].chars().collect(); relative_locations(&header, offsets) .into_iter() - .map(|(name, relative_line, relative_column, length)| { + .map(|location| { + let RelativeLocation { + name, + line: relative_line, + column: relative_column, + length, + } = location; let line = header_line + relative_line - 1; let column = relative_column + if relative_line == 1 { @@ -39,11 +45,19 @@ pub(crate) fn header_column_locations( .collect() } +/// One header column key's line and column relative to the header, and its length. +struct RelativeLocation { + name: String, + line: usize, + column: usize, + length: usize, +} + /// Python's `_header_column_relative_locations`, including its forward-only cursor. fn relative_locations( header: &[char], offsets: &[(String, usize, usize)], -) -> Vec<(String, usize, usize, usize)> { +) -> Vec { let mut locations = Vec::with_capacity(offsets.len()); let (mut line, mut column, mut cursor) = (1, 1, 0); for (name, position, length) in offsets { @@ -63,7 +77,12 @@ fn relative_locations( } } } - locations.push((name.clone(), line, column, *length)); + locations.push(RelativeLocation { + name: name.clone(), + line, + column, + length: *length, + }); } locations } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs index 00b92ba4c4..a7db77b4eb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs @@ -9,7 +9,10 @@ use pyo3::prelude::{ use pyo3::types::{PyBool, PyDict, PyMapping, PyTuple}; use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_analysis::semantic_validation::models as validation; -use sqlbuild_analysis::semantic_validation::types::{BindingRequest, DiagnosticRow, Relations}; +use sqlbuild_analysis::semantic_validation::models::DiagnosticRow; + +use crate::bindings::types::IntoPositional; +use sqlbuild_analysis::semantic_validation::types::{BindingRequest, Relations}; use crate::bindings::_helpers::analysis::conversions; use crate::bindings::_helpers::boundary::panics::compiler_error; @@ -57,7 +60,13 @@ pub(crate) struct BindingCatalog { } /// A catalog's dialect, case mode, known functions and known types. -pub(crate) type CatalogKey = (String, bool, Vec, Vec); +#[derive(Clone, PartialEq, Eq)] +pub(crate) struct CatalogKey { + pub(crate) dialect: String, + pub(crate) quoted_ignore_case: bool, + pub(crate) known_functions: Vec, + pub(crate) known_types: Vec, +} impl BindingCatalog { /// Whether `schemas` holds exactly the relations it was created with and later given. @@ -68,9 +77,15 @@ impl BindingCatalog { /// A catalog for `key` that starts with `relations`. pub(crate) fn create( py: Python<'_>, - (dialect, quoted_ignore_case, known_functions, known_types): CatalogKey, + key: CatalogKey, relations: &Bound<'_, PyAny>, ) -> PyResult { + let CatalogKey { + dialect, + quoted_ignore_case, + known_functions, + known_types, + } = key; let input: CatalogInput = CatalogInput { dialect: dialect.clone(), quoted_ignore_case, @@ -160,12 +175,12 @@ impl BindingCatalog { )) }) }; - let key: CatalogKey = ( - field("dialect")?.extract()?, - field("quoted_ignore_case")?.extract()?, - field("known_functions")?.extract()?, - field("known_types")?.extract()?, - ); + let key: CatalogKey = CatalogKey { + dialect: field("dialect")?.extract()?, + quoted_ignore_case: field("quoted_ignore_case")?.extract()?, + known_functions: field("known_functions")?.extract()?, + known_types: field("known_types")?.extract()?, + }; Self::create(py, key, &field("relations")?) } @@ -309,12 +324,13 @@ pub(crate) fn schema_validations<'py>( let mut created: Vec<(CatalogKey, Py)> = Vec::new(); let mut groups: Vec<(Py, bool, Vec)> = Vec::new(); for (index, request) in requests.iter().enumerate() { - let key: CatalogKey = ( - native_dialect(&native_field(request, "dialect")?)?, - native_field(request, "quoted_identifiers_ignore_case")?.extract()?, - native_field(request, "known_functions")?.extract()?, - native_field(request, "known_types")?.extract()?, - ); + let key: CatalogKey = CatalogKey { + dialect: native_dialect(&native_field(request, "dialect")?)?, + quoted_ignore_case: native_field(request, "quoted_identifiers_ignore_case")? + .extract()?, + known_functions: native_field(request, "known_functions")?.extract()?, + known_types: native_field(request, "known_types")?.extract()?, + }; let explicit: Bound<'py, PyAny> = native_field(request, "catalog")?; let own: bool = !explicit.is_none(); let catalog: Py = if own { @@ -363,7 +379,14 @@ pub(crate) fn schema_validations<'py>( for (index, diagnostics) in members.into_iter().zip(rows) { let built: Vec> = diagnostics .into_iter() - .map(|row| new_record(py, RecordId::SqlBindingDiagnostic, row, None)) + .map(|row| { + new_record( + py, + RecordId::SqlBindingDiagnostic, + row.into_positional(py)?, + None, + ) + }) .collect::>()?; let keywords: Bound<'py, PyDict> = PyDict::new(py); keywords.set_item("diagnostics", PyTuple::new(py, built)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs index 4ac680f26c..52f7e0d07d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs @@ -4,27 +4,44 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::binding_locations::main::locate_binding_diagnostics::locate_binding_diagnostics; -use sqlbuild_analysis::binding_locations::models::{BindingSource, LocatedDiagnostic}; -use sqlbuild_analysis::semantic_validation::types::{DiagnosticRow, Expansion}; +use sqlbuild_analysis::binding_locations::models::{ + AuthoredSpan, BindingSource, LocatedDiagnostic, +}; +use sqlbuild_analysis::semantic_validation::models::{DiagnosticRow, Expansion}; -/// `(code, message, severity, line, column, (line, column, end line, end column))`. -type LocatedRow = ( - String, - String, - String, - Option, - Option, - Option<(usize, usize, Option, Option)>, -); +use crate::bindings::models::Positional; + +positional_record! { + into; + /// One located diagnostic as Python reads it, its span `(line, column, end line, end column)`. + pub(crate) struct LocatedRow { + pub(crate) code: String, + pub(crate) message: String, + pub(crate) severity: String, + pub(crate) line: Option, + pub(crate) column: Option, + pub(crate) span: Option>, + } +} + +/// The expanded SQL and the expansion passes Python sends for a located model. +pub(crate) type ExpansionRow = Option<(String, Vec>>)>; /// Each reported diagnostic of `rows` through `(authored, query, cleaned)` and the expansion. #[pyfunction] pub(crate) fn locate_model_binding_diagnostics( - rows: Vec, + rows: Vec>, texts: (String, String, String), - expansion: Option<(String, Vec>)>, + expansion: ExpansionRow, ) -> PyResult> { let (authored, query, cleaned) = texts; + let rows: Vec = Positional::unwrap_all(rows); + let expansion: Option<(String, Vec>)> = expansion.map(|(expanded, passes)| { + ( + expanded, + passes.into_iter().map(Positional::unwrap_all).collect(), + ) + }); let source: BindingSource<'_> = BindingSource { authored: &authored, query: &query, @@ -39,17 +56,14 @@ pub(crate) fn locate_model_binding_diagnostics( } fn located_row(item: LocatedDiagnostic) -> LocatedRow { - let span = item - .location - .map(|span| (span.line, span.column, span.end_line, span.end_column)); - ( - item.code, - item.message, - item.severity, - item.line, - item.column, - span, - ) + LocatedRow { + code: item.code, + message: item.message, + severity: item.severity, + line: item.line, + column: item.column, + span: item.location.map(Positional), + } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/catalog.rs index 37314dacda..52c663d6b0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/catalog.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/catalog.rs @@ -3,6 +3,7 @@ use crate::bindings::_helpers::analysis::conversions; use crate::bindings::_helpers::analysis::normalization_results::normalization_results; use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::models::Positional; use crate::bindings::models::{CatalogInput, CompactAnalysisJob, ProjectCatalog}; use crate::bindings::types::CompilerDetach; use pyo3::exceptions::PyValueError; @@ -10,8 +11,9 @@ use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use pyo3::pymethods; use pyo3::types::PyBytes; use sqlbuild_analysis::semantic_validation::models as validation; +use sqlbuild_analysis::semantic_validation::models::DiagnosticRow; use sqlbuild_analysis::semantic_validation::types::{ - BindingRequest, DiagnosticRow, NormalizationRequest, Relations, + BindingRequest, NormalizationRequest, Relations, }; use std::collections::HashMap; @@ -90,9 +92,11 @@ impl ProjectCatalog { &self, py: Python<'_>, #[pyo3(from_py_with = conversions::binding_requests)] requests: Vec, - ) -> PyResult>> { - py.compiler_detach(|| self.inner.binding_results(requests)) - .map_err(compiler_error) + ) -> PyResult>>> { + let rows: Vec> = py + .compiler_detach(|| self.inner.binding_results(requests)) + .map_err(compiler_error)?; + Ok(rows.into_iter().map(Positional::wrap_all).collect()) } /// Normalize a preparation batch on the idle analysis pool, else in turn, in one detached call. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs index 47c47892ed..221a27d6de 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs @@ -423,16 +423,23 @@ pub(crate) fn old_name_reference_message( old_name_message(owner_label, written, method, model_name) } -/// `(audit name, target, gated, read)` for one attached-audit ordering edge. -pub(crate) type GateRow = (String, ResourceKey, ResourceKey, ResourceKey); -/// `(hook name, gated, read, HookReadType value, path)` for one hook-read ordering edge. -pub(crate) type HookRow = ( - String, - ResourceKey, - ResourceKey, - &'static str, - Option, -); +/// One attached-audit ordering edge: the audit, its target, and the gated and read resources. +pub(crate) struct GateRow { + pub(crate) audit_name: String, + pub(crate) target: ResourceKey, + pub(crate) gated: ResourceKey, + pub(crate) read: ResourceKey, +} + +/// One hook-read ordering edge: the hook, the gated and read resources, its type and path. +pub(crate) struct HookRow { + pub(crate) hook_name: String, + pub(crate) gated: ResourceKey, + pub(crate) read: ResourceKey, + /// The `HookReadType` value. + pub(crate) hook_type: &'static str, + pub(crate) path: Option, +} /// The attached-audit gate edges and hook-read edges of one project, in Python's order. pub(crate) fn native_execution_gate_edges( @@ -452,7 +459,12 @@ pub(crate) fn native_execution_gate_edges( } fn gate_row(edge: GateEdge) -> GateRow { - (edge.audit_name, edge.target, edge.gated, edge.read) + GateRow { + audit_name: edge.audit_name, + target: edge.target, + gated: edge.gated, + read: edge.read, + } } fn hook_row(edge: HookEdge) -> HookRow { @@ -461,7 +473,13 @@ fn hook_row(edge: HookEdge) -> HookRow { HookType::Sql => "SQL", HookType::InlineSql => "inline SQL", }; - (edge.hook_name, edge.gated, edge.read, hook_type, edge.path) + HookRow { + hook_name: edge.hook_name, + gated: edge.gated, + read: edge.read, + hook_type, + path: edge.path, + } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs index e1012641f5..9c449309a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs @@ -7,8 +7,14 @@ use crate::bindings::_helpers::compiled_project::compiled_model_view::{ }; use crate::bindings::_helpers::records::fields::field as native_field; -/// `(output, transform kind, confidence, [(resource type, resource, column)])` of one lineage output. -pub(crate) type LineageFactRow = (String, String, String, Vec<(String, String, String)>); +/// One lineage output with its transform kind and confidence values, and its sources. +pub(crate) struct LineageFactRow { + pub(crate) output_column: String, + pub(crate) transform: String, + pub(crate) confidence: String, + /// `(resource type, resource, column)` of each source. + pub(crate) sources: Vec<(String, String, String)>, +} const TRANSFORM_VALUES: [&str; 6] = [ "direct", @@ -139,12 +145,12 @@ pub(crate) fn lineage_facts(model: &Bound<'_, PyAny>) -> PyResult = Vec::with_capacity(rows.len()); for row in rows { - facts.push(( - row.output_column, - coded(&TRANSFORM_VALUES, row.transform_code)?, - coded(&CONFIDENCE_VALUES, row.confidence_code)?, - row.sources, - )); + facts.push(LineageFactRow { + output_column: row.output_column, + transform: coded(&TRANSFORM_VALUES, row.transform_code)?, + confidence: coded(&CONFIDENCE_VALUES, row.confidence_code)?, + sources: row.sources, + }); } return Ok(Some(facts)); } @@ -164,12 +170,12 @@ pub(crate) fn lineage_facts(model: &Bound<'_, PyAny>) -> PyResult, sql: &str, dialect: &str, - rows: Vec, -) -> PyResult> { + rows: Vec>, +) -> PyResult>> { + let rows: Vec = Positional::unwrap_all(rows); py.compiler_detach(|| { sqlbuild_analysis::semantic_validation::main::diagnostics::binding_diagnostics( sql, dialect, rows, ) }) + .map(Positional::wrap_all) .map_err(value_error) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index 2610876a8a..840a961495 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -19,11 +19,14 @@ use crate::bindings::_helpers::analysis_session::session::{ use crate::bindings::_helpers::analysis_session::stage_functions::family_rows; use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; use crate::bindings::_helpers::cache::native_store::{content_digest, current_code_identity}; -use crate::bindings::_helpers::compile_driver::analysis_cache::record_analysis_cache_metrics; +use crate::bindings::_helpers::compile_driver::analysis_cache::{ + AnalysisCacheCounts, record_analysis_cache_metrics, +}; use crate::bindings::_helpers::compiled_project::model_input_view::{ AttachedSchema, NativeModelInput, attached_schema, input_model_name, input_query_sql, input_reference_rows, }; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::construct::enum_member; @@ -106,36 +109,46 @@ fn session_request<'py>( let (name, declared): (String, Bound<'py, PyAny>) = item?.extract()?; families.push((name, family_rows(&declared)?)); } - let (types, nullability, complete, schemas): (Shapes, Shapes, Shapes, Shapes) = - match declared_rows(request, catalog)? { - Some(rows) => rows, - None => ( - shape_rows(&request.column_types)?, - shape_rows(&request.nullability)?, - shape_rows(&request.complete_schemas)?, - shape_rows(&native_field(catalog, "schemas")?)?, - ), - }; - Ok(( - profile_dialect(profile)?, - case_sensitive_shapes(profile, &native_field(profile, "sql_analysis_dialect")?)?, - return_types(profile)?, - Some(rules), - request.rich_type_inference, - types, - nullability, - complete, - schemas, - families, - model_rows(profile.py(), &request.model_inputs)?, - )) + let shapes: RelationShapes = match declared_rows(request, catalog)? { + Some(rows) => rows, + None => RelationShapes { + types: shape_rows(&request.column_types)?, + nullability: shape_rows(&request.nullability)?, + complete: shape_rows(&request.complete_schemas)?, + schemas: shape_rows(&native_field(catalog, "schemas")?)?, + }, + }; + Ok(RequestRow { + dialect: profile_dialect(profile)?, + case_sensitive_shapes: case_sensitive_shapes( + profile, + &native_field(profile, "sql_analysis_dialect")?, + )?, + function_return_types: return_types(profile)?, + nullability_rules: Some(rules), + rich_type_inference: request.rich_type_inference, + column_types: shapes.types, + column_nullability: shapes.nullability, + complete_schemas: shapes.complete, + catalog_schemas: shapes.schemas, + dynamic_families_by_table: families, + models: model_rows(profile.py(), &request.model_inputs)?, + }) +} + +/// The column types, nullability, complete schemas and catalog schemas a session reads. +struct RelationShapes { + types: Shapes, + nullability: Shapes, + complete: Shapes, + schemas: Shapes, } /// The caller's native declaration rows when they still match the maps and catalog they mirror. fn declared_rows( request: &ModelAnalysisRequest<'_, '_>, catalog: &Bound<'_, PyAny>, -) -> PyResult> { +) -> PyResult> { let Some(declared) = request.declared else { return Ok(None); }; @@ -151,12 +164,12 @@ fn declared_rows( if !unchanged { return Ok(None); } - Ok(Some(( - declared.types.rows().clone(), - declared.nullability.rows().clone(), - complete.clone(), - complete.clone(), - ))) + Ok(Some(RelationShapes { + types: declared.types.rows().clone(), + nullability: declared.nullability.rows().clone(), + complete: complete.clone(), + schemas: complete.clone(), + })) } /// The analysis store's path and environment, or None where Python bypasses its cache. @@ -276,7 +289,13 @@ fn record_cache( python: &[Bound<'_, PyAny>], stats: Option, ) -> PyResult<()> { - let Some((hits, misses, _, failure)) = stats else { + let Some(CacheStatsRow { + hits, + misses, + failure, + .. + }) = stats + else { return record_metrics((0, 0, python.len())); }; if let Some(failure) = failure { @@ -294,7 +313,12 @@ fn record_cache( /// Record the analysis cache entries a native session read, missed and bypassed. fn record_metrics((hits, misses, bypasses): (usize, usize, usize)) -> PyResult<()> { - record_analysis_cache_metrics((0, hits, misses, bypasses)); + record_analysis_cache_metrics(AnalysisCacheCounts { + batch_hits: 0, + entry_hits: hits, + misses, + bypasses, + }); Ok(()) } @@ -334,15 +358,15 @@ fn model_rows<'py>(py: Python<'py>, model_inputs: &[Bound<'py, PyAny>]) -> PyRes Some(schema) => family_rows(&schema.dynamic_columns)?, None => Vec::new(), }; - rows.push(( - input_model_name(model_input)?, + rows.push(ModelRow { + name: input_model_name(model_input)?, query_sql, placeholders, - references, - recovers, + reference_rows: references, + recover_cte_facts: recovers, snapshot_columns, - families, - )); + dynamic_families: families, + }); } Ok(rows) } @@ -352,7 +376,7 @@ fn analysed_references(model_input: &Bound<'_, PyAny>) -> PyResult() { return Ok(input_reference_rows(model_input)? .into_iter() - .map(|(kind, name, _, _)| (kind, name)) + .map(|ReferenceRow { kind, name, .. }| (kind, name)) .collect()); } let mut references: Vec<(String, String)> = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/oracle.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/oracle.rs index 5d597675aa..e195edbd4b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/oracle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/oracle.rs @@ -10,15 +10,36 @@ use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::types::CompilerDetach; -/// `(cleaned SQL, dialect, input schemas, return types, nullability rules, recover, null filter)`. -type RecoveryRow = (String, String, Shapes, Pairs, Option, bool, bool); -/// `(deferral, types, nullability, sorted direct outputs, sorted non-null outputs)`. -type FactsRow = (Option, Pairs, Pairs, Vec, Vec); +positional_record! { + extract; + /// One query's CTE fact recovery request, as the oracle tests send it. + struct RecoveryRow { + cleaned_sql: String, + dialect: String, + input_schemas: Shapes, + function_return_types: Pairs, + nullability_rules: Option, + recover: bool, + null_filter: bool, + } +} + +positional_record! { + into; + /// The deferral, types, nullability, sorted direct outputs and sorted non-null outputs. + struct FactsRow { + deferral: Option, + types: Pairs, + nullability: Pairs, + direct_outputs: Vec, + non_null_outputs: Vec, + } +} /// The facts native enrichment recovers for one query, or the reason it defers to Python. #[pyfunction] fn _oracle_cte_fact_recovery(py: Python<'_>, request: RecoveryRow) -> PyResult { - let ( + let RecoveryRow { cleaned_sql, dialect, input_schemas, @@ -26,7 +47,7 @@ fn _oracle_cte_fact_recovery(py: Python<'_>, request: RecoveryRow) -> PyResult, request: RecoveryRow) -> PyResult ( - None, - facts.types, - facts.nullability, - facts.direct_outputs, - facts.non_null_outputs, - ), - Err(deferral) => ( - Some(deferral), - Vec::new(), - Vec::new(), - Vec::new(), - Vec::new(), - ), + Ok(facts) => FactsRow { + deferral: None, + types: facts.types, + nullability: facts.nullability, + direct_outputs: facts.direct_outputs, + non_null_outputs: facts.non_null_outputs, + }, + Err(deferral) => FactsRow { + deferral: Some(deferral), + types: Vec::new(), + nullability: Vec::new(), + direct_outputs: Vec::new(), + non_null_outputs: Vec::new(), + }, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs index 02ae2be7b0..4143d02422 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs @@ -15,27 +15,45 @@ use crate::bindings::_helpers::records::ids::RecordId; const LINEAGE_NATIVE: &str = "native"; type Object<'py> = Bound<'py, PyAny>; -/// `(succeeded, columns, lineage source, lineage rows, star, resolved, bindings, validated, SQL)`. -type OutcomeObjects<'py> = ( - Object<'py>, - Object<'py>, - String, - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, -); -/// `(output proven, fixed columns, families, input relations, failure reason, bare pivot)`. -type ProofObjects<'py> = ( - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, -); +positional_record! { + extract; + /// A session outcome row: its analysis, lineage, binding diagnostics and cleaned SQL. + struct OutcomeObjects<'py> { + succeeded: Object<'py>, + columns: Object<'py>, + source: String, + rows: Object<'py>, + has_star: Object<'py>, + star_resolved: Object<'py>, + raw: Object<'py>, + validated: Object<'py>, + cleaned: Object<'py>, + } +} + +positional_record! { + extract; + /// A dynamic pivot proof row. + struct ProofObjects<'py> { + output_proven: Object<'py>, + columns: Object<'py>, + families: Object<'py>, + inputs: Object<'py>, + failure_reason: Object<'py>, + bare: Object<'py>, + } +} + +positional_record! { + extract; + /// A lineage row: output column, transform and confidence codes and its sources. + struct LineageObjects<'py> { + output_column: Object<'py>, + transform_code: Object<'py>, + confidence_code: Object<'py>, + sources: Object<'py>, + } +} /// The outcome at `position` of a finished session as `ModelSqlAnalysis`, for hand-built inputs. #[pyfunction] @@ -49,7 +67,17 @@ pub(crate) fn session_outcome_analysis<'py>( .get("session")? .call_method1("outcome_rows", (request.get("position")?,))? .extract()?; - let (succeeded, columns, source, rows, has_star, star_resolved, raw, validated, cleaned): OutcomeObjects<'py> = outcome.extract()?; + let OutcomeObjects { + succeeded, + columns, + source, + rows, + has_star, + star_resolved, + raw, + validated, + cleaned, + } = outcome.extract()?; let lineage: Bound<'py, PyAny> = if source != LINEAGE_NATIVE { lineage_facts(&rows)? } else if succeeded.is_truthy()? { @@ -101,8 +129,14 @@ fn proof<'py>(row: &Bound<'py, PyAny>) -> PyResult> { if row.is_none() { return Ok(row.clone()); } - let (output_proven, columns, families, inputs, failure_reason, bare): ProofObjects<'py> = - row.extract()?; + let ProofObjects { + output_proven, + columns, + families, + inputs, + failure_reason, + bare, + } = row.extract()?; let family_class: Bound<'py, PyAny> = record_class(py, RecordId::DynamicColumnFamilyProof)?; let mut proofs: Vec> = Vec::new(); for family in families.try_iter()? { @@ -174,12 +208,12 @@ fn compact_lineage<'py>(rows: &Bound<'py, PyAny>) -> PyResult> }; let mut compact: Vec> = Vec::new(); for row in rows.try_iter()? { - let (output_column, transform_code, confidence_code, sources): ( - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - ) = row?.extract()?; + let LineageObjects { + output_column, + transform_code, + confidence_code, + sources, + } = row?.extract()?; let mut source_indexes: Vec> = Vec::new(); for source in sources.try_iter()? { let (resource_type, resource_name, column_name): ( @@ -225,12 +259,12 @@ fn lineage_facts<'py>(rows: &Bound<'py, PyAny>) -> PyResult> { let source_class: Bound<'py, PyAny> = record_class(py, RecordId::CompiledLineageSourceFact)?; let mut facts: Vec> = Vec::new(); for row in rows.try_iter()? { - let (output_column, transform_code, confidence_code, sources): ( - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - ) = row?.extract()?; + let LineageObjects { + output_column, + transform_code, + confidence_code, + sources, + } = row?.extract()?; let mut upstream: Vec> = Vec::new(); for source in sources.try_iter()? { let (resource_type, resource_name, column_name): ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs index 7348587e6b..bdf1ddf33d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs @@ -5,7 +5,7 @@ use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; use crate::bindings::_helpers::analysis::binding_catalog::{ - BindingCatalog, CASE_SENSITIVE_BINDING_DIALECTS, NATIVE_DIALECT_ALIASES, + BindingCatalog, CASE_SENSITIVE_BINDING_DIALECTS, CatalogKey, NATIVE_DIALECT_ALIASES, }; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; @@ -135,12 +135,13 @@ pub(crate) fn profile_catalog<'py>( let py: Python<'py> = profile.py(); let catalog: BindingCatalog = BindingCatalog::create( py, - ( - dialect.to_owned(), - native_field(profile, "quoted_identifiers_ignore_case")?.extract()?, - Vec::new(), - Vec::new(), - ), + CatalogKey { + dialect: dialect.to_owned(), + quoted_ignore_case: native_field(profile, "quoted_identifiers_ignore_case")? + .extract()?, + known_functions: Vec::new(), + known_types: Vec::new(), + }, PyDict::new(py).as_any(), )?; Ok(Bound::new(py, catalog)?.into_any()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs index 13b2eecf79..e54f8b3274 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs @@ -23,17 +23,32 @@ use crate::bindings::types::CompilerDetach; const QUERY_COLUMNS_CONTEXT: &str = "native query column analysis"; const PROJECTION_FACTS_CONTEXT: &str = "native fixture projection facts"; -/// `(sql, placeholders, analysis names, lineage references, recover CTE facts, mode)`. -pub(crate) type QueryRow = ( - String, - Pairs, - Vec, - Vec<(String, String, String)>, - bool, - String, -); -/// `(dialect, return types, rules, types, nullability, queries, normalization dialect)`. -type RequestRow = (String, Pairs, Pairs, Shapes, Shapes, Vec, String); +positional_record! { + extract; + /// `(sql, placeholders, analysis names, lineage references, recover CTE facts, mode)`. + pub(crate) struct QueryRow { + pub(crate) sql: String, + pub(crate) placeholders: Pairs, + pub(crate) analysis_names: Vec, + pub(crate) lineage_references: Vec<(String, String, String)>, + pub(crate) recover_cte_facts: bool, + pub(crate) mode: String, + } +} + +positional_record! { + extract; + /// `(dialect, return types, rules, types, nullability, queries, normalization dialect)`. + pub(crate) struct QueryRequestRow { + pub(crate) dialect: String, + pub(crate) function_return_types: Pairs, + pub(crate) nullability_rules: Pairs, + pub(crate) types: Shapes, + pub(crate) nullability: Shapes, + pub(crate) queries: Vec, + pub(crate) normalization_dialect: String, + } +} /// `([(name, type)], NULL literal names, quoted names)`. type ProjectionFactsRow = (Vec<(String, String)>, Vec, Vec); /// `(succeeded, [(name, type, nullability)] or None, has star)`. @@ -45,10 +60,10 @@ pub(crate) type ColumnsRow = (bool, Option, String)> pub(crate) fn infer_query_columns( py: Python<'_>, catalog: Option>, - request: RequestRow, + request: QueryRequestRow, adapter_rules: Option<(Py, Py)>, ) -> PyResult> { - let ( + let QueryRequestRow { dialect, function_return_types, nullability_rules, @@ -56,7 +71,7 @@ pub(crate) fn infer_query_columns( nullability, queries, normalization_dialect, - ) = request; + } = request; let queries: Vec = queries .into_iter() .map(query_input) @@ -99,7 +114,14 @@ pub(crate) fn query_columns_error(failure: QueryColumnsFailure) -> PyErr { } fn query_input(row: QueryRow) -> Result { - let (sql, placeholders, analysis_names, lineage_references, recover_cte_facts, mode) = row; + let QueryRow { + sql, + placeholders, + analysis_names, + lineage_references, + recover_cte_facts, + mode, + } = row; let mode: QueryColumnsMode = match mode.as_str() { "batch" => QueryColumnsMode::Batch, "reanalysis" => QueryColumnsMode::Reanalysis, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs index daffde72a8..30dd79b759 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs @@ -23,7 +23,7 @@ use sqlbuild_analysis::assembly::analysis_session::models::{ SessionStep, }; use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; -use sqlbuild_analysis::semantic_validation::types::DiagnosticRow; +use sqlbuild_analysis::semantic_validation::models::DiagnosticRow; use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; use sqlbuild_cache::store::models::NativeStore; @@ -38,77 +38,138 @@ use crate::bindings::_helpers::analysis_session::nullability_rules::{ }; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::cache::native_store::{open_store, save_store}; +use crate::bindings::models::Positional; use crate::bindings::models::ProjectCatalog; use crate::bindings::types::CompilerDetach; /// `(name, type, nullability)`. type ColumnRow = (String, Option, String); -/// `(output column, transform code, confidence code, [(resource type, resource, column)])`. -type LineageItem = (String, u8, u8, Vec<(String, String, String)>); -/// `(name, pivot column, value column, aggregate, type, name pattern)`. -pub(crate) type FamilyRow = (String, String, String, String, String, Option); +positional_record! { + into; + /// One lineage column: output column, transform and confidence codes and its sources. + struct LineageItem { + output_column: String, + transform_code: u8, + confidence_code: u8, + /// `(resource type, resource, column)` of each source. + sources: Vec<(String, String, String)>, + } +} + +positional_record! { + extract; + /// One declared dynamic column family. + pub(crate) struct FamilyRow { + pub(crate) name: String, + pub(crate) pivot_column: String, + pub(crate) value_column: String, + pub(crate) aggregate: String, + pub(crate) data_type: String, + pub(crate) name_pattern: Option, + } +} + /// A model's name, SQL, placeholders, references, lineage, required names, flags and pivot. -pub(crate) type ModelRow = ( - String, - String, - Pairs, - Vec<(String, String)>, - bool, - Option<(String, String)>, - Vec, -); +pub(crate) struct ModelRow { + pub(crate) name: String, + pub(crate) query_sql: String, + pub(crate) placeholders: Pairs, + pub(crate) reference_rows: Vec<(String, String)>, + pub(crate) recover_cte_facts: bool, + pub(crate) snapshot_columns: Option<(String, String)>, + pub(crate) dynamic_families: Vec, +} + /// The profile settings, relation shapes and models one session analyses. -pub(crate) type RequestRow = ( - String, - bool, - Pairs, - Option, - bool, - Shapes, - Shapes, - Shapes, - Shapes, - Vec<(String, Vec)>, - Vec, -); +pub(crate) struct RequestRow { + pub(crate) dialect: String, + pub(crate) case_sensitive_shapes: bool, + pub(crate) function_return_types: Pairs, + pub(crate) nullability_rules: Option, + pub(crate) rich_type_inference: bool, + pub(crate) column_types: Shapes, + pub(crate) column_nullability: Shapes, + pub(crate) complete_schemas: Shapes, + pub(crate) catalog_schemas: Shapes, + pub(crate) dynamic_families_by_table: Vec<(String, Vec)>, + pub(crate) models: Vec, +} + /// `(publications, failures)`. pub(crate) type StepRow = (Shapes, Vec); -/// One model's analysis, lineage source and cleaned SQL. -type OutcomeRow = ( - bool, - Option>, - &'static str, - Vec, - bool, - bool, - Vec, - bool, - String, -); -/// `(output proven, fixed columns, (family, inferred type), inputs, failure, bare pivot)`. -type ProofRow = ( - bool, - Vec, - Vec<(String, Option)>, - Vec, - Option, - bool, -); + +positional_record! { + into; + /// One model's analysis, lineage source and cleaned SQL. + struct OutcomeRow { + analysis_succeeded: bool, + columns: Option>, + lineage_source: &'static str, + lineage: Vec, + has_star: bool, + star_resolved: bool, + binding_diagnostics: Vec>, + binding_validated: bool, + cleaned_sql: String, + } +} + +positional_record! { + into; + /// One dynamic pivot proof. + pub(crate) struct ProofRow { + output_proven: bool, + fixed_columns: Vec, + /// `(family, inferred type)` of each family. + families: Vec<(String, Option)>, + input_relations: Vec, + failure_reason: Option, + bare_dynamic_pivot: bool, + } +} + /// One model's pivot SQL and declared families. pub(crate) type PivotModelRow = (String, Vec); -/// The dialect, relation facts and families by table every proof reads, and the models. -type PivotRequestRow = ( - String, - Shapes, - Shapes, - Shapes, - Vec<(String, Vec)>, - Vec, -); + +positional_record! { + extract; + /// The dialect, relation facts and families by table every proof reads, and the models. + pub(crate) struct PivotRequestRow { + pub(crate) dialect: String, + pub(crate) column_types: Shapes, + pub(crate) authoritative_types: Shapes, + pub(crate) column_nullability: Shapes, + pub(crate) families_by_table: Vec<(String, Vec)>, + pub(crate) models: Vec, + } +} + /// `("absent" | "deferred" | "proof", proof)`. pub(crate) type ContractRow = (&'static str, Option); -/// `(hits, misses, stored, why the store was not read or saved)`. -pub(crate) type CacheStatsRow = (usize, usize, usize, Option); + +positional_record! { + into; + /// The analysis cache's hits, misses, stored entries and why it was not read or saved. + #[derive(Clone)] + pub(crate) struct CacheStatsRow { + pub(crate) hits: usize, + pub(crate) misses: usize, + pub(crate) stored: usize, + pub(crate) failure: Option, + } +} + +positional_record! { + extract; + /// The settings and expressions one expression-source shape request infers. + pub(crate) struct ExpressionShapesRow { + pub(crate) dialect: String, + pub(crate) case_sensitive_shapes: bool, + pub(crate) function_return_types: Pairs, + pub(crate) nullability_rules: Option, + pub(crate) expressions: Vec, + } +} /// `(catalog schema additions, analysis-shape names, whether each model's analysis succeeded)`. pub(crate) type FinishRow = (Shapes, Vec, Vec); @@ -188,7 +249,12 @@ impl NativeModelAnalysisSession { let saved: Option = save_store(py, &store, path, &[]) .err() .map(|error| error.to_string()); - self.cache_stats = Some((stats.hits, stats.misses, stats.stored, saved)); + self.cache_stats = Some(CacheStatsRow { + hits: stats.hits, + misses: stats.misses, + stored: stats.stored, + failure: saved, + }); } let SessionOutcome { models, @@ -217,10 +283,10 @@ impl NativeModelAnalysisSession { } /// Each requested model's raw binding diagnostic rows, in request order. - pub(crate) fn binding_rows(&self) -> Vec> { + pub(crate) fn binding_rows(&self) -> Vec>> { self.outcomes .iter() - .map(|(outcome, _)| outcome.analysis.binding_diagnostics.clone()) + .map(|(outcome, _)| Positional::wrap_all(outcome.analysis.binding_diagnostics.clone())) .collect() } @@ -295,7 +361,14 @@ pub(crate) fn start_model_analysis_session( attach_analysis_cache(&mut session, store); cache_path = Some(path); } - Err(error) => cache_stats = Some((0, 0, 0, Some(error.to_string()))), + Err(error) => { + cache_stats = Some(CacheStatsRow { + hits: 0, + misses: 0, + stored: 0, + failure: Some(error.to_string()), + }); + } } } Ok(NativeModelAnalysisSession { @@ -315,11 +388,16 @@ pub(crate) fn start_model_analysis_session( pub(crate) fn infer_expression_source_shapes( py: Python<'_>, catalog: PyRef<'_, ProjectCatalog>, - request: (String, bool, Pairs, Option, Vec), + request: ExpressionShapesRow, adapter_rules: Option<(Py, Py)>, ) -> PyResult>> { - let (dialect, case_sensitive_shapes, function_return_types, nullability_rules, expressions) = - request; + let ExpressionShapesRow { + dialect, + case_sensitive_shapes, + function_return_types, + nullability_rules, + expressions, + } = request; let rule_failure: RuleFailure = RuleFailure::default(); let request = ExpressionShapeRequest { dialect, @@ -347,7 +425,7 @@ pub(crate) fn infer_expression_source_shapes( } fn session_request(request: RequestRow) -> SessionRequest { - let ( + let RequestRow { dialect, case_sensitive_shapes, function_return_types, @@ -359,7 +437,7 @@ fn session_request(request: RequestRow) -> SessionRequest { catalog_schemas, dynamic_families_by_table, models, - ) = request; + } = request; SessionRequest { dialect, case_sensitive_shapes, @@ -379,7 +457,7 @@ fn session_request(request: RequestRow) -> SessionRequest { } fn model_request(row: ModelRow) -> ModelRequest { - let ( + let ModelRow { name, query_sql, placeholders, @@ -387,7 +465,7 @@ fn model_request(row: ModelRow) -> ModelRequest { recover_cte_facts, snapshot_columns, dynamic_families, - ) = row; + } = row; let has_set_operation = names_set_operation(&query_sql); let facts = model_reference_facts(&reference_rows); let pivot_sql = if placeholders.is_empty() { @@ -420,13 +498,11 @@ fn column_row(column: ColumnFact) -> ColumnRow { fn lineage_items(rows: Vec) -> Vec { rows.into_iter() - .map(|row| { - ( - row.output_column, - row.transform_code, - row.confidence_code, - row.sources, - ) + .map(|row| LineageItem { + output_column: row.output_column, + transform_code: row.transform_code, + confidence_code: row.confidence_code, + sources: row.sources, }) .collect() } @@ -438,7 +514,14 @@ fn families_by_table(rows: Vec<(String, Vec)>) -> Vec<(String, Vec DynamicFamily { - let (name, pivot_column, value_column, aggregate, data_type, name_pattern) = row; + let FamilyRow { + name, + pivot_column, + value_column, + aggregate, + data_type, + name_pattern, + } = row; DynamicFamily { name, pivot_column, @@ -454,14 +537,14 @@ fn contract_row(outcome: PivotOutcome) -> ContractRow { PivotOutcome::Absent => ("absent", None), PivotOutcome::Proof(proof) => ( "proof", - Some(( - proof.output_proven, - proof.fixed_columns.into_iter().map(column_row).collect(), - proof.families, - proof.input_relations, - proof.failure_reason, - proof.bare_dynamic_pivot, - )), + Some(ProofRow { + output_proven: proof.output_proven, + fixed_columns: proof.fixed_columns.into_iter().map(column_row).collect(), + families: proof.families, + input_relations: proof.input_relations, + failure_reason: proof.failure_reason, + bare_dynamic_pivot: proof.bare_dynamic_pivot, + }), ), } } @@ -472,19 +555,19 @@ fn outcome_row(outcome: ModelOutcome, cleaned_sql: String) -> OutcomeRow { LineageFacts::Native(rows) => ("native", lineage_items(rows)), LineageFacts::NativeFacts(rows) => ("facts", lineage_items(rows)), }; - ( - analysis.analysis_succeeded, - analysis + OutcomeRow { + analysis_succeeded: analysis.analysis_succeeded, + columns: analysis .columns .map(|columns| columns.into_iter().map(column_row).collect()), - source, + lineage_source: source, lineage, - analysis.has_star, - analysis.star_resolved, - analysis.binding_diagnostics, - analysis.binding_validated, + has_star: analysis.has_star, + star_resolved: analysis.star_resolved, + binding_diagnostics: Positional::wrap_all(analysis.binding_diagnostics), + binding_validated: analysis.binding_validated, cleaned_sql, - ) + } } /// Each model's dynamic pivot proof outside any session. @@ -493,8 +576,14 @@ pub(crate) fn prove_dynamic_column_contracts( py: Python<'_>, request: PivotRequestRow, ) -> PyResult> { - let (dialect, column_types, authoritative_types, column_nullability, by_table, models) = - request; + let PivotRequestRow { + dialect, + column_types, + authoritative_types, + column_nullability, + families_by_table: by_table, + models, + } = request; let request = PivotBatchRequest { tables: PivotTables { dialect, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs index 34104f8412..d4025bf278 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs @@ -13,11 +13,11 @@ use crate::bindings::_helpers::analysis_session::profile_inputs::{ profile_dialect, return_types, shape_rows, }; use crate::bindings::_helpers::analysis_session::query_columns::{ - ColumnsRow, QueryRow, infer_query_columns, + ColumnsRow, QueryRequestRow, QueryRow, infer_query_columns, }; use crate::bindings::_helpers::analysis_session::session::{ - ContractRow, FamilyRow, NativeModelAnalysisSession, PivotModelRow, - infer_expression_source_shapes, prove_dynamic_column_contracts, + ContractRow, ExpressionShapesRow, FamilyRow, NativeModelAnalysisSession, PivotModelRow, + PivotRequestRow, infer_expression_source_shapes, prove_dynamic_column_contracts, }; use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::compile_driver::stage_work::{ @@ -154,13 +154,13 @@ pub(crate) fn expression_source_shapes<'py>( let shapes: Vec> = infer_expression_source_shapes( py, native, - ( + ExpressionShapesRow { dialect, - case_sensitive, - return_types(&profile)?, - Some(rules), - pending_names, - ), + case_sensitive_shapes: case_sensitive, + function_return_types: return_types(&profile)?, + nullability_rules: Some(rules), + expressions: pending_names, + }, adapter_rules, )?; for (expression, shape) in pending.keys().iter().zip(shapes) { @@ -223,17 +223,20 @@ pub(crate) fn dynamic_contract_proofs<'py>( } prove_dynamic_column_contracts( py, - ( - dialect_or_generic(&native_field(tables, "dialect")?)?, - shape_rows(&native_field(tables, "column_types_by_table")?)?, - shape_rows(&native_field( + PivotRequestRow { + dialect: dialect_or_generic(&native_field(tables, "dialect")?)?, + column_types: shape_rows(&native_field(tables, "column_types_by_table")?)?, + authoritative_types: shape_rows(&native_field( tables, "authoritative_column_types_by_table", )?)?, - shape_rows(&native_field(tables, "column_nullability_by_table")?)?, - families, - rows, - ), + column_nullability: shape_rows(&native_field( + tables, + "column_nullability_by_table", + )?)?, + families_by_table: families, + models: rows, + }, )? } else { let session: PyRef<'py, NativeModelAnalysisSession> = session.extract()?; @@ -251,14 +254,14 @@ pub(crate) fn family_rows(families: &Bound<'_, PyAny>) -> PyResult = Vec::new(); for family in families.try_iter()? { let family: Bound<'_, PyAny> = family?; - rows.push(( - native_field(&family, "name")?.extract()?, - native_field(&family, "pivot_column")?.extract()?, - native_field(&family, "value_column")?.extract()?, - native_field(&family, "aggregate")?.extract()?, - native_field(&family, "type")?.extract()?, - native_field(&family, "name_pattern")?.extract()?, - )); + rows.push(FamilyRow { + name: native_field(&family, "name")?.extract()?, + pivot_column: native_field(&family, "pivot_column")?.extract()?, + value_column: native_field(&family, "value_column")?.extract()?, + aggregate: native_field(&family, "aggregate")?.extract()?, + data_type: native_field(&family, "type")?.extract()?, + name_pattern: native_field(&family, "name_pattern")?.extract()?, + }); } Ok(rows) } @@ -310,15 +313,15 @@ pub(crate) fn query_column_rows( infer_query_columns( py, catalog, - ( + QueryRequestRow { dialect, - return_types(profile)?, - rules, - shape_rows(types)?, - shape_rows(nullability)?, - rows, + function_return_types: return_types(profile)?, + nullability_rules: rules, + types: shape_rows(types)?, + nullability: shape_rows(nullability)?, + queries: rows, normalization_dialect, - ), + }, adapter_rules, ) } @@ -359,14 +362,14 @@ fn query_row(query: &Bound<'_, PyAny>) -> PyResult { } } } - Ok(( - native_field(query, "sql")?.extract()?, + Ok(QueryRow { + sql: native_field(query, "sql")?.extract()?, placeholders, analysis_names, - lineage, - native_field(query, "recover_cte_facts")?.extract()?, - native_field(query, "mode")?.extract()?, - )) + lineage_references: lineage, + recover_cte_facts: native_field(query, "recover_cte_facts")?.extract()?, + mode: native_field(query, "mode")?.extract()?, + }) } trait OrderedDict { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs index 56be30d4df..ae146797c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs @@ -1,6 +1,10 @@ +#[macro_use] +mod positional; + mod allocator; pub(crate) mod config_values; pub(crate) mod expected_errors; pub(crate) mod keyword_request; pub(crate) mod oracles; pub(crate) mod panics; +mod positional_records; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/boundary/positional.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/positional.rs new file mode 100644 index 0000000000..0485f9623e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/boundary/positional.rs @@ -0,0 +1,114 @@ +//! Macros declaring named records that cross the Python boundary as positional tuples. + +/// Implement the positional traits for another crate's struct, its fields in tuple order. +macro_rules! positional_fields { + ($($mode:ident),+; $name:ident $body:tt) => { + $( + positional_fields!( + @$mode $name $body + ); + )+ + }; + (@extract $name:ident { $($field:ident : $ty:ty),+ $(,)? }) => { + impl $crate::bindings::types::FromPositional for $name { + fn from_positional(value: &pyo3::Bound<'_, pyo3::PyAny>) -> pyo3::PyResult { + let ($($field,)+): ($($ty,)+) = pyo3::types::PyAnyMethods::extract(value)?; + Ok($name { $($field),+ }) + } + } + }; + (@into $name:ident { $($field:ident : $ty:ty),+ $(,)? }) => { + impl $crate::bindings::types::IntoPositional for $name { + fn into_positional<'py>( + self, + py: pyo3::Python<'py>, + ) -> pyo3::PyResult> { + let $name { $($field),+ } = self; + pyo3::IntoPyObject::into_pyobject(($($field,)+), py) + } + } + }; +} + +/// Declare a struct naming the positions of a tuple Python sends (`extract`) or receives (`into`). +macro_rules! positional_record { + ( + $($mode:ident),+; + $(#[$meta:meta])* + $vis:vis struct $name:ident <$lifetime:lifetime> $body:tt + ) => { + $(#[$meta])* + $vis struct $name<$lifetime> $body + + $( + positional_record!( + @$mode $name <$lifetime> $body + ); + )+ + }; + ( + $($mode:ident),+; + $(#[$meta:meta])* + $vis:vis struct $name:ident $body:tt + ) => { + $(#[$meta])* + $vis struct $name $body + + $( + positional_record!( + @$mode $name $body + ); + )+ + }; + (@extract $name:ident { + $($(#[$field_meta:meta])* $field_vis:vis $field:ident : $ty:ty),+ $(,)? + }) => { + impl<'py> pyo3::FromPyObject<'py> for $name { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let ($($field,)+): ($($ty,)+) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { $($field),+ }) + } + } + }; + (@extract $name:ident <$lifetime:lifetime> { + $($(#[$field_meta:meta])* $field_vis:vis $field:ident : $ty:ty),+ $(,)? + }) => { + impl<$lifetime> pyo3::FromPyObject<$lifetime> for $name<$lifetime> { + fn extract_bound( + value: &pyo3::Bound<$lifetime, pyo3::PyAny>, + ) -> pyo3::PyResult { + let ($($field,)+): ($($ty,)+) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { $($field),+ }) + } + } + }; + (@into $name:ident { + $($(#[$field_meta:meta])* $field_vis:vis $field:ident : $ty:ty),+ $(,)? + }) => { + impl<'py> pyo3::IntoPyObject<'py> for $name { + type Target = pyo3::types::PyTuple; + type Output = pyo3::Bound<'py, pyo3::types::PyTuple>; + type Error = pyo3::PyErr; + + fn into_pyobject(self, py: pyo3::Python<'py>) -> Result { + pyo3::IntoPyObject::into_pyobject(($(self.$field,)+), py) + } + } + }; + (@into $name:ident <$lifetime:lifetime> { + $($(#[$field_meta:meta])* $field_vis:vis $field:ident : $ty:ty),+ $(,)? + }) => { + impl<$lifetime> pyo3::IntoPyObject<$lifetime> for $name<$lifetime> { + type Target = pyo3::types::PyTuple; + type Output = pyo3::Bound<$lifetime, pyo3::types::PyTuple>; + type Error = pyo3::PyErr; + + fn into_pyobject( + self, + py: pyo3::Python<$lifetime>, + ) -> Result { + pyo3::IntoPyObject::into_pyobject(($(self.$field,)+), py) + } + } + }; +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/boundary/positional_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/positional_records.rs new file mode 100644 index 0000000000..3162c55985 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/boundary/positional_records.rs @@ -0,0 +1,50 @@ +//! The tuple positions of records from other crates that Python exchanges as tuples. + +use sqlbuild_analysis::binding_locations::models::AuthoredSpan; +use sqlbuild_analysis::semantic_validation::models::{DiagnosticRow, Expansion}; +use sqlbuild_rules::lint_text::models::InterpolationSite; + +positional_fields! { + extract, into; + InterpolationSite { + sentinel: String, + neutralized_start: usize, + neutralized_end: usize, + original_start: usize, + original_end: usize, + original_text: String, + } +} + +positional_fields! { + extract, into; + DiagnosticRow { + code: String, + message: String, + line: Option, + column: Option, + start: Option, + end: Option, + severity: String, + } +} + +positional_fields! { + extract; + Expansion { + source_start: usize, + source_end: usize, + output_start: usize, + output_end: usize, + } +} + +positional_fields! { + into; + AuthoredSpan { + line: usize, + column: usize, + end_line: Option, + end_column: Option, + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs index e14811cbdd..92558b69c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs @@ -17,13 +17,13 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; /// The project directory, discovered inputs, adapter context, colour and progress stream. -type InvocationContext<'py> = ( - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - bool, - Bound<'py, PyAny>, -); +struct InvocationContext<'py> { + project_dir: Bound<'py, PyAny>, + discovered: Bound<'py, PyAny>, + adapter_context: Bound<'py, PyAny>, + use_color: bool, + stream: Bound<'py, PyAny>, +} fn boolean(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { PyBool::new(py, value).to_owned().into_any() @@ -97,13 +97,25 @@ fn invocation_context<'py>( let stream: Bound<'py, PyAny> = py.import("sys")? .getattr(if machine_output { "stderr" } else { "stdout" })?; - Ok((project_dir, discovered, adapter_context, use_color, stream)) + Ok(InvocationContext { + project_dir, + discovered, + adapter_context, + use_color, + stream, + }) } fn context_fields<'py>( context: &InvocationContext<'py>, ) -> PyResult)>> { - let (project_dir, discovered, adapter_context, use_color, stream) = context; + let InvocationContext { + project_dir, + discovered, + adapter_context, + use_color, + stream, + } = context; let py: Python<'py> = project_dir.py(); Ok(vec![ ("effective_project_dir", project_dir.clone()), @@ -139,7 +151,12 @@ pub(crate) fn resolve_reported_command_invocation<'py>( ) -> PyResult> { let py: Python<'py> = request.py(); let context: InvocationContext<'py> = invocation_context(request, require_connection)?; - let (_, _, adapter_context, use_color, stream) = &context; + let InvocationContext { + adapter_context, + use_color, + stream, + .. + } = &context; let mut fields: Vec<(&str, Bound<'py, PyAny>)> = context_fields(&context)?; fields.push(( "connection_progress", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs index 7b662cb091..68a6c51b19 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs @@ -5,6 +5,7 @@ use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; use crate::bindings::_helpers::compiled_project::model_input_view::input_reference_rows; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; @@ -86,7 +87,7 @@ pub(crate) fn resolve_cursor_input_roles<'py>( Err(_) => { let cursor: Bound<'py, PyAny> = mapping_get(&values, "cursor")?; if cursor.is_instance_of::() { - for (_, name, _, _) in input_reference_rows(model)? { + for ReferenceRow { name, .. } in input_reference_rows(model)? { filter.set_item(name, &cursor)?; } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs index 7b371a1eb9..ee385e3913 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -12,7 +12,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; use crate::bindings::_helpers::compiled_project::loop_classes::model_loop_classes; use crate::bindings::_helpers::compiled_project::model_input_view::NativeModelInput; -use crate::bindings::_helpers::compiled_project::model_loop::NativeModelLoop; +use crate::bindings::_helpers::compiled_project::model_loop::{LoopSwitches, NativeModelLoop}; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::model_config::contract_classes::contract_classes; @@ -275,12 +275,13 @@ pub(crate) fn build_model_inputs<'py>( native_field(&syntax, "native_mapping")?.extract()?, reserved_cursor_markers(), ( - ( - native_field(&settings, "sql_analysis")?.extract()?, + LoopSwitches { + sql_analysis: native_field(&settings, "sql_analysis")?.extract()?, no_sql_validation, - defer_model_sql_validation, - native_field(&settings, "require_sql_analysis")?.extract()?, - ), + defer_sql_validation: defer_model_sql_validation, + require_sql_analysis: native_field(&settings, "require_sql_analysis")? + .extract()?, + }, project_config_path.str()?.extract()?, ), ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs index dd32ec6f5e..6a44449683 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs @@ -252,13 +252,40 @@ fn section<'a>(sections: &'a ImageSections, name: &str) -> Option<&'a [u8]> { .map(|(_, bytes)| bytes.as_slice()) } -/// The diagnostics and report lines an image keeps as JSON, being small and varied. -type StoredMisc = ( - Vec, - Option>, - Vec, - Option>, -); +/// The diagnostics and report lines an image keeps as one JSON array, being small and varied. +struct StoredMisc { + diagnostics: Vec, + text_models: Option>, + environment_names: Vec, + coverage: Option>, +} + +impl<'de> serde::Deserialize<'de> for StoredMisc { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_tuple(4, StoredMiscVisitor) + } +} + +/// Reads [`StoredMisc`] from its four-element array. +struct StoredMiscVisitor; + +impl<'de> serde::de::Visitor<'de> for StoredMiscVisitor { + type Value = StoredMisc; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a tuple of size 4") + } + + fn visit_seq>(self, mut seq: A) -> Result { + let missing = |index: usize| serde::de::Error::invalid_length(index, &self); + Ok(StoredMisc { + diagnostics: seq.next_element()?.ok_or_else(|| missing(0))?, + text_models: seq.next_element()?.ok_or_else(|| missing(1))?, + environment_names: seq.next_element()?.ok_or_else(|| missing(2))?, + coverage: seq.next_element()?.ok_or_else(|| missing(3))?, + }) + } +} fn put_shapes<'a>(encoder: &mut Encoder<'a>, shapes: &'a Shapes) { encoder.count(shapes.len()); @@ -458,8 +485,12 @@ impl StoredFacts { .list(|decoder| Some((decoder.string()?, decoder.strings()?)))? .into_iter() .collect(); - let (diagnostics, text_models, environment_names, coverage): StoredMisc = - from_json(decoder.bytes()?)?; + let StoredMisc { + diagnostics, + text_models, + environment_names, + coverage, + }: StoredMisc = from_json(decoder.bytes()?)?; decoder.finish()?; Some(Self { usages, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs index 3e92952ae7..73dc556a63 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs @@ -120,9 +120,22 @@ fn build_analysis_cache_context<'py>( ) } +/// Semantic analysis cache outcome counts for one invocation. +pub(crate) struct AnalysisCacheCounts { + pub(crate) batch_hits: usize, + pub(crate) entry_hits: usize, + pub(crate) misses: usize, + pub(crate) bypasses: usize, +} + /// Record semantic analysis cache outcomes for one invocation. pub(crate) fn record_analysis_cache_metrics( - (batch_hits, entry_hits, misses, bypasses): (usize, usize, usize, usize), + AnalysisCacheCounts { + batch_hits, + entry_hits, + misses, + bypasses, + }: AnalysisCacheCounts, ) { for (metric, value) in [ ("analysis_batch_cache_hits", batch_hits), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index 60ec1d1a43..0f447a7cf2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -7,7 +7,9 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple, use sqlbuild_analysis::assembly::analysis_session::main::native_dialect::native_dialect; use sqlbuild_analysis::assembly::analysis_session::types::Pairs; -use crate::bindings::_helpers::analysis::binding_catalog::{BindingCatalog, ignores_quoted_case}; +use crate::bindings::_helpers::analysis::binding_catalog::{ + BindingCatalog, CatalogKey, ignores_quoted_case, +}; use crate::bindings::_helpers::analysis_session::model_analyses::{ ModelAnalyses, ModelAnalysisRequest, analyze_models, }; @@ -93,8 +95,12 @@ pub(crate) fn analysis_context<'py>( &native_field(inputs, "effective_settings")?.getattr("sql_analysis")?, request.skip_column_inference.bind(py), )?; - let (column_types, nullability, complete_schemas, declared): DeclaredRelations<'py> = - declared_relations(inputs.clone())?; + let DeclaredRelations { + column_types, + nullability, + complete_schemas, + declared, + }: DeclaredRelations<'py> = declared_relations(inputs.clone())?; let (column_types, nullability, complete_schemas): ( Bound<'py, PyAny>, Bound<'py, PyAny>, @@ -218,12 +224,13 @@ fn profile_with_catalog<'py>( py, BindingCatalog::create( py, - ( + CatalogKey { dialect, - native_field(&profile, "quoted_identifiers_ignore_case")?.extract()?, - native_field(&profile, "semantic_known_functions")?.extract()?, - native_field(&profile, "semantic_known_types")?.extract()?, - ), + quoted_ignore_case: native_field(&profile, "quoted_identifiers_ignore_case")? + .extract()?, + known_functions: native_field(&profile, "semantic_known_functions")?.extract()?, + known_types: native_field(&profile, "semantic_known_types")?.extract()?, + }, PyDict::new(py).as_any(), )?, )? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs index 02f990d47b..5caef70872 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs @@ -270,7 +270,7 @@ pub(crate) fn binding_rows<'py>( .flatten() { let position: usize = position.extract()?; - rows_by_name.set_item(name, rows.get(position).into_pyobject(py)?)?; + rows_by_name.set_item(name, rows.get(position).cloned().into_pyobject(py)?)?; } Ok(rows_by_name) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs index 2c61cac5ed..6563157e15 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs @@ -24,7 +24,17 @@ use sqlbuild_cache::project_snapshot::models::SnapshotRules; use crate::bindings::_helpers::boundary::panics::compiler_guard; -type SettingsRow = (bool, Vec, Vec, Vec, Vec); +positional_record! { + extract; + /// Settings case sensitivity, names, prefixes, env files and secrets directories. + struct SettingsRow { + case_sensitive: bool, + names: Vec, + prefixes: Vec, + env_files: Vec, + secrets_dirs: Vec, + } +} type ReplayRow = (String, Vec, i64); type LaunchIdentityRow = (String, BTreeMap, Vec); /// A written artifact's digest, or a kept artifact's size and mtime. @@ -254,7 +264,13 @@ impl NativeReuseAttempt { } fn settings_inputs(row: SettingsRow) -> SettingsInputs { - let (case_sensitive, names, prefixes, env_files, secrets_dirs) = row; + let SettingsRow { + case_sensitive, + names, + prefixes, + env_files, + secrets_dirs, + } = row; SettingsInputs { case_sensitive, names, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs index 1b899e2c31..e733e70446 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs @@ -28,7 +28,13 @@ const PYTHON_CHECK_KIND: &str = "python_check"; const JSON_INDENT: i64 = 2; /// One test's identity as the test runtime target matches entries to results. -type TestKey = (String, Option, Option, Option); +#[derive(Hash, PartialEq, Eq)] +struct TestKey { + name: String, + source_path: Option, + block_index: Option, + case_name: Option, +} fn posix(path: &Bound<'_, PyAny>) -> PyResult> { if path.is_none() { @@ -42,16 +48,21 @@ fn path_object<'py>(py: Python<'py>, text: &str) -> PyResult> } fn test_key(value: &Bound<'_, PyAny>, name_field: &str) -> PyResult { - Ok(( - native_field(value, name_field)?.extract()?, - posix(&native_field(value, "source_path")?)?, - native_field(value, "block_index")?.extract()?, - native_field(value, "case_name")?.extract()?, - )) + Ok(TestKey { + name: native_field(value, name_field)?.extract()?, + source_path: posix(&native_field(value, "source_path")?)?, + block_index: native_field(value, "block_index")?.extract()?, + case_name: native_field(value, "case_name")?.extract()?, + }) } fn path_facts(test: &Bound<'_, PyAny>, model_names: Vec) -> PyResult { - let (name, source_path, block_index, case_name) = test_key(test, "name")?; + let TestKey { + name, + source_path, + block_index, + case_name, + } = test_key(test, "name")?; Ok(SqlTestPathFacts { name, source_path, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs index 5adc9d2492..ea17c67eb1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs @@ -12,7 +12,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::compiled_project::audit_loop::{AuditBlockRow, AuditFileRow}; -use crate::bindings::_helpers::compiled_project::function_loop::FunctionRow; +use crate::bindings::_helpers::compiled_project::function_loop::{FunctionRow, FunctionSettings}; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_project::resource_diagnostics::{ DiagnosticSettings, LoaderRow, resource_diagnostics, @@ -92,19 +92,19 @@ fn function_rows( items(discovered, field)? .into_iter() .map(|file| { - Ok(( - text(&file, "file_path")?, - text(&file, "relative_path")?, - stem(&file, "file_path")?, - file.clone().unbind(), - attr(&file, "header_values")?, - attr(&file, body)?, - if python { + Ok(FunctionRow { + file_path: text(&file, "file_path")?, + relative_path: text(&file, "relative_path")?, + name: stem(&file, "file_path")?, + function_file: file.clone().unbind(), + header_values: attr(&file, "header_values")?, + body: attr(&file, body)?, + contents: if python { None } else { Some(attr(&file, "contents")?) }, - )) + }) }) .collect() } @@ -187,17 +187,17 @@ pub(crate) fn attach_resources<'py>( true, )?, ), - ( + FunctionSettings { adapter_name, validate, - ( + defaults: ( either(&defaults, ("function_database", "database"))?, either(&defaults, ("function_schema", "schema"))?, ), - target_namespace(&target_config)?, - python_functions_inherit_default_namespace, - function_kind.unbind(), - ), + target: target_namespace(&target_config)?, + inherit: python_functions_inherit_default_namespace, + kind: function_kind.unbind(), + }, )?; let mut source_rows: Vec = Vec::new(); for source_file in items(discovered_inputs, "source_files")? { @@ -241,46 +241,46 @@ fn test_case_row(case: Bound<'_, PyAny>) -> PyResult { } fn test_block_row(block: Bound<'_, PyAny>) -> PyResult { - Ok(( - block.clone().unbind(), - attr(&block, "name")?, - attr(&block, "sql_body")?, - attr(&native_field(&block, "mode")?, "value")?, - items(&block, "parameters")? + Ok(TestBlockRow { + block: block.clone().unbind(), + name: attr(&block, "name")?, + sql_body: attr(&block, "sql_body")?, + mode: attr(&native_field(&block, "mode")?, "value")?, + parameters: items(&block, "parameters")? .iter() .map(|parameter| attr(parameter, "name")) .collect::>()?, - items(&block, "cases")? + cases: items(&block, "cases")? .into_iter() .map(test_case_row) .collect::>()?, - attr(&block, "sql_body_span")?, - )) + sql_body_span: attr(&block, "sql_body_span")?, + }) } fn test_file_row(file: Bound<'_, PyAny>) -> PyResult { - Ok(( - text(&file, "file_path")?, - text(&file, "relative_path")?, - stem(&file, "relative_path")?, - file.clone().unbind(), - attr(&file, "contents")?, - items(&file, "blocks")? + Ok(TestFileRow { + file_path: text(&file, "file_path")?, + relative_path: text(&file, "relative_path")?, + stem: stem(&file, "relative_path")?, + file: file.clone().unbind(), + contents: attr(&file, "contents")?, + blocks: items(&file, "blocks")? .into_iter() .map(test_block_row) .collect::>()?, - )) + }) } fn scenario_row(scenario: Bound<'_, PyAny>) -> PyResult { - Ok(( - text(&scenario, "file_path")?, - text(&scenario, "relative_path")?, - scenario.clone().unbind(), - attr(&scenario, "contents")?, - attr(&scenario, "sql_body")?, - attr(&scenario, "name")?, - )) + Ok(ScenarioRow { + file_path: text(&scenario, "file_path")?, + relative_path: text(&scenario, "relative_path")?, + scenario_file: scenario.clone().unbind(), + contents: attr(&scenario, "contents")?, + sql_body: attr(&scenario, "sql_body")?, + name: attr(&scenario, "name")?, + }) } /// The adapter's typed scalar renderer as `(text, None)`, or `(None, error)` for a bad value. @@ -378,31 +378,31 @@ fn audit_block_row(block: Bound<'_, PyAny>) -> PyResult { let py: Python<'_> = block.py(); let measurement: Bound<'_, PyAny> = host::entry(py, FacadeHost::AuditEvaluationMode)?.getattr(MEASUREMENT_MODE)?; - Ok(( - block.clone().unbind(), - attr(&block, "name")?, - attr(&block, "sql_body")?, - attr(&block, "evidence_sql")?, - native_field(&block, "evaluation_mode")?.eq(measurement)?, - attr(&block, "header_values")?, - )) + Ok(AuditBlockRow { + block: block.clone().unbind(), + name: attr(&block, "name")?, + sql_body: attr(&block, "sql_body")?, + evidence_sql: attr(&block, "evidence_sql")?, + measurement: native_field(&block, "evaluation_mode")?.eq(measurement)?, + header: attr(&block, "header_values")?, + }) } fn audit_file_row(file: Bound<'_, PyAny>) -> PyResult { let py: Python<'_> = file.py(); let generic: Bound<'_, PyAny> = enum_member(py, RecordId::DeclarationKind, "AUDIT")?; - Ok(( - text(&file, "file_path")?, - text(&file, "relative_path")?, - stem(&file, "file_path")?, - file.clone().unbind(), - native_field(&file, "declaration_kind")?.is(&generic), - attr(&file, "contents")?, - items(&file, "blocks")? + Ok(AuditFileRow { + file_path: text(&file, "file_path")?, + relative_path: text(&file, "relative_path")?, + stem: stem(&file, "file_path")?, + file: file.clone().unbind(), + generic: native_field(&file, "declaration_kind")?.is(&generic), + contents: attr(&file, "contents")?, + blocks: items(&file, "blocks")? .into_iter() .map(audit_block_row) .collect::>()?, - )) + }) } /// The discovered file of one built-in generic audit, as project discovery would describe it. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs index f748dffb8e..22a82e9e5b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs @@ -61,26 +61,32 @@ const MODEL_TARGET: &str = "model"; const SOURCE_TARGET: &str = "source"; const SEED_TARGET: &str = "seed"; -/// An audit block, its name, SQL body, evidence SQL, measurement flag and header values. -pub(crate) type AuditBlockRow = ( - Py, - Option, - String, - Option, - bool, - Py, -); - -/// An audit file's paths, stem, discovered file, generic flag, contents and blocks. -pub(crate) type AuditFileRow = ( - String, - String, - String, - Py, - bool, - String, - Vec, -); +positional_record! { + extract, into; + /// An audit block, its name, SQL body, evidence SQL, measurement flag and header values. + pub(crate) struct AuditBlockRow { + pub(crate) block: Py, + pub(crate) name: Option, + pub(crate) sql_body: String, + pub(crate) evidence_sql: Option, + pub(crate) measurement: bool, + pub(crate) header: Py, + } +} + +positional_record! { + extract, into; + /// An audit file's paths, stem, discovered file, generic flag, contents and blocks. + pub(crate) struct AuditFileRow { + pub(crate) file_path: String, + pub(crate) relative_path: String, + pub(crate) stem: String, + pub(crate) file: Py, + pub(crate) generic: bool, + pub(crate) contents: String, + pub(crate) blocks: Vec, + } +} /// What every audit in one compile resolves against, and the Python values it builds with. #[derive(FromPyObject)] @@ -132,8 +138,8 @@ pub(crate) fn attached_audits<'py>( settings, }; let mut audits: Vec = Vec::new(); - for file in files.iter().filter(|file| !file.4) { - for block in &file.6 { + for file in files.iter().filter(|file| !file.generic) { + for block in &file.blocks { audits.push(scope.singular(file, block)?); } } @@ -158,41 +164,41 @@ fn generic_definitions<'a, 'py>( ) -> PyResult> { let py: Python<'py> = context.host.py(); let mut project: BTreeMap> = BTreeMap::new(); - for file in files.iter().filter(|file| file.4) { - let [block] = file.6.as_slice() else { + for file in files.iter().filter(|file| file.generic) { + let [block] = file.blocks.as_slice() else { let message: String = format!( "Generic audit definition {} must contain exactly one AUDIT block", - file.1 + file.relative_path ); return Err(compile_error(context.classes, &message, None)); }; - let header: &Bound<'_, PyDict> = block.5.bind(py); - if block.4 + let header: &Bound<'_, PyDict> = block.header.bind(py); + if block.measurement && (header.contains(MEASUREMENT_HEADER_KEYS[0])? || header.contains(MEASUREMENT_HEADER_KEYS[1])?) { let message: String = format!( "Generic measurement audit definition {} must not define thresholds or \ minimum_samples in its AUDIT header; attachment owns policy", - file.1 + file.relative_path ); return Err(compile_error(context.classes, &message, None)); } - if let Some(existing) = project.get(&file.2) { + if let Some(existing) = project.get(&file.stem) { let message: String = format!( "Duplicate generic audit definition found for '{}' in {} and {}", - file.2, existing.file.1, file.1 + file.stem, existing.file.relative_path, file.relative_path ); return Err(compile_error(context.classes, &message, None)); } - project.insert(file.2.clone(), Definition { file, block }); + project.insert(file.stem.clone(), Definition { file, block }); } let mut shadows: Vec> = Vec::new(); let mut definitions: HashMap> = HashMap::new(); for builtin in builtins { - if let Some(block) = builtin.6.first() { + if let Some(block) = builtin.blocks.first() { definitions.insert( - builtin.2.clone(), + builtin.stem.clone(), Definition { file: builtin, block, @@ -202,7 +208,11 @@ fn generic_definitions<'a, 'py>( } for (name, definition) in project { if definitions.contains_key(&name) { - shadows.push(shadow_warning(context, &name, &definition.file.1)?); + shadows.push(shadow_warning( + context, + &name, + &definition.file.relative_path, + )?); } definitions.insert(name, definition); } @@ -251,8 +261,20 @@ impl<'py> AuditScope<'_, 'py> { /// One singular audit block, as `_build_singular_audit_inputs` builds it. fn singular(&self, file: &AuditFileRow, block: &AuditBlockRow) -> PyResult { let py: Python<'py> = self.context.host.py(); - let (_, relative_path, stem, audit_file, _, _, _) = file; - let (audit_block, block_name, sql_body, evidence_sql, measurement, header) = block; + let AuditFileRow { + relative_path, + stem, + file: audit_file, + .. + } = file; + let AuditBlockRow { + block: audit_block, + name: block_name, + sql_body, + evidence_sql, + measurement, + header, + } = block; let audit_name: String = block_name.clone().unwrap_or_else(|| stem.clone()); let identity: Bound<'py, PyAny> = self .settings @@ -394,7 +416,7 @@ impl<'py> AuditScope<'_, 'py> { let minimum_samples: Bound<'py, PyAny> = native_field(instance, "minimum_samples")?; let instance_severity: Option = native_field(instance, "severity")?.extract()?; let label: String = format!("{owner_file} audit '{definition_name}'"); - let measurement: bool = definition.block.4; + let measurement: bool = definition.block.measurement; if measurement { if instance_severity.is_some() { let message: String = format!( @@ -425,8 +447,8 @@ impl<'py> AuditScope<'_, 'py> { let rendering: AuditRendering = render_attached_audit(&AuditAttachment { owner_label: owner_file.to_owned(), definition_name: definition_name.clone(), - sql_body: definition.block.2.clone(), - evidence_sql: definition.block.3.clone(), + sql_body: definition.block.sql_body.clone(), + evidence_sql: definition.block.evidence_sql.clone(), implicit_arguments: implicit, explicit_arguments: argument_pairs(native_field(instance, "arguments")?.downcast()?)?, has_thresholds: !thresholds.is_none(), @@ -455,7 +477,7 @@ impl<'py> AuditScope<'_, 'py> { &format!("Audit '{definition_name}'"), )?; let calls: Vec<(&str, &str)> = reference_calls(&expanded.references); - self.checked_references(&definition.file.1, &calls)?; + self.checked_references(&definition.file.relative_path, &calls)?; let (severity, source) = rendered .policies .map_err(|message| compile_error(self.context.classes, &message, None))?; @@ -471,13 +493,13 @@ impl<'py> AuditScope<'_, 'py> { } usages.extend(expanded.usages); Ok(AttachedAudit { - audit_file: definition.file.3.clone_ref(py), - audit_block: definition.block.0.clone_ref(py), + audit_file: definition.file.file.clone_ref(py), + audit_block: definition.block.block.clone_ref(py), definition_name: definition .block - .1 + .name .clone() - .unwrap_or_else(|| definition.file.2.clone()), + .unwrap_or_else(|| definition.file.stem.clone()), sql_body: expanded.sql_body, measurement, evidence_sql: expanded.evidence_sql, @@ -508,7 +530,12 @@ impl<'py> AuditScope<'_, 'py> { identity: &Bound<'py, PyAny>, label: &str, ) -> PyResult> { - let (file_path, relative_path, _, _, _, contents, _) = file; + let AuditFileRow { + file_path, + relative_path, + contents, + .. + } = file; let declarations: Bound<'py, PyAny> = file_declarations(self.context, file_path, identity)?; let body: AuthoredExpansion<'py> = expanded_authored_sql(self.context, (sql.0, file_path), &declarations)?; @@ -682,7 +709,7 @@ impl<'py> AuditScope<'_, 'py> { fn reference_calls(references: &[ReferenceRow]) -> Vec<(&str, &str)> { references .iter() - .map(|(kind, name, _, _)| (kind.as_str(), name.as_str())) + .map(|ReferenceRow { kind, name, .. }| (kind.as_str(), name.as_str())) .collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs index 98af5da878..7ddaf506dc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs @@ -331,7 +331,14 @@ audit_getters!(NativeAuditInput, { attached .references .iter() - .map(|(kind, name, package, _)| (kind.clone(), name.clone(), package.clone())) + .map( + |ReferenceRow { + kind, + name, + package, + .. + }| (kind.clone(), name.clone(), package.clone()), + ) .collect(), attached .target diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs index 8871aa9d1d..f42801729b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs @@ -14,7 +14,7 @@ use sqlbuild_analysis::assembly::analysis_session::models::{ }; use sqlbuild_analysis::binding_locations::main::locate_binding_diagnostics::locate_binding_diagnostics; use sqlbuild_analysis::binding_locations::models::{BindingSource, LocatedDiagnostic}; -use sqlbuild_analysis::semantic_validation::types::{DiagnosticRow, Expansion}; +use sqlbuild_analysis::semantic_validation::models::{DiagnosticRow, Expansion}; use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; use crate::bindings::_helpers::boundary::panics::compiler_error; @@ -30,15 +30,18 @@ use crate::bindings::_helpers::record_behaviours::lineage_facts::{ const COMPILE_PHASE: &str = "compile"; const MODEL_RESOURCE: &str = "model"; -/// `(output proven, fixed columns, (family, inferred type), inputs, failure, bare pivot)`. -pub(crate) type ProofArgument = ( - bool, - Vec, - Vec<(String, Option)>, - Vec, - Option, - bool, -); +positional_record! { + extract; + /// `(output proven, fixed columns, (family, inferred type), inputs, failure, bare pivot)`. + pub(crate) struct ProofArgument { + output_proven: bool, + fixed: Vec, + families: Vec<(String, Option)>, + input_relations: Vec, + failure_reason: Option, + bare_dynamic_pivot: bool, + } +} /// The session analysis a model was assembled from: the session and its request position. pub(crate) type SessionArgument = (Py, usize); @@ -81,8 +84,14 @@ fn inferred_row(column: &ColumnFact) -> InferredRow { } fn proof_fact(argument: ProofArgument) -> ContractProof { - let (output_proven, fixed, families, input_relations, failure_reason, bare_dynamic_pivot) = - argument; + let ProofArgument { + output_proven, + fixed, + families, + input_relations, + failure_reason, + bare_dynamic_pivot, + } = argument; ContractProof { output_proven, fixed_columns: fixed @@ -157,8 +166,18 @@ fn located(input: &ModelInputRecord, outcome: &ModelOutcome) -> PyResult> = vec![ - expansion.declaration_spans.clone(), - expansion.macro_spans.clone(), + expansion + .declaration_spans + .iter() + .copied() + .map(Expansion::from) + .collect(), + expansion + .macro_spans + .iter() + .copied() + .map(Expansion::from) + .collect(), ]; let source: BindingSource<'_> = BindingSource { authored, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs index cd7cf04bb6..fc157b5c91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs @@ -161,6 +161,14 @@ impl Default for ViewProjections { } } +/// A model's name, compiled query, authored file, and whether it calls a function. +pub(crate) struct ModelMetadataTexts { + pub(crate) name: String, + pub(crate) query_sql: String, + pub(crate) authored_sql: String, + pub(crate) calls_functions: bool, +} + /// Deps, session analysis, failed analysis's columns, standalone proof, and macro deps. #[derive(FromPyObject)] pub(crate) struct CompiledArguments( @@ -444,17 +452,17 @@ impl NativeCompiledModel { &self, py: Python<'_>, function_kinds: &[&str], - ) -> PyResult<(String, String, String, bool)> { + ) -> PyResult { self.read(py, |record, _| { - Ok(( - model_name(record), - record.query_sql.to_string(), - record.model_file.get().contents_text().to_string(), - record + Ok(ModelMetadataTexts { + name: model_name(record), + query_sql: record.query_sql.to_string(), + authored_sql: record.model_file.get().contents_text().to_string(), + calls_functions: record .references .iter() - .any(|(kind, _, _, _)| function_kinds.contains(&kind.as_str())), - )) + .any(|reference| function_kinds.contains(&reference.kind.as_str())), + }) }) } @@ -904,12 +912,12 @@ pub(crate) fn model_graph_row( } } let folder: String = record.model_file.get().folder(py)?; - Ok(( - (MODEL_RESOURCE.to_owned(), model_name(record)), - compiled.deps.clone(), + Ok(ResourceRow { + key: (MODEL_RESOURCE.to_owned(), model_name(record)), + deps: compiled.deps.clone(), tags, - Some(folder), - )) + folder: Some(folder), + }) }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs index ea653300d5..31748115b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs @@ -38,7 +38,9 @@ use crate::bindings::_helpers::compiled_project::function_views::{ use crate::bindings::_helpers::compiled_project::model_loop::{ LoopContext, NativeModelLoop, compile_error, }; -use crate::bindings::_helpers::compiled_project::model_loop_calls::report_invalid_calls; +use crate::bindings::_helpers::compiled_project::model_loop_calls::{ + CallOrigin, report_invalid_calls, +}; use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::model_loop_inputs::reference_row; use crate::bindings::_helpers::compiled_project::source_loop::{ @@ -55,26 +57,32 @@ const SCHEMA_KEY: &str = "schema"; /// The reference kind of a source. const SOURCE_KIND: &str = "source"; -/// A function's paths, stem, discovered file, header values, body and SQL file contents. -pub(crate) type FunctionRow = ( - String, - String, - String, - Py, - Py, - String, - Option, -); +positional_record! { + extract; + /// A function's paths, stem, discovered file, header values, body and SQL file contents. + pub(crate) struct FunctionRow { + pub(crate) file_path: String, + pub(crate) relative_path: String, + pub(crate) name: String, + pub(crate) function_file: Py, + pub(crate) header_values: Py, + pub(crate) body: String, + pub(crate) contents: Option, + } +} -/// Adapter, validation, raw default and target namespaces, inheritance and resource kind. -pub(crate) type FunctionSettings = ( - String, - bool, - (Option, Option), - Option<(Option, Option)>, - bool, - Py, -); +positional_record! { + extract; + /// Adapter, validation, raw default and target namespaces, inheritance and resource kind. + pub(crate) struct FunctionSettings { + pub(crate) adapter_name: String, + pub(crate) validate: bool, + pub(crate) defaults: (Option, Option), + pub(crate) target: Option<(Option, Option)>, + pub(crate) inherit: bool, + pub(crate) kind: Py, + } +} /// What every function in one compile resolves against. struct FunctionScope<'a, 'py> { @@ -97,8 +105,14 @@ pub(crate) fn attached_functions<'py>( settings: FunctionSettings, ) -> PyResult> { let py: Python<'py> = context.host.py(); - let (adapter_name, validate, (default_database, default_schema), target, inherit, kind) = - settings; + let FunctionSettings { + adapter_name, + validate, + defaults: (default_database, default_schema), + target, + inherit, + kind, + } = settings; let defaults = ( optional_template(context, default_database, "default function database")?, optional_template(context, default_schema, "default function schema")?, @@ -130,15 +144,15 @@ pub(crate) fn attached_functions<'py>( let mut attached: Vec = Vec::with_capacity(sql_rows.len() + python_rows.len()); for row in sql_rows { - if !names.insert(row.2.clone()) { - let message: String = format!("Duplicate SQL function name '{}'", row.2); + if !names.insert(row.name.clone()) { + let message: String = format!("Duplicate SQL function name '{}'", row.name); return Err(compile_error(context.classes, &message, None)); } attached.push(sql_function(&scope, row)?); } for row in python_rows { - if !names.insert(row.2.clone()) { - let message: String = format!("Duplicate function name '{}'", row.2); + if !names.insert(row.name.clone()) { + let message: String = format!("Duplicate function name '{}'", row.name); return Err(compile_error(context.classes, &message, None)); } attached.push(python_function(&scope, row)?); @@ -150,7 +164,15 @@ pub(crate) fn attached_functions<'py>( fn sql_function(scope: &FunctionScope<'_, '_>, row: FunctionRow) -> PyResult { let context: &LoopContext<'_, '_> = scope.context; let py: Python<'_> = context.host.py(); - let (file_path, relative_path, name, function_file, header_values, body, contents) = row; + let FunctionRow { + file_path, + relative_path, + name, + function_file, + header_values, + body, + contents, + } = row; let header_values: Bound<'_, PyDict> = header_values.into_bound(py); let label: String = format!("SQL function {relative_path}"); let header: FunctionHeader = @@ -238,7 +260,14 @@ fn sql_function(scope: &FunctionScope<'_, '_>, row: FunctionRow) -> PyResult, row: FunctionRow) -> PyResult { let context: &LoopContext<'_, '_> = scope.context; let py: Python<'_> = context.host.py(); - let (_, relative_path, name, function_file, header_values, body, _) = row; + let FunctionRow { + relative_path, + name, + function_file, + header_values, + body, + .. + } = row; let header_values: Bound<'_, PyDict> = header_values.into_bound(py); let label: String = format!("Python function {relative_path}"); let header: FunctionHeader = @@ -505,7 +534,12 @@ fn function_references( context, syntax, &scanned.invalid_calls, - (None, relative_path, file_path, contents), + CallOrigin { + name: None, + relative_path, + file_path, + contents, + }, )?; let references: Vec = merge_call_site_references( scanned.references.iter().map(reference_row).collect(), @@ -520,7 +554,7 @@ fn function_references( let known: KnownResources<'_> = validator.get().known_resources(); let calls: Vec<(&str, &str)> = references .iter() - .map(|(kind, name, _, _)| (kind.as_str(), name.as_str())) + .map(|ReferenceRow { kind, name, .. }| (kind.as_str(), name.as_str())) .collect(); record_source_reads( calls @@ -585,7 +619,14 @@ pub(crate) fn checked_table_function_arities( for (owner, references) in owners { let calls: Vec<(&str, &str, Option)> = references .iter() - .map(|(kind, name, _, count)| (kind.as_str(), name.as_str(), *count)) + .map( + |ReferenceRow { + kind, + name, + call_argument_count: count, + .. + }| (kind.as_str(), name.as_str(), *count), + ) .collect(); if let Some(message) = table_function_arity_error(&owner, &calls, &counts) { return Err(compile_error(context.classes, &message, None)); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs index ca5bc2aeb9..80158fa8b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs @@ -38,13 +38,16 @@ pub(crate) struct FunctionClasses<'py> { /// `(name, type)` of an argument or return column, its types template-expanded. pub(crate) type NamedTypeRow = (String, String); -/// `(node type, name, tags, references as (kind, name, package))`. -pub(crate) type FunctionAssemblyRow = ( - &'static str, - String, - Vec, - Vec<(String, String, Option)>, -); +positional_record! { + into; + /// `(node type, name, tags, references as (kind, name, package))`. + pub(crate) struct FunctionAssemblyRow { + node_type: &'static str, + name: String, + tags: Vec, + references: Vec<(String, String, Option)>, + } +} /// `(kind, name)` of a compiled object key. pub(crate) type ObjectKeyRow = (String, String); @@ -396,16 +399,23 @@ function_getters!(NativeFunctionInput, { pub(crate) fn assembly_row(&self, py: Python<'_>) -> PyResult { read_function(&self.owner, self.index, py, |function, _| { let attached: &AttachedFunction = &function.attached; - Ok(( - attached.node_type(), - attached.name.clone(), - attached.tags.clone(), - attached + Ok(FunctionAssemblyRow { + node_type: attached.node_type(), + name: attached.name.clone(), + tags: attached.tags.clone(), + references: attached .references .iter() - .map(|(kind, name, package, _)| (kind.clone(), name.clone(), package.clone())) + .map( + |ReferenceRow { + kind, + name, + package, + .. + }| (kind.clone(), name.clone(), package.clone()), + ) .collect(), - )) + }) }) } }); @@ -433,15 +443,15 @@ impl NativeCompiledFunction { .deps .as_deref() .ok_or_else(|| PyRuntimeError::new_err(NOT_ASSEMBLED))?; - Ok(( - ( + Ok(ResourceRow { + key: ( function.attached.node_type().to_owned(), function.attached.name.clone(), ), - deps.to_vec(), - function.attached.tags.clone(), - None, - )) + deps: deps.to_vec(), + tags: function.attached.tags.clone(), + folder: None, + }) }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs index bc37f09b55..9e37af932f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs @@ -17,7 +17,7 @@ use crate::bindings::_helpers::compiled_project::schema_facts::python_column_fac use crate::bindings::_helpers::discovery::model_file_record::{NativeModelFile, projected_text}; use crate::bindings::_helpers::model_config::header_metadata::HeaderColumns; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::render::declaration_expansion::SpanRow; +use crate::bindings::_helpers::render::macro_call_rows::SpanRow; use crate::bindings::_helpers::rules::relation_keys::{RelationKeyRow, relation_key_row}; const MODEL_RELATION_KIND: &str = "ref"; @@ -382,7 +382,13 @@ pub(crate) fn reference_tuple( cache .get_or_try_init(py, || { let mut references: Vec> = Vec::with_capacity(rows.len()); - for (kind, name, package, count) in rows { + for ReferenceRow { + kind, + name, + package, + call_argument_count: count, + } in rows + { let fields: Bound<'_, PyDict> = PyDict::new(py); fields.set_item("ref_kind", classes.reference_kind.call1((kind,))?)?; fields.set_item("ref_name", name)?; @@ -561,10 +567,10 @@ pub(crate) fn input_reference_rows(model_input: &Bound<'_, PyAny>) -> PyResult) -> PyResult { - Ok(( - native_field(reference, "ref_kind")?.str()?.extract()?, - native_field(reference, "ref_name")?.extract()?, - native_field(reference, "ref_package")?.extract()?, - native_field(reference, "call_argument_count")?.extract()?, - )) + Ok(ReferenceRow { + kind: native_field(reference, "ref_kind")?.str()?.extract()?, + name: native_field(reference, "ref_name")?.extract()?, + package: native_field(reference, "ref_package")?.extract()?, + call_argument_count: native_field(reference, "call_argument_count")?.extract()?, + }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs index bdea620787..e6d8cd6bb8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs @@ -9,7 +9,9 @@ use crate::bindings::_helpers::compiled_project::function_loop::{ }; use crate::bindings::_helpers::compiled_project::function_views::AttachedFunction; use crate::bindings::_helpers::compiled_project::model_input_view::NativeModelInput; -use crate::bindings::_helpers::compiled_project::model_loop_calls::report_invalid_calls; +use crate::bindings::_helpers::compiled_project::model_loop_calls::{ + CallOrigin, report_invalid_calls, +}; use crate::bindings::_helpers::compiled_project::model_loop_config::{ConfigClasses, ModelConfigs}; use crate::bindings::_helpers::compiled_project::model_loop_external::external_errors; use crate::bindings::_helpers::compiled_project::model_loop_header::{ @@ -63,10 +65,11 @@ use crate::bindings::_helpers::model_config::header_metadata::ContractClasses; use crate::bindings::_helpers::model_config::model_validator::NativeModelValidator; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::render::declaration_expansion::{ - DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, SpanRow, + DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, expand_declarations, }; use crate::bindings::_helpers::render::declaration_values::ConstantRendering; +use crate::bindings::_helpers::render::macro_call_rows::SpanRow; use crate::bindings::_helpers::render::macro_call_rows::span_row; use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; @@ -118,8 +121,19 @@ const COMPILE_ERROR_KEY: &str = "compile_error"; /// The native validator, header contract classes, and the loop's Python classes. type LoopChecks = (Py, Py, Py); +positional_record! { + extract; + /// Whether SQL analysis runs, is skipped, is deferred, and is required. + pub(crate) struct LoopSwitches { + pub(crate) sql_analysis: bool, + pub(crate) no_sql_validation: bool, + pub(crate) defer_sql_validation: bool, + pub(crate) require_sql_analysis: bool, + } +} + /// `(analysis switches, project config path)`. -type LoopSettingsInput = ((bool, bool, bool, bool), String); +type LoopSettingsInput = (LoopSwitches, String); /// The Python classes and error factory the loop builds usages, hooks and errors with. #[derive(FromPyObject)] @@ -216,7 +230,12 @@ impl NativeModelLoop { ) -> PyResult { let (validator, contract_classes, loop_classes) = checks; let ( - (sql_analysis, no_sql_validation, defer_sql_validation, require_sql_analysis), + LoopSwitches { + sql_analysis, + no_sql_validation, + defer_sql_validation, + require_sql_analysis, + }, project_config_path, ) = settings; Python::attach(|py| index_sql_hook_definitions(loop_classes.bind(py)))?; @@ -291,19 +310,30 @@ fn prepared_rows<'py>( &sqls, )?; let mut rows: Vec = Vec::with_capacity(model_files.len()); - for (((model_file, paths), (authored, _)), (rendered, spans, reads, error)) in model_files + for (((model_file, paths), (authored, _)), interpolated) in model_files .into_iter() .zip(paths) .zip(sqls) .zip(interpolated) { + let SqlInterpolationRow { + sql: rendered, + spans, + reads, + error, + } = interpolated; let sql: String = rendered.unwrap_or(authored); let scan: Option = error.is_none().then(|| declaration_scan(&sql)); let reads: Vec<(String, String)> = reads .into_iter() .map(|(kind, name)| (kind.to_owned(), name)) .collect(); - let interpolation: InterpolationRow = (sql, reads, error, spans); + let interpolation: InterpolationRow = InterpolationRow { + sql, + reads, + error, + spans, + }; rows.push(ModelLoopRow::new((model_file, paths), scan, interpolation)); } Ok(rows) @@ -673,7 +703,12 @@ impl NativeModelLoop { context, &self.syntax, &scanned.invalid_calls, - (Some(name), relative_path, file_path, contents), + CallOrigin { + name: Some(name), + relative_path, + file_path, + contents, + }, )?; let analysis: (bool, Option>) = ( analysed && scanned.invalid_calls.is_empty(), @@ -807,7 +842,9 @@ impl NativeModelLoop { let rows: Vec<(&str, &str, bool)> = references .iter() .zip(&external) - .map(|((kind, name, _, _), error)| (kind.as_str(), name.as_str(), error.is_some())) + .map(|(ReferenceRow { kind, name, .. }, error)| { + (kind.as_str(), name.as_str(), error.is_some()) + }) .collect(); let unmanaged: Bound<'_, PyAny> = native_field( &native_field(config, RETENTION_ATTRIBUTE)?, @@ -851,7 +888,9 @@ fn applied_interpolation( context: &LoopContext<'_, '_>, interpolation: &InterpolationRow, ) -> PyResult { - let (sql, reads, error, _) = interpolation; + let InterpolationRow { + sql, reads, error, .. + } = interpolation; record_reads(context, reads)?; match error { Some(message) => Err(compile_error(context.classes, message, None)), @@ -919,13 +958,11 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { /// Span rows as the spliced spans the scan-fault locator maps through. fn spliced_spans(rows: &[SpanRow]) -> Vec { rows.iter() - .map( - |&(source_start, source_end, output_start, output_end)| SplicedSpan { - source_start, - source_end, - output_start, - output_end, - }, - ) + .map(|row| SplicedSpan { + source_start: row.source_start, + source_end: row.source_end, + output_start: row.output_start, + output_end: row.output_end, + }) .collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_calls.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_calls.rs index f80603a5f8..0403d11316 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_calls.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_calls.rs @@ -19,8 +19,14 @@ use crate::bindings::_helpers::compiled_project::model_loop::{LoopContext, compi const REFERENCE_CALL_SYNTAX_CODE: &str = "P012"; const MODEL_RESOURCE: &str = "model"; -/// `(model name or none for a function, relative path, file path, contents)` of a file. -pub(crate) type CallOrigin<'a> = (Option<&'a str>, &'a str, &'a str, &'a str); +/// The model name, or none for a function, and the relative path, file path and contents of a file. +#[derive(Clone, Copy)] +pub(crate) struct CallOrigin<'a> { + pub(crate) name: Option<&'a str>, + pub(crate) relative_path: &'a str, + pub(crate) file_path: &'a str, + pub(crate) contents: &'a str, +} /// Report each rejected call in order, at its authored location when the authored text has it. pub(crate) fn report_invalid_calls( @@ -32,7 +38,7 @@ pub(crate) fn report_invalid_calls( if invalid_calls.is_empty() { return Ok(()); } - let (_, _, _, contents) = origin; + let CallOrigin { contents, .. } = origin; let authored: HashMap> = match extract_sql_references(contents, syntax) { ReferenceExtraction::Extracted(scan) => { let mut starts: HashMap> = HashMap::new(); @@ -62,7 +68,12 @@ pub(crate) fn report_invalid_call( start: Option, origin: CallOrigin<'_>, ) -> PyResult<()> { - let (name, relative_path, file_path, contents) = origin; + let CallOrigin { + name, + relative_path, + file_path, + contents, + } = origin; let py: Python<'_> = context.host.py(); let path: Bound<'_, PyAny> = context.classes.path_class.call1((relative_path,))?; let fields: Bound<'_, PyDict> = PyDict::new(py); @@ -152,7 +163,12 @@ pub(crate) fn origin_references( context, syntax, &scanned.invalid_calls, - (None, relative_path, file_path, contents), + CallOrigin { + name: None, + relative_path, + file_path, + contents, + }, )?; Ok(scanned.references) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_external.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_external.rs index d968d3fae7..e6b96c419c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_external.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_external.rs @@ -20,7 +20,13 @@ pub(crate) fn external_errors( relative_path: &str, ) -> PyResult>> { let mut errors: Vec> = Vec::with_capacity(references.len()); - for (kind, name, package, _) in references { + for ReferenceRow { + kind, + name, + package, + .. + } in references + { errors.push(if kind == DBT_REF_KIND { external_error(context, (kind, name, package.as_deref()), relative_path)? } else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs index 2f27b2dd4e..eb86febd9a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs @@ -23,7 +23,9 @@ use crate::bindings::_helpers::compiled_project::model_loop::{ LoopContext, NativeModelLoop, VisibleDeclarations, compile_error, record_reads, validation_outcome, }; -use crate::bindings::_helpers::compiled_project::model_loop_calls::report_invalid_calls; +use crate::bindings::_helpers::compiled_project::model_loop_calls::{ + CallOrigin, report_invalid_calls, +}; use crate::bindings::_helpers::compiled_project::model_loop_external::external_errors; use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::model_loop_inputs::{ @@ -145,7 +147,9 @@ pub(crate) fn expanded_hooks<'py>( let rows: Vec<(&str, &str, bool)> = references .iter() .zip(&external) - .map(|((kind, name, _, _), error)| (kind.as_str(), name.as_str(), error.is_some())) + .map(|(ReferenceRow { kind, name, .. }, error)| { + (kind.as_str(), name.as_str(), error.is_some()) + }) .collect(); let outcome: Bound<'py, PyAny> = model_loop .validator @@ -448,12 +452,12 @@ impl<'py> HookRun<'_, '_, 'py> { self.context, &self.model_loop.syntax, &scanned.invalid_calls, - ( - Some(self.model.name), - self.model.relative_path, - self.model.file_path, - self.model.contents, - ), + CallOrigin { + name: Some(self.model.name), + relative_path: self.model.relative_path, + file_path: self.model.file_path, + contents: self.model.contents, + }, )?; self.references .borrow_mut() @@ -496,7 +500,7 @@ impl<'py> HookRun<'_, '_, 'py> { let py: Python<'py> = hooks.explicit_kinds.py(); let mut seen: Vec<(String, String)> = Vec::new(); let mut reads: Vec> = Vec::new(); - for (kind, name, _, _) in self.references.borrow().iter().skip(start) { + for ReferenceRow { kind, name, .. } in self.references.borrow().iter().skip(start) { let Some(resource_kind) = hooks.explicit_kinds.get_item(kind)? else { continue; }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs index 38a437789d..85574127c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs @@ -25,11 +25,10 @@ use crate::bindings::_helpers::compiled_project::project::{ }; use crate::bindings::_helpers::discovery::model_file_record::{ModelFileTexts, NativeModelFile}; use crate::bindings::_helpers::records::fields::{field as native_field, replaced_keywords}; -use crate::bindings::_helpers::render::declaration_expansion::{ - ExpandedDeclarations, ScannedReference, span_rows, -}; +use crate::bindings::_helpers::render::declaration_expansion::{ExpandedDeclarations, span_rows}; use crate::bindings::_helpers::render::macro_call_rows::{SpanRow, span_row}; use crate::bindings::_helpers::render::macro_expansion::ExpansionOutput; +use crate::bindings::_helpers::sqltext::declaration_references::ScannedReference; use sqlbuild_discovery::model_files::models::HeaderFacts; const HEADER_KEYS_ATTRIBUTE: &str = "model_header_keys"; @@ -77,10 +76,25 @@ impl ModelLoopRow { } } } -/// `(interpolated query, template reads, error, substitution spans)`. -pub(crate) type InterpolationRow = (String, Vec<(String, String)>, Option, Vec); -/// `(kind, name, package, call argument count)`. -pub(crate) type ReferenceRow = (String, String, Option, Option); +/// A model query's interpolation: the interpolated query, template reads, error and spans. +pub(crate) struct InterpolationRow { + pub(crate) sql: String, + pub(crate) reads: Vec<(String, String)>, + pub(crate) error: Option, + /// The substitution spans. + pub(crate) spans: Vec, +} +positional_record! { + extract, into; + /// `(kind, name, package, call argument count)`. + #[derive(Clone, Debug, PartialEq, Eq)] + pub(crate) struct ReferenceRow { + pub(crate) kind: String, + pub(crate) name: String, + pub(crate) package: Option, + pub(crate) call_argument_count: Option, + } +} /// One model the native loop finished, before the host builds its Python view. pub(crate) struct FinishedModel<'py> { @@ -130,7 +144,12 @@ pub(crate) fn finished_record<'py>( }, authored_query: authored_query_sql, contents: authored_sql, - interpolation: (interpolated_sql, _, _, interpolation_spans), + interpolation: + InterpolationRow { + sql: interpolated_sql, + spans: interpolation_spans, + .. + }, model_file, .. } = row; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_inputs.rs index 8574f80e70..82a3173139 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_inputs.rs @@ -68,15 +68,22 @@ pub(crate) fn stripped_values<'py>( } pub(crate) fn reference_row(reference: &SqlReference) -> ReferenceRow { - ( - reference.kind.to_owned(), - reference.name.clone(), - reference.package.clone(), - reference.call_argument_count, - ) + ReferenceRow { + kind: reference.kind.to_owned(), + name: reference.name.clone(), + package: reference.package.clone(), + call_argument_count: reference.call_argument_count, + } } -pub(crate) fn reference_fact((kind, name, package, _): ReferenceRow) -> ModelReferenceFact { +pub(crate) fn reference_fact( + ReferenceRow { + kind, + name, + package, + .. + }: ReferenceRow, +) -> ModelReferenceFact { ModelReferenceFact { kind, name, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_macros.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_macros.rs index 64a85b3c40..56eef6ba1d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_macros.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_macros.rs @@ -17,7 +17,7 @@ use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::model_loop_inputs::reference_row; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::render::macro_expansion::{ - ExpansionOutput, NativeMacroExpansion, expanded, + ExpansionOutput, NativeMacroExpansion, StateInput, expanded, }; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; @@ -112,13 +112,13 @@ pub(crate) fn expanded_with<'py>( let py: Python<'py> = consumer.py(); let expansion: NativeMacroExpansion = NativeMacroExpansion::new( &context.classes.path_class.call1((file_path,))?, - ( - classes.loaded_macros.clone_ref(py), - PyDict::new(py).unbind(), - classes.macro_context.clone().unbind(), - declarations.clone().unbind(), - consumer.clone().unbind(), - ), + StateInput { + loaded_macros: classes.loaded_macros.clone_ref(py), + overrides: PyDict::new(py).unbind(), + macro_context: classes.macro_context.clone().unbind(), + declarations: declarations.clone().unbind(), + consumer: consumer.clone().unbind(), + }, native_field(&classes.macro_context, ENFORCE_ATTRIBUTE)?.is_truthy()?, classes.expansion.clone_ref(py), )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs index ef803053de..6189299d1d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs @@ -49,7 +49,7 @@ use crate::bindings::_helpers::compiled_project::model_loop::NativeModelLoop; use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::project_view::NativeProjectView; use crate::bindings::_helpers::compiled_project::seed_views::{ - NativeCompiledSeed, NativeSeedInput, SeedNamespace, SeedRecord, paired_seeds, + NativeCompiledSeed, NativeSeedInput, SeedNamespace, SeedParts, SeedRecord, paired_seeds, }; use crate::bindings::_helpers::compiled_project::source_loop::{SourceRow, SourceSettings}; use crate::bindings::_helpers::compiled_project::source_views::{ @@ -70,11 +70,42 @@ const MATERIALIZED_KEY: &str = "materialized"; const TABLE_MATERIALIZATION: &str = "table"; /// `(name, type)`. -type NamedType = (String, Option); -/// `(output proven, bare dynamic pivot, (family key, inferred type), failure, (family, type))`. -type ProofPy = (bool, bool, Vec, Option, Vec); -/// `(inferred, (name, type), nullable per column, has star, unchecked output columns)`. -type AnalysisPy = (bool, Vec, Vec, bool, Vec); +pub(crate) type NamedType = (String, Option); +positional_record! { + extract; + /// `(output proven, bare dynamic pivot, (family key, inferred type), failure, (family, type))`. + pub(crate) struct ProofPy { + pub(crate) output_proven: bool, + pub(crate) bare_dynamic_pivot: bool, + pub(crate) families: Vec, + pub(crate) failure_reason: Option, + pub(crate) named_families: Vec, + } +} + +positional_record! { + extract; + /// `(inferred, (name, type), nullable per column, has star, unchecked output columns)`. + pub(crate) struct AnalysisPy { + pub(crate) inferred: bool, + pub(crate) inferred_columns: Vec, + pub(crate) nullable_columns: Vec, + pub(crate) fast_lineage_has_star: bool, + pub(crate) unchecked: Vec, + } +} + +positional_record! { + extract; + /// A model's name, relative path, compiled and authored queries, and authored SQL file. + pub(crate) struct ModelTextsPy { + name: String, + relative_path: String, + query_sql: String, + authored_query_sql: String, + authored_sql: String, + } +} /// Encoded config values and the encoder's error, if any. pub(crate) type ConfigValues = (BTreeMap, Option); @@ -82,7 +113,7 @@ pub(crate) type ConfigValues = (BTreeMap, Option); /// One model as the compile-input stage produced it. #[derive(FromPyObject)] pub(crate) struct ModelInputPy<'py>( - (String, String, String, String, String), + ModelTextsPy, (Bound<'py, PyDict>, Vec, Option), Vec<(String, String, Option)>, Vec, @@ -194,7 +225,12 @@ impl NativeCompiledProject { )?); } for seed in &self.seeds { - let (name, seed_file, schema_entry, schema_file) = seed.parts(); + let SeedParts { + name, + seed_file, + schema_entry, + schema_file, + } = seed.parts(); let schema_entry: &Bound<'py, PyAny> = schema_entry.bind(py); owners.push(resource_owner( SEED_OWNER, @@ -335,19 +371,28 @@ impl NativeCompiledProject { columns: AnalysisPy, dynamic_proof: Option, ) -> bool { - let (inferred, inferred_columns, nullable_columns, fast_lineage_has_star, unchecked) = - columns; + let AnalysisPy { + inferred, + inferred_columns, + nullable_columns, + fast_lineage_has_star, + unchecked, + } = columns; let analysis: ModelAnalysisFact = ModelAnalysisFact { inferred_columns, dynamic_proof: dynamic_proof.map( - |(output_proven, bare_dynamic_pivot, families, failure_reason, named_families)| { - DynamicProofFact { - output_proven, - bare_dynamic_pivot, - families, - failure_reason, - named_families, - } + |ProofPy { + output_proven, + bare_dynamic_pivot, + families, + failure_reason, + named_families, + }| DynamicProofFact { + output_proven, + bare_dynamic_pivot, + families, + failure_reason, + named_families, }, ), inferred, @@ -620,7 +665,13 @@ impl NativeCompiledProject { pub(crate) fn model_facts(model: ModelInputPy<'_>) -> PyResult { let ModelInputPy( - (name, relative_path, query_sql, authored_query_sql, authored_sql), + ModelTextsPy { + name, + relative_path, + query_sql, + authored_query_sql, + authored_sql, + }, (values, header_keys, layer_schema), references, enum_columns, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs index a7bf443dcf..966d655754 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs @@ -576,12 +576,12 @@ fn resource_row( tag_names.push(tag?.str()?.extract()?); } } - Ok(( - key_row(&native_field(resource, KEY_ATTRIBUTE)?)?, + Ok(ResourceRow { + key: key_row(&native_field(resource, KEY_ATTRIBUTE)?)?, deps, - tag_names, - None, - )) + tags: tag_names, + folder: None, + }) } /// `(resource type, name)` of a `CompiledObjectKey`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs index 503d7026a5..4d095ca181 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs @@ -13,7 +13,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::compiled_project::compiled_model_view::NativeCompiledModel; use crate::bindings::_helpers::compiled_project::inputs_view::NativeInputsView; use crate::bindings::_helpers::compiled_project::model_input_view::NativeModelInput; -use crate::bindings::_helpers::compiled_project::project::{NativeCompiledProject, model_facts}; +use crate::bindings::_helpers::compiled_project::project::{ + AnalysisPy, NativeCompiledProject, ProofPy, model_facts, +}; use crate::bindings::_helpers::compiled_project::project_view::NativeProjectView; use crate::bindings::_helpers::compiled_project::schema_facts::entry_schema_fact; use crate::bindings::_helpers::records::construct::{enum_member, new_record, record_class}; @@ -397,28 +399,30 @@ pub(crate) fn record_model_analyses<'py>( None } else { let families: Bound<'py, PyAny> = native_field(&proof, "families")?; - Some(( - native_field(&proof, "output_proven")?.extract()?, - native_field(&proof, "bare_dynamic_pivot")?.extract()?, - family_types(&families, true)?, - native_field(&proof, "failure_reason")?.extract()?, - family_types(&families, false)?, - )) + Some(ProofPy { + output_proven: native_field(&proof, "output_proven")?.extract()?, + bare_dynamic_pivot: native_field(&proof, "bare_dynamic_pivot")?.extract()?, + families: family_types(&families, true)?, + failure_reason: native_field(&proof, "failure_reason")?.extract()?, + named_families: family_types(&families, false)?, + }) }; - let analysis = ( - !inferred.is_none(), - rows.iter() + let analysis = AnalysisPy { + inferred: !inferred.is_none(), + inferred_columns: rows + .iter() .map(|(name, column_type, _)| (name.clone(), column_type.clone())) .collect(), - rows.iter() + nullable_columns: rows + .iter() .map(|(_, _, nullability)| nullability.eq(&nullable)) .collect::>()?, - native_field(model, "fast_lineage_has_star")?.is_truthy()?, - native_field(model, "unchecked_output_columns")? + fast_lineage_has_star: native_field(model, "fast_lineage_has_star")?.is_truthy()?, + unchecked: native_field(model, "unchecked_output_columns")? .try_iter()? .map(|column| column?.extract()) .collect::>()?, - ); + }; let name: String = native_field(model, "name")?.extract()?; transferred += usize::from( project diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs index efd61f55c2..9dd9ba24b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs @@ -21,7 +21,9 @@ use sqlbuild_analysis::resource_diagnostics::main::missing_description::missing_ use sqlbuild_analysis::resource_diagnostics::main::yaml_help::yaml_help; use sqlbuild_analysis::resource_diagnostics::models::{CursorModel, HeaderHelp}; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::seed_views::SeedParts; use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::records::fields::field as native_field; @@ -36,8 +38,39 @@ const PYTHON_UDF_DECORATOR: &str = "udf"; /// A loader's description, relative path and whether that path is a YAML file. pub(crate) type LoaderRow = (Option, String, bool); -/// A Python resource's kind, name, description (a `str` that may hold surrogates), location, help. -type PythonResourceRow = (String, String, Option>, Py, String); +positional_record! { + extract; + /// A Python resource's kind, name, description (a `str` that may hold surrogates), location, help. + struct PythonResourceRow { + kind: String, + name: String, + description: Option>, + location: Py, + help: String, + } +} + +positional_record! { + extract; + /// A scenario's name, relative path, contents and authored header description. + struct ScenarioDescriptionRow { + name: String, + path: String, + contents: String, + raw: Py, + } +} + +positional_record! { + extract; + /// A SQL hook file's name, relative path, contents and description. + struct HookDescriptionRow { + name: String, + path: String, + contents: String, + description: Option, + } +} /// What the resource diagnostics read beyond the owner, and the Python values they build. #[derive(FromPyObject)] @@ -53,9 +86,9 @@ pub(crate) struct DiagnosticSettings { /// Each discovered audit factory's name, relative path and line. audit_factories: Vec<(String, String, usize)>, /// Each scenario's name, relative path, contents and authored header description. - scenarios: Vec<(String, String, String, Py)>, + scenarios: Vec, /// Each SQL hook file's name, relative path, contents and description. - sql_hooks: Vec<(String, String, String, Option)>, + sql_hooks: Vec, /// Each loader by name. loaders: HashMap, /// Python-defined resources: kind, name, description, location and help, read by Python. @@ -277,7 +310,13 @@ fn missing_descriptions<'py>( &help, )?); } - for (name, path, contents, raw) in &diagnostics.settings.scenarios { + for ScenarioDescriptionRow { + name, + path, + contents, + raw, + } in &diagnostics.settings.scenarios + { let raw: &Bound<'py, PyAny> = raw.bind(py); let description: Option = if raw.is_instance_of::() { Some(raw.extract()?) @@ -301,7 +340,11 @@ fn missing_descriptions<'py>( )?); } for seed in project.seeds() { - let (_, _, entry, schema_file) = seed.parts(); + let SeedParts { + schema_entry: entry, + schema_file, + .. + } = seed.parts(); let entry: &Bound<'py, PyAny> = entry.bind(py); let schema_file: &Bound<'py, PyAny> = schema_file.bind(py); let name: String = native_field(entry, "name")?.extract()?; @@ -324,7 +367,13 @@ fn missing_descriptions<'py>( for function in project.functions() { found.push(function_description(diagnostics, function)?); } - for (name, path, contents, description) in &diagnostics.settings.sql_hooks { + for HookDescriptionRow { + name, + path, + contents, + description, + } in &diagnostics.settings.sql_hooks + { let entry: String = format!("description \"What {name} does\""); let help: String = header_help(&HeaderHelp { kind: "hook", @@ -341,7 +390,14 @@ fn missing_descriptions<'py>( &help, )?); } - for (kind, name, description, location, help) in &diagnostics.settings.python_resources { + for PythonResourceRow { + kind, + name, + description, + location, + help, + } in &diagnostics.settings.python_resources + { let text = description .as_ref() .map(|description| docstring_text(description.bind(py))); @@ -484,7 +540,7 @@ fn cursor_without_inputs<'py>( let kinds: Vec<&str> = record .references .iter() - .map(|(kind, _, _, _)| kind.as_str()) + .map(|ReferenceRow { kind, .. }| kind.as_str()) .collect(); let materialized: Option = config_text(&values, "materialized")?; let cursor: Option = config_text(&values, CURSOR_KEY)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs index 11af682292..2e8c4f2db4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs @@ -22,8 +22,18 @@ use crate::bindings::_helpers::compiled_project::sql_test_loop::{ FileOrigin, TestScope, assertion_targets, checked_body, source_names, unique, }; -/// A scenario file's paths, discovered file, contents, SQL body and name. -pub(crate) type ScenarioRow = (String, String, Py, String, String, String); +positional_record! { + extract; + /// A scenario file's paths, discovered file, contents, SQL body and name. + pub(crate) struct ScenarioRow { + pub(crate) file_path: String, + pub(crate) relative_path: String, + pub(crate) scenario_file: Py, + pub(crate) contents: String, + pub(crate) sql_body: String, + pub(crate) name: String, + } +} impl<'py> TestScope<'_, 'py> { /// Every scenario expanded, classified and checked, in discovery order. @@ -33,7 +43,14 @@ impl<'py> TestScope<'_, 'py> { fn scenario(&self, row: &ScenarioRow) -> PyResult> { let py: Python<'py> = self.context.host.py(); - let (file_path, relative_path, scenario_file, contents, sql_body, name) = row; + let ScenarioRow { + file_path, + relative_path, + scenario_file, + contents, + sql_body, + name, + } = row; let resource: Bound<'py, PyAny> = self .context .classes diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs index 4e775cf3cb..52486ab28b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs @@ -24,13 +24,24 @@ const FILE_PATH_ATTRIBUTE: &str = "file_path"; const STEM_ATTRIBUTE: &str = "stem"; const RELATIVE_PATH_ATTRIBUTE: &str = "relative_path"; -/// `(database, schema, logical database, logical schema)` of a seed's destination. -pub(crate) type SeedNamespace = ( - Option, - Option, - Option, - Option, -); +positional_record! { + extract; + /// `(database, schema, logical database, logical schema)` of a seed's destination. + pub(crate) struct SeedNamespace { + database: Option, + schema: Option, + logical_database: Option, + logical_schema: Option, + } +} + +/// A seed's name, CSV file, declaration and declaring schema file. +pub(crate) struct SeedParts<'a> { + pub(crate) name: &'a str, + pub(crate) seed_file: &'a Py, + pub(crate) schema_entry: &'a Py, + pub(crate) schema_file: &'a Py, +} /// One seed CSV paired with its declaration; the discovered objects are discovery's. pub(crate) struct SeedRecord { @@ -100,18 +111,23 @@ pub(crate) fn paired_seeds( impl SeedRecord { /// The seed's name, CSV file, declaration and declaring schema file. - pub(crate) fn parts(&self) -> (&str, &Py, &Py, &Py) { - ( - &self.name, - &self.seed_file, - &self.schema_entry, - &self.schema_file, - ) + pub(crate) fn parts(&self) -> SeedParts<'_> { + SeedParts { + name: &self.name, + seed_file: &self.seed_file, + schema_entry: &self.schema_entry, + schema_file: &self.schema_file, + } } /// Keep the destination native project assembly resolved for the seed. pub(crate) fn set_namespace(&mut self, namespace: SeedNamespace) { - let (database, schema, logical_database, logical_schema) = namespace; + let SeedNamespace { + database, + schema, + logical_database, + logical_schema, + } = namespace; self.destination = Some(DestinationFact { database, schema, @@ -201,12 +217,12 @@ impl NativeCompiledSeed { for tag in native_field(seed.schema_entry.bind(py), TAGS_ATTRIBUTE)?.try_iter()? { tags.push(tag?.str()?.extract()?); } - Ok(( - (SEED_RESOURCE.to_owned(), seed.name.clone()), - Vec::new(), + Ok(ResourceRow { + key: (SEED_RESOURCE.to_owned(), seed.name.clone()), + deps: Vec::new(), tags, - None, - )) + folder: None, + }) }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs index 48690285f0..e3537d16b5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs @@ -20,6 +20,17 @@ use crate::bindings::_helpers::rules::relation_keys::audit_key_rows; const SOURCE_RESOURCE: &str = "source"; +positional_record! { + into; + /// `(name, has a loader, database, schema)` of one source. + pub(crate) struct SourceAssemblyRow { + name: String, + managed: bool, + database: Option, + schema: Option, + } +} + /// `(database, schema)` native project assembly resolved for a source, or `None`. pub(crate) type SourceNamespace = Option<(Option, Option)>; @@ -210,18 +221,15 @@ impl NativeSourceInput { /// `(name, has a loader, database, schema)`, what project assembly resolves a namespace from. #[getter] - pub(crate) fn assembly_row( - &self, - py: Python<'_>, - ) -> PyResult<(String, bool, Option, Option)> { + pub(crate) fn assembly_row(&self, py: Python<'_>) -> PyResult { read_source(&self.owner, self.index, py, |source, py| { let attached: &AttachedSource = &source.attached; - Ok(( - attached.name.clone(), - !native_field(attached.entry.bind(py), "loader")?.is_none(), - attached.database.clone(), - attached.schema.clone(), - )) + Ok(SourceAssemblyRow { + name: attached.name.clone(), + managed: !native_field(attached.entry.bind(py), "loader")?.is_none(), + database: attached.database.clone(), + schema: attached.schema.clone(), + }) }) } } @@ -241,12 +249,12 @@ impl NativeCompiledSource { /// The source's project graph row: its key, with no dependencies, tags or folder. pub(crate) fn graph_row(&self, py: Python<'_>) -> PyResult { read_source(&self.owner, self.index, py, |source, _| { - Ok(( - (SOURCE_RESOURCE.to_owned(), source.attached.name.clone()), - Vec::new(), - Vec::new(), - None, - )) + Ok(ResourceRow { + key: (SOURCE_RESOURCE.to_owned(), source.attached.name.clone()), + deps: Vec::new(), + tags: Vec::new(), + folder: None, + }) }) } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs index 07563842eb..40aa64302a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs @@ -42,7 +42,7 @@ use crate::bindings::_helpers::compiled_project::model_loop::{ LoopContext, NativeModelLoop, compile_error, helped_compile_error, }; use crate::bindings::_helpers::compiled_project::model_loop_calls::{ - origin_references, report_invalid_call, + CallOrigin, origin_references, report_invalid_call, }; use crate::bindings::_helpers::compiled_project::source_loop::{ AuthoredExpansion, expanded_authored_sql, scoped_declarations, @@ -70,19 +70,32 @@ const MACRO_VISIBILITY: &str = "macro_visibility"; /// A test case's discovered case, name and parameter values in authored order. pub(crate) type TestCaseRow = (Py, String, Vec<(String, Py)>); -/// A test block, its name, SQL, mode, declared parameter names, cases and authored SQL span. -pub(crate) type TestBlockRow = ( - Py, - Option, - String, - String, - Vec, - Vec, - Option<(usize, usize)>, -); +positional_record! { + extract; + /// A test block, its name, SQL, mode, declared parameter names, cases and authored SQL span. + pub(crate) struct TestBlockRow { + pub(crate) block: Py, + pub(crate) name: Option, + pub(crate) sql_body: String, + pub(crate) mode: String, + pub(crate) parameters: Vec, + pub(crate) cases: Vec, + pub(crate) sql_body_span: Option<(usize, usize)>, + } +} -/// A test file's paths, stem, discovered file, contents and blocks. -pub(crate) type TestFileRow = (String, String, String, Py, String, Vec); +positional_record! { + extract; + /// A test file's paths, stem, discovered file, contents and blocks. + pub(crate) struct TestFileRow { + pub(crate) file_path: String, + pub(crate) relative_path: String, + pub(crate) stem: String, + pub(crate) file: Py, + pub(crate) contents: String, + pub(crate) blocks: Vec, + } +} /// What every SQL test and scenario of one compile resolves and records with. #[derive(FromPyObject)] @@ -151,11 +164,11 @@ impl<'py> TestScope<'_, 'py> { pub(crate) fn attached(&self, files: &[TestFileRow]) -> PyResult>> { let mut expanded: Vec> = Vec::new(); for file in files { - for block in &file.5 { - if block.5.is_empty() { + for block in &file.blocks { + if block.cases.is_empty() { expanded.push(self.expanded(file, block, None)?); } - for case in &block.5 { + for case in &block.cases { expanded.push(self.expanded(file, block, Some(case))?); } } @@ -178,8 +191,20 @@ impl<'py> TestScope<'_, 'py> { case: Option<&'a TestCaseRow>, ) -> PyResult> { let py: Python<'py> = self.py(); - let (file_path, relative_path, stem, _, contents, _) = file; - let (_, name, sql_body, mode, parameters, _, _) = block; + let TestFileRow { + file_path, + relative_path, + stem, + contents, + .. + } = file; + let TestBlockRow { + name, + sql_body, + mode, + parameters, + .. + } = block; let test_name: &str = name.as_deref().unwrap_or(stem); let resource: Bound<'py, PyAny> = self .context @@ -329,7 +354,7 @@ impl<'py> TestScope<'_, 'py> { let (file, block) = test; let request: SqlTestRequest = SqlTestRequest { sql: sql.to_owned(), - file_label: file.1.clone(), + file_label: file.relative_path.clone(), mode: mode.to_owned(), raw: true, authored: false, @@ -348,7 +373,7 @@ impl<'py> TestScope<'_, 'py> { .. } = &classification { - let origin: FileOrigin<'_> = (&file.0, &file.1, &file.4); + let origin: FileOrigin<'_> = (&file.file_path, &file.relative_path, &file.contents); if self.authored_invalid_calls(file, block, &actual.0)? == 0 { for cte in helpers.iter().chain([expected]) { origin_references(self.context, &self.model_loop.syntax, &cte.1, origin)?; @@ -367,8 +392,8 @@ impl<'py> TestScope<'_, 'py> { ) -> PyResult { let syntax = &self.model_loop.syntax; let request: SqlTestRequest = SqlTestRequest { - sql: block.2.clone(), - file_label: file.1.clone(), + sql: block.sql_body.clone(), + file_label: file.relative_path.clone(), mode: MODEL_MODE.to_owned(), raw: false, authored: true, @@ -394,16 +419,24 @@ impl<'py> TestScope<'_, 'py> { ReferenceExtraction::Failed(failure) => return Err(self.error(&failure.message)), }; for invalid in &scan.invalid_calls { - let Some(offset) = - authored_test_offset(&file.4, &block.2, block.6, body_start + invalid.start) - else { + let Some(offset) = authored_test_offset( + &file.contents, + &block.sql_body, + block.sql_body_span, + body_start + invalid.start, + ) else { return Ok(reported); }; report_invalid_call( self.context, invalid, Some(offset), - (None, &file.1, &file.0, &file.4), + CallOrigin { + name: None, + relative_path: &file.relative_path, + file_path: &file.file_path, + contents: &file.contents, + }, )?; reported += 1; } @@ -611,11 +644,11 @@ impl<'py> TestScope<'_, 'py> { let mut files: Vec<(&str, Vec)> = Vec::new(); let mut positions: HashMap<&str, usize> = HashMap::new(); for (index, test) in tests.iter().enumerate() { - match positions.get(test.file.1.as_str()) { + match positions.get(test.file.relative_path.as_str()) { Some(position) => files[*position].1.push(index), None => { - positions.insert(&test.file.1, files.len()); - files.push((&test.file.1, vec![index])); + positions.insert(&test.file.relative_path, files.len()); + files.push((&test.file.relative_path, vec![index])); } } } @@ -635,8 +668,8 @@ impl<'py> TestScope<'_, 'py> { let mut modes: Vec<&str> = Vec::with_capacity(indexes.len()); for index in indexes { parts.push(tests[*index].sql.clone()); - parts.push(tests[*index].block.3.clone()); - modes.push(&tests[*index].block.3); + parts.push(tests[*index].block.mode.clone()); + modes.push(&tests[*index].block.mode); } let stored: Option> = match &store { Some(store) => store @@ -662,8 +695,8 @@ impl<'py> TestScope<'_, 'py> { .iter() .map(|index| SqlTestRequest { sql: tests[*index].sql.clone(), - file_label: tests[*index].file.1.clone(), - mode: tests[*index].block.3.clone(), + file_label: tests[*index].file.relative_path.clone(), + mode: tests[*index].block.mode.clone(), raw: false, authored: false, scenario: false, @@ -713,9 +746,21 @@ impl<'py> TestScope<'_, 'py> { let offset: Option = failure .token .as_ref() - .filter(|_| sql == block.2) - .and_then(|(_, offset)| authored_test_offset(&file.4, &block.2, block.6, *offset)); - located_error(self.context, failure, (&file.1, &file.4), offset) + .filter(|_| sql == block.sql_body) + .and_then(|(_, offset)| { + authored_test_offset( + &file.contents, + &block.sql_body, + block.sql_body_span, + *offset, + ) + }); + located_error( + self.context, + failure, + (&file.relative_path, &file.contents), + offset, + ) } /// Check one classified test; `None` when a P013 problem was reported instead. @@ -725,8 +770,20 @@ impl<'py> TestScope<'_, 'py> { classification: SqlTestClassification, ) -> PyResult>> { let py: Python<'py> = self.py(); - let (file_path, relative_path, stem, test_file, contents, _) = test.file; - let (test_block, name, _, mode, _, _, _) = test.block; + let TestFileRow { + file_path, + relative_path, + stem, + file: test_file, + contents, + .. + } = test.file; + let TestBlockRow { + block: test_block, + name, + mode, + .. + } = test.block; let test_name: &str = name.as_deref().unwrap_or(stem); let origin: FileOrigin<'_> = (file_path, relative_path, contents); let payload: Bound<'py, PyAny> = match &classification { @@ -911,14 +968,14 @@ impl<'py> TestScope<'_, 'py> { ) -> PyResult<()> { let py: Python<'py> = self.py(); let located: CteLocation = sql_test_cte_location( - &file.4, - &block.2, + &file.contents, + &block.sql_body, &problem.cte_name, problem.call.as_deref(), ); let (line, column) = (located.line, located.column); let arguments: Bound<'py, PyDict> = PyDict::new(py); - arguments.set_item("path", native_field(file.3.bind(py), "relative_path")?)?; + arguments.set_item("path", native_field(file.file.bind(py), "relative_path")?)?; arguments.set_item("line", line)?; arguments.set_item("column", column)?; arguments.set_item("end_line", located.end_line)?; @@ -941,7 +998,7 @@ impl<'py> TestScope<'_, 'py> { py, [ HELPER_REFERENCE_CODE.to_owned(), - file.1.replace('\\', "/"), + file.relative_path.replace('\\', "/"), format!("{line:09}:{column:09}"), problem.message.clone(), ], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs index 2dba2e534a..f9b65d4618 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs @@ -56,10 +56,16 @@ fn output_column_location<'py>( } let implicit: bool = native_field(model, "extract_implicit_alias_columns")?.is_truthy()?; let rows: Vec = location_rows(model_output_columns(&authored, implicit)); - let Some((_, line, column, end_line, end_column)) = rows + let Some(ColumnLocation { + line, + column, + end_line, + end_column, + .. + }) = rows .into_iter() .rev() - .find(|(name, ..)| name == column_name) + .find(|location| location.name == column_name) else { return Ok(None); }; @@ -126,7 +132,16 @@ fn contract_diagnostic<'py>( row: DiagnosticRow, ) -> PyResult> { let py: Python<'py> = model.py(); - let (code, is_error, message, column_name, declared_index, output_column, related, help) = row; + let DiagnosticRow { + code, + is_error, + message, + column_name, + declared_index, + output_column, + related, + help, + } = row; let severity: &str = if is_error { "error" } else { "warning" }; keyword_call( &record_class(py, RecordId::CompilerDiagnostic)?, @@ -228,7 +243,14 @@ pub(crate) fn promotion_conflict_diagnostics<'py>( let diagnostic_type: Bound<'py, PyAny> = record_class(py, RecordId::CompilerDiagnostic)?; let path_type: Bound<'py, PyAny> = py.import(PATHLIB_MODULE)?.getattr("Path")?; let mut diagnostics: Vec> = Vec::with_capacity(conflicts.len()); - for (name, relative_path, code, message, help) in conflicts { + for ConflictRow { + name, + relative_path, + code, + message, + help, + } in conflicts + { diagnostics.push(keyword_call( &diagnostic_type, vec![ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs index 4b840c0e71..d3d532a8c0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs @@ -20,17 +20,18 @@ use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::type_system::normalization::normalization_error; use crate::bindings::types::CompilerDetach; -/// `(code, is_error, message, column, declared_index, output_column, related, help)`. -pub(crate) type DiagnosticRow = ( - &'static str, - bool, - String, - Option, - Option, - Option, - Option<(String, String)>, - String, -); +/// One contract diagnostic: its code, severity, message, column, location, related output and help. +#[derive(Clone)] +pub(crate) struct DiagnosticRow { + pub(crate) code: &'static str, + pub(crate) is_error: bool, + pub(crate) message: String, + pub(crate) column_name: Option, + pub(crate) declared_index: Option, + pub(crate) output_column: Option, + pub(crate) related: Option<(String, String)>, + pub(crate) help: String, +} /// Evaluate the column contract of every model the compile's owner recorded, in its order. pub(crate) fn evaluate_project_model_contracts( @@ -133,8 +134,14 @@ fn fact_dynamic_proof(proof: &DynamicProofFact) -> DynamicContractProof { } } -/// `(model name, model path, code, message, help)`. -pub(crate) type ConflictRow = (String, String, &'static str, String, String); +/// One K011 conflict: the model name and path, and the code, message and help. +pub(crate) struct ConflictRow { + pub(crate) name: String, + pub(crate) relative_path: String, + pub(crate) code: &'static str, + pub(crate) message: String, + pub(crate) help: String, +} /// The K011 conflicts of enforced contracts under immediate table promotion, by model name. pub(crate) fn project_promotion_conflicts( @@ -166,29 +173,29 @@ pub(crate) fn project_promotion_conflicts( .into_iter() .map(|conflict| { let model: &CompiledModelFacts = &models[conflict.model_index]; - ( - model.name.clone(), - model.relative_path.clone(), - conflict.code, - conflict.message, - conflict.help, - ) + ConflictRow { + name: model.name.clone(), + relative_path: model.relative_path.clone(), + code: conflict.code, + message: conflict.message, + help: conflict.help, + } }) .collect()) } fn diagnostic_row(diagnostic: ContractDiagnostic) -> DiagnosticRow { let (declared_index, output_column) = diagnostic.location.into_parts(); - ( - diagnostic.code, - diagnostic.severity == ContractSeverity::Error, - diagnostic.message, - diagnostic.column_name, + DiagnosticRow { + code: diagnostic.code, + is_error: diagnostic.severity == ContractSeverity::Error, + message: diagnostic.message, + column_name: diagnostic.column_name, declared_index, output_column, - diagnostic + related: diagnostic .related_output .map(|related| (related.column_name, related.message)), - diagnostic.help, - ) + help: diagnostic.help, + } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs index 405832ce4d..bd8dd810b2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs @@ -162,6 +162,17 @@ fn read_only_query( Ok(normalized.to_owned()) } +positional_record! { + extract; + /// The target name, target, database and schema a raw-query diff resolves. + struct ResolvedNamespace<'py> { + target_name: Bound<'py, PyAny>, + _target: Bound<'py, PyAny>, + database: Bound<'py, PyAny>, + schema: Bound<'py, PyAny>, + } +} + /// Resolve raw SQL, the active target namespace, the adapter and its connection. pub(crate) fn prepare_query_diff<'py>( request: &Bound<'py, PyAny>, @@ -206,12 +217,12 @@ pub(crate) fn prepare_query_diff<'py>( )?), ) .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))?; - let (target_name, _target, database, schema): ( - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - ) = namespace.extract()?; + let ResolvedNamespace { + target_name, + database, + schema, + .. + }: ResolvedNamespace<'py> = namespace.extract()?; if schema.is_none() { return Err(failure( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs index 605837e2c8..623fd46ff4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs @@ -103,7 +103,14 @@ fn source_locations<'py>( ) -> PyResult> { let py: Python<'py> = relative_path.py(); let locations: Bound<'py, PyDict> = PyDict::new(py); - for (name, line, column, end_line, end_column) in rows { + for ColumnLocation { + name, + line, + column, + end_line, + end_column, + } in rows + { let arguments: Bound<'py, PyDict> = PyDict::new(py); arguments.set_item("path", relative_path)?; arguments.set_item("line", line)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs index 2ebfe2d8b7..ac8620b4b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs @@ -39,7 +39,13 @@ fn attached_audit_gate_edges<'py>( let cache: KeyCache<'py> = KeyCache::new(py)?; let edge_type: Bound<'py, PyAny> = record_class(py, RecordId::AttachedAuditGateEdge)?; let mut edges: Vec> = Vec::with_capacity(gates.len()); - for (audit_name, target, gated, read) in gates { + for GateRow { + audit_name, + target, + gated, + read, + } in gates + { edges.push(keyword_call( &edge_type, &[ @@ -65,7 +71,14 @@ fn hook_read_edges<'py>( let edge_type: Bound<'py, PyAny> = record_class(py, RecordId::HookReadEdge)?; let path_type: Bound<'py, PyAny> = py.import(PATHLIB_MODULE)?.getattr("Path")?; let mut edges: Vec> = Vec::with_capacity(hooks.len()); - for (hook_name, gated, read, kind, path) in hooks { + for HookRow { + hook_name, + gated, + read, + hook_type: kind, + path, + } in hooks + { let hook_path: Bound<'py, PyAny> = match path { None => py.None().into_bound(py), Some(path) => path_type.call1((path,))?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs index 62e7a80269..6e880c4db6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs @@ -24,17 +24,28 @@ use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::graph::views::lineage_graph_views; use crate::bindings::_helpers::records::fields::field as native_field; -/// `(key, deps, tags, model folder)`. -pub(crate) type ResourceRow = (ObjectKey, Vec, Vec, Option); +positional_record! { + extract; + /// `(key, deps, tags, model folder)`. + pub(crate) struct ResourceRow { + pub(crate) key: ObjectKey, + pub(crate) deps: Vec, + pub(crate) tags: Vec, + pub(crate) folder: Option, + } +} type Edges = Vec<(ObjectKey, Vec)>; -/// `(names, upstream, downstream, tags, paths)` in Python's dict order. -type IndexesRow = ( - Vec<(String, ObjectKey)>, - Edges, - Edges, - Vec<(String, Vec)>, - Vec<(ObjectKey, String)>, -); +positional_record! { + extract, into; + /// `(names, upstream, downstream, tags, paths)` in Python's dict order. + struct IndexesRow { + names: Vec<(String, ObjectKey)>, + upstream: Edges, + downstream: Edges, + tags: Vec<(String, Vec)>, + paths: Vec<(ObjectKey, String)>, + } +} /// `(unique id, neighbour unique ids)` in graph order. type UniqueIdMap = Vec<(String, Vec)>; /// `PlannerInputError`'s `(code, message, help)`. @@ -167,12 +178,19 @@ pub(crate) fn project_graph(resources: Vec) -> PyResult = resources .into_iter() - .map(|(key, deps, tags, folder)| GraphResource { - key, - deps, - tags, - folder, - }) + .map( + |ResourceRow { + key, + deps, + tags, + folder, + }| GraphResource { + key, + deps, + tags, + folder, + }, + ) .collect(); Ok(NativeProjectGraph::new(build_project_graph(&resources))) }) @@ -190,7 +208,13 @@ impl NativeProjectGraph { #[staticmethod] fn from_indexes(indexes: IndexesRow) -> PyResult { compiler_guard(|| { - let (names, upstream, downstream, tags, paths) = indexes; + let IndexesRow { + names, + upstream, + downstream, + tags, + paths, + } = indexes; Ok(Self::new(graph_from_indexes(GraphIndexes { names, upstream, @@ -204,13 +228,13 @@ impl NativeProjectGraph { /// `(names, upstream, downstream, tags, paths)` for Python's `ProjectGraph`. fn indexes(&self) -> IndexesRow { let graph: &ProjectGraph = &self.graph; - ( - graph.names.clone(), - graph.upstream.clone(), - graph.downstream.clone(), - graph.tags.clone(), - graph.paths.clone(), - ) + IndexesRow { + names: graph.names.clone(), + upstream: graph.upstream.clone(), + downstream: graph.downstream.clone(), + tags: graph.tags.clone(), + paths: graph.paths.clone(), + } } /// The key's direct upstream lineage keys, in authored dependency order. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs index 44539a45cf..63aa2fd6ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs @@ -24,16 +24,28 @@ use crate::bindings::_helpers::records::fields::mapping_items; use crate::bindings::_helpers::records::ids::ErrorId; type NameRows = Vec<(String, Vec)>; -/// `(name, tags, POSIX relative path, upstream node names)` of one discovered node. -type InventoryRow = (String, Vec, String, Vec); -/// `(edges, upstream, downstream, tags, paths)` in Python's dict order. -type IndexesRow = ( - Vec<(String, String)>, - NameRows, - NameRows, - NameRows, - Vec<(String, String)>, -); +positional_record! { + extract; + /// `(name, tags, POSIX relative path, upstream node names)` of one discovered node. + struct InventoryRow { + name: String, + tags: Vec, + relative_path: String, + upstream: Vec, + } +} + +positional_record! { + into; + /// `(edges, upstream, downstream, tags, paths)` in Python's dict order. + struct IndexesRow { + edges: Vec<(String, String)>, + upstream: NameRows, + downstream: NameRows, + tags: NameRows, + paths: Vec<(String, String)>, + } +} /// The `PlannerInputError` Python raises for a selector failure. fn planner_error(py: Python<'_>, error: SelectorError) -> PyErr { @@ -189,12 +201,19 @@ fn python_node_graph_indexes(nodes: Vec) -> PyResult { compiler_guard(|| { let nodes: Vec = nodes .into_iter() - .map(|(name, tags, relative_path, upstream)| InventoryNode { - name, - tags, - relative_path, - upstream, - }) + .map( + |InventoryRow { + name, + tags, + relative_path, + upstream, + }| InventoryNode { + name, + tags, + relative_path, + upstream, + }, + ) .collect(); let InventoryIndexes { edges, @@ -203,7 +222,13 @@ fn python_node_graph_indexes(nodes: Vec) -> PyResult { tags, paths, } = graph_indexes(&nodes); - Ok((edges, upstream, downstream, tags, paths)) + Ok(IndexesRow { + edges, + upstream, + downstream, + tags, + paths, + }) }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs index c950f2c2f7..d2b894de74 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs @@ -1,15 +1,19 @@ //! A `CompiledModel` for a model input built outside the native compile-input stage. -use crate::bindings::_helpers::analysis::binding_locations::locate_model_binding_diagnostics; +use crate::bindings::_helpers::analysis::binding_locations::{ + ExpansionRow, LocatedRow, locate_model_binding_diagnostics, +}; use crate::bindings::_helpers::analysis_session::outcome_objects::dynamic_contract_proof; use crate::bindings::_helpers::analysis_session::outcome_objects::session_outcome_analysis; use crate::bindings::_helpers::records::construct::{enum_from_value, enum_member, record_class}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::ids::RecordId; +use crate::bindings::models::Positional; +use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyList, PyString, PyTuple}; -use sqlbuild_analysis::semantic_validation::types::{DiagnosticRow, Expansion}; +use sqlbuild_analysis::semantic_validation::models::DiagnosticRow; type Object<'py> = Bound<'py, PyAny>; @@ -264,53 +268,43 @@ fn located<'py>( )? .into_any() }; - let located: Object<'py> = locate_model_binding_diagnostics( - PyList::new(py, rows)?.extract::>()?, + let located: Vec = locate_model_binding_diagnostics( + PyList::new(py, rows)?.extract::>>()?, texts.extract::<(String, String, String)>()?, - expansion_row.extract::>)>>()?, - ) - .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))?; + expansion_row.extract::()?, + )?; let diagnostic_class: Object<'py> = record_class(py, RecordId::CompilerDiagnostic)?; let location_class: Object<'py> = record_class(py, RecordId::SourceLocation)?; let phase: Object<'py> = enum_member(py, RecordId::DiagnosticPhase, "COMPILE")?; let model: Object<'py> = enum_member(py, RecordId::CompiledResourceType, "MODEL")?; let name: Object<'py> = native_field(&model_file, "file_path")?.getattr("stem")?; let mut built: Vec> = Vec::new(); - for row in located.try_iter()? { - let (code, message, severity, line, column, span): ( - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, - ) = row?.extract()?; - let location: Object<'py> = if span.is_none() { - span - } else { - call( + for row in located { + let location: Object<'py> = match row.span { + None => py.None().into_bound(py), + Some(Positional(span)) => call( &location_class, &[ ("path", path.clone()), - ("line", span.get_item(0)?), - ("column", span.get_item(1)?), - ("end_line", span.get_item(2)?), - ("end_column", span.get_item(3)?), + ("line", span.line.into_bound_py_any(py)?), + ("column", span.column.into_bound_py_any(py)?), + ("end_line", span.end_line.into_bound_py_any(py)?), + ("end_column", span.end_column.into_bound_py_any(py)?), ], - )? + )?, }; built.push(call( &diagnostic_class, &[ ("phase", phase.clone()), - ("severity", severity), - ("code", code), - ("message", message), + ("severity", row.severity.into_bound_py_any(py)?), + ("code", row.code.into_bound_py_any(py)?), + ("message", row.message.into_bound_py_any(py)?), ("resource_type", model.clone()), ("resource_name", name.clone()), ("path", path.clone()), - ("line", line), - ("column", column), + ("line", row.line.into_bound_py_any(py)?), + ("column", row.column.into_bound_py_any(py)?), ("location", location), ], )?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs index 91df11f629..e588c0c463 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs @@ -58,6 +58,17 @@ enum EdgeParts { }, } +positional_record! { + extract; + /// One compact lineage row: output name index, transform and confidence codes, sources. + struct CompactColumnRow<'py> { + name_index: Bound<'py, PyAny>, + transform_code: i64, + confidence_code: i64, + sources: Bound<'py, PyAny>, + } +} + /// Indexed edge records, materialised edges shared between equal records. struct EdgeRecords { keys: Vec, @@ -442,12 +453,12 @@ impl RecordBuilder { || native_field(columns, "resource_name_indexes")?.is_truthy()?; let mut members = CodeMembers::default(); for row in native_field(columns, "rows")?.try_iter()? { - let (name_index, transform_code, confidence_code, sources): ( - Bound<'_, PyAny>, - i64, - i64, - Bound<'_, PyAny>, - ) = row?.extract()?; + let CompactColumnRow { + name_index, + transform_code, + confidence_code, + sources, + }: CompactColumnRow<'_> = row?.extract()?; let output_column: String = pool.get_item(name_index)?.extract()?; for source in sources.try_iter()? { let (kind, name, column): (Bound<'_, PyAny>, Bound<'_, PyAny>, Bound<'_, PyAny>) = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs index a83d2f1dd9..e17daf4485 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs @@ -70,8 +70,27 @@ const SCHEMA_SQL: &str = " ); "; -type ResourceRow = (String, String, Option, Option); -type EdgeRow = (String, String, String, String); +positional_record! { + extract; + /// One cached resource: its type, name, relative path and qualified name. + struct ResourceRow { + resource_type: String, + name: String, + relative_path: Option, + qualified_name: Option, + } +} + +positional_record! { + extract; + /// One cached lineage edge: its upstream and downstream types and names. + struct EdgeRow { + upstream_type: String, + upstream_name: String, + downstream_type: String, + downstream_name: String, + } +} type TripleRow = (String, String, String); fn cache_path(project_dir: &Path) -> PathBuf { @@ -178,7 +197,13 @@ fn read_index(connection: &Bound<'_, PyAny>) -> PyResult> { return Ok(None); } let mut nodes: HashMap = HashMap::new(); - for (resource_type, name, relative_path, qualified_name) in resources { + for ResourceRow { + resource_type, + name, + relative_path, + qualified_name, + } in resources + { let key: LineageKey = LineageKey { resource_type, name, @@ -201,7 +226,13 @@ fn read_index(connection: &Bound<'_, PyAny>) -> PyResult> { }; let mut upstream: Groups = Groups::new(); let mut downstream: Groups = Groups::new(); - for (upstream_type, upstream_name, downstream_type, downstream_name) in edges { + for EdgeRow { + upstream_type, + upstream_name, + downstream_type, + downstream_name, + } in edges + { let (Some(parent), Some(child)) = ( known(upstream_type, upstream_name), known(downstream_type, downstream_name), @@ -379,12 +410,15 @@ fn write_index( "INSERT INTO lookups VALUES (?, ?, ?)", PyList::new(py, lookups)?, )?; - let mut edges: Vec<(&str, &str, &str, &str)> = Vec::new(); + let mut edges: Vec> = Vec::new(); for (child, parents) in &index.upstream { for parent in parents { let ((parent_type, parent_name), (child_type, child_name)) = (key_parts(parent), key_parts(child)); - edges.push((parent_type, parent_name, child_type, child_name)); + edges.push(PyTuple::new( + py, + [parent_type, parent_name, child_type, child_name], + )?); } } insert( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs index 49c73bacfc..a5fca8538c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs @@ -237,7 +237,7 @@ fn fast_lineage<'py>( if model.expands_star { let star_columns: Vec = outcomes .next() - .map(|outcome| outcome.into_parts().1) + .map(|outcome| outcome.into_parts().columns) .unwrap_or_default(); let mut expanded: Vec> = columns.try_iter()?.collect::>()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs index 1447ef485b..0255c9390e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs @@ -20,13 +20,16 @@ use crate::bindings::_helpers::records::fields::mapping_get_or; const DLT_ERROR: ThirdPartyHost = ThirdPartyHost::DltIntegrationError; -/// `(factory, arguments, options, dataset_name)` of one dlt destination. -type DestinationCall<'py> = ( - &'static str, - Bound<'py, PyDict>, - Bound<'py, PyDict>, - Bound<'py, PyAny>, -); +positional_record! { + into; + /// `(factory, arguments, options, dataset_name)` of one dlt destination. + struct DestinationCall<'py> { + factory: &'static str, + arguments: Bound<'py, PyDict>, + options: Bound<'py, PyDict>, + dataset: Bound<'py, PyAny>, + } +} /// A `DltIntegrationError`, or the failure to construct one. fn dlt_error(py: Python<'_>, message: String) -> PyErr { @@ -110,12 +113,12 @@ fn dlt_destination_arguments<'py>( DltDataset::Given => dataset_name.clone(), DltDataset::Default(name) => PyString::new(py, name).into_any(), }; - Ok(( - plan.factory, - arguments(&plan.arguments, connection_config)?, + Ok(DestinationCall { + factory: plan.factory, + arguments: arguments(&plan.arguments, connection_config)?, options, dataset, - )) + }) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 0db8c1a73f..06dd803e7b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -1,10 +1,12 @@ +#[macro_use] +pub(crate) mod boundary; + pub(crate) mod analysis; pub(crate) mod analysis_session; pub(crate) mod attachments; pub(crate) mod audit_command; pub(crate) mod auditing; pub(crate) mod authored_values; -pub(crate) mod boundary; pub(crate) mod build_command; pub(crate) mod build_progress; pub(crate) mod cache; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs index 68a0ba0b93..2e4d8740f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs @@ -24,17 +24,28 @@ const RETURNS_KEY: &str = "returns"; /// The only key of a table function's `returns` mapping. const TABLE_RETURNS_KEY: &str = "table"; -/// Model, seed, source, function and table function names. -type ResourceNames = ( - HashSet, - HashSet, - HashSet, - HashSet, - HashSet, -); - -/// References `(kind, name, rejected)`, schema columns, unmanaged retention and table type. -type ModelFacts = (Vec<(String, String, bool)>, Option>, bool, bool); +positional_record! { + extract; + /// Model, seed, source, function and table function names. + struct ResourceNames { + models: HashSet, + seeds: HashSet, + sources: HashSet, + functions: HashSet, + table_functions: HashSet, + } +} + +positional_record! { + extract; + /// References `(kind, name, rejected)`, schema columns, unmanaged retention and table type. + struct ModelFacts { + references: Vec<(String, String, bool)>, + declared_columns: Option>, + retention_unmanaged: bool, + table_type_declared: bool, + } +} /// The project facts every model's validation reads, captured once per compile. #[pyclass(module = "sqlbuild._native", frozen)] @@ -83,7 +94,13 @@ fn discovered_names(inputs: &Bound<'_, PyAny>) -> PyResult { functions.insert(name); } functions.extend(each(inputs, "python_function_files", model_stem)?); - Ok((models, seeds, sources, functions, table_functions)) + Ok(ResourceNames { + models, + seeds, + sources, + functions, + table_functions, + }) } /// Whether a SQL function's header `returns` is a mapping whose only key is `table`. @@ -126,7 +143,13 @@ impl NativeModelValidator { custom_materializations: HashSet, microbatch_concurrency: bool, ) -> PyResult { - let (models, seeds, sources, functions, table_functions) = names; + let ResourceNames { + models, + seeds, + sources, + functions, + table_functions, + } = names; Ok(Self { project: ProjectValidationFacts { custom_materializations, @@ -150,7 +173,12 @@ impl NativeModelValidator { ) -> PyResult> { compiler_guard(|| { let (model_name, query_sql, relative_path) = model; - let (references, declared_columns, retention_unmanaged, table_type_declared) = facts; + let ModelFacts { + references, + declared_columns, + retention_unmanaged, + table_type_declared, + } = facts; let references: Vec = references .into_iter() .map(|(kind, name, externally_rejected)| ModelReference { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs index 64a3add7ec..f3d60591e4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs @@ -443,13 +443,13 @@ fn python_entry_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult = ( - String, - (String, String, String), - Bound<'py, PyAny>, - Bound<'py, PyAny>, -); +/// One provider usage: provider, `(kind, name, parameter)` consumer, annotation class and module. +struct Usage<'py> { + provider: String, + consumer: (String, String, String), + class_name: Bound<'py, PyAny>, + module: Bound<'py, PyAny>, +} fn providers_json<'py>( plan: &Bound<'py, PyAny>, @@ -458,37 +458,45 @@ fn providers_json<'py>( let py: Python<'py> = plan.py(); let mut usages: Vec> = Vec::new(); for usage in items(&native_field(plan, "provider_usages")?)? { - usages.push(( - text(&native_field(&usage, "provider_name")?)?, - ( + usages.push(Usage { + provider: text(&native_field(&usage, "provider_name")?)?, + consumer: ( text(&native_field(&usage, "consumer_kind")?)?, text(&native_field(&usage, "consumer_name")?)?, text(&native_field(&usage, "parameter_name")?)?, ), - native_field(&usage, "annotation_class_name")?, - native_field(&usage, "annotation_module")?, - )); + class_name: native_field(&usage, "annotation_class_name")?, + module: native_field(&usage, "annotation_module")?, + }); } for entry in python_entries { let kind: String = text(&enum_value(entry, "kind")?)?; let name: String = text(&native_field(entry, "name")?)?; for usage in items(&native_field(entry, "provider_usages")?)? { - usages.push(( - text(&native_field(&usage, "provider_name")?)?, - ( + usages.push(Usage { + provider: text(&native_field(&usage, "provider_name")?)?, + consumer: ( kind.clone(), name.clone(), text(&native_field(&usage, "parameter_name")?)?, ), - native_field(&usage, "annotation_class_name")?, - native_field(&usage, "annotation_module")?, - )); + class_name: native_field(&usage, "annotation_class_name")?, + module: native_field(&usage, "annotation_module")?, + }); } } - usages.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1))); + usages.sort_by(|left, right| { + (&left.provider, &left.consumer).cmp(&(&right.provider, &right.consumer)) + }); let providers: Bound<'py, PyList> = PyList::empty(py); let mut current: Option<(String, Bound<'py, PyList>)> = None; - for (provider, (kind, name, parameter), class_name, module) in usages { + for Usage { + provider, + consumer: (kind, name, parameter), + class_name, + module, + } in usages + { if current.as_ref().is_none_or(|(open, _)| *open != provider) { let used_by: Bound<'py, PyList> = PyList::empty(py); providers.append(dict( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs index 61289d856c..d2e2734323 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs @@ -23,6 +23,9 @@ use crate::bindings::_helpers::planner_phases::identities::{ use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; use crate::bindings::_helpers::planner_phases::models::PlannerRuntime; use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; +use crate::bindings::_helpers::planner_phases::models::{ + PlanOutputContext, PlanWarningContext, PlannedModelEntries, StoragePolicyContext, +}; use crate::bindings::_helpers::planner_phases::output_assembly::{ add_plan_metadata, add_plan_warnings, add_storage_policies, base_plan_output, }; @@ -158,19 +161,37 @@ fn planned_output<'py>( )?; let planned_sql_tests = test_planning.result(py)?.into_bound(py); let output = base_plan_output( - (runtime, &warehouse, &identities, overrides), - ( - &inspection_scope, - &reconciliation, + PlanOutputContext { + runtime, + warehouse: &warehouse, + identities: &identities, + overrides, + }, + PlannedModelEntries { + execution_scope: &inspection_scope, + reconciliation: &reconciliation, entries, - &source_freshness, - ), + source_freshness: &source_freshness, + }, &planned_sql_tests, )?; - let output = add_storage_policies(output, (runtime, &warehouse, scopes, &policies))?; + let output = add_storage_policies( + output, + StoragePolicyContext { + runtime, + warehouse: &warehouse, + scopes, + policies: &policies, + }, + )?; let output = add_plan_warnings( output, - (scopes, &warehouse, &identities, &stale_warning_changes), + PlanWarningContext { + scopes, + warehouse: &warehouse, + identities: &identities, + stale_warning_changes: &stale_warning_changes, + }, (&source_freshness, &policies), )?; let plan_output = add_plan_metadata(output, &source_freshness, &policies)?.build(py)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs index d97d516142..35ebf4b34e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs @@ -142,11 +142,11 @@ pub(crate) fn build_execution_upstream_deps<'py>( let (gates, hooks): (Vec, Vec) = native_execution_gate_edges(project)?; let cache: KeyCache<'py> = KeyCache::new(py)?; let mut extra: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); - for (_, _, gated, read) in &gates { - extra.push((cache.key(gated)?, cache.key(read)?)); + for gate in &gates { + extra.push((cache.key(&gate.gated)?, cache.key(&gate.read)?)); } - for (_, gated, read, _, _) in &hooks { - extra.push((cache.key(gated)?, cache.key(read)?)); + for hook in &hooks { + extra.push((cache.key(&hook.gated)?, cache.key(&hook.read)?)); } for (gated, read) in extra { let gated_upstream: Bound<'py, PyAny> = @@ -192,7 +192,13 @@ pub(crate) fn build_execution_edge_origins<'py>( .call_method1("upstream", (gated.clone(),))? .contains(read.clone()) }; - for (audit_name, target, gated, read) in &gates { + for GateRow { + audit_name, + target, + gated, + read, + } in &gates + { if is_lineage(gated, read)? { continue; } @@ -204,7 +210,14 @@ pub(crate) fn build_execution_edge_origins<'py>( ), )?; } - for (hook_name, gated, read, kind, _) in &hooks { + for HookRow { + hook_name, + gated, + read, + hook_type: kind, + .. + } in &hooks + { if is_lineage(gated, read)? { continue; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs index dcb5eee1d3..ced5ea9dd5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs @@ -26,6 +26,42 @@ pub(crate) struct ModelEntryResults<'py> { pub(crate) warnings: Vec>, } +/// The runtime, warehouse state, identities and overrides the base plan output reads. +#[derive(Debug)] +pub(crate) struct PlanOutputContext<'a, 'py> { + pub(crate) runtime: &'a Bound<'py, PyAny>, + pub(crate) warehouse: &'a Bound<'py, PyAny>, + pub(crate) identities: &'a Bound<'py, PyAny>, + pub(crate) overrides: &'a Bound<'py, PyAny>, +} + +/// The execution scope, reconciled changes, planned entries and source freshness of a plan. +#[derive(Debug)] +pub(crate) struct PlannedModelEntries<'a, 'py> { + pub(crate) execution_scope: &'a Bound<'py, PyAny>, + pub(crate) reconciliation: &'a ChangeReconciliation<'py>, + pub(crate) entries: ModelEntryResults<'py>, + pub(crate) source_freshness: &'a Bound<'py, PyAny>, +} + +/// The runtime, warehouse state, scopes and policies storage policy planning reads. +#[derive(Debug)] +pub(crate) struct StoragePolicyContext<'a, 'py> { + pub(crate) runtime: &'a Bound<'py, PyAny>, + pub(crate) warehouse: &'a Bound<'py, PyAny>, + pub(crate) scopes: &'a Bound<'py, PyAny>, + pub(crate) policies: &'a Bound<'py, PyAny>, +} + +/// The scopes, warehouse state, identities and changes stale-selection warnings read. +#[derive(Debug)] +pub(crate) struct PlanWarningContext<'a, 'py> { + pub(crate) scopes: &'a Bound<'py, PyAny>, + pub(crate) warehouse: &'a Bound<'py, PyAny>, + pub(crate) identities: &'a Bound<'py, PyAny>, + pub(crate) stale_warning_changes: &'a Bound<'py, PyAny>, +} + /// Resolved execution environment shared by planner phases. #[pyclass(module = "sqlbuild._native", frozen, get_all)] #[derive(Debug)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs index 0dfe501080..71e3e09c26 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs @@ -10,7 +10,8 @@ use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; use crate::bindings::_helpers::planner_phases::models::{ - ChangeReconciliation, ModelEntryResults, PlanOutputFields, + PlanOutputContext, PlanOutputFields, PlanWarningContext, PlannedModelEntries, + StoragePolicyContext, }; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::objects::{ @@ -69,18 +70,18 @@ impl<'py> PlanOutputFields<'py> { /// The plan output's base fields from the planned entries, relations and source freshness. pub(crate) fn base_plan_output<'py>( - (runtime, warehouse, identities, overrides): ( - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - ), - (execution_scope, reconciliation, entries, source_freshness): ( - &Bound<'py, PyAny>, - &ChangeReconciliation<'py>, - ModelEntryResults<'py>, - &Bound<'py, PyAny>, - ), + PlanOutputContext { + runtime, + warehouse, + identities, + overrides, + }: PlanOutputContext<'_, 'py>, + PlannedModelEntries { + execution_scope, + reconciliation, + entries, + source_freshness, + }: PlannedModelEntries<'_, 'py>, planned_sql_tests: &Bound<'py, PyAny>, ) -> PyResult> { let py = runtime.py(); @@ -163,12 +164,12 @@ pub(crate) fn with_column_rename_hints<'py>( /// Add migration fingerprints, rename hints, migration, table-type and retention entries. pub(crate) fn add_storage_policies<'py>( mut output: PlanOutputFields<'py>, - (runtime, warehouse, scopes, policies): ( - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - ), + StoragePolicyContext { + runtime, + warehouse, + scopes, + policies, + }: StoragePolicyContext<'_, 'py>, ) -> PyResult> { let py = runtime.py(); let mut entries = output.field("model_entries")?; @@ -242,12 +243,12 @@ pub(crate) fn add_storage_policies<'py>( /// Add unknown source freshness warnings and, with selection diagnostics, stale-selection warnings. pub(crate) fn add_plan_warnings<'py>( mut output: PlanOutputFields<'py>, - (scopes, warehouse, identities, stale_warning_changes): ( - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - ), + PlanWarningContext { + scopes, + warehouse, + identities, + stale_warning_changes, + }: PlanWarningContext<'_, 'py>, (source_freshness, policies): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { let py = scopes.py(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs index 71d74dd497..bd33f206d1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs @@ -6,16 +6,17 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_fixtures::main::required_columns::required_columns; use sqlbuild_analysis::planner_fixtures::main::star_keys::star_keys; use sqlbuild_analysis::planner_fixtures::models::{ - FixtureReadKey, FixtureReadModel, FixtureReadRequirements, FixtureReadSchemas, - FixtureStarRelation, + FixtureReadKey, FixtureReadModel, FixtureReadReference, FixtureReadRequirements, + FixtureReadSchemas, FixtureStarRelation, }; -use sqlbuild_analysis::planner_fixtures::types::FixtureReadReference; use sqlbuild_analysis::required_columns::models::ColumnRead; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use std::collections::{HashMap, HashSet}; use crate::bindings::_helpers::analysis::model_reads; -use crate::bindings::_helpers::analysis::model_reads::{lineage_facts, proven_inputs, star_flags}; +use crate::bindings::_helpers::analysis::model_reads::{ + LineageFactRow, lineage_facts, proven_inputs, star_flags, +}; use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::planner_strategy::constants::HIGH_CONFIDENCE; use crate::bindings::_helpers::planner_strategy::fixture_context::work_counts; @@ -63,12 +64,12 @@ pub(crate) fn required_fixture_columns( let mut references: Vec = Vec::new(); for reference in native_field(&model, "references")?.try_iter()? { let reference = reference?; - references.push(( - native_field(&reference, "ref_kind")?.extract()?, - native_field(&reference, "ref_name")?.extract()?, - native_field(&reference, "ref_package")?.extract()?, - native_field(&reference, "call_argument_count")?.extract()?, - )); + references.push(FixtureReadReference { + kind: native_field(&reference, "ref_kind")?.extract()?, + name: native_field(&reference, "ref_name")?.extract()?, + package: native_field(&reference, "ref_package")?.extract()?, + argument_count: native_field(&reference, "call_argument_count")?.extract()?, + }); } models.push(FixtureReadModel { name: name.clone(), @@ -102,7 +103,12 @@ pub(crate) fn required_fixture_columns( pub(crate) fn lineage_reads(model: &Bound<'_, PyAny>) -> PyResult> { let mut reads: Vec = Vec::new(); - for (_, _, confidence, sources) in lineage_facts(model)?.unwrap_or_default() { + for LineageFactRow { + confidence, + sources, + .. + } in lineage_facts(model)?.unwrap_or_default() + { if confidence != HIGH_CONFIDENCE { continue; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs index 3a6a1dd758..1fe436cfdf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs @@ -9,7 +9,7 @@ use sqlbuild_analysis::planner_fixtures::models::{ use sqlbuild_core::text::main::python_casefold::python_casefold; -use crate::bindings::_helpers::analysis::model_reads::lineage_facts; +use crate::bindings::_helpers::analysis::model_reads::{LineageFactRow, lineage_facts}; use crate::bindings::_helpers::planner_strategy::constants::{ DIRECT_TRANSFORM, MODEL_REFERENCE_KIND, }; @@ -88,7 +88,13 @@ pub(crate) fn expected_fixture_types<'a>( pub(crate) fn projections(model: &Bound<'_, PyAny>) -> PyResult> { let mut projections: Vec = Vec::new(); - for (output_column, transform, _, sources) in lineage_facts(model)?.unwrap_or_default() { + for LineageFactRow { + output_column, + transform, + sources, + .. + } in lineage_facts(model)?.unwrap_or_default() + { if transform != DIRECT_TRANSFORM { continue; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs index 4059788791..16afa8f495 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs @@ -31,7 +31,14 @@ const INSERT_SQL: &str = "INSERT OR REPLACE INTO migration_fingerprint (cache_key, payload) VALUES (?, ?)"; const SELECT_SQL: &str = "SELECT payload FROM migration_fingerprint WHERE cache_key = ?"; -type MemoKey = (String, String, Option, Vec<(String, String)>); +/// A fingerprint request: query SQL, metadata JSON, dialect and visible renames. +#[derive(Hash, PartialEq, Eq)] +struct MemoKey { + query_sql: String, + metadata_json: String, + dialect: Option, + visible_renames: Vec<(String, String)>, +} static FINGERPRINT_WORK: PyOnceLock> = PyOnceLock::new(); @@ -160,10 +167,10 @@ impl MigrationFingerprintCache { } fingerprint_cache_key(&FingerprintCacheKey { shared_digest: self.shared_digest.clone(), - query_sql: key.0.clone(), - metadata_json: key.1.clone(), - dialect: key.2.clone(), - visible_renames: key.3.clone(), + query_sql: key.query_sql.clone(), + metadata_json: key.metadata_json.clone(), + dialect: key.dialect.clone(), + visible_renames: key.visible_renames.clone(), }) .map_err(|error| PyValueError::new_err(format!("{error:?}"))) } @@ -339,7 +346,12 @@ impl MigrationFingerprintCache { visible.push((name, identity)); } } - let key: MemoKey = (query_sql, metadata_json, dialect, visible); + let key: MemoKey = MemoKey { + query_sql, + metadata_json, + dialect, + visible_renames: visible, + }; if let Some(found) = self.fingerprints.get(&key) { return Ok(found.clone()); } @@ -348,9 +360,14 @@ impl MigrationFingerprintCache { Some(stored) => stored, None => { count_computation(py, &key)?; - let identities: BTreeMap = key.3.iter().cloned().collect(); - let computed: Option = - fingerprint(&key.0, &key.1, &identities, key.2.as_deref())?; + let identities: BTreeMap = + key.visible_renames.iter().cloned().collect(); + let computed: Option = fingerprint( + &key.query_sql, + &key.metadata_json, + &identities, + key.dialect.as_deref(), + )?; self.remember(&cache_key, computed.as_deref())?; computed } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs index effacd0eb4..f308830cb4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs @@ -27,13 +27,13 @@ const UNKNOWN_MEMBER: &str = "UNKNOWN"; /// `str(None)`, the text of an exact declaration without a type. const NONE_TEXT: &str = "None"; -/// The three per-relation declaration maps the analysis starts from. -pub(crate) type DeclaredRelations<'py> = ( - Bound<'py, PyDict>, - Bound<'py, PyDict>, - Bound<'py, PyDict>, - DeclaredShapes, -); +/// The three per-relation declaration maps the analysis starts from, and the declared shapes. +pub(crate) struct DeclaredRelations<'py> { + pub(crate) column_types: Bound<'py, PyDict>, + pub(crate) nullability: Bound<'py, PyDict>, + pub(crate) complete_schemas: Bound<'py, PyDict>, + pub(crate) declared: DeclaredShapes, +} /// Relation shapes as rows in a dict's insertion order, each value as `str(value)`. #[derive(Debug, Clone, Default)] @@ -327,5 +327,10 @@ pub(crate) fn declared_relations<'py>( declared.complete(&maps, &name, (&columns, true))?; } let (types, nulls, complete) = maps; - Ok((types, nulls, complete, declared.shapes)) + Ok(DeclaredRelations { + column_types: types, + nullability: nulls, + complete_schemas: complete, + declared: declared.shapes, + }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs index 267a7b8c1d..0fceed1307 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs @@ -20,6 +20,7 @@ use crate::bindings::_helpers::authored_values::project_vars::project_var_text; use crate::bindings::_helpers::boundary::panics::{NativeCompilerError, compiler_error}; use crate::bindings::_helpers::compile_driver::python_calls::{imported, keywords}; use crate::bindings::_helpers::compiled_project::model_input_view::input_reference_rows; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::model_config::template_data::record_reads; use crate::bindings::_helpers::project_assembly::models::{ NativeProjectResources, NativeResourceFailure, @@ -43,13 +44,42 @@ const CONTEXT_READ: &str = "ctx"; const NATIVE_FAILURE_PREFIX: &str = "NativeCompilerError: "; /// `(kind, name, package)`. -type ReferenceRow = (String, String, Option); -/// `(name, managed, database, schema)`. -type SourceRow = (String, bool, Option, Option); -/// `(kind, name, tags, references)`. -type FunctionRow = (String, String, Vec, Vec); +type ResourceReferenceRow = (String, String, Option); /// `(references, attached (kind, name))`. -type AuditRow = (Vec, Option<(String, String)>); +type AuditRow = (Vec, Option<(String, String)>); + +positional_record! { + extract; + /// `(name, managed, database, schema)`. + struct SourceRow { + name: String, + managed: bool, + database: Option, + schema: Option, + } +} + +positional_record! { + extract; + /// `(kind, name, tags, references)`. + struct FunctionRow { + kind: String, + name: String, + tags: Vec, + references: Vec, + } +} + +positional_record! { + into; + /// `(database, schema, logical database, logical schema)` of one seed. + struct NamespaceRow { + database: Option, + schema: Option, + logical_database: Option, + logical_schema: Option, + } +} /// The request's named variables and the Python exceptions opaque ones raise, by index. type RequestVariables<'py> = (Vec<(String, Variable)>, Vec>); @@ -143,7 +173,7 @@ fn environment_rows( .collect() } -fn references(rows: Vec) -> Vec { +fn references(rows: Vec) -> Vec { rows.into_iter() .map(|(kind, name, package)| Reference { kind, @@ -166,9 +196,16 @@ fn tags(value: &Bound<'_, PyAny>) -> PyResult> { fn model_facts(model_input: &Bound<'_, PyAny>, checks: &Bound<'_, PyAny>) -> PyResult { let model_file: Bound<'_, PyAny> = native_field(model_input, "model_file")?; - let reference_rows: Vec = input_reference_rows(model_input)? + let reference_rows: Vec = input_reference_rows(model_input)? .into_iter() - .map(|(kind, name, package, _)| (kind, name, package)) + .map( + |ReferenceRow { + kind, + name, + package, + .. + }| (kind, name, package), + ) .collect(); let mut syntax_checks: Vec = Vec::new(); for check in checks.try_iter()? { @@ -280,7 +317,12 @@ fn project_request<'py>( let sources: Vec = rows("source_inputs")? .into_iter() .map(|row| { - let (name, managed, database, schema): SourceRow = row.extract()?; + let SourceRow { + name, + managed, + database, + schema, + }: SourceRow = row.extract()?; Ok(SourceFacts { name, managed, @@ -292,7 +334,12 @@ fn project_request<'py>( let functions: Vec = rows("sql_function_inputs")? .into_iter() .map(|row| { - let (kind, name, tags, reference_rows): FunctionRow = row.extract()?; + let FunctionRow { + kind, + name, + tags, + references: reference_rows, + }: FunctionRow = row.extract()?; Ok(FunctionFacts { kind, name, @@ -383,20 +430,13 @@ fn failure_object<'py>( .into_any()) } -fn namespace_row( - namespace: Namespace, -) -> ( - Option, - Option, - Option, - Option, -) { - ( - namespace.database, - namespace.schema, - namespace.logical_database, - namespace.logical_schema, - ) +fn namespace_row(namespace: Namespace) -> NamespaceRow { + NamespaceRow { + database: namespace.database, + schema: namespace.schema, + logical_database: namespace.logical_database, + logical_schema: namespace.logical_schema, + } } /// `NativeProjectResources` for the native result, replaying its template reads first. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lineage_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lineage_facts.rs index bbba79328d..32f93faf4f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lineage_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lineage_facts.rs @@ -70,16 +70,27 @@ pub(crate) fn facts_resource_name( Ok(resource_name(instance, &index)?.unbind()) } +positional_record! { + extract; + /// One compact lineage row: output name index, transform and confidence codes, sources. + struct CompactColumnRow<'py> { + name_index: Bound<'py, PyAny>, + transform_code: usize, + confidence_code: usize, + sources: Bound<'py, PyAny>, + } +} + /// One compact row projected to a `CompiledLineageColumnFact`. fn project_row(facts: &Bound<'_, PyAny>, row: &Bound<'_, PyAny>) -> PyResult> { let py: Python<'_> = facts.py(); let pool: Bound<'_, PyAny> = native_field(facts, "string_pool")?; - let (name_index, transform_code, confidence_code, sources): ( - Bound<'_, PyAny>, - usize, - usize, - Bound<'_, PyAny>, - ) = row.extract()?; + let CompactColumnRow { + name_index, + transform_code, + confidence_code, + sources, + }: CompactColumnRow<'_> = row.extract()?; let mut upstream: Vec> = Vec::new(); for source in sources.try_iter()? { let source: Bound<'_, PyTuple> = source?.cast_into::()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs index 7e6704149a..a2fe347ae6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs @@ -131,12 +131,12 @@ fn resource_rows( for resource in resources.try_iter()? { let resource: Bound<'_, PyAny> = resource?; let (tags, folder) = describe(&resource)?; - rows.push(( - graph_key(&native_field(&resource, "key")?)?, - graph_keys(&native_field(&resource, "deps")?)?, + rows.push(ResourceRow { + key: graph_key(&native_field(&resource, "key")?)?, + deps: graph_keys(&native_field(&resource, "deps")?)?, tags, folder, - )); + }); } Ok(rows) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs index df38754cff..666f5985c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs @@ -196,27 +196,61 @@ fn block_bodies(blocks: &Bound<'_, PyAny>) -> PyResult> { .collect() } +/// One discovered file group: its role, attribute, body field and whether it has blocks. +struct FileGroup { + role: SqlFileRole, + attribute: &'static str, + body: Option<&'static str>, + blocks: bool, +} + fn authored_files(discovered: &Bound<'_, PyAny>) -> PyResult> { - let groups: [(SqlFileRole, &str, Option<&str>, bool); 6] = [ - (SqlFileRole::Test, "test_files", None, true), - ( - SqlFileRole::Scenario, - "scenario_files", - Some("sql_body"), - false, - ), - (SqlFileRole::Audit, "audit_files", None, true), - (SqlFileRole::Hook, "sql_hook_files", Some("sql_body"), false), - ( - SqlFileRole::Function, - "sql_function_files", - Some("body_sql"), - false, - ), - (SqlFileRole::Schema, "model_schema_files", None, false), + let groups: [FileGroup; 6] = [ + FileGroup { + role: SqlFileRole::Test, + attribute: "test_files", + body: None, + blocks: true, + }, + FileGroup { + role: SqlFileRole::Scenario, + attribute: "scenario_files", + body: Some("sql_body"), + blocks: false, + }, + FileGroup { + role: SqlFileRole::Audit, + attribute: "audit_files", + body: None, + blocks: true, + }, + FileGroup { + role: SqlFileRole::Hook, + attribute: "sql_hook_files", + body: Some("sql_body"), + blocks: false, + }, + FileGroup { + role: SqlFileRole::Function, + attribute: "sql_function_files", + body: Some("body_sql"), + blocks: false, + }, + FileGroup { + role: SqlFileRole::Schema, + attribute: "model_schema_files", + body: None, + blocks: false, + }, ]; let mut files: Vec = Vec::new(); - for (role, attribute, body, blocks) in groups { + for FileGroup { + role, + attribute, + body, + blocks, + } in groups + { for item in native_field(discovered, attribute)?.try_iter()? { let item: Bound<'_, PyAny> = item?; let texts: Vec = match (body, blocks) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs index 498e99529b..2f52c18305 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs @@ -1,7 +1,7 @@ //! Authored SQL expanded through project vars, declarations and macros, with each pass's spans. use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, + Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -16,15 +16,18 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::render::declaration_expansion::{ - DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, SpanRow, + DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, expand_declarations, span_rows, }; use crate::bindings::_helpers::render::declaration_values::ConstantRendering; use crate::bindings::_helpers::render::declaration_values::resolved_declaration_context; +use crate::bindings::_helpers::render::macro_call_rows::SpanRow; use crate::bindings::_helpers::render::macro_call_rows::span_row; use crate::bindings::_helpers::render::macro_expansion_result::{ExpansionRequest, macro_output}; use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan; -use crate::bindings::_helpers::sqltext::static_sql::{InterpolationSources, interpolation_batch}; +use crate::bindings::_helpers::sqltext::static_sql::{ + InterpolationRow, InterpolationSources, interpolation_batch, +}; const DECLARATION_FIELDS: [&str; 4] = [ "enums", @@ -53,7 +56,13 @@ fn substituted<'py>( let text: String = sql.extract()?; let path: String = file_path.str()?.extract()?; let rows = interpolation_batch(sources, &[(text.clone(), path)])?; - let Some((rendered, spans, reads, error)) = rows.into_iter().next() else { + let Some(InterpolationRow { + sql: rendered, + spans, + reads, + error, + }) = rows.into_iter().next() + else { return Ok((text, Vec::new())); }; record_reads(py, reads.iter().map(|(kind, name)| (*kind, name.as_str())))?; @@ -69,7 +78,14 @@ fn substituted<'py>( fn span_objects<'py>(py: Python<'py>, spans: &[SpanRow]) -> PyResult> { let objects: Vec> = spans .iter() - .map(|span| new_record(py, RecordId::ExpansionSpan, *span, None)) + .map(|span| { + new_record( + py, + RecordId::ExpansionSpan, + IntoPyObject::into_pyobject(*span, py)?, + None, + ) + }) .collect::>()?; PyTuple::new(py, objects) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs index 57f1fa829d..57a9ea530c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs @@ -1,5 +1,6 @@ //! Enum and constant references expanded natively, for model queries and every other caller. +use crate::bindings::_helpers::render::macro_call_rows::SpanRow; use pyo3::FromPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyInt, PyTuple}; @@ -12,6 +13,7 @@ use sqlbuild_analysis::compiled_project::models::{ }; use crate::bindings::_helpers::render::declaration_values::ConstantRendering; +use crate::bindings::_helpers::sqltext::declaration_references::ScannedReference; const ENUM_REFERENCE_KIND: u8 = 0; const ENUM_KIND: &str = "enum"; @@ -39,11 +41,6 @@ const THROUGH: &str = "through"; const CONSUMER: &str = "consumer"; const ENUM_MEMBER: &str = "enum_member"; -/// `(kind, name, member, start, end)` for one natively scanned reference. -pub(crate) type ScannedReference = (u8, String, Option, usize, usize); -/// `(source start, source end, output start, output end)`. -pub(crate) type SpanRow = (usize, usize, usize, usize); - /// The Python classes expansion builds usages and errors from. #[derive(FromPyObject)] #[pyo3(from_item_all)] @@ -82,7 +79,14 @@ pub(crate) fn expand_declarations<'py>( ) -> PyResult> { let mut splices: Vec = Vec::with_capacity(request.references.len()); let mut usages: Vec> = Vec::new(); - for (kind, name, member, start, end) in request.references { + for ScannedReference { + kind, + name, + member, + start, + end, + } in request.references + { let (replacement, visibility) = if *kind == ENUM_REFERENCE_KIND { enum_text(request, name, member.as_deref())? } else { @@ -281,13 +285,11 @@ pub(crate) fn span_rows(spliced: &SplicedSql) -> Vec { spliced .spans .iter() - .map(|span| { - ( - span.source_start, - span.source_end, - span.output_start, - span.output_end, - ) + .map(|span| SpanRow { + source_start: span.source_start, + source_end: span.source_end, + output_start: span.output_start, + output_end: span.output_end, }) .collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_rows.rs index 33941bf858..fb1a6f7986 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_rows.rs @@ -1,28 +1,71 @@ //! Python rows of macro call sites and splice spans. +use sqlbuild_analysis::semantic_validation::models::Expansion; use sqlbuild_render::macro_calls::models::{MacroCallSite, MacroSpliceSpan}; +use sqlbuild_sqltext::compiler::models::CharSpan; -/// One call site: start, arguments start, end, name, tree names if scanned and whether typed reference text appears. -pub(crate) type SiteRow = (usize, usize, usize, String, Option>, bool); -/// One substitution span: source start and end, output start and end. -pub(crate) type SpanRow = (usize, usize, usize, usize); +positional_record! { + into; + /// One call site: start, arguments start, end, name, tree names if scanned and whether typed reference text appears. + pub(crate) struct SiteRow { + start: usize, + args_start: usize, + end: usize, + name: String, + tree_names: Option>, + typed_reference_text: bool, + } +} +positional_record! { + extract, into; + /// One substitution span: source start and end, output start and end. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub(crate) struct SpanRow { + pub(crate) source_start: usize, + pub(crate) source_end: usize, + pub(crate) output_start: usize, + pub(crate) output_end: usize, + } +} + +impl From for SpanRow { + fn from(span: CharSpan) -> Self { + Self { + source_start: span.source_start, + source_end: span.source_end, + output_start: span.output_start, + output_end: span.output_end, + } + } +} + +impl From for Expansion { + fn from(span: SpanRow) -> Self { + Self { + source_start: span.source_start, + source_end: span.source_end, + output_start: span.output_start, + output_end: span.output_end, + } + } +} pub(crate) fn site_row(site: MacroCallSite) -> SiteRow { - ( - site.start, - site.args_start, - site.end, - site.name, - site.tree_names, - site.typed_reference_text, - ) + SiteRow { + start: site.start, + args_start: site.args_start, + end: site.end, + name: site.name, + tree_names: site.tree_names, + typed_reference_text: site.typed_reference_text, + } } pub(crate) fn span_row(span: MacroSpliceSpan) -> SpanRow { - ( - span.source_start, - span.source_end, - span.output_start, - span.output_end, - ) + SpanRow { + source_start: span.source_start, + source_end: span.source_end, + output_start: span.output_start, + output_end: span.output_end, + } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs index c44bc2db8f..c605e8c5ab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs @@ -33,14 +33,17 @@ const RELATION_SUFFIX: &str = "__"; const CONSTANT_KIND: &str = "constant"; const RELATIONSHIP_CONSUMER_KINDS: [&str; 2] = ["test", "scenario"]; -/// `(sql, spans, argument references, dependencies, usages)` of one expansion. -type ExpansionRow<'py> = ( - String, - Vec, - Vec, - Vec>, - Vec>, -); +positional_record! { + into; + /// `(sql, spans, argument references, dependencies, usages)` of one expansion. + struct ExpansionRow { + sql: String, + spans: Vec, + argument_references: Vec, + dependencies: Vec>, + usages: Vec>, + } +} /// The Python classes, enums and helpers an expansion builds facts, errors and contexts from. #[derive(FromPyObject)] @@ -87,8 +90,17 @@ pub(crate) struct NativeMacroExpansion { pub(crate) facts: RefCell, } -/// The `(macros, overrides, context, declarations, consumer)` one expansion reads. -type StateInput = (Py, Py, Py, Py, Py); +positional_record! { + extract; + /// The `(macros, overrides, context, declarations, consumer)` one expansion reads. + pub(crate) struct StateInput { + pub(crate) loaded_macros: Py, + pub(crate) overrides: Py, + pub(crate) macro_context: Py, + pub(crate) declarations: Py, + pub(crate) consumer: Py, + } +} #[pymethods] impl NativeMacroExpansion { @@ -99,7 +111,13 @@ impl NativeMacroExpansion { enforce_explicit: bool, classes: Py, ) -> PyResult { - let (loaded_macros, overrides, macro_context, declarations, consumer) = state; + let StateInput { + loaded_macros, + overrides, + macro_context, + declarations, + consumer, + } = state; Ok(Self { file_path: file_path.clone().unbind(), file_label: file_path.str()?.to_string(), @@ -119,15 +137,15 @@ impl NativeMacroExpansion { slf: &Bound<'py, Self>, sql: &Bound<'py, PyString>, bridge: &Bound<'py, PyAny>, - ) -> PyResult> { + ) -> PyResult { let output: ExpansionOutput = expanded(slf, sql.to_str()?, bridge)?; - Ok(( - output.sql, - output.spans.into_iter().map(span_row).collect(), - output.argument_references, - output.dependencies, - output.usages, - )) + Ok(ExpansionRow { + sql: output.sql, + spans: output.spans.into_iter().map(span_row).collect(), + argument_references: output.argument_references, + dependencies: output.dependencies, + usages: output.usages, + }) } /// `render_relation`: an approved call-site reference's placeholder, else its reference call. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs index 5bdfb03605..56347e672a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs @@ -19,9 +19,9 @@ use crate::bindings::_helpers::records::construct::{ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::render::declaration_values::resolved_declaration_context; -use crate::bindings::_helpers::render::macro_call_rows::span_row; +use crate::bindings::_helpers::render::macro_call_rows::{SpanRow, span_row}; use crate::bindings::_helpers::render::macro_expansion::{ - ExpansionOutput, NativeMacroExpansion, expanded, + ExpansionOutput, NativeMacroExpansion, StateInput, expanded, }; use crate::bindings::_helpers::sqltext::sql_references::extract_references; @@ -189,16 +189,16 @@ pub(crate) fn macro_output(request: ExpansionRequest<'_>) -> PyResult(request: ExpansionRequest<'py>) -> PyResult< }; let mut spans: Vec> = Vec::with_capacity(output.spans.len()); for span in output.spans { - let (source_start, source_end, output_start, output_end) = span_row(span); + let SpanRow { + source_start, + source_end, + output_start, + output_end, + } = span_row(span); spans.push(new_record_with( py, RecordId::ExpansionSpan, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_modules.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_modules.rs index d19447b762..215f7a4a2d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_modules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_modules.rs @@ -14,20 +14,44 @@ use sqlbuild_pysource::macro_modules::types::MacroScopeKind; use crate::bindings::_helpers::boundary::panics::{NativeCompilerError, compiler_error}; use crate::bindings::types::CompilerDetach; -/// Path as `str`, `as_posix` and parts, root and owner parts, scope, source and CPython error. -type SourceRow = ( - String, - String, - Vec, - Option>, - Option>, - String, - String, - Option, -); - -/// `(name, file index, parameters, line, source digest, dependency macro names)`. -type ExportRow = (String, usize, Vec, usize, String, Vec); +positional_record! { + extract; + /// Path as `str`, `as_posix` and parts, root and owner parts, scope, source and CPython error. + pub(crate) struct SourceRow { + display: String, + posix: String, + parts: Vec, + declaration_root: Option>, + owning_path: Option>, + scope_kind: String, + contents: String, + syntax_error: Option, + } +} + +positional_record! { + into; + /// `(name, file index, parameters, line, source digest, dependency macro names)`. + pub(crate) struct ExportRow { + name: String, + file_index: usize, + parameters: Vec, + line: usize, + source_digest: String, + dependencies: Vec, + } +} + +positional_record! { + extract; + /// `(name, path, contents, dependencies)` of one macro module to load. + struct ModuleSpecRow { + name: String, + file_path: String, + contents: String, + dependencies: Vec, + } +} /// `(file index, message)`. type FaultRow = (usize, String); @@ -49,8 +73,16 @@ fn scope_kind_of(value: &str) -> MacroScopeKind { } fn source(row: SourceRow) -> MacroModuleSource { - let (display, posix, parts, declaration_root, owning_path, scope_kind, contents, syntax_error) = - row; + let SourceRow { + display, + posix, + parts, + declaration_root, + owning_path, + scope_kind, + contents, + syntax_error, + } = row; MacroModuleSource { relative_path: MacroModulePath { display, @@ -66,14 +98,14 @@ fn source(row: SourceRow) -> MacroModuleSource { } fn export_row(export: MacroExport) -> ExportRow { - ( - export.name, - export.file_index, - export.parameters, - export.line, - export.source_digest, - export.dependencies, - ) + ExportRow { + name: export.name, + file_index: export.file_index, + parameters: export.parameters, + line: export.line, + source_digest: export.source_digest, + dependencies: export.dependencies, + } } fn fault_row(fault: MacroFault) -> FaultRow { @@ -112,13 +144,16 @@ pub(crate) fn inventory_macro_modules( /// `(synthetic root, packages shallowest first, module execution order)` of `(name, path, contents, dependencies)` rows. #[pyfunction] -fn macro_module_load_plan( - modules: Vec<(String, String, String, Vec)>, -) -> (String, Vec, Vec) { +fn macro_module_load_plan(modules: Vec) -> (String, Vec, Vec) { let specs: Vec = modules .into_iter() .map( - |(name, file_path, contents, dependencies)| MacroModuleSpec { + |ModuleSpecRow { + name, + file_path, + contents, + dependencies, + }| MacroModuleSpec { name, file_path, contents, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs index 52734f0837..d3a30786e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs @@ -102,13 +102,24 @@ pub(crate) fn custom_rule_implementation_fingerprint( Ok(implementation_fingerprint(&source, &closure, &root)?) } +positional_record! { + into; + /// `(path, line, column, owner)` of one public-harness case. + struct EvidenceRow { + path: String, + line: usize, + column: usize, + owner: String, + } +} + /// `(path, line, column, owner)` of each public-harness case that statically evaluates the rule. #[pyfunction] #[pyo3(signature = (*, rule, project_dir))] fn custom_rule_test_evidence<'py>( rule: &Bound<'py, PyAny>, project_dir: &Bound<'py, PyAny>, -) -> PyResult> { +) -> PyResult> { let py: Python<'py> = rule.py(); let Some(source) = resolved_source(rule)? else { return Ok(Vec::new()); @@ -128,7 +139,12 @@ fn custom_rule_test_evidence<'py>( }; Ok(evidence .into_iter() - .map(|item: RuleTestEvidence| (item.path, item.line, item.column, item.owner)) + .map(|item: RuleTestEvidence| EvidenceRow { + path: item.path, + line: item.line, + column: item.column, + owner: item.owner, + }) .collect()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs index 03d70f111e..932c2a997c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs @@ -461,12 +461,12 @@ impl<'py> FixSession<'py> { .map(|(path, reason)| { violation( &path, - ( - "format-fix-skipped", - &format!("Rule fixes skipped this file: {}", reason.str()?), - "warning", - "Fix the reported problem, then rerun `sqb format --fix`.", - ), + ViolationDetails { + code: "format-fix-skipped", + message: &format!("Rule fixes skipped this file: {}", reason.str()?), + severity: "warning", + remediation: "Fix the reported problem, then rerun `sqb format --fix`.", + }, ) }) .collect() @@ -681,9 +681,17 @@ fn refused<'py>(report: &Bound<'py, PyAny>, reason: &str) -> PyResult { + code: &'a str, + message: &'a str, + severity: &'a str, + remediation: &'a str, +} + fn violation<'py>( path: &Bound<'py, PyAny>, - details: (&str, &str, &str, &str), + details: ViolationDetails<'_>, ) -> PyResult> { let py = path.py(); new_record_with( @@ -693,11 +701,14 @@ fn violation<'py>( ("file_path", path.clone()), ("line", 1_i64.into_pyobject(py)?.into_any()), ("column", 1_i64.into_pyobject(py)?.into_any()), - ("code", details.0.into_pyobject(py)?.into_any()), - ("message", details.1.into_pyobject(py)?.into_any()), - ("severity", details.2.into_pyobject(py)?.into_any()), + ("code", details.code.into_pyobject(py)?.into_any()), + ("message", details.message.into_pyobject(py)?.into_any()), + ("severity", details.severity.into_pyobject(py)?.into_any()), ("engine", "native".into_pyobject(py)?.into_any()), - ("remediation", details.3.into_pyobject(py)?.into_any()), + ( + "remediation", + details.remediation.into_pyobject(py)?.into_any(), + ), ], ) } @@ -705,12 +716,12 @@ fn violation<'py>( fn verification_fault<'py>(path: &Bound<'py, PyAny>, reason: &str) -> PyResult> { violation( path, - ( - "format-fix-verification-failed", - reason, - "fault", - "Restructure the SQL manually and recompile; the original file was retained.", - ), + ViolationDetails { + code: "format-fix-verification-failed", + message: reason, + severity: "fault", + remediation: "Restructure the SQL manually and recompile; the original file was retained.", + }, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_verdict.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_verdict.rs index d58ccb5322..f1449ca5cd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_verdict.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_verdict.rs @@ -6,7 +6,9 @@ use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use sqlbuild_rules::fix_verification::main::compare::compare; -use sqlbuild_rules::fix_verification::models::{DiagnosticComparison, ModelComparison}; +use sqlbuild_rules::fix_verification::models::{ + DiagnosticComparison, DiagnosticIdentity, ModelComparison, +}; use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field as native_field; @@ -107,8 +109,39 @@ fn lineage<'py>(model: &Bound<'py, PyAny>) -> PyResult> { } } -fn counts( - value: Option>, -) -> PyResult> { - value.map_or_else(|| Ok(BTreeMap::new()), |value| value.extract()) +positional_record! { + extract; + /// The `(code, resource type, resource name, message)` key `compile_facts` counts. + #[derive(PartialEq, Eq, PartialOrd, Ord)] + struct IdentityKey { + code: String, + resource_type: String, + resource_name: String, + message: String, + } +} + +fn counts(value: Option>) -> PyResult> { + let Some(value) = value else { + return Ok(BTreeMap::new()); + }; + let counted: BTreeMap = value.extract()?; + Ok(counted + .into_iter() + .map(|(key, count)| { + let IdentityKey { + code, + resource_type, + resource_name, + message, + } = key; + let identity = DiagnosticIdentity { + code, + resource_type, + resource_name, + message, + }; + (identity, count) + }) + .collect()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs index 54e7413b52..c28d595f73 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs @@ -2,9 +2,12 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_rules::lint_text::models::InterpolationSite; +use sqlbuild_rules::sql_lint::types::PreparedSql; use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; use crate::bindings::_helpers::rules::relation_keys; +use crate::bindings::models::Positional; use crate::bindings::types::CompilerDetach; #[pyfunction] @@ -23,20 +26,31 @@ struct LintPreparationRequest { dialect: String, } +/// [`PreparedSql`] as Python reads it, each site a positional tuple. +type PreparedLintSql = (String, Vec>, Vec); + #[pyfunction] fn prepare_lint_sql( py: Python<'_>, request: LintPreparationRequest, -) -> PyResult> { - py.compiler_detach(|| { - sqlbuild_rules::sql_lint::main::preparation::prepare( - &request.expanded, - &request.before_expansion, - &request.prior_sites, - &request.dialect, +) -> PyResult> { + let prepared: Option = py + .compiler_detach(|| { + sqlbuild_rules::sql_lint::main::preparation::prepare( + &request.expanded, + &request.before_expansion, + &request.prior_sites, + &request.dialect, + ) + }) + .map_err(value_error)?; + Ok(prepared.map(|(text, sites, referenced)| { + ( + text, + sites.into_iter().map(Positional).collect(), + referenced, ) - }) - .map_err(value_error) + })) } #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs index 7e6a2607ee..164895dc88 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs @@ -12,6 +12,7 @@ use crate::bindings::_helpers::compiled_project::model_input_view::{ }; use crate::bindings::_helpers::compiled_project::project_view::NativeProjectView; use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::render::macro_call_rows::SpanRow; use crate::bindings::_helpers::rules::lint_pipeline::{ CompiledExpansion, CompiledSource, span_pass, }; @@ -39,10 +40,7 @@ fn record_path(record: &ModelInputRecord) -> PathBuf { } /// The record's shared texts with the given passes, read only when their file is linted. -fn expansion( - expansion: &ExpansionRecord, - passes: &[&Vec<(usize, usize, usize, usize)>], -) -> CompiledSource { +fn expansion(expansion: &ExpansionRecord, passes: &[&Vec]) -> CompiledSource { CompiledSource::Shared { authored_sql: Arc::clone(&expansion.authored_sql), expanded_sql: Arc::clone(&expansion.expanded_sql), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs index 5c06143522..934ad63333 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs @@ -43,7 +43,7 @@ use crate::bindings::_helpers::records::construct::{new_record_with, record_clas use crate::bindings::_helpers::records::fields::field as record_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::render::authored_sql::{AuthoredExpansion, expand_with_inputs}; -use crate::bindings::_helpers::render::declaration_expansion::SpanRow; +use crate::bindings::_helpers::render::macro_call_rows::SpanRow; use crate::bindings::_helpers::rules::dsl_headers::header_violations; use crate::bindings::_helpers::rules::native_header_parser::{ HeaderFailure, NativeHeaderParser, external_identifiers, @@ -213,14 +213,12 @@ impl CompiledSource { /// Expansion span rows as lint spans. pub(crate) fn span_pass(rows: &[SpanRow]) -> Vec { rows.iter() - .map( - |&(source_start, source_end, output_start, output_end)| ExpansionSpan { - source_start: offset(source_start), - source_end: offset(source_end), - output_start: offset(output_start), - output_end: offset(output_end), - }, - ) + .map(|row| ExpansionSpan { + source_start: offset(row.source_start), + source_end: offset(row.source_end), + output_start: offset(row.output_start), + output_end: offset(row.output_end), + }) .collect() } @@ -809,12 +807,10 @@ pub(crate) fn prepare_single_body(request: &SingleBodyRequest<'_, '_>) -> PyResu role, dialect, } = request; - let (context, file_path, range, dialect): ( - &SqlExpansionInputs, - &Bound<'_, PyAny>, - (usize, usize), - &str, - ) = (context, file_path, *range, dialect); + let context: &SqlExpansionInputs = context; + let file_path: &Bound<'_, PyAny> = file_path; + let range: (usize, usize) = *range; + let dialect: &str = dialect; let py: Python<'_> = file_path.py(); let text: Vec = contents.chars().collect(); let planned: PlannedBody = plan_body(&text, range, neutralizer(role), None); @@ -1071,28 +1067,6 @@ fn lint_body( body: PendingBody, (lint_text, prepared_sites, externally_referenced_ctes): PreparedSql, ) -> PyResult<(LintSqlBody, Vec<(String, String)>)> { - let prepared_sites: Vec = prepared_sites - .into_iter() - .map( - |( - sentinel, - neutralized_start, - neutralized_end, - original_start, - original_end, - original_text, - )| { - InterpolationSite { - sentinel, - neutralized_start, - neutralized_end, - original_start, - original_end, - original_text, - } - }, - ) - .collect(); let dependencies = dependency_sites(&prepared_sites).map_err(value_error)?; let mut passes: Vec> = Vec::with_capacity(body.passes.len() + 2); passes.push(sentinel_spans(&body.sites)); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs index 7efa8a6682..9f6a9d83b6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs @@ -30,30 +30,14 @@ use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::rules::lint_pipeline::{ LintViolationRow, NATIVE_ENGINE, ViolationClasses, }; +use crate::bindings::models::Positional; use crate::bindings::types::CompilerDetach; const FORMAT_UNSAFE_CODE: &str = "format-unsafe"; const FORMAT_UNSAFE_REMEDIATION: &str = "Rewrite the unsupported SQL shape or report a minimal formatter reproduction. No header, SQL body, or fixture changes were written to this file."; -/// One interpolation site as `InterpolationSite` fields. -type SiteTuple = (String, usize, usize, usize, usize, String); - -fn site_tuples(neutralized: Neutralized) -> (String, Vec) { - let sites: Vec = neutralized - .sites - .into_iter() - .map(|site| { - ( - site.sentinel, - site.neutralized_start, - site.neutralized_end, - site.original_start, - site.original_end, - site.original_text, - ) - }) - .collect(); - (neutralized.text, sites) +fn site_tuples(neutralized: Neutralized) -> (String, Vec>) { + (neutralized.text, Positional::wrap_all(neutralized.sites)) } #[pyfunction] @@ -62,7 +46,7 @@ fn neutralize_lint_interpolation( body: &str, dialect: &str, for_formatting: bool, -) -> PyResult<(String, Vec)> { +) -> PyResult<(String, Vec>)> { compiler_guard(|| { neutralize_interpolation(body, dialect, for_formatting) .map(site_tuples) @@ -71,29 +55,26 @@ fn neutralize_lint_interpolation( } #[pyfunction] -fn neutralize_lint_context_interpolation(body: &str) -> PyResult<(String, Vec)> { +fn neutralize_lint_context_interpolation( + body: &str, +) -> PyResult<(String, Vec>)> { compiler_guard(|| Ok(site_tuples(neutralize_context_interpolation(body)))) } #[pyfunction] -fn neutralize_lint_audit_parameters(body: &str) -> PyResult<(String, Vec)> { +fn neutralize_lint_audit_parameters( + body: &str, +) -> PyResult<(String, Vec>)> { compiler_guard(|| Ok(site_tuples(neutralize_audit_parameters(body)))) } /// Restored text, or `(False, reason)` when a sentinel is missing or repeated. #[pyfunction] -fn restore_lint_interpolation(fixed: &str, sites: Vec) -> PyResult<(bool, String)> { - let sites: Vec = sites - .into_iter() - .map(|site| InterpolationSite { - sentinel: site.0, - neutralized_start: site.1, - neutralized_end: site.2, - original_start: site.3, - original_end: site.4, - original_text: site.5, - }) - .collect(); +fn restore_lint_interpolation( + fixed: &str, + sites: Vec>, +) -> PyResult<(bool, String)> { + let sites: Vec = Positional::unwrap_all(sites); match compiler_guard(|| Ok(restore_interpolation(fixed, &sites)))? { Ok(restored) => Ok((true, restored)), Err(LintTextError::Restoration(message)) => Ok((false, message)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs index 989f88060c..406440f71f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs @@ -23,18 +23,21 @@ const UNIQUE_AUDIT_COLUMN_ARGUMENT: &str = "column"; const RELATION_KIND_SEED: &str = "seed"; const RELATION_KIND_SOURCE: &str = "source"; -/// `(kind, name, materialized, mode, strategy, cursor, unique key, columns, audits)`. -pub(crate) type RelationKeyRow = ( - String, - String, - String, - String, - String, - bool, - Option>, - Vec<(String, Vec, bool)>, - Vec<(String, Option)>, -); +positional_record! { + extract, into; + /// `(kind, name, materialized, mode, strategy, cursor, unique key, columns, audits)`. + pub(crate) struct RelationKeyRow { + kind: String, + name: String, + materialized: String, + mode: String, + strategy: String, + has_cursor: bool, + unique_key: Option>, + columns: Vec<(String, Vec, bool)>, + audits: Vec<(String, Option)>, + } +} /// The key row of one relation from its config values and declared column and audit specs. pub(crate) fn relation_key_row( @@ -55,19 +58,19 @@ pub(crate) fn relation_key_row( } audits = audit_key_rows(declared_audits)?; } - Ok(( - kind.to_owned(), + Ok(RelationKeyRow { + kind: kind.to_owned(), name, - config_text(values, MATERIALIZED_CONFIG)?, - config_text(values, INCREMENTAL_MODE_CONFIG)?, - config_text(values, INCREMENTAL_STRATEGY_CONFIG)?, - values + materialized: config_text(values, MATERIALIZED_CONFIG)?, + mode: config_text(values, INCREMENTAL_MODE_CONFIG)?, + strategy: config_text(values, INCREMENTAL_STRATEGY_CONFIG)?, + has_cursor: values .get_item(CURSOR_CONFIG)? .is_some_and(|value| value.is_truthy().unwrap_or(false)), - unique_key(values)?, + unique_key: unique_key(values)?, columns, audits, - )) + }) } /// Each audit spec's `(definition name, column argument when it is text)`. @@ -93,15 +96,15 @@ pub(crate) fn audit_key_rows(audits: &Bound<'_, PyAny>) -> PyResult) -> Option> { read_source_view(view, |source, py| { let facts: &SourceKeyFacts = source.key_facts(py)?; - Ok(( - RELATION_KIND_SOURCE.to_owned(), - source.attached().name.clone(), - String::new(), - String::new(), - String::new(), - false, - None, - facts + Ok(RelationKeyRow { + kind: RELATION_KIND_SOURCE.to_owned(), + name: source.attached().name.clone(), + materialized: String::new(), + mode: String::new(), + strategy: String::new(), + has_cursor: false, + unique_key: None, + columns: facts .columns .iter() .map(|column| { @@ -112,8 +115,8 @@ pub(crate) fn source_view_key_row(view: &Bound<'_, PyAny>) -> Option(project: &Bound<'py, PyAny>) -> PyResult DeclaredRelation { - let (kind, name, materialized, mode, strategy, has_cursor, unique_key, columns, audits) = row; + let RelationKeyRow { + kind, + name, + materialized, + mode, + strategy, + has_cursor, + unique_key, + columns, + audits, + } = row; DeclaredRelation { kind, name, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index 94c3848ffd..e90de02503 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -63,6 +63,16 @@ struct DeclarationRowPy<'py>( Option, ); +/// The code, path, line, column and message that identify one fault. +#[derive(Hash, PartialEq, Eq)] +struct FaultIdentity { + code: String, + path: String, + line: u64, + column: u64, + message: String, +} + /// The request's project facts, which models, tests and scenarios are built beside. struct ProjectRows<'py> { memo: Option<(String, String)>, @@ -410,7 +420,7 @@ fn finalized<'py>( .str()? .to_str()? .to_owned(); - let mut fixable: HashSet<(String, String, u64, u64, String)> = HashSet::new(); + let mut fixable: HashSet = HashSet::new(); let mut faults: Vec = Vec::with_capacity(findings.len()); for finding in &findings { let fault: Fault = finding_fault(finding)?; @@ -518,14 +528,14 @@ fn declaration_member((name, value): (String, Bound<'_, PyAny>)) -> PyResult (String, String, u64, u64, String) { - ( - fault.code.clone(), - fault.path.clone(), - fault.line, - fault.column, - fault.message.clone(), - ) +fn fault_identity(fault: &Fault) -> FaultIdentity { + FaultIdentity { + code: fault.code.clone(), + path: fault.path.clone(), + line: fault.line, + column: fault.column, + message: fault.message.clone(), + } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs index 7334eb244e..f9e58d2580 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs @@ -52,6 +52,17 @@ fn sql_rule_lint_key(config: &Bound<'_, PyAny>) -> PyResult { Ok(settings_lint_key(&LintSettings::from_record(config)?)) } +positional_record! { + extract; + /// One model's path, authored SQL, query SQL and contract. + pub(crate) struct ModelRuleRow { + pub(crate) path: String, + pub(crate) authored_sql: String, + pub(crate) query_sql: String, + pub(crate) contract: Option, + } +} + /// Each model path's SQL rule cache key from `(path, authored_sql, query_sql, contract)`. #[pyfunction] #[pyo3(signature = (*, lint_key, codes, models))] @@ -59,7 +70,7 @@ fn sql_model_rule_keys( py: Python<'_>, lint_key: &str, codes: Vec, - models: Vec<(String, String, String, Option)>, + models: Vec, ) -> BTreeMap { model_rule_keys(py, lint_key, &codes, &models) } @@ -69,20 +80,20 @@ pub(crate) fn model_rule_keys( py: Python<'_>, lint_key: &str, codes: &[String], - models: &[(String, String, String, Option)], + models: &[ModelRuleRow], ) -> BTreeMap { let inputs: Vec> = models .iter() - .map(|(_, authored_sql, query_sql, contract)| ModelKeyInput { - authored_sql, - query_sql, - contract: contract.as_deref(), + .map(|model| ModelKeyInput { + authored_sql: &model.authored_sql, + query_sql: &model.query_sql, + contract: model.contract.as_deref(), }) .collect(); let keys: Vec = py.detach(|| parallel_model_rule_keys(lint_key, codes, &inputs)); models .iter() - .map(|(path, ..)| path.clone()) + .map(|model| model.path.clone()) .zip(keys) .collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index ed839bc92a..6330b98eac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -43,8 +43,8 @@ use crate::bindings::_helpers::rules::lint_pipeline::{ }; use crate::bindings::_helpers::rules::relation_keys::project_relation_keys; use crate::bindings::_helpers::rules::rule_cache::{ - StoredEntries, cached_rule_findings, encoded_findings, model_rule_keys, settings_lint_key, - stored_entries, write_rule_cache, + ModelRuleRow, StoredEntries, cached_rule_findings, encoded_findings, model_rule_keys, + settings_lint_key, stored_entries, write_rule_cache, }; const DATACLASSES_MODULE: &str = "dataclasses"; @@ -293,17 +293,22 @@ fn model_rule_identities(run: &SqlRun<'_, '_>, lint_key: &str) -> PyResult)> = Vec::new(); + let mut models: Vec = Vec::new(); for model in sql_rule_models(&run.project, true)? { if let Some(key) = model.key { - models.push((model.path, key.authored_sql, key.query_sql, key.contract)); + models.push(ModelRuleRow { + path: model.path, + authored_sql: key.authored_sql, + query_sql: key.query_sql, + contract: key.contract, + }); } } let keys: BTreeMap = model_rule_keys(run.project.py(), lint_key, run.codes, &models); let fresh: Vec<(String, String)> = models .into_iter() - .map(|(path, ..)| { + .map(|ModelRuleRow { path, .. }| { let key: String = keys[&path].clone(); (path, key) }) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs index 6cc919eaf7..470ab6f736 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs @@ -15,8 +15,8 @@ use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::scenario_command::conversions::succeeded; use crate::bindings::_helpers::scenario_command::presentation::{ - CompletionKind, SectionStatus, activity, begin_section, connection_hooks, progress_relay, - write_completion, + CompletionKind, RelayContext, SectionStatus, activity, begin_section, connection_hooks, + progress_relay, write_completion, }; use crate::bindings::_helpers::scenario_command::setup::{ CompileInputs, ScenarioProject, capture_limits, capture_settings, cli_user_error, compile, @@ -101,12 +101,12 @@ pub(crate) fn run_scenario_capture_command( let relay = progress_relay( &stream, §ion, - ( - &project.project_dir, - &adapter, - CompletionKind::Capture, + RelayContext { + project_dir: &project.project_dir, + adapter: &adapter, + kind: CompletionKind::Capture, style, - ), + }, )?; let results: Vec> = call_with( &host::entry(py, ExecutorHost::RunScenarioCapturePipeline)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs index 1d40c492fb..0762d7b65e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs @@ -233,16 +233,24 @@ pub(crate) fn begin_section<'py>( }) } +/// The project directory, adapter, completion kind and style a progress relay reports with. +pub(crate) struct RelayContext<'a, 'py> { + pub(crate) project_dir: &'a Bound<'py, PyAny>, + pub(crate) adapter: &'a Bound<'py, PyAny>, + pub(crate) kind: CompletionKind, + pub(crate) style: CliStyle, +} + /// The relay reporting one section's scenarios. pub(crate) fn progress_relay<'py>( stream: &Bound<'py, PyAny>, section: &SectionStatus<'py>, - (project_dir, adapter, kind, style): ( - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - CompletionKind, - CliStyle, - ), + RelayContext { + project_dir, + adapter, + kind, + style, + }: RelayContext<'_, 'py>, ) -> PyResult> { Bound::new( stream.py(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs index 69f78e3f35..d125ca59c2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs @@ -17,7 +17,7 @@ use crate::bindings::_helpers::result_output::documents::format_scenario_snapsho use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use crate::bindings::_helpers::scenario_command::conversions::succeeded; use crate::bindings::_helpers::scenario_command::presentation::{ - CompletionKind, SectionStatus, begin_section, connection_hooks, progress_relay, + CompletionKind, RelayContext, SectionStatus, begin_section, connection_hooks, progress_relay, }; use crate::bindings::_helpers::scenario_command::setup::{ CompileInputs, boolean, capture_limits, capture_settings, compile, flush, optional_text, @@ -161,12 +161,12 @@ pub(crate) fn sync_local_snapshots<'py>( let relay = progress_relay( &run.stream, §ion, - ( - &run.project.project_dir, - &run.project_adapter, - CompletionKind::Sync { refresh }, - run.style, - ), + RelayContext { + project_dir: &run.project.project_dir, + adapter: &run.project_adapter, + kind: CompletionKind::Sync { refresh }, + style: run.style, + }, )?; let settings: Bound<'py, PyAny> = capture_settings( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs index 14018b8ddf..adadca3f31 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs @@ -23,8 +23,8 @@ use crate::bindings::_helpers::result_output::documents::format_scenario_executi use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use crate::bindings::_helpers::scenario_command::conversions::succeeded; use crate::bindings::_helpers::scenario_command::presentation::{ - CompletionKind, SectionStatus, activity, begin_section, connection_hooks, progress_relay, - write_completion, + CompletionKind, RelayContext, SectionStatus, activity, begin_section, connection_hooks, + progress_relay, write_completion, }; use crate::bindings::_helpers::scenario_command::setup::{ CompileInputs, ScenarioProject, boolean, cli_user_error, compile, discover_project, flush, int, @@ -213,12 +213,12 @@ fn run_scenarios( let relay = progress_relay( &run.stream, §ion, - ( - &run.project.project_dir, - &run.adapter, - CompletionKind::Run { target_dir }, - run.style, - ), + RelayContext { + project_dir: &run.project.project_dir, + adapter: &run.adapter, + kind: CompletionKind::Run { target_dir }, + style: run.style, + }, )?; let project_name: Bound<'_, PyAny> = native_field( &native_field(&run.project.discovered, "project_config")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs index 46e1d892fa..58e7eddeb9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs @@ -33,21 +33,30 @@ type IdentityRow = (String, String); /// A declaration's kind, name and private owner; a resource's kind and name with no owner. type QualifiedRow = (String, String, Option); /// Consumer, declaration, usage kind, what it is used through, and the enum member. -pub(crate) type UsageRow = ( - QualifiedRow, - QualifiedRow, - String, - Option, - Option, -); +#[derive(Clone, Hash, PartialEq, Eq)] +pub(crate) struct UsageRow { + pub(crate) consumer: QualifiedRow, + pub(crate) declaration: QualifiedRow, + pub(crate) kind: String, + pub(crate) through: Option, + pub(crate) enum_member: Option, +} /// Path, scope, private owner, owning path and ownership root path of a named record. -type RecordRow = (String, String, Option, Option, String); +struct RecordRow { + path: String, + scope: String, + owner: Option, + owning_path: Option, + ownership_root: String, +} /// Name, the schema it extends, and each column's name and type. type SchemaRow = (String, Option, Vec<(String, Option)>); -/// Report key, message, path and help of one S006. -type DiagnosticRow = (Vec, String, String, String); /// Usages as `(consumer, declaration)` record positions, S006s, else a scope path error. -type SchemaUsageRows = (Vec<(usize, usize)>, Vec, Option); +type SchemaUsageRows = ( + Vec<(usize, usize)>, + Vec, + Option, +); /// `(own, position)`: an own dependency's position, else a distinct usage's. pub(crate) type SourceRow = (bool, usize); /// Distinct collected positions, merged sources per declaration, and final usage positions. @@ -58,7 +67,13 @@ fn invalid(what: &str) -> pyo3::PyErr { } fn named_record(handle: usize, row: RecordRow) -> PyResult { - let (path, scope, owner, owning_path, ownership_root) = row; + let RecordRow { + path, + scope, + owner, + owning_path, + ownership_root, + } = row; let owner: Option = match owner { None => None, Some((kind, name)) => Some(ResourceIdentity { @@ -78,15 +93,6 @@ fn named_record(handle: usize, row: RecordRow) -> PyResult { }) } -fn diagnostic_row(diagnostic: InaccessibleDiagnostic) -> DiagnosticRow { - ( - diagnostic.key, - diagnostic.message, - diagnostic.path, - diagnostic.help, - ) -} - /// `model_schema_scope_usages`: each schema's parent and enum `UsageRecord`s; S006s are reported. #[pyfunction] pub(crate) fn model_schema_scope_usages<'py>( @@ -163,10 +169,10 @@ pub(crate) fn model_schema_scope_usages<'py>( fn record_row(record: &Bound<'_, PyAny>) -> PyResult { let identity: Bound<'_, PyAny> = native_field(record, "identity")?; let owner: Bound<'_, PyAny> = native_field(&identity, "owner")?; - Ok(( - native_field(record, "path")?.extract()?, - native_field(&native_field(record, "scope")?, "value")?.extract()?, - if owner.is_none() { + Ok(RecordRow { + path: native_field(record, "path")?.extract()?, + scope: native_field(&native_field(record, "scope")?, "value")?.extract()?, + owner: if owner.is_none() { None } else { Some(( @@ -174,18 +180,25 @@ fn record_row(record: &Bound<'_, PyAny>) -> PyResult { native_field(&owner, "name")?.extract()?, )) }, - native_field(record, "owning_path")?.extract()?, - native_field(&native_field(record, "ownership_root")?, "path")?.extract()?, - )) + owning_path: native_field(record, "owning_path")?.extract()?, + ownership_root: native_field(&native_field(record, "ownership_root")?, "path")? + .extract()?, + }) } /// Report each S006 through the active compile diagnostic collector. -fn report_inaccessible(py: Python<'_>, diagnostics: Vec) -> PyResult<()> { +fn report_inaccessible(py: Python<'_>, diagnostics: Vec) -> PyResult<()> { if diagnostics.is_empty() { return Ok(()); } let path_class = py.import("pathlib")?.getattr("Path")?; - for (key, message, path, help) in diagnostics { + for InaccessibleDiagnostic { + key, + message, + path, + help, + } in diagnostics + { let fields: Bound<'_, PyDict> = PyDict::new(py); fields.set_item("phase", "compile")?; fields.set_item("severity", "error")?; @@ -230,11 +243,7 @@ fn scope_schema_usages( Ok(SchemaUsages { usages, diagnostics, - }) => ( - usages, - diagnostics.into_iter().map(diagnostic_row).collect(), - None, - ), + }) => (usages, diagnostics, None), Err(rejected) => ( Vec::new(), Vec::new(), @@ -265,7 +274,11 @@ pub(crate) fn merged_usage_rows( let mut consumed: Vec<(Option, QualifiedRow)> = Vec::with_capacity(distinct.len()); for position in &distinct { - let (consumer, declaration, _, _, _) = &collected[*position]; + let UsageRow { + consumer, + declaration, + .. + } = &collected[*position]; let by_declaration: Option = DeclarationKind::parse(&consumer.0) .is_some() .then(|| consumer.clone()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/placement.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/placement.rs index 2644315669..8f9d194b3c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/placement.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/placement.rs @@ -13,15 +13,15 @@ pub(crate) type QualifiedRow = (String, String, Option); /// An ownership root's path, kind and resource kind. pub(crate) type RootRow = (String, String, Option); /// Identity, path, line, column, scope, ownership root and owning path. -pub(crate) type DeclarationRow = ( - QualifiedRow, - String, - i64, - i64, - String, - RootRow, - Option, -); +pub(crate) struct DeclarationRow { + pub(crate) identity: QualifiedRow, + pub(crate) path: String, + pub(crate) line: i64, + pub(crate) column: i64, + pub(crate) scope: String, + pub(crate) root: RootRow, + pub(crate) owning_path: Option, +} fn invalid(what: &str) -> pyo3::PyErr { PyValueError::new_err(format!("invalid scope placement {what}")) @@ -73,7 +73,15 @@ pub(crate) fn scope_kind(value: &str) -> PyResult { /// A declaration row as its record, without macro dependencies. pub(crate) fn declaration_record(row: DeclarationRow) -> PyResult { - let (identity, path, line, column, scope, root, owning_path) = row; + let DeclarationRow { + identity, + path, + line, + column, + scope, + root, + owning_path, + } = row; Ok(DeclarationRecord { identity: declaration_identity(identity)?, path, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs index e0e672910c..564051531b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs @@ -24,8 +24,8 @@ use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; use crate::bindings::_helpers::scopes::scope_objects::ScopeClasses; use crate::bindings::_helpers::scopes::scope_report_objects::read_result; use crate::bindings::_helpers::scopes::scope_reports::{ - scope_browse, scope_explain, scope_list, scope_move_preview, scope_report, - serialize_scope_report, + ExplainArguments, ListArguments, ReportArguments, scope_browse, scope_explain, scope_list, + scope_move_preview, scope_report, serialize_scope_report, }; const TARGET_ERROR_CODE: &str = "C960"; @@ -249,12 +249,12 @@ fn query_result<'py>( return Ok(scope_list( py, lookup, - ( + ListArguments { folder, - Some(filters), - optional_text(py, &target), - prospective, - ), + filters: Some(filters), + target: optional_text(py, &target), + target_is_prospective: prospective, + }, )? .into_bound(py)); } @@ -264,12 +264,12 @@ fn query_result<'py>( let mut report: Bound<'_, PyAny> = scope_report( py, lookup, - ( - optional_text(py, &target), - optional_text(py, &at), + ReportArguments { + target: optional_text(py, &target), + at: optional_text(py, &at), directory, - Some(filters), - ), + filters: Some(filters), + }, )? .into_bound(py); let explain: Option = native_field(request, "explain")?.extract()?; @@ -277,12 +277,12 @@ fn query_result<'py>( let (explanation, diagnostics): (Bound<'_, PyAny>, Bound<'_, PyAny>) = scope_explain( py, lookup, - ( - PyString::new(py, &declaration).into_any(), - optional_text(py, &target), - optional_text(py, &at), + ExplainArguments { + declaration: PyString::new(py, &declaration).into_any(), + target: optional_text(py, &target), + at: optional_text(py, &at), directory, - ), + }, ) .map(|(explanation, diagnostics)| { (explanation.into_bound(py), diagnostics.into_bound(py)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs index 0cc73f0663..3ee4cd952a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs @@ -29,7 +29,12 @@ use crate::bindings::types::CompilerDetach; const TYPES_MODULE: &str = "types"; /// `(declarations, canonical positions)` matched, else identity and path error messages. -pub(crate) type TargetRows = (bool, Vec, Option, Option); +pub(crate) struct TargetRows { + pub(crate) declarations: bool, + pub(crate) positions: Vec, + pub(crate) identity_error: Option, + pub(crate) path_error: Option, +} /// Visible `(position, reason, grant)`, inaccessible `(position, reason)`, else the path error. pub(crate) type ResolvedRows = ( Vec<(usize, &'static str, Option)>, @@ -119,27 +124,30 @@ impl NativeScopeLookup { None => ReportTarget::Text(text.to_owned()), }; Ok(match report_target(&self.records, &self.lookup, &target) { - Ok(TargetMatch::Resources(positions)) => ( - false, - canonical(&self.canonical.resources, positions), - None, - None, - ), - Ok(TargetMatch::Declarations(positions)) => ( - true, - canonical(&self.canonical.declarations, positions), - None, - None, - ), - Err(TargetFailure::Identity(failure)) => ( - false, - Vec::new(), - Some(identity_message(&failure, text)), - None, - ), - Err(TargetFailure::Path(rejected)) => { - (false, Vec::new(), None, Some(path_message(&rejected))) - } + Ok(TargetMatch::Resources(positions)) => TargetRows { + declarations: false, + positions: canonical(&self.canonical.resources, positions), + identity_error: None, + path_error: None, + }, + Ok(TargetMatch::Declarations(positions)) => TargetRows { + declarations: true, + positions: canonical(&self.canonical.declarations, positions), + identity_error: None, + path_error: None, + }, + Err(TargetFailure::Identity(failure)) => TargetRows { + declarations: false, + positions: Vec::new(), + identity_error: Some(identity_message(&failure, text)), + path_error: None, + }, + Err(TargetFailure::Path(rejected)) => TargetRows { + declarations: false, + positions: Vec::new(), + identity_error: None, + path_error: Some(path_message(&rejected)), + }, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs index ab0f6eb8d3..8f9205a9c8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs @@ -21,8 +21,8 @@ use crate::bindings::_helpers::records::construct::{new_record, record_class}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::scopes::placement::{ - IdentityRow, QualifiedRow, RootRow, declaration_identity, declaration_record, ownership_root, - qualified_identity, resource_identity, + DeclarationRow, IdentityRow, QualifiedRow, RootRow, declaration_identity, declaration_record, + ownership_root, qualified_identity, resource_identity, }; use crate::bindings::_helpers::scopes::scope_lookup::grant_kind; @@ -179,15 +179,15 @@ fn read_constant(facts: &Bound<'_, PyAny>) -> PyResult { /// One `DeclarationRecord` with its value metadata. pub(crate) fn read_declaration(record: &Bound<'_, PyAny>) -> PyResult { Ok(CachedDeclaration { - record: declaration_record(( - qualified_row(&native_field(record, "identity")?)?, - field(record, "path")?, - field(record, "line")?, - field(record, "column")?, - value_of(record, "scope")?, - root_row(&native_field(record, "ownership_root")?)?, - field(record, "owning_path")?, - ))?, + record: declaration_record(DeclarationRow { + identity: qualified_row(&native_field(record, "identity")?)?, + path: field(record, "path")?, + line: field(record, "line")?, + column: field(record, "column")?, + scope: value_of(record, "scope")?, + root: root_row(&native_field(record, "ownership_root")?)?, + owning_path: field(record, "owning_path")?, + })?, macro_facts: optional_item(record, "macro")? .map(|facts| read_macro(&facts)) .transpose()?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs index 9583d93bff..bb699daa8c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs @@ -40,26 +40,26 @@ use crate::bindings::_helpers::scopes::scope_report_objects::{ type PyObject = Py; /// Target, location, whether it is a directory, and the filters. -pub(crate) type ReportArguments<'py> = ( - Option>, - Option>, - bool, - Option>, -); +pub(crate) struct ReportArguments<'py> { + pub(crate) target: Option>, + pub(crate) at: Option>, + pub(crate) directory: bool, + pub(crate) filters: Option>, +} /// Declaration, target, location and whether it is a directory. -pub(crate) type ExplainArguments<'py> = ( - Bound<'py, PyAny>, - Option>, - Option>, - bool, -); +pub(crate) struct ExplainArguments<'py> { + pub(crate) declaration: Bound<'py, PyAny>, + pub(crate) target: Option>, + pub(crate) at: Option>, + pub(crate) directory: bool, +} /// Folder, filters, target and whether the target is prospective. -pub(crate) type ListArguments<'py> = ( - String, - Option>, - Option>, - bool, -); +pub(crate) struct ListArguments<'py> { + pub(crate) folder: String, + pub(crate) filters: Option>, + pub(crate) target: Option>, + pub(crate) target_is_prospective: bool, +} const CURRENT_FOLDER: &str = "."; @@ -148,7 +148,12 @@ pub(crate) fn scope_report( lookup: &Bound<'_, PyAny>, request: ReportArguments<'_>, ) -> PyResult { - let (target, at, directory, filters) = request; + let ReportArguments { + target, + at, + directory, + filters, + } = request; let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; let selected: Bound<'_, PyAny> = match filters.filter(|filters| !filters.is_none()) { Some(filters) => filters, @@ -178,7 +183,12 @@ pub(crate) fn scope_explain( lookup: &Bound<'_, PyAny>, request: ExplainArguments<'_>, ) -> PyResult<(PyObject, PyObject)> { - let (declaration, target, at, directory) = request; + let ExplainArguments { + declaration, + target, + at, + directory, + } = request; let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; let reports: LookupReports = lookup_reports(lookup)?; let request: ExplainRequest = ExplainRequest { @@ -240,7 +250,12 @@ pub(crate) fn scope_list( lookup: &Bound<'_, PyAny>, request: ListArguments<'_>, ) -> PyResult { - let (folder, filters, target, target_is_prospective) = request; + let ListArguments { + folder, + filters, + target, + target_is_prospective, + } = request; let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; let reports: LookupReports = lookup_reports(lookup)?; let request: FolderRequest = @@ -354,12 +369,12 @@ fn query_scope_report_binding<'py>( scope_report( py, &arguments.value("lookup")?, - ( - optional(&arguments, "target")?, - optional(&arguments, "at")?, - arguments.flag("directory")?, - optional(&arguments, "filters")?, - ), + ReportArguments { + target: optional(&arguments, "target")?, + at: optional(&arguments, "at")?, + directory: arguments.flag("directory")?, + filters: optional(&arguments, "filters")?, + }, ) } @@ -379,12 +394,12 @@ fn explain_scope_declaration_binding<'py>( scope_explain( py, &arguments.value("lookup")?, - ( - arguments.value("declaration")?, - optional(&arguments, "target")?, - optional(&arguments, "at")?, - arguments.flag("directory")?, - ), + ExplainArguments { + declaration: arguments.value("declaration")?, + target: optional(&arguments, "target")?, + at: optional(&arguments, "at")?, + directory: arguments.flag("directory")?, + }, ) } @@ -435,12 +450,12 @@ fn list_scope_declarations_binding<'py>( scope_list( py, &arguments.value("lookup")?, - ( - arguments.value("folder")?.extract()?, - optional(&arguments, "filters")?, - optional(&arguments, "target")?, - arguments.flag("target_is_prospective")?, - ), + ListArguments { + folder: arguments.value("folder")?.extract()?, + filters: optional(&arguments, "filters")?, + target: optional(&arguments, "target")?, + target_is_prospective: arguments.flag("target_is_prospective")?, + }, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs index 8338ebd9bd..8ddc2ef87f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs @@ -57,15 +57,15 @@ fn tuple_from<'py>(py: Python<'py>, items: Vec>) -> PyResult) -> PyResult { - Ok(( - qualified_row(&native_field(usage, "consumer")?)?, - qualified_row(&native_field(usage, "declaration")?)?, - value_of(usage, "kind")?, - optional_item(usage, "through")? + Ok(UsageRow { + consumer: qualified_row(&native_field(usage, "consumer")?)?, + declaration: qualified_row(&native_field(usage, "declaration")?)?, + kind: value_of(usage, "kind")?, + through: optional_item(usage, "through")? .map(|through| qualified_row(&through)) .transpose()?, - field(usage, "enum_member")?, - )) + enum_member: field(usage, "enum_member")?, + }) } fn dependency_row( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs index 40d9e18fbf..46b48fc41a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs @@ -47,8 +47,12 @@ fn target_query<'py>( } else { None }; - let (declarations, positions, identity_error, path_error): TargetRows = - native_lookup(lookup)?.get().target(row, &raw)?; + let TargetRows { + declarations, + positions, + identity_error, + path_error, + }: TargetRows = native_lookup(lookup)?.get().target(row, &raw)?; if let Some(message) = identity_error { return Err(PyErr::from_value( error_class(py, ErrorId::InvalidQualifiedIdentity)?.call1((message,))?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs index 4be1bbc2fa..02e58040ce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs @@ -16,7 +16,9 @@ use sqlbuild_analysis::semantic_validation::models as validation; use crate::bindings::_helpers::analysis::model_reads::binding_validated; use crate::bindings::_helpers::boundary::panics::compiler_error; -use crate::bindings::_helpers::compiled_project::compiled_model_view::NativeCompiledModel; +use crate::bindings::_helpers::compiled_project::compiled_model_view::{ + ModelMetadataTexts, NativeCompiledModel, +}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::fields::mapping_get_or; @@ -282,7 +284,12 @@ fn model_diagnostics<'py>( } else { &classes.warning }; - let row: FindingRow<'_> = (&audit.code, &audit.message, audit.line, audit.column); + let row: FindingRow<'_> = FindingRow { + code: &audit.code, + message: &audit.message, + line: audit.line, + column: audit.column, + }; diagnostics.push(model_error(classes, model, &row, severity)?); } for finding in findings.references { @@ -340,17 +347,32 @@ fn pairs(mapping: &Bound<'_, PyAny>) -> PyResult> { .collect() } -/// `(code, message, line, column)`. -type FindingRow<'a> = (&'a str, &'a str, i64, i64); +/// The code, message, line and column of one finding. +struct FindingRow<'a> { + code: &'a str, + message: &'a str, + line: i64, + column: i64, +} fn finding_row(finding: &MetadataFinding) -> FindingRow<'_> { - (finding.code, &finding.message, finding.line, finding.column) + FindingRow { + code: finding.code, + message: &finding.message, + line: finding.line, + column: finding.column, + } } fn model_error<'py>( classes: &StageClasses<'py>, model: &Bound<'py, PyAny>, - &(code, message, line, column): &FindingRow<'_>, + &FindingRow { + code, + message, + line, + column, + }: &FindingRow<'_>, severity: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = classes.py(); @@ -446,8 +468,12 @@ fn metadata_model(classes: &StageClasses<'_>, model: &Bound<'_, PyAny>) -> PyRes upstreams.push((upstream.str()?.extract()?, column_names(&columns)?)); } } - let (name, query_sql, authored_sql, calls_functions) = match model.cast::() - { + let ModelMetadataTexts { + name, + query_sql, + authored_sql, + calls_functions, + } = match model.cast::() { Ok(native) => native .get() .metadata_texts(model.py(), &FUNCTION_REFERENCE_KINDS)?, @@ -460,12 +486,12 @@ fn metadata_model(classes: &StageClasses<'_>, model: &Bound<'_, PyAny>) -> PyRes break; } } - ( - native_field(model, "name")?.extract()?, - native_field(model, "query_sql")?.extract()?, - native_field(model, "authored_sql")?.extract()?, + ModelMetadataTexts { + name: native_field(model, "name")?.extract()?, + query_sql: native_field(model, "query_sql")?.extract()?, + authored_sql: native_field(model, "authored_sql")?.extract()?, calls_functions, - ) + } } }; Ok(MetadataModel { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs index 42644f97bf..cdf45c3cc2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs @@ -9,8 +9,8 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::semantic_checks::main::resource_sql_diagnostics::resource_sql_diagnostics; use sqlbuild_analysis::semantic_checks::models::{ - AuditSqlInput, HookModelInput, HookSqlInput, ResourceSqlDiagnostic, ResourceSqlFailure, - ResourceSqlKind, ResourceSqlRequest, SqlTestSqlInput, + AuditSqlInput, HookModelInput, HookSqlInput, ResourceSpan, ResourceSqlDiagnostic, + ResourceSqlFailure, ResourceSqlKind, ResourceSqlRequest, SqlTestSqlInput, }; use crate::bindings::_helpers::boundary::panics::compiler_error; @@ -272,7 +272,7 @@ fn opts_out(values: &Bound<'_, PyAny>) -> PyResult { fn row_relations(rows: Vec) -> Vec { let mut names: Vec = rows .into_iter() - .filter_map(|(kind, name, _, _)| relation_name(&kind, name)) + .filter_map(|ReferenceRow { kind, name, .. }| relation_name(&kind, name)) .collect(); names.sort_unstable(); names.dedup(); @@ -318,7 +318,12 @@ fn python_diagnostic<'py>( classes: &DiagnosticClasses<'py>, diagnostic: ResourceSqlDiagnostic, ) -> PyResult> { - let (line, column, end_line, end_column) = diagnostic.span; + let ResourceSpan { + line, + column, + end_line, + end_column, + } = diagnostic.span; let location: Bound<'py, PyDict> = PyDict::new(py); location.set_item("path", classes.path.call1((&diagnostic.path,))?)?; location.set_item("line", line)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs index e5dfd2cf62..04336f22b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs @@ -5,7 +5,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::analysis::binding_catalog::{ - BindingCatalog, CASE_SENSITIVE_BINDING_DIALECTS, + BindingCatalog, CASE_SENSITIVE_BINDING_DIALECTS, CatalogKey, }; use crate::bindings::_helpers::analysis::model_reads::{ declared_rows, dynamic_columns_declared, inferred_rows, star_flags, unchecked_columns, @@ -74,7 +74,7 @@ fn model_relation_names(model: &Bound<'_, PyAny>) -> PyResult<(Vec, bool let referenced: bool = !rows.is_empty(); let names: Vec = relation_names( rows.into_iter() - .map(|(kind, name, _, _)| (kind, name)) + .map(|ReferenceRow { kind, name, .. }| (kind, name)) .collect(), ); return Ok((names, referenced)); @@ -155,12 +155,12 @@ fn binding_shape<'py>( if !catalog.is_truthy()? { let created: BindingCatalog = BindingCatalog::create( py, - ( - or_generic(&dialect)?.extract()?, - false, - native_field(profile, "semantic_known_functions")?.extract()?, - native_field(profile, "semantic_known_types")?.extract()?, - ), + CatalogKey { + dialect: or_generic(&dialect)?.extract()?, + quoted_ignore_case: false, + known_functions: native_field(profile, "semantic_known_functions")?.extract()?, + known_types: native_field(profile, "semantic_known_types")?.extract()?, + }, inputs.as_any(), )?; catalog = Bound::new(py, created)?.into_any(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs index 9dcd6a4f87..e35b5a5709 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs @@ -14,6 +14,7 @@ use sqlbuild_core::panics::main::native_failure::native_failure; use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::compiled_project::model_input_view::NativeModelInput; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::records::fields::{ field as native_field, path_posix_text, path_stem, }; @@ -26,19 +27,11 @@ const INTERNAL_FAILURE: &str = "internal"; const DECIMAL_OVERFLOW: &str = "decimal_overflow"; const SQL_TEST_ASSEMBLY_CONTEXT: &str = "native SQL test assembly"; -/// `(line, column, end line, end column, message, help)` of one P013 diagnostic. -pub(crate) type DiagnosticRow = (usize, usize, usize, usize, String, String); -/// `(name, scope deps, target models, case fingerprint, diagnostic resource, diagnostics)`. -pub(crate) type FactsRow = ( - String, - Vec<(&'static str, String)>, - Vec, - Option, - String, - Vec, -); /// The test's facts, and the `(kind, message)` of the error its assembly raises, if any. -pub(crate) type AssemblyRow = (Option, Option<(&'static str, String)>); +pub(crate) type AssemblyRow = ( + Option, + Option<(&'static str, String)>, +); /// The compile inputs assembly reads: model inputs, SQL test inputs and lexical rules. pub(crate) struct AssemblyRequest<'py> { @@ -86,11 +79,18 @@ fn assembly_model(model: &Bound<'_, PyAny>) -> PyResult { references: record .references .iter() - .map(|(kind, name, package, _)| SqlTestAssemblyReference { - kind: kind.clone(), - name: name.clone(), - package: package.clone(), - }) + .map( + |ReferenceRow { + kind, + name, + package, + .. + }| SqlTestAssemblyReference { + kind: kind.clone(), + name: name.clone(), + package: package.clone(), + }, + ) .collect(), }) }); @@ -240,11 +240,10 @@ fn parameter_values(items: &Bound<'_, PyAny>) -> PyResult AssemblyRow { match outcome { - SqlTestAssemblyOutcome::Assembled(facts) => (Some(facts_row(facts)), None), - SqlTestAssemblyOutcome::FingerprintOverflow(facts) => ( - Some(facts_row(facts)), - Some((DECIMAL_OVERFLOW, String::new())), - ), + SqlTestAssemblyOutcome::Assembled(facts) => (Some(facts), None), + SqlTestAssemblyOutcome::FingerprintOverflow(facts) => { + (Some(facts), Some((DECIMAL_OVERFLOW, String::new()))) + } SqlTestAssemblyOutcome::Failed(SqlTestAssemblyFailure::Input(message)) => { (None, Some((INPUT_FAILURE, message))) } @@ -257,27 +256,3 @@ fn assembly_row(outcome: SqlTestAssemblyOutcome) -> AssemblyRow { ), } } - -fn facts_row(facts: AssembledSqlTestFacts) -> FactsRow { - ( - facts.name, - facts.scope_deps, - facts.target_model_names, - facts.case_fingerprint, - facts.diagnostic_resource_name, - facts - .diagnostics - .into_iter() - .map(|diagnostic| { - ( - diagnostic.line, - diagnostic.column, - diagnostic.end_line, - diagnostic.end_column, - diagnostic.message, - diagnostic.help, - ) - }) - .collect(), - ) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs index 029f6f42a9..4cec1db815 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs @@ -3,13 +3,14 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::compiler::models::{AssembledSqlTestFacts, SqlTestHelperDiagnostic}; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::records::construct::{enum_member, record_class}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::sql_test_glue::assembly::{ - AssemblyRequest, AssemblyRow, DiagnosticRow, FactsRow, assemble_compiled_sql_tests, + AssemblyRequest, AssemblyRow, assemble_compiled_sql_tests, }; /// Fields a direct-logic compiled payload copies from its input payload, in order. @@ -159,7 +160,14 @@ fn compiled_payload<'py>( fn keyed_diagnostic<'py>( classes: &Classes<'py>, (test_input, resource_name): (&Bound<'py, PyAny>, &str), - (line, column, end_line, end_column, message, help): DiagnosticRow, + SqlTestHelperDiagnostic { + line, + column, + end_line, + end_column, + message, + help, + }: SqlTestHelperDiagnostic, ) -> PyResult> { let py: Python<'py> = classes.py(); let path: Bound<'py, PyAny> = @@ -203,11 +211,17 @@ fn keyed_diagnostic<'py>( fn compiled_test<'py>( classes: &Classes<'py>, test_input: &Bound<'py, PyAny>, - facts: FactsRow, + facts: AssembledSqlTestFacts, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyTuple>)> { let py: Python<'py> = classes.py(); - let (name, scope_rows, target_model_names, case_fingerprint, resource_name, diagnostics) = - facts; + let AssembledSqlTestFacts { + name, + scope_deps: scope_rows, + target_model_names, + case_fingerprint, + diagnostic_resource_name: resource_name, + diagnostics, + } = facts; let scope_deps: Vec> = scope_rows .iter() .map(|(resource_type, dep)| classes.object_key(resource_type, dep)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs index 51dac5c47a..45d8a0755b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs @@ -2,13 +2,24 @@ use crate::bindings::_helpers::records::construct::{enum_from_value, new_record_with}; use crate::bindings::_helpers::records::ids::RecordId; -use crate::bindings::_helpers::sql_test_glue::planning::{AssertionRow, ChainRow, PlanRow}; +use crate::bindings::_helpers::sql_test_glue::planning::PlanRow; use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyTuple; +use sqlbuild_analysis::compiler::models::{ + SqlTestAssertionStep, SqlTestChainStep, SqlTestPlan, SqlTestPlanWarning, +}; -fn chain_step<'py>(py: Python<'py>, row: ChainRow) -> PyResult> { - let (model, sql, expected_sql, lifted, body, expected_columns, expected_lifted) = row; +fn chain_step<'py>(py: Python<'py>, step: SqlTestChainStep) -> PyResult> { + let SqlTestChainStep { + model_name: model, + resolved_sql: sql, + expected_cte_sql: expected_sql, + expected_lifted_ctes: expected_lifted, + lifted_ctes: lifted, + comparison_body_sql: body, + expected_columns, + } = step; new_record_with( py, RecordId::ChainStep, @@ -33,8 +44,13 @@ fn chain_step<'py>(py: Python<'py>, row: ChainRow) -> PyResult ) } -fn assertion_step<'py>(py: Python<'py>, row: AssertionRow) -> PyResult> { - let (name, sql, lifted, body) = row; +fn assertion_step<'py>(py: Python<'py>, step: SqlTestAssertionStep) -> PyResult> { + let SqlTestAssertionStep { + name, + resolved_sql: sql, + lifted_ctes: lifted, + comparison_body_sql: body, + } = step; new_record_with( py, RecordId::SqlTestAssertionStep, @@ -48,7 +64,17 @@ fn assertion_step<'py>(py: Python<'py>, row: AssertionRow) -> PyResult, row: PlanRow) -> PyResult> { - let (sql, chain_rows, assertion_rows, model_names, warning_rows, error_messages) = row; + let PlanRow { + plan: + SqlTestPlan { + sql, + chain: chain_rows, + assertions: assertion_rows, + model_names, + warnings: warning_rows, + }, + error_messages, + } = row; let mut chain: Vec> = Vec::new(); for value in chain_rows { chain.push(chain_step(py, value)?); @@ -58,7 +84,12 @@ pub(super) fn plan(py: Python<'_>, row: PlanRow) -> PyResult> { assertions.push(assertion_step(py, value)?); } let mut warnings: Vec> = Vec::new(); - for (model_name, level, message) in warning_rows { + for SqlTestPlanWarning { + model_name, + severity: level, + message, + } in warning_rows + { warnings.push(new_record_with( py, RecordId::PlanWarning, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs index df61cf369a..fa0241d2d1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs @@ -7,9 +7,8 @@ use sqlbuild_analysis::compiler::main::sql_test_chains::resolve_sql_test_chains; use sqlbuild_analysis::compiler::main::sql_test_glue::plan_sql_test_batch; use sqlbuild_analysis::compiler::main::sql_test_plan_errors::sql_test_plan_error_messages; use sqlbuild_analysis::compiler::models::{ - SqlTestAssertionStep, SqlTestChainBatch, SqlTestChainModel, SqlTestChainStep, SqlTestCte, - SqlTestPlan, SqlTestPlanBatch, SqlTestPlanFunction, SqlTestPlanModel, SqlTestPlanPayload, - SqlTestPlanTest, + SqlTestChainBatch, SqlTestChainModel, SqlTestCte, SqlTestPlan, SqlTestPlanBatch, + SqlTestPlanFunction, SqlTestPlanModel, SqlTestPlanPayload, SqlTestPlanTest, }; use crate::bindings::_helpers::analysis::model_reads; @@ -21,37 +20,30 @@ use crate::bindings::types::CompilerDetach; const MODEL_RESOURCE_TYPE: &str = "model"; const PLANNING_WORKERS: usize = 4; -pub(crate) type CtePair = (String, String); -/// `(model, SQL, expected SQL, lifted CTEs, body, expected columns, expected lifted CTEs)`. -pub(crate) type ChainRow = ( - String, - String, - Option, - Vec, - Option, - Option>, - Vec, -); -/// `(name, resolved SQL, lifted CTEs, comparison body)`. -pub(crate) type AssertionRow = (String, String, Vec, Option); -/// `(model, severity, message)`. -pub(crate) type WarningRow = (Option, &'static str, String); -/// `(SQL, chain, assertions, model names, warnings, distinct error messages)`. -pub(crate) type PlanRow = ( - Option, - Vec, - Vec, - Vec, - Vec, - Vec, -); +/// A native SQL-test plan with its distinct error messages. +pub(crate) struct PlanRow { + pub(crate) plan: SqlTestPlan, + pub(crate) error_messages: Vec, +} + +positional_record! { + extract; + /// An SQL function's name with its rendered UDF and table-function call affixes. + pub(crate) struct FunctionRow { + pub(crate) name: String, + pub(crate) udf_prefix: String, + pub(crate) udf_suffix: String, + pub(crate) table_function_prefix: String, + pub(crate) table_function_suffix: String, + } +} /// The planning request: compiled models and tests, with the adapter's facts. pub(crate) struct PlanningRequest<'py> { pub(crate) models: Vec>, /// Cursor-rendered SQL by model index, for models whose SQL uses cursor intrinsics. pub(crate) rendered_model_sql: HashMap, - pub(crate) functions: Vec<(String, String, String, String, String)>, + pub(crate) functions: Vec, pub(crate) tests: Vec>, /// Cursor-rendered model query overrides by test index, replacing the test's own. pub(crate) rendered_test_overrides: HashMap>, @@ -171,13 +163,13 @@ fn model_dependencies(model: &Bound<'_, PyAny>) -> PyResult> { } fn plan_function( - (name, udf_prefix, udf_suffix, table_function_prefix, table_function_suffix): ( - String, - String, - String, - String, - String, - ), + FunctionRow { + name, + udf_prefix, + udf_suffix, + table_function_prefix, + table_function_suffix, + }: FunctionRow, ) -> SqlTestPlanFunction { SqlTestPlanFunction { name, @@ -242,36 +234,8 @@ fn cte(value: &Bound<'_, PyAny>) -> PyResult { fn plan_row(plan: SqlTestPlan) -> PlanRow { let error_messages: Vec = sql_test_plan_error_messages(&plan); - ( - plan.sql, - plan.chain.into_iter().map(chain_row).collect(), - plan.assertions.into_iter().map(assertion_row).collect(), - plan.model_names, - plan.warnings - .into_iter() - .map(|warning| (warning.model_name, warning.severity, warning.message)) - .collect(), + PlanRow { + plan, error_messages, - ) -} - -fn chain_row(step: SqlTestChainStep) -> ChainRow { - ( - step.model_name, - step.resolved_sql, - step.expected_cte_sql, - step.lifted_ctes, - step.comparison_body_sql, - step.expected_columns, - step.expected_lifted_ctes, - ) -} - -fn assertion_row(step: SqlTestAssertionStep) -> AssertionRow { - ( - step.name, - step.resolved_sql, - step.lifted_ctes, - step.comparison_body_sql, - ) + } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs index f4eed0434b..c1ca49eb60 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs @@ -6,10 +6,17 @@ use sqlbuild_sqltext::compiler::models::{ DeclarationReferenceStop, }; -/// `(kind code, name, member, start, end)`: kind 0 is an enum member, 1 a constant. -type ReferenceRow = (u8, String, Option, usize, usize); +/// One natively scanned reference: kind 0 is an enum member, 1 a constant. +#[derive(Clone)] +pub(crate) struct ScannedReference { + pub(crate) kind: u8, + pub(crate) name: String, + pub(crate) member: Option, + pub(crate) start: usize, + pub(crate) end: usize, +} /// References and stop code: unclosed quote 0, block comment 1, invalid enum 2, constant 3. -type ScanRow = (Vec, Option); +type ScanRow = (Vec, Option); /// One SQL string's references and stop code, scanned without walking past malformed ones. pub(crate) fn declaration_scan(sql: &str) -> ScanRow { @@ -30,16 +37,16 @@ fn scan_row(scan: DeclarationReferenceScan) -> ScanRow { ) } -fn reference_row(reference: DeclarationReference) -> ReferenceRow { +fn reference_row(reference: DeclarationReference) -> ScannedReference { let kind: u8 = match reference.kind { DeclarationReferenceKind::Enum => 0, DeclarationReferenceKind::Constant => 1, }; - ( + ScannedReference { kind, - reference.name, - reference.member, - reference.start, - reference.end, - ) + name: reference.name, + member: reference.member, + start: reference.start, + end: reference.end, + } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs index 8d2aab2a9f..881982dc82 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs @@ -54,21 +54,29 @@ fn tokenize_model_header(header: &str) -> PyResult> { }) } -pub(crate) type ColumnLocation = (String, usize, usize, usize, usize); +positional_record! { + into; + /// A column name and its authored line, column, end line and end column. + pub(crate) struct ColumnLocation { + pub(crate) name: String, + pub(crate) line: usize, + pub(crate) column: usize, + pub(crate) end_line: usize, + pub(crate) end_column: usize, + } +} pub(crate) fn location_rows( locations: Vec<(String, sqlbuild_sqltext::compiler::models::TextLocation)>, ) -> Vec { locations .into_iter() - .map(|(name, location)| { - ( - name, - location.line, - location.column, - location.end_line, - location.end_column, - ) + .map(|(name, location)| ColumnLocation { + name, + line: location.line, + column: location.column, + end_line: location.end_line, + end_column: location.end_column, }) .collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs index a365dde1bc..ffd2158bac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs @@ -23,10 +23,29 @@ use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; const REFERENCE_CALL_SYNTAX_CODE: &str = "P012"; -/// `(kind, name, package, call_argument_count)` for one reference. -type ReferenceRow = (&'static str, String, Option, Option); -/// `(kind, call, start, message, help, corrected_call)` for one rejected call. -type InvalidCallRow = (&'static str, String, usize, String, String, String); +positional_record! { + into; + /// `(kind, name, package, call_argument_count)` for one reference. + struct ReferenceRow { + kind: &'static str, + name: String, + package: Option, + call_argument_count: Option, + } +} + +positional_record! { + into; + /// `(kind, call, start, message, help, corrected_call)` for one rejected call. + struct InvalidCallRow { + kind: &'static str, + call: String, + start: usize, + message: String, + help: String, + corrected_call: String, + } +} /// Python references and rejected calls of one scan. type ScannedObjects<'py> = (Vec>, Vec>); /// The references and rejected calls, or the scan error and its start. @@ -73,23 +92,23 @@ impl SqlReferenceScanner { } fn reference_row(reference: SqlReference) -> ReferenceRow { - ( - reference.kind, - reference.name, - reference.package, - reference.call_argument_count, - ) + ReferenceRow { + kind: reference.kind, + name: reference.name, + package: reference.package, + call_argument_count: reference.call_argument_count, + } } fn invalid_call_row(call: InvalidReferenceCall) -> InvalidCallRow { - ( - call.kind, - call.call, - call.start, - call.message, - call.help, - call.corrected_call, - ) + InvalidCallRow { + kind: call.kind, + call: call.call, + start: call.start, + message: call.message, + help: call.help, + corrected_call: call.corrected_call, + } } fn text<'py>(py: Python<'py>, value: impl IntoPyObject<'py>) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs index 415807dfa7..687b20670c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs @@ -10,16 +10,19 @@ use sqlbuild_sqltext::compiler::main::sql_interpolation::interpolate_sql; use sqlbuild_sqltext::compiler::models::InterpolationRead; use sqlbuild_sqltext::compiler::types::InterpolationHost; +use crate::bindings::_helpers::render::macro_call_rows::SpanRow; + const ENVIRONMENT_READ: &str = "env"; const CONTEXT_READ: &str = "ctx"; -/// `(sql or None when unchanged, spans, reads, error message or None)` for one SQL text. -pub(crate) type InterpolationRow = ( - Option, - Vec<(usize, usize, usize, usize)>, - Vec<(&'static str, String)>, - Option, -); +/// One SQL text interpolated: its SQL, substitution spans, reads and error message. +pub(crate) struct InterpolationRow { + /// The interpolated SQL, or `None` when unchanged. + pub(crate) sql: Option, + pub(crate) spans: Vec, + pub(crate) reads: Vec<(&'static str, String)>, + pub(crate) error: Option, +} /// Project variables, the environment, context values (or `None`) and the variable renderer. #[derive(FromPyObject)] @@ -154,13 +157,18 @@ pub(crate) fn interpolation_batch( let mut rows = Vec::with_capacity(sqls.len()); for (sql, file_path) in sqls { let row = match interpolate_sql(&host, sql, file_path) { - Ok(result) => (result.sql, result.spans, read_rows(result.reads), None), - Err(failure) => ( - None, - Vec::new(), - read_rows(failure.reads), - Some(failure.message), - ), + Ok(result) => InterpolationRow { + sql: result.sql, + spans: result.spans.into_iter().map(SpanRow::from).collect(), + reads: read_rows(result.reads), + error: None, + }, + Err(failure) => InterpolationRow { + sql: None, + spans: Vec::new(), + reads: read_rows(failure.reads), + error: Some(failure.message), + }, }; if let Some(error) = host.raised.borrow_mut().take() { return Err(error); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/normalization.rs b/crates/sqlbuild-python/src/bindings/_helpers/type_system/normalization.rs index 6007f4e19d..74fe591603 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/type_system/normalization.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/type_system/normalization.rs @@ -10,14 +10,17 @@ use sqlbuild_analysis::type_system::models::{NormalizedType, TypeNormalization}; use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::records::fields::field as native_field; -/// `(name, family, precision, scale, length)` of one normalized type. -type NormalizedRow = ( - String, - &'static str, - Option, - Option, - Option, -); +positional_record! { + into; + /// `(name, family, precision, scale, length)` of one normalized type. + struct NormalizedRow { + name: String, + family: &'static str, + precision: Option, + scale: Option, + length: Option, + } +} /// Normalize one type: `(normalized row, Polyglot parse error)`, or Python's exception. #[pyfunction] @@ -51,13 +54,13 @@ pub(crate) fn normalization_error(python: Python<'_>, error: &TypeNormalizationE } fn normalized_row(normalized: NormalizedType) -> NormalizedRow { - ( - normalized.normalized_name, - normalized.family.as_str(), - normalized.precision.map(|value| value.as_str().to_owned()), - normalized.scale.map(|value| value.as_str().to_owned()), - normalized.length.map(|value| value.as_str().to_owned()), - ) + NormalizedRow { + name: normalized.normalized_name, + family: normalized.family.as_str(), + precision: normalized.precision.map(|value| value.as_str().to_owned()), + scale: normalized.scale.map(|value| value.as_str().to_owned()), + length: normalized.length.map(|value| value.as_str().to_owned()), + } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/models.rs b/crates/sqlbuild-python/src/bindings/models.rs index 08e37dd711..cf4f54599c 100644 --- a/crates/sqlbuild-python/src/bindings/models.rs +++ b/crates/sqlbuild-python/src/bindings/models.rs @@ -1,8 +1,13 @@ //! Python-facing classes and mapping requests, converted to native models at the boundary. +use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyErr, PyResult, Python}; +use pyo3::types::PyTuple; use pyo3::{FromPyObject, pyclass}; use sqlbuild_analysis::semantic_validation::models as validation; -use sqlbuild_analysis::semantic_validation::types::{Expansion, Relations}; +use sqlbuild_analysis::semantic_validation::models::Expansion; +use sqlbuild_analysis::semantic_validation::types::Relations; + +use crate::bindings::types::{FromPositional, IntoPositional}; use std::collections::HashMap; #[derive(FromPyObject, Debug)] @@ -32,7 +37,7 @@ pub(crate) struct PositionInput { pub(crate) query: String, pub(crate) expanded: String, pub(crate) cleaned: String, - pub(crate) passes: Vec>, + pub(crate) passes: Vec>>, } #[pyclass(module = "sqlbuild._native")] @@ -83,7 +88,11 @@ impl From for validation::PositionInput { query: input.query, expanded: input.expanded, cleaned: input.cleaned, - passes: input.passes, + passes: input + .passes + .into_iter() + .map(Positional::unwrap_all) + .collect(), } } } @@ -148,3 +157,35 @@ pub(crate) struct DeclarationTextRequest { pub(crate) audit_keys: Vec, pub(crate) hook_keys: Vec, } + +/// A record of another crate exchanged with Python as its positional tuple. +#[derive(Clone, Debug)] +pub(crate) struct Positional(pub(crate) T); + +impl Positional { + /// Each record of `items`, for Python. + pub(crate) fn wrap_all(items: Vec) -> Vec { + items.into_iter().map(Self).collect() + } + + /// Each record Python sent in `items`. + pub(crate) fn unwrap_all(items: Vec) -> Vec { + items.into_iter().map(|item| item.0).collect() + } +} + +impl<'py, T: FromPositional> FromPyObject<'py> for Positional { + fn extract_bound(value: &Bound<'py, PyAny>) -> PyResult { + T::from_positional(value).map(Self) + } +} + +impl<'py, T: IntoPositional> IntoPyObject<'py> for Positional { + type Target = PyTuple; + type Output = Bound<'py, PyTuple>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + self.0.into_positional(py) + } +} diff --git a/crates/sqlbuild-python/src/bindings/types.rs b/crates/sqlbuild-python/src/bindings/types.rs index f0038d008c..367b8e9c5e 100644 --- a/crates/sqlbuild-python/src/bindings/types.rs +++ b/crates/sqlbuild-python/src/bindings/types.rs @@ -1,6 +1,19 @@ +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyTuple; + pub(crate) trait CompilerDetach { fn compiler_detach Result + Send>( self, operation: F, ) -> Result; } + +/// A named record read from the positional tuple Python sends. +pub(crate) trait FromPositional: Sized { + fn from_positional(value: &Bound<'_, PyAny>) -> PyResult; +} + +/// A named record converted into the positional tuple Python receives. +pub(crate) trait IntoPositional { + fn into_positional<'py>(self, py: Python<'py>) -> PyResult>; +} diff --git a/crates/sqlbuild-recall/src/clickhouse/constants.rs b/crates/sqlbuild-recall/src/clickhouse/constants.rs index 46bf41da3d..d4a540e63a 100644 --- a/crates/sqlbuild-recall/src/clickhouse/constants.rs +++ b/crates/sqlbuild-recall/src/clickhouse/constants.rs @@ -6,24 +6,50 @@ use crate::schema::models::Table; pub const BATCH_ROWS: usize = 5_000; /// Longest wait for a batch to fill, so each table gets at most about one part per second per writer. pub const BATCH_WINDOW_MS: u64 = 1_000; +/// ClickHouse `READONLY`. +pub const READONLY_CODE: u64 = 164; +/// ClickHouse `DATABASE_ACCESS_DENIED`. +pub const DATABASE_ACCESS_DENIED_CODE: u64 = 291; +/// ClickHouse `ACCESS_DENIED`. +pub const ACCESS_DENIED_CODE: u64 = 497; +/// ClickHouse `AUTHENTICATION_FAILED`. +pub const AUTHENTICATION_FAILED_CODE: u64 = 516; +/// ClickHouse `TIMEOUT_EXCEEDED`. +pub const TIMEOUT_EXCEEDED_CODE: u64 = 159; +/// ClickHouse `TOO_MANY_SIMULTANEOUS_QUERIES`. +pub const TOO_MANY_SIMULTANEOUS_QUERIES_CODE: u64 = 202; +/// ClickHouse `SOCKET_TIMEOUT`. +pub const SOCKET_TIMEOUT_CODE: u64 = 209; +/// ClickHouse `NETWORK_ERROR`. +pub const NETWORK_ERROR_CODE: u64 = 210; +/// ClickHouse `MEMORY_LIMIT_EXCEEDED`. +pub const MEMORY_LIMIT_EXCEEDED_CODE: u64 = 241; +/// ClickHouse `TABLE_IS_READ_ONLY`. +pub const TABLE_IS_READ_ONLY_CODE: u64 = 242; +/// ClickHouse `TOO_MANY_PARTS`. +pub const TOO_MANY_PARTS_CODE: u64 = 252; +/// ClickHouse `UNKNOWN_STATUS_OF_INSERT`. +pub const UNKNOWN_STATUS_OF_INSERT_CODE: u64 = 319; +/// ClickHouse `SYSTEM_ERROR`. +pub const SYSTEM_ERROR_CODE: u64 = 425; /// Server error codes meaning the user may not create or write the store. pub const PERMISSION_CODES: [u64; 4] = [ - 164, // READONLY - 291, // DATABASE_ACCESS_DENIED - 497, // ACCESS_DENIED - 516, // AUTHENTICATION_FAILED + READONLY_CODE, + DATABASE_ACCESS_DENIED_CODE, + ACCESS_DENIED_CODE, + AUTHENTICATION_FAILED_CODE, ]; /// Server error codes worth retrying with the same rows and deduplication tokens. pub const TRANSIENT_CODES: [u64; 9] = [ - 159, // TIMEOUT_EXCEEDED - 202, // TOO_MANY_SIMULTANEOUS_QUERIES - 209, // SOCKET_TIMEOUT - 210, // NETWORK_ERROR - 241, // MEMORY_LIMIT_EXCEEDED - 242, // TABLE_IS_READ_ONLY - 252, // TOO_MANY_PARTS - 319, // UNKNOWN_STATUS_OF_INSERT - 425, // SYSTEM_ERROR + TIMEOUT_EXCEEDED_CODE, + TOO_MANY_SIMULTANEOUS_QUERIES_CODE, + SOCKET_TIMEOUT_CODE, + NETWORK_ERROR_CODE, + MEMORY_LIMIT_EXCEEDED_CODE, + TABLE_IS_READ_ONLY_CODE, + TOO_MANY_PARTS_CODE, + UNKNOWN_STATUS_OF_INSERT_CODE, + SYSTEM_ERROR_CODE, ]; /// UNKNOWN_DATABASE: Recall never creates databases. pub const UNKNOWN_DATABASE_CODE: u64 = 81; diff --git a/crates/sqlbuild-recall/src/store/main/read_output_transcript.rs b/crates/sqlbuild-recall/src/store/main/read_output_transcript.rs index c21864bb78..44be378a54 100644 --- a/crates/sqlbuild-recall/src/store/main/read_output_transcript.rs +++ b/crates/sqlbuild-recall/src/store/main/read_output_transcript.rs @@ -52,7 +52,7 @@ pub fn read_output_transcript( .optional(), "read Recall invocation", )?; - let completeness: Option<(i64, i64, i64, bool, bool, Option)> = checked( + checked( store .connection .query_row( @@ -60,26 +60,17 @@ pub fn read_output_transcript( gap_reason FROM output_invocations WHERE invocation_id = ?1", [invocation_id], |row| { - Ok(( - row.get(0)?, - row.get(1)?, - row.get(2)?, - row.get(3)?, - row.get(4)?, - row.get(5)?, - )) + transcript.missing_sequences = row.get(0)?; + transcript.reported_drops = row.get(1)?; + transcript.filtered = row.get(2)?; + transcript.flush_complete = row.get(3)?; + transcript.complete = row.get(4)?; + transcript.gap_reason = row.get(5)?; + Ok(()) }, ) .optional(), "read Recall output completeness", )?; - if let Some((missing, drops, filtered, flushed, complete, reason)) = completeness { - transcript.missing_sequences = missing; - transcript.reported_drops = drops; - transcript.filtered = filtered; - transcript.flush_complete = flushed; - transcript.complete = complete; - transcript.gap_reason = reason; - } Ok(transcript) } diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/sql_sites.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/sql_sites.rs index 9a3becdfd7..fd68155fc5 100644 --- a/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/sql_sites.rs +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/sql_sites.rs @@ -46,8 +46,16 @@ pub(crate) struct ModelBody { pub(crate) passes: Vec>, } -/// `RESOURCE_CALL_PATTERN` on a site's text: `(kind, name, name_start, name_end)`. -fn resource_call(text: &[char]) -> Option<(String, String, usize, usize)> { +/// A resource call a site's text holds, with its name span within that text. +struct ResourceCall { + kind: String, + name: String, + name_start: usize, + name_end: usize, +} + +/// `RESOURCE_CALL_PATTERN` on a site's text. +fn resource_call(text: &[char]) -> Option { if !starts_with(text, 0, "__") { return None; } @@ -82,12 +90,12 @@ fn resource_call(text: &[char]) -> Option<(String, String, usize, usize)> { if !(rest.is_empty() || rest == ['\n']) { return None; } - Some(( - kind.to_owned(), - slice(text, name_start, name_end), + Some(ResourceCall { + kind: kind.to_owned(), + name: slice(text, name_start, name_end), name_start, name_end, - )) + }) } fn skip_space(text: &[char], mut index: usize) -> usize { @@ -102,7 +110,12 @@ pub(crate) fn resource_sites(text: &[char], context: &ScanContext) -> Vec AnalysisSql let call = resource_call(&chars(&site.text)); let kind: char = call .as_ref() - .and_then(|(kind, ..)| kind.chars().next()) + .and_then(|call| call.kind.chars().next()) .unwrap_or(GENERIC_PLACEHOLDER_KIND); let placeholder = placeholder(kind, index, site.end - site.start); - if let Some((kind, name, ..)) = call { + if let Some(ResourceCall { kind, name, .. }) = call { let key = (kind, name); match tables.iter_mut().find(|(item, _)| *item == key) { Some((_, placeholders)) => { diff --git a/crates/sqlbuild-rules/src/engine/_helpers/rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/rows.rs index fc694bf75b..eda014fbd7 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/rows.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/rows.rs @@ -68,9 +68,14 @@ fn build_model(row: ModelRow, inputs: &ModelInputs<'_, '_>) -> Result schema_facts(&row, schema, inputs)?, - None => (Vec::new(), Vec::new(), false, 0), + None => SchemaFacts::default(), }; Ok(Model { sql_analysis_disabled, @@ -97,7 +102,14 @@ fn build_model(row: ModelRow, inputs: &ModelInputs<'_, '_>) -> Result, Vec, bool, u32); +/// A model's declared columns and dynamic families with their proofs and audit count. +#[derive(Default)] +struct SchemaFacts { + columns: Vec, + dynamic_columns: Vec, + dynamic_columns_proven: bool, + audit_count: u32, +} fn schema_facts( row: &ModelRow, @@ -141,7 +153,12 @@ fn schema_facts( .map(|column| column.audit_count) .sum::(); if schema.dynamic_columns.is_empty() { - return Ok((columns, Vec::new(), false, audit_count)); + return Ok(SchemaFacts { + columns, + dynamic_columns: Vec::new(), + dynamic_columns_proven: false, + audit_count, + }); } let proof_types: HashMap<&str, Option<&str>> = row .dynamic_proof @@ -172,7 +189,12 @@ fn schema_facts( .dynamic_proof .as_ref() .is_some_and(|proof| proof.output_proven); - Ok((columns, dynamic_columns, proven, audit_count)) + Ok(SchemaFacts { + columns, + dynamic_columns, + dynamic_columns_proven: proven, + audit_count, + }) } /// Python's `types_equal`, normalizing each type once; Python raises what normalization rejects. diff --git a/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs b/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs index 92bc83d6e9..b39a5d3784 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs @@ -208,7 +208,7 @@ fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scann prepared .1 .first() - .map(|site| site.5.clone()) + .map(|site| site.original_text.clone()) .unwrap_or_default() }); assert_eq!( @@ -277,7 +277,7 @@ fn given_lint_dialect_when_preparing_macro_sites_then_backticks_follow_rules_quo for test_case in test_cases { let lint_sql = format!("SELECT @m({} x) AS y", test_case.fragment); let lint_site = prepare(&lint_sql, &lint_sql, &[], test_case.dialect)? - .and_then(|prepared| prepared.1.first().map(|site| site.5.clone())) + .and_then(|prepared| prepared.1.first().map(|site| site.original_text.clone())) .unwrap_or_default(); assert_eq!( lint_site, test_case.expected_lint_site, @@ -311,7 +311,13 @@ fn given_dollar_quoted_apostrophe_when_preparing_lint_sites_then_later_reference let sql = "SELECT $$Customer's order$$ AS note, $tag$it's$tag$ AS tag FROM __ref(\"orders\") o"; for test_case in test_cases { let sites: Vec = prepare(sql, sql, &[], test_case.dialect)? - .map(|prepared| prepared.1.into_iter().map(|site| site.5).collect()) + .map(|prepared| { + prepared + .1 + .into_iter() + .map(|site| site.original_text) + .collect() + }) .unwrap_or_default(); assert_eq!(sites, test_case.expected_sites, "{}", test_case.description); } diff --git a/crates/sqlbuild-rules/src/fix_verification/main/compare.rs b/crates/sqlbuild-rules/src/fix_verification/main/compare.rs index 5e6fa94418..4eff333623 100644 --- a/crates/sqlbuild-rules/src/fix_verification/main/compare.rs +++ b/crates/sqlbuild-rules/src/fix_verification/main/compare.rs @@ -51,7 +51,7 @@ pub fn compare( .iter() .filter_map(|(identity, count)| { (*count > *diagnostic.before.get(identity).unwrap_or(&0)) - .then_some(identity.0.as_str()) + .then_some(identity.code.as_str()) }) .collect(); if !codes.is_empty() { diff --git a/crates/sqlbuild-rules/src/fix_verification/models.rs b/crates/sqlbuild-rules/src/fix_verification/models.rs index 3d5cc6dc99..8f0d439363 100644 --- a/crates/sqlbuild-rules/src/fix_verification/models.rs +++ b/crates/sqlbuild-rules/src/fix_verification/models.rs @@ -29,13 +29,22 @@ pub struct ModelComparison { pub lineage_equal: bool, } +/// A diagnostic's location-insensitive identity. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct DiagnosticIdentity { + pub code: String, + pub resource_type: String, + pub resource_name: String, + pub message: String, +} + /// Location-insensitive diagnostic multiplicities for one owner. #[derive(Debug)] pub struct DiagnosticComparison { pub owner: usize, pub edited: bool, - pub before: BTreeMap<(String, String, String, String), i64>, - pub after: BTreeMap<(String, String, String, String), i64>, + pub before: BTreeMap, + pub after: BTreeMap, } /// Failures attributed to edited owners and changes outside those owners. diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs index 9599fb1e2b..43cb230326 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs @@ -217,11 +217,20 @@ fn keepends_lengths(text: &str) -> Vec { .collect() } +/// The fields of one native diagnostic, its span in lint-text characters. +struct DiagnosticFields { + code: String, + message: String, + remediation: String, + start: i64, + end: i64, +} + /// The fields of one diagnostic: code, message, remediation, start and end. fn diagnostic_fields( diagnostic: &Map, body_length: i64, -) -> Result<(String, String, String, i64, i64), String> { +) -> Result { let text = |key: &str| { diagnostic .get(key) @@ -242,7 +251,13 @@ fn diagnostic_fields( if start < 0 || end < start || end > body_length { return Err(INVALID_DIAGNOSTIC.to_string()); } - Ok((code, message, remediation, start, end)) + Ok(DiagnosticFields { + code, + message, + remediation, + start, + end, + }) } /// Map one native diagnostic onto authored source, or drop it when it belongs to generated SQL. @@ -255,7 +270,13 @@ pub(crate) fn authored_diagnostic( return Err(NOT_AN_OBJECT.to_string()); }; let body_length: i64 = body.lint_text.chars().count() as i64; - let (code, message, remediation, start, end) = diagnostic_fields(diagnostic, body_length)?; + let DiagnosticFields { + code, + message, + remediation, + start, + end, + } = diagnostic_fields(diagnostic, body_length)?; let reason: Option = match diagnostic.get("fix_unavailable_reason") { None | Some(Value::Null) => None, Some(Value::String(reason)) => Some(reason.clone()), diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs index 36bb34d459..aa80d6d1ab 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs @@ -54,6 +54,14 @@ fn skippable(line: &str) -> bool { stripped.is_empty() || stripped.starts_with("--") } +/// Where a retained violation sits, its code and whether a native rule reported it. +struct SuppressedTarget<'a> { + file: usize, + line: usize, + code: &'a str, + native: bool, +} + /// Whether a retained violation is the native finding one directive suppresses. pub(crate) fn matches_directive( item: &RetainedViolation, @@ -61,23 +69,28 @@ pub(crate) fn matches_directive( file: usize, directive: &SuppressionDirective, ) -> bool { - let (item_file, line, code, native): (usize, usize, &str, bool) = match item { + let target: SuppressedTarget<'_> = match item { RetainedViolation::Existing(index) => { let violation: &SuppressibleViolation = &violations[*index]; - ( - violation.file, - violation.line, - &violation.code, - violation.native, - ) + SuppressedTarget { + file: violation.file, + line: violation.line, + code: &violation.code, + native: violation.native, + } } - RetainedViolation::Added(finding) => (finding.file, finding.line, SUPPRESSION_CODE, true), + RetainedViolation::Added(finding) => SuppressedTarget { + file: finding.file, + line: finding.line, + code: SUPPRESSION_CODE, + native: true, + }, }; - item_file == file - && line == directive.target - && code == directive.code - && native - && code.starts_with(NATIVE_RULE_PREFIX) + target.file == file + && target.line == directive.target + && target.code == directive.code + && target.native + && target.code.starts_with(NATIVE_RULE_PREFIX) } /// Findings for every directive line when the project forbids suppressions. diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/preparation.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/preparation.rs index f514682f67..4ef2ad23dc 100644 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/preparation.rs +++ b/crates/sqlbuild-rules/src/sql_lint/_helpers/preparation.rs @@ -7,7 +7,7 @@ use regex::Regex; use crate::lint_text::main::neutralize_interpolation::neutralize_interpolation; use crate::lint_text::models::Neutralized; -use crate::sql_lint::types::{InterpolationSite, PreparedSql}; +use crate::sql_lint::types::PreparedSql; use sqlbuild_sqltext::sql_scan::main::quote_policy::quote_policy; static CTES: LazyLock> = LazyLock::new(|| { @@ -68,21 +68,7 @@ pub(crate) fn prepare( } let mut referenced: Vec = referenced.into_iter().collect(); referenced.sort(); - let sites: Vec = neutralized - .sites - .into_iter() - .map(|site| { - ( - site.sentinel, - site.neutralized_start, - site.neutralized_end, - site.original_start, - site.original_end, - site.original_text, - ) - }) - .collect(); - Ok(Some((neutralized.text, sites, referenced))) + Ok(Some((neutralized.text, neutralized.sites, referenced))) } /// The byte offset of character position `position`, or `None` past the end. diff --git a/crates/sqlbuild-rules/src/sql_lint/types.rs b/crates/sqlbuild-rules/src/sql_lint/types.rs index 1cb260e2c3..ef7601e9a1 100644 --- a/crates/sqlbuild-rules/src/sql_lint/types.rs +++ b/crates/sqlbuild-rules/src/sql_lint/types.rs @@ -1,2 +1,4 @@ -pub(crate) type InterpolationSite = (String, usize, usize, usize, usize, String); +use crate::lint_text::models::InterpolationSite; + +/// Lint text with sentinels, its interpolation sites and the CTEs interpolation references. pub type PreparedSql = (String, Vec, Vec); diff --git a/crates/sqlbuild-scopes/src/scope_index/_helpers/grants.rs b/crates/sqlbuild-scopes/src/scope_index/_helpers/grants.rs index cde7244b9a..3f09e406fa 100644 --- a/crates/sqlbuild-scopes/src/scope_index/_helpers/grants.rs +++ b/crates/sqlbuild-scopes/src/scope_index/_helpers/grants.rs @@ -26,12 +26,14 @@ pub(crate) fn unique_grants(index: &ScopeIndex, grants: Vec) -> Vec .collect() } -type GrantKey<'a> = ( - &'a ResourceIdentity, - &'a DeclarationIdentity, - GrantThroughKey<'a>, - GrantKind, -); +/// What makes two grants the same: resource, declaration, what grants it and its kind. +#[derive(PartialEq, Eq, Hash)] +struct GrantKey<'a> { + resource: &'a ResourceIdentity, + declaration: &'a DeclarationIdentity, + through: GrantThroughKey<'a>, + kind: GrantKind, +} #[derive(PartialEq, Eq, Hash)] enum GrantThroughKey<'a> { @@ -40,17 +42,17 @@ enum GrantThroughKey<'a> { } fn grant_key<'a>(index: &'a ScopeIndex, grant: &'a GrantRecord) -> GrantKey<'a> { - ( - &grant.resource, - &index.declarations[grant.declaration].identity, - match &grant.through { + GrantKey { + resource: &grant.resource, + declaration: &index.declarations[grant.declaration].identity, + through: match &grant.through { GrantThrough::Model(name) => GrantThroughKey::Model(name), GrantThrough::Declaration(declaration) => { GrantThroughKey::Declaration(&index.declarations[*declaration].identity) } }, - grant.kind, - ) + kind: grant.kind, + } } /// Visibility and identity lookups over one index, as the Python grant functions use them. diff --git a/crates/sqlbuild-scopes/src/scope_index/_helpers/ordering.rs b/crates/sqlbuild-scopes/src/scope_index/_helpers/ordering.rs index 0a9859764e..fb3f786713 100644 --- a/crates/sqlbuild-scopes/src/scope_index/_helpers/ordering.rs +++ b/crates/sqlbuild-scopes/src/scope_index/_helpers/ordering.rs @@ -11,14 +11,23 @@ pub(crate) fn resource_key(record: &ResourceRecord) -> (String, String) { (record.path.clone(), resource_text(&record.identity)) } +/// A declaration sort key: identity text, path, then line and column as `P` compares them. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub(crate) struct DeclarationKey

{ + identity: String, + path: String, + line: P, + column: P, +} + /// The builder's `_declaration_key`, which compares line and column as integers. -pub(crate) fn declaration_key(record: &DeclarationRecord) -> (String, String, i64, i64) { - ( - declaration_text(&record.identity), - record.path.clone(), - record.line, - record.column, - ) +pub(crate) fn declaration_key(record: &DeclarationRecord) -> DeclarationKey { + DeclarationKey { + identity: declaration_text(&record.identity), + path: record.path.clone(), + line: record.line, + column: record.column, + } } /// The lookup's `resource_sort_key`. @@ -34,13 +43,13 @@ pub(crate) fn canonical_resource_key( /// The lookup's `declaration_sort_key`, which compares line and column as text. pub(crate) fn canonical_declaration_key( record: &DeclarationRecord, -) -> Result<(String, String, String, String), ScopeDeferral> { - Ok(( - declaration_text(&record.identity), - normalize_path(&record.path)?, - record.line.to_string(), - record.column.to_string(), - )) +) -> Result, ScopeDeferral> { + Ok(DeclarationKey { + identity: declaration_text(&record.identity), + path: normalize_path(&record.path)?, + line: record.line.to_string(), + column: record.column.to_string(), + }) } /// Stable sort of `members` by a fallible key, as Python's `sorted(..., key=...)`. diff --git a/crates/sqlbuild-scopes/src/scope_report/_helpers/declarations.rs b/crates/sqlbuild-scopes/src/scope_report/_helpers/declarations.rs index 50af50a8f9..9a6e8a92eb 100644 --- a/crates/sqlbuild-scopes/src/scope_report/_helpers/declarations.rs +++ b/crates/sqlbuild-scopes/src/scope_report/_helpers/declarations.rs @@ -14,7 +14,12 @@ use crate::scope_report::models::{ }; /// `_declaration_container`: role, visibility label, role root and bucket path. -pub(crate) type Container = (Option, &'static str, Option, Option); +pub(crate) struct Container { + pub(crate) role: Option, + pub(crate) visibility: &'static str, + pub(crate) role_root: Option, + pub(crate) bucket: Option, +} pub(crate) fn declaration_text(record: &DeclarationRecord) -> String { format_identity(&QualifiedIdentity::Declaration(record.identity.clone())) @@ -50,7 +55,12 @@ fn visibility_label(scope: ScopeKind) -> &'static str { pub(crate) fn container(record: &DeclarationRecord) -> Container { let visibility: &'static str = visibility_label(record.scope); if record.scope == ScopeKind::Private { - return (None, visibility, None, None); + return Container { + role: None, + visibility, + role_root: None, + bucket: None, + }; } let role_parts: &[&str] = record.identity.kind.role_parts(); let safe: String = safe_scope_path(&record.path); @@ -65,12 +75,12 @@ pub(crate) fn container(record: &DeclarationRecord) -> Container { let role_end: usize = (role_index + role_parts.len()).min(parts.len()); let bucket_end: usize = parts.len().saturating_sub(1).max(role_end); let bucket: String = parts[role_end..bucket_end].join("/"); - ( - Some(role_parts.join("/")), + Container { + role: Some(role_parts.join("/")), visibility, - Some(parts[..role_end].join("/")), - (!bucket.is_empty()).then_some(bucket), - ) + role_root: Some(parts[..role_end].join("/")), + bucket: (!bucket.is_empty()).then_some(bucket), + } } /// The report metadata of one declaration's values and container. @@ -112,7 +122,12 @@ fn metadata(values: Option<&DeclarationValues>, container: Container) -> Vec<(St push("render_as", optional_text(constant.render_as.as_deref())); } } - let (role, visibility, role_root, bucket) = container; + let Container { + role, + visibility, + role_root, + bucket, + } = container; push("role", optional_text(role.as_deref())); push("visibility_label", text(visibility)); push("role_root", optional_text(role_root.as_deref())); diff --git a/crates/sqlbuild-scopes/src/scope_report/_helpers/projection.rs b/crates/sqlbuild-scopes/src/scope_report/_helpers/projection.rs index 07e79ead32..04f4774014 100644 --- a/crates/sqlbuild-scopes/src/scope_report/_helpers/projection.rs +++ b/crates/sqlbuild-scopes/src/scope_report/_helpers/projection.rs @@ -6,7 +6,9 @@ use crate::scope_index::main::format_identity::format_identity; use crate::scope_index::models::{ DeclarationIdentity, DeclarationRecord, QualifiedIdentity, ResourceIdentity, }; -use crate::scope_report::_helpers::declarations::{container, integer, optional_text, text, texts}; +use crate::scope_report::_helpers::declarations::{ + Container, container, integer, optional_text, text, texts, +}; use crate::scope_report::_helpers::json::{diagnostic_json, object}; use crate::scope_report::_helpers::paths::safe_scope_path; use crate::scope_report::constants::METADATA_SCHEMA_VERSION; @@ -91,7 +93,12 @@ fn declaration_projection( record: &DeclarationRecord, values: Option<&DeclarationValues>, ) -> JsonValue { - let (role, visibility, role_root, bucket) = container(record); + let Container { + role, + visibility, + role_root, + bucket, + } = container(record); object(vec![ ("identity", text(&declaration_key(&record.identity))), ("kind", text(record.identity.kind.as_str())), diff --git a/crates/sqlbuild-scopes/src/scope_text/_helpers/results.rs b/crates/sqlbuild-scopes/src/scope_text/_helpers/results.rs index ea40a71473..ba1f16bef2 100644 --- a/crates/sqlbuild-scopes/src/scope_text/_helpers/results.rs +++ b/crates/sqlbuild-scopes/src/scope_text/_helpers/results.rs @@ -80,28 +80,47 @@ fn chain_lines( lines } +/// One availability section: its title, declarations, report section name and whether optional. +struct AvailabilityGroup<'a> { + title: &'a str, + declarations: &'a [DeclarationReport], + name: &'a str, + optional: bool, +} + fn availability_lines( (style, request): (CliStyle, &ScopeTextRequest), report: &ScopeReport, used: &HashSet, ) -> Vec { - let groups: [(&str, &[DeclarationReport], &str, bool); 3] = [ - ("Available", &report.available, "available", false), - ( - "Relationship grants", - &report.relationship_scope, - "relationship_scope", - true, - ), - ( - "Nearby unavailable", - &report.nearby_unavailable, - "nearby_unavailable", - true, - ), + let groups: [AvailabilityGroup<'_>; 3] = [ + AvailabilityGroup { + title: "Available", + declarations: &report.available, + name: "available", + optional: false, + }, + AvailabilityGroup { + title: "Relationship grants", + declarations: &report.relationship_scope, + name: "relationship_scope", + optional: true, + }, + AvailabilityGroup { + title: "Nearby unavailable", + declarations: &report.nearby_unavailable, + name: "nearby_unavailable", + optional: true, + }, ]; let mut lines: Vec = Vec::new(); - for (title, declarations, name, optional) in groups { + for AvailabilityGroup { + title, + declarations, + name, + optional, + } in groups + { let section: Option<&ScopeSection> = report_section(report, name); let collapsed_count: i64 = section.map_or(0, |section| section.collapsed_count); if optional && !request.verbose && declarations.is_empty() && collapsed_count == 0 { diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs index da4557cda7..6d583b0502 100644 --- a/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs @@ -1,7 +1,8 @@ //! `@@name`, `@@ENV:NAME` and `@@CTX:name` interpolation of authored SQL. +use crate::compiler::models::CharSpan; use crate::compiler::models::{InterpolatedSql, InterpolationFailure, InterpolationRead}; -use crate::compiler::types::{CharSpan, InterpolationHost}; +use crate::compiler::types::InterpolationHost; use crate::sql_scan::main::non_code_end::non_code_end; use crate::sql_scan::models::{QuotePolicy, Unclosed}; use sqlbuild_core::text::main::is_letter::is_letter; @@ -11,8 +12,13 @@ const TOKEN: &str = "@@"; const ENVIRONMENT_PREFIX: &str = "ENV:"; const CONTEXT_PREFIX: &str = "CTX:"; -/// One substitution in byte offsets: `(source start, source end, output start, output end)`. -type ByteSpan = (usize, usize, usize, usize); +/// One substitution in bytes: source start and end, output start and end. +struct ByteSpan { + source_start: usize, + source_end: usize, + output_start: usize, + output_end: usize, +} struct Interpolation<'run, H: InterpolationHost> { host: &'run H, @@ -112,8 +118,12 @@ impl Interpolation<'_, H> { let text = &sql[..limit]; let output_start = self.output.len(); let end = self.rendered_token(text, start)?; - self.spans - .push((start, end, output_start, self.output.len())); + self.spans.push(ByteSpan { + source_start: start, + source_end: end, + output_start, + output_end: self.output.len(), + }); Ok(end) } @@ -256,13 +266,11 @@ fn character_spans(source: &str, output: &str, spans: &[ByteSpan]) -> Vec, - pub spans: Vec, + pub spans: Vec, pub reads: Vec, } diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs b/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs index a8225ae81c..0326016fed 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs @@ -4,12 +4,12 @@ use crate::compiler::_helpers::model_headers::tokenization::{ use crate::compiler::main::declaration_references::scan_declaration_references; use crate::compiler::main::model_header_single_parsing::parse_one; use crate::compiler::main::sql_interpolation::interpolate_sql; +use crate::compiler::models::CharSpan; use crate::compiler::models::{ AuthoredValue, DeclarationReferenceKind, DeclarationReferenceScan, DeclarationReferenceStop, InterpolationRead, }; use crate::compiler::tests::test_types::{HeaderNestingTestCase, MapHost}; -use crate::compiler::types::CharSpan; use std::collections::BTreeMap; use std::thread; @@ -251,13 +251,22 @@ pub(crate) fn interpolated(host: &MapHost, sql: &str) -> Result pub(crate) fn interpolation_facts( host: &MapHost, sql: &str, -) -> (Vec, Vec) { +) -> (Vec<[usize; 4]>, Vec) { interpolate_sql(host, sql, "models/orders.sql").map_or_else( |failure| (Vec::new(), failure.reads), - |result| (result.spans, result.reads), + |result| (result.spans.iter().map(span_parts).collect(), result.reads), ) } +fn span_parts(span: &CharSpan) -> [usize; 4] { + [ + span.source_start, + span.source_end, + span.output_start, + span.output_end, + ] +} + /// Named locations from `[line, column, end_line, end_column]` rows. pub(crate) fn locations( columns: Vec<(&str, [usize; 4])>, diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs b/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs index 7a2eafc102..308d2501e4 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs @@ -136,10 +136,10 @@ fn given_tokens_when_interpolating_then_spans_count_code_points_and_reads_keep_o description: "variables, environment, context and a bare @@ after non-ASCII text", sql: "SELECT 'ü', @@größe, @@ENV:REGION, @@CTX:this @@", expected_spans: vec![ - (12, 19, 12, 17), - (21, 33, 19, 21), - (35, 45, 23, 29), - (46, 48, 30, 32), + [12, 19, 12, 17], + [21, 33, 19, 21], + [35, 45, 23, 29], + [46, 48, 30, 32], ], expected_reads: vec![ InterpolationRead::Environment("REGION".to_owned()), diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs b/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs index 319db3016a..d116ba54d6 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs @@ -1,5 +1,5 @@ use crate::compiler::models::{InterpolationRead, NestingFailure}; -use crate::compiler::types::{CharSpan, InterpolationHost}; +use crate::compiler::types::InterpolationHost; use std::collections::BTreeMap; pub(crate) struct ModelHeaderTokenizationTestCase { @@ -51,7 +51,7 @@ pub(crate) struct InterpolationTestCase { pub(crate) struct InterpolationFactsTestCase { pub(crate) description: &'static str, pub(crate) sql: &'static str, - pub(crate) expected_spans: Vec, + pub(crate) expected_spans: Vec<[usize; 4]>, pub(crate) expected_reads: Vec, } diff --git a/crates/sqlbuild-sqltext/src/compiler/types.rs b/crates/sqlbuild-sqltext/src/compiler/types.rs index a16c3efdff..95b3a0bf9d 100644 --- a/crates/sqlbuild-sqltext/src/compiler/types.rs +++ b/crates/sqlbuild-sqltext/src/compiler/types.rs @@ -1,6 +1,3 @@ -/// One substitution in code points: `(source start, source end, output start, output end)`. -pub type CharSpan = (usize, usize, usize, usize); - /// The project variables, process environment and context values one interpolation reads. pub trait InterpolationHost { /// The text of project variable `name`, `Err(message)` when it cannot be text, or `None`. diff --git a/fensu.toml b/fensu.toml index 154893a523..a58a5bdf7f 100644 --- a/fensu.toml +++ b/fensu.toml @@ -21,6 +21,1107 @@ require_cacheable = true [targets.python.evaluation] exclude = ["tests/e2e/fixtures/**"] +[targets.python.dead_code] +enabled = true + +[[targets.python.dead_code.roots]] +modules = ["scripts.*"] +symbols = ["main"] +reason = "Repository tooling runs these modules with `python -m scripts.` from the Makefile, CI and pre-commit." + +[[targets.python.dead_code.roots]] +modules = ["scripts.fensu_policy.rules.*"] +symbols = ["*"] +reason = "Fensu loads these custom rule modules from rule_modules in this file." + +[[targets.python.dead_code.roots]] +modules = [ + "sqlbuild.rule_engine._helpers.host.custom_host", + "sqlbuild.rule_engine._helpers.host.custom_host_launch", +] +symbols = ["main"] +reason = "The custom-rule host processes run these modules with `python -m`; native code starts the launcher, which starts the host." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild._native_host"] +symbols = ["*"] +reason = "Native code imports this module by path and reads its attributes by name." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.contract.classes.base_adapter"] +symbols = ["BaseAdapter"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.contract.exceptions"] +symbols = ["AdapterUserError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.contract.models"] +symbols = ["ColumnInfo", "ExpressionInferenceProfile", "RelationInfo", "RelationLookup", "RetentionRequest", "RowDiffSampling", "RowDiffTolerance", "RowDiffTolerances", "TableFreshnessMetadata", "TableFreshnessRequest"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.contract.types"] +symbols = ["BuiltinAdapter", "FrameworkType", "LifeCycleEventKind", "MigrationTransfer", "RetentionScope", "TableFreshnessStatus"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.discovery.main._session_warehouse_support"] +symbols = ["adapter_supports_session_warehouse"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.discovery.main.builtins"] +symbols = ["builtin_adapter_classes"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.discovery.main.project_adapters"] +symbols = ["discover_project_adapters"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.discovery.main.resolve_adapter"] +symbols = ["resolve_adapter"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main._get_columns_for_inspection"] +symbols = ["get_columns_for_inspection"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main._list_relations_for_inspection"] +symbols = ["list_relations_for_inspection"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main.active_inspection_catalog"] +symbols = ["active_inspection_catalog"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main.inspection_query_sink"] +symbols = ["inspection_query_sink"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main.open_inspection_catalog"] +symbols = ["open_inspection_catalog"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main.record_inspection_query"] +symbols = ["record_inspection_query"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main.relation_lookup"] +symbols = ["build_relation_lookup"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name"] +symbols = ["resolve_relation_location_qualified_name"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main.run_bounded_inspections"] +symbols = ["run_bounded_inspections"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main.run_recorded_inspection_query"] +symbols = ["run_recorded_inspection_query"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.models"] +symbols = ["InspectionQueryRecord", "StatementMetadataEffect"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.type_system.main.conditional_result_nullability"] +symbols = ["conditional_result_nullability"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.type_system.main.first_arg_nullability"] +symbols = ["first_arg_nullability"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.type_system.main.normalize_relation_type"] +symbols = ["normalize_relation_type"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.type_system.main.normalize_type"] +symbols = ["normalize_type"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.type_system.main.safe_cast_nullability"] +symbols = ["safe_cast_nullability"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.clickhouse._helpers.recall_store"] +symbols = ["close_recall_store", "connect_recall_store", "insert_recall_rows", "run_recall_command", "run_recall_query"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.cli.commands.main.entrypoint._dispatch_with_compute_logs"] +symbols = ["dispatch_with_compute_logs"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.cli.compile_reuse._helpers.provider_settings"] +symbols = ["provider_settings_inputs"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.cli.compile_reuse._helpers.startup_inputs"] +symbols = ["startup_inputs"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.cli.compile_reuse.models"] +symbols = ["CompileReuseRequest"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.auditing.types"] +symbols = ["AuditEvaluationMode", "AuditOutcome", "AuditSeverity", "ThresholdOperator"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.compile._helpers.diagnostics.python_resources"] +symbols = ["python_resource_rows"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.compile._helpers.macro_bridge.call_store"] +symbols = ["target_context_with_macro_call_store"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.compile._helpers.render.user_macros"] +symbols = ["inventory_project_macros", "load_project_macros", "macro_invocation_context"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.compile.types"] +symbols = ["AttachedAuditTargetKind", "FunctionLanguage", "SqlTestMode"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks"] +symbols = ["discover_command_output_sink_functions"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.discovery._helpers.filesystem.project_modules"] +symbols = ["import_python_root_module", "install_python_root_package", "load_materialize_function", "load_provider_module", "load_python_node_module", "load_sink_module", "module_functions", "module_provider_classes", "parameter_annotations", "provider_usages", "requires_arguments", "restore_python_root_modules", "source_line", "validate_event_exporter_signature"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.discovery._helpers.validation.discovery"] +symbols = ["validate_discovered_inputs"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.discovery.main.runtime_extensions"] +symbols = ["discover_runtime_extensions"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.macro_bridge.classes.macro_bridge"] +symbols = ["run_with_macro_bridge"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.macro_bridge.main._macro_store_module_paths"] +symbols = ["macro_store_module_paths"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.macro_bridge.main.active_macro_bridge"] +symbols = ["active_macro_bridge"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.pipeline._helpers.materializations"] +symbols = ["load_custom_materializations"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.python_nodes._helpers.user_inventory"] +symbols = ["python_node_dependency_key", "python_node_identity"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.python_nodes.main._hook_identities"] +symbols = ["build_hook_identities"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.python_nodes.types"] +symbols = ["PythonNodeKind", "PythonNodeStatus"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.references.models"] +symbols = ["ProjectRelation", "ProjectRelationIndex", "RelationName"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.cost.models"] +symbols = ["CostResourceContext", "RunCostSummary"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.cost.types"] +symbols = ["CostCapability", "CostStatus"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.cursor_algebra.exceptions"] +symbols = ["CursorAlgebraError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.cursor_algebra.types"] +symbols = ["BoundSentinel"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.diagnostics._helpers.process_resources"] +symbols = ["read_process_resources"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.diagnostics.constants"] +symbols = ["SQL_PRIVATE_RECORD_MARKER"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.diagnostics.main.diagnostics_context"] +symbols = ["diagnostics_context"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.diagnostics.main.log_debug_event"] +symbols = ["log_debug_event"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.diagnostics.main.log_sql"] +symbols = ["log_sql"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.diagnostics.main.process_resource_reporting"] +symbols = ["process_resource_reporting"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.diagnostics.models"] +symbols = ["ProcessResourceUsage"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.errors.contracts.exceptions"] +symbols = ["ExecutorInputError", "SharedInputError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.auditing.main._current_audit_result_projection"] +symbols = ["current_audit_result_projection"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.auditing.models"] +symbols = ["AuditExecutionResult"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.build.models"] +symbols = ["BuildCallbacks", "BuildCustomizations", "BuildExecutionResult", "BuildInitialState", "BuildRuntimeParams", "FunctionExecutionResult", "SeedExecutionResult"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.build.types"] +symbols = ["BuildStatus"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.clone.main._build_destination_transient_models"] +symbols = ["build_destination_transient_models"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.clone.main._build_retention_requests"] +symbols = ["build_destination_retention_requests"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.clone.main._fingerprinting"] +symbols = ["copy_clone_fingerprints"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.clone.main._run_prephase_clone_stream"] +symbols = ["run_prephase_clone_stream"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.clone.main.execute"] +symbols = ["execute_clone"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.clone.models"] +symbols = ["CloneExecutionInput", "CloneItemResult", "CloneSourceEntries"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.clone.types"] +symbols = ["CloneStatus"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.diff.classes.query_artifact_lifecycle"] +symbols = ["QueryDiffArtifactLifecycle"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.diff.exceptions"] +symbols = ["FullDiffSizeGuardError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.diff.main._execute_query"] +symbols = ["execute_query_diff"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.diff.main.execute"] +symbols = ["execute_diff"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.diff.models"] +symbols = ["DiffExecutionOptions", "DiffExecutionResult", "FullDiffSizeLimits", "RowDiffSamplingOverride"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.janitor.main.execute"] +symbols = ["execute_janitor_plan"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.janitor.models"] +symbols = ["JanitorArchiveCandidate", "JanitorArchivedRelation", "JanitorBlockedSchema", "JanitorDeleteCandidate", "JanitorDirectStatePruneCandidate", "JanitorOldNameView", "JanitorOldNameViewPlanning", "JanitorPlan", "JanitorQueryDiffArtifactCandidate", "JanitorRelationKey", "JanitorSkippedRelation", "JanitorSkippedSchema"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.load.main.run"] +symbols = ["run_load_pipeline"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.load.models"] +symbols = ["LoadCallbacks", "LoadExecutionResult", "LoadRuntimeParams"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.migrations._helpers.relation_for_location"] +symbols = ["migration_relation_for_location"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.old_name_views._helpers.janitor"] +symbols = ["is_compatibility_view"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline._helpers.auditing"] +symbols = ["run_audit_pipeline_with_callbacks"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline._helpers.scenario"] +symbols = ["run_scenario_capture_pipeline", "run_scenario_local_test_pipeline", "run_scenario_test_pipeline", "select_scenario_snapshot_capture_candidates"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline._helpers.seeding"] +symbols = ["run_seed_pipeline"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline._helpers.testing"] +symbols = ["run_test_pipeline"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline.exceptions"] +symbols = ["AuditExecutionError", "TemporaryTestFunctionError", "WarehouseConnectionError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline.main.raise_warehouse_connection_failure"] +symbols = ["raise_warehouse_connection_failure"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline.main.run"] +symbols = ["run_build_pipeline"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline.models"] +symbols = ["AuditPipelineCallbacks", "TestPipelineCallbacks"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.python_nodes._helpers.ingress_execution"] +symbols = ["execute_ingress_python_loader_nodes"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.python_nodes.classes.run_state"] +symbols = ["PythonNodeRunState"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.python_nodes.main._checks"] +symbols = ["execute_python_checks"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.python_nodes.main._ingress"] +symbols = ["run_ingress_python_loader_nodes"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.python_nodes.main.read_side"] +symbols = ["create_read_side_python_execution_tracker"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.python_nodes.models"] +symbols = ["IngressCallbacks", "PythonCheckCallbacks", "PythonCheckExecutionResult", "PythonNodeExecutionResult", "PythonNodeRuntime"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.run.models"] +symbols = ["ModelExecutionResult"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.scenario.models"] +symbols = ["ScenarioCaptureSettings", "ScenarioLocalReplaySource", "ScenarioSnapshotCaptureLimits", "ScenarioTestSettings"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.scheduling.types"] +symbols = ["ExecutionStatus"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.testing.models"] +symbols = ["SqlTestExecutionResult"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dagster.constants"] +symbols = ["ASSET_SELECTION_COMMANDS", "CHECK_METADATA_EXCLUDED_KEYS", "CLONE_COMMAND", "COMPLETED_EXECUTION_STATUSES", "DAGSTER_ASSET_ENABLED_FIELD", "DAGSTER_ASSET_NODE_KINDS", "DAGSTER_CHECK_ENABLED_FIELD", "DAGSTER_CHECK_NAME_FIELD", "DAGSTER_DIRECT_KIND_NODE_KINDS", "DEFAULT_SELECTABLE_NODE_KINDS", "EXPLICIT_SELECTION_FLAGS", "INSUFFICIENT_CHECK_STATUS", "LIVE_EVENT_COMMANDS", "LOADER_NODE_KIND", "LOAD_SELECTABLE_NODE_KINDS", "MATERIALIZABLE_NODE_KINDS", "MODEL_NODE_KIND", "SCENARIO_CHECK_KIND", "SOURCE_NODE_KIND", "STANDALONE_CHECK_COMMANDS", "SUCCESS_EXECUTION_STATUS", "VIEW_MATERIALIZATION_TYPE", "WARNING_CHECK_SEVERITY"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dagster.exceptions"] +symbols = ["DagsterDagInputError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dbt._helpers.profile.load"] +symbols = ["load_dbt_yaml"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dbt._helpers.profile.render"] +symbols = ["render_selected_dbt_profile_output"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dbt.exceptions"] +symbols = ["DbtProfileError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dlt.constants"] +symbols = ["DLT_INTEGRATION_KIND"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dlt.exceptions"] +symbols = ["DltIntegrationError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dlt.main._loaders"] +symbols = ["dlt_loader_function"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dlt.models"] +symbols = ["DltResourceConfig", "DltSourceConfig"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.ingestr.constants"] +symbols = ["INGESTR_INTEGRATION_KIND"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.ingestr.exceptions"] +symbols = ["IngestrIntegrationError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.ingestr.main._loaders"] +symbols = ["ingestr_loader_function"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.ingestr.models"] +symbols = ["IngestrSourceConfig"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.rivers.constants"] +symbols = ["RIVERS_ASSET_NODE_KINDS"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.microbatches.classes.event_codec"] +symbols = ["MicrobatchEventCodec"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.microbatches.constants"] +symbols = ["MICROBATCH_COLUMNS", "MICROBATCH_COLUMN_TYPES", "MICROBATCH_DIRECT_INDEXES", "MICROBATCH_GENERATION_WILDCARD", "MICROBATCH_TABLE_NAME"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.microbatches.exceptions"] +symbols = ["MicrobatchStateError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.microbatches.models"] +symbols = ["MicrobatchCoverageProjection", "MicrobatchInterval", "ProjectedMicrobatchInterval", "ReplayRequirementProjection"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.microbatches.types"] +symbols = ["ReplayRequirementState"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.provider.exceptions"] +symbols = ["ProviderInputError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.provider.main.session"] +symbols = ["build_provider_session"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.constants"] +symbols = ["RELATION_RENDERER"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.main.read_asset_definition"] +symbols = ["read_asset_definition"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.main.read_audit_factory_definition"] +symbols = ["read_audit_factory_definition"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.main.read_check_definition"] +symbols = ["read_check_definition"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.main.read_factory_definition"] +symbols = ["read_factory_definition"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.main.read_hook_definition"] +symbols = ["read_hook_definition"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.main.read_loader_definition"] +symbols = ["read_loader_definition"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.main.read_task_definition"] +symbols = ["read_task_definition"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.models"] +symbols = ["AuditCase", "SqlResourceRef"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.types"] +symbols = ["PythonCheckSeverity", "SqlResourceRefKind"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.rule_engine._helpers.engine._custom_catalogue"] +symbols = ["load_custom_rules"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.rule_engine._helpers.engine.custom_payloads"] +symbols = ["custom_rule_payloads"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.compute_logs.classes.text_tee"] +symbols = ["TextComputeLogTee"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.compute_logs.exceptions"] +symbols = ["CaptureAlreadyExistsError", "CaptureNotFoundError", "CaptureStateError", "ComputeLogMetadataError", "ComputeLogPathError", "InvalidCaptureIdError", "InvalidComputeLogCursorError", "InvalidComputeLogLimitError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.compute_logs.models"] +symbols = ["CaptureByteCounts", "CaptureInventory", "CaptureInventoryItem", "CaptureMetadata", "ComputeLogReadChunk", "FinalCaptureMetadata", "PruneResult"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.compute_logs.types"] +symbols = ["ComputeLogStream"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.contracts.models"] +symbols = ["ConnectionHooks"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.contracts.types"] +symbols = ["ExecutionResourceKind"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.event_exporting.exceptions"] +symbols = ["EventExporterInputError", "EventExporterStateError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.event_exporting.main._invoke_lifecycle_event_sink"] +symbols = ["invoke_lifecycle_event_sink"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.event_exporting.main._invoke_recall_sink"] +symbols = ["invoke_recall_sink"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.event_exporting.main._run_command_output_sinks"] +symbols = ["run_command_output_sinks"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.event_exporting.main.get_lifecycle_event_sink_definition"] +symbols = ["get_lifecycle_event_sink_definition"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.execution_history.exceptions"] +symbols = ["ExecutionHistoryStorageError", "IntegrityConflictError", "InvalidCursorError", "InvalidEventError", "InvalidFilterError", "InvalidLimitError", "ProjectionConsistencyError", "UnsupportedSchemaVersionError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.execution_history.models"] +symbols = ["EventPage", "RunPage", "RunRecord", "StoredEvent"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.execution_history.types"] +symbols = ["RunStatus"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.execution_limits.exceptions"] +symbols = ["ExecutionDurationLimitError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.execution_limits.main.execution_deadline_scope"] +symbols = ["execution_deadline_scope"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.execution_limits.models"] +symbols = ["RunItemTimeouts"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.classes.event_dispatcher"] +symbols = ["EventDispatcher"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.classes.operation_lifecycle"] +symbols = ["OperationLifecycle"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.exceptions"] +symbols = ["ObservabilityValidationError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.create_lifecycle_event"] +symbols = ["create_lifecycle_event"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.current_execution_identity"] +symbols = ["current_execution_identity"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.diagnostic_log_to_json"] +symbols = ["diagnostic_log_to_json"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.dispatcher_scope"] +symbols = ["dispatcher_scope"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.invocation_external_context_scope"] +symbols = ["invocation_external_context_scope"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.invocation_scope"] +symbols = ["invocation_scope"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.lifecycle_event_from_json"] +symbols = ["lifecycle_event_from_json"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.lifecycle_event_to_json"] +symbols = ["lifecycle_event_to_json"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.run_scope"] +symbols = ["run_scope"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.models"] +symbols = ["DiagnosticLog", "LifecycleEvent", "OperationAttributes"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.output_capture.exceptions"] +symbols = ["CommandOutputValidationError", "OutputCaptureInputError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.output_capture.main.configured_output_capture_scope"] +symbols = ["configured_output_capture_scope"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.output_capture.models"] +symbols = ["BoundCommandOutputSink", "CommandOutputCaptureSummary", "CommandOutputRecord", "OutputCaptureContext"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.output_capture.types"] +symbols = ["CommandOutputStream", "OutputRecordPriority"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.sql_values.exceptions"] +symbols = ["SqlValueRenderingError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.bigquery.classes.bigquery_adapter"] +symbols = ["BigQueryAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.clickhouse.classes.clickhouse_adapter"] +symbols = ["ClickHouseAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.databricks.classes.databricks_adapter"] +symbols = ["DatabricksAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.duckdb.classes.duckdb_adapter"] +symbols = ["DuckDbAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.motherduck.classes.motherduck_adapter"] +symbols = ["MotherDuckAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.postgres.classes.postgres_adapter"] +symbols = ["PostgresAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.snowflake.classes.snowflake_adapter"] +symbols = ["SnowflakeAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.sqlite.classes.sqlite_adapter"] +symbols = ["SqliteAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.sqlserver.classes.sqlserver_adapter"] +symbols = ["SqlServerAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance._helpers.dense_project"] +symbols = ["dense_model_path", "dense_test_path"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance._helpers.diagnostic_measurement"] +symbols = ["measure_diagnostic_compile", "measure_function_name_scan", "measure_position_mapping"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance._helpers.diagnostic_project"] +symbols = ["write_cascade_project", "write_diagnostic_project", "write_function_name_project", "write_single_model_diagnostic_project", "write_undescribed_project"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance._helpers.varied_benchmark"] +symbols = ["run_varied_cache_benchmark"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance._helpers.varied_project"] +symbols = ["write_varied_compile_project"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance._helpers.varied_statistics"] +symbols = ["varied_project_statistics"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance.main.assert_required_cgroup_memory_limit"] +symbols = ["assert_required_cgroup_memory_limit"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance.main.read_compile_measurement"] +symbols = ["read_compile_measurement"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance.main.semantic_compile_fingerprint"] +symbols = ["semantic_compile_fingerprint"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance.models"] +symbols = ["VariedProjectStatistics"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.compiler_differential._helpers.corpus.emitted_codes"] +symbols = ["analysis_error_codes", "case_emitted_codes", "discovery_error_codes", "emitted_failure_codes", "inline_compile_codes", "render_error_codes", "unaccounted_inline_codes"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.compiler_differential._helpers.coverage.analysis"] +symbols = ["analysis_capture_problems"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.compiler_differential._helpers.coverage.discovery"] +symbols = ["discovery_capture_problems"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.compiler_differential._helpers.coverage.render"] +symbols = ["render_capture_problems"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.compiler_differential.constants"] +symbols = ["ANALYSIS_UNREACHABLE_CODES", "DISCOVERY_UNREACHABLE_CODES", "RENDER_UNREACHABLE_CODES"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.compiler_differential.main.failure_cases"] +symbols = ["failure_cases"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.compiler_differential.main.generate_project"] +symbols = ["generate_project"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.compiler_differential.models"] +symbols = ["EmittedCodes"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.delta_compile_series._helpers.outcomes"] +symbols = ["full_compile_difference"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.delta_compile_series.constants"] +symbols = ["DELTA_COMPILE_LINE", "DOUBLE_CAST", "DOUBLE_COLUMN", "FLOAT_CAST", "FLOAT_COLUMN", "PROJECT_CONFIG"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.delta_compile_series.main.run_delta_compile_sequence"] +symbols = ["run_delta_compile_sequence"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.delta_compile_series.main.run_delta_compile_series"] +symbols = ["run_delta_compile_series"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.delta_compile_series.main.write_delta_compile_project"] +symbols = ["write_delta_compile_project"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.delta_compile_series.models"] +symbols = ["DeltaCompileOutcome", "DeltaCompileSequence", "DeltaCompileSeries"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.delta_compile_series.types"] +symbols = ["SequenceEdit"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.docs_skill._helpers.navigation"] +symbols = ["list_ordered_page_paths"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.format_performance._helpers.model_project"] +symbols = ["write_model_format_project"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.native_module_paths._helpers.crates"] +symbols = ["native_module_path_errors", "scan_native_crates"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.native_module_paths._helpers.host_roots"] +symbols = ["host_dead_code_roots", "host_targets"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.native_module_paths._helpers.scan"] +symbols = ["scan_native_source"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.native_module_paths.models"] +symbols = ["NativeModulePathScan"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.setting_help.main.find_settings_without_snippet"] +symbols = ["find_settings_without_snippet"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.setting_help.main.parse_setting_snippet"] +symbols = ["parse_setting_snippet"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dagster.constants", "sqlbuild.integrations.rivers.constants"] +symbols = ["*_EXCLUSIONS", "*_MEMBERS"] +reason = "Exhaustiveness guards: tests prove every enum member is either a member or an exclusion of each derived set." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.compile.types"] +symbols = ["AuditInput", "TypedSqlValueRenderer"] +reason = "src/sqlbuild/_native.pyi annotates native bindings with these protocols; Fensu does not read stubs." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.contract._helpers.raw_statement_audit"] +symbols = ["audit_raw_statement_calls"] +reason = "A unit test runs this repository audit over the adapter sources." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.clone.main._prephase_row_from_clone_item"] +symbols = ["prephase_row_from_clone_item"] +reason = "The clone prephase-row tests use this entry point; production calls the helper directly." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline._helpers.auditing"] +symbols = ["run_audit_pipeline"] +reason = "The audit pipeline tests drive this callback-free form; production calls run_audit_pipeline_with_callbacks." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline._helpers.scenario"] +symbols = ["_ScenarioPipelineResult"] +reason = "PEP 695 type-parameter bounds in this module use the protocol; Fensu does not count bounds as references." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.rule_engine._helpers.host.module_state"] +symbols = ["module_state_token"] +reason = "The custom-rule host tests compute expected module-state tokens with it." + [targets.python.thresholds] max_positional_args = 1 @@ -50,11 +1151,6 @@ rule = "FFR309" path = "src/sqlbuild/cost/models.py" reason = "Native bindings own run-cost allocation, Snowflake cost collection, the statement ledger and the cost context; this package only declares the cost models and types they use." -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/lint/__init__.py" -reason = "Native bindings own lint entry points, expansion preparation and compiler-verified fix planning; this package retains boundary models, custom-rule and macro callbacks." - [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/cli/compile_reuse/__init__.py" @@ -67,33 +1163,38 @@ reason = "Native bindings own microbatch coverage and replay projection, event I [[targets.python.rule_exceptions]] rule = "FFR309" -path = "src/sqlbuild/cursor_algebra/__init__.py" -reason = "The native cursor algebra in sqlbuild-cursor owns cursor behavior; this package only declares the cursor value models, sentinels and grain metadata callers import." +path = "src/sqlbuild/lint/__init__.py" +reason = "Empty package kept so the tests of the native lint, expansion and fix planning keep their mirrored location; it declares nothing." [[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/spec/contracts/__init__.py" -reason = "Native bindings own configuration resolution; this package only declares the configuration models, constants and errors callers import." +rule = "FFL108" +path = "src/sqlbuild/lint/__init__.py" +reason = "Empty package kept so the tests of the native lint, expansion and fix planning keep their mirrored location; it declares nothing." [[targets.python.rule_exceptions]] rule = "FFR309" -path = "src/sqlbuild/integrations/dbt/__init__.py" -reason = "Native bindings own the dbt interop commands, the dbt runner, its JSON event stream and project initialisation; this package keeps the dbt models, types and errors and the PyYAML and Jinja profile glue they call." +path = "src/sqlbuild/presentation/__init__.py" +reason = "Empty package kept so the tests of the native CLI presentation keep their mirrored location; it declares nothing." + +[[targets.python.rule_exceptions]] +rule = "FFL108" +path = "src/sqlbuild/presentation/__init__.py" +reason = "Empty package kept so the tests of the native CLI presentation keep their mirrored location; it declares nothing." [[targets.python.rule_exceptions]] rule = "FFR309" -path = "src/sqlbuild/compiler/frontier/__init__.py" -reason = "Native bindings own the compile frontier and the installed code identity; this package only declares the frontier constants, types and errors they read and raise." +path = "src/sqlbuild/cursor_algebra/__init__.py" +reason = "The native cursor algebra in sqlbuild-cursor owns cursor behavior; this package only declares the cursor value models, sentinels and grain metadata callers import." [[targets.python.rule_exceptions]] rule = "FFR309" -path = "src/sqlbuild/compiler/planner/__init__.py" -reason = "Native bindings own planning, plan entries, plan output, command plans and the planner value records; this package only declares the plan models, types, constants and errors the executor, integrations and pipeline import." +path = "src/sqlbuild/spec/contracts/__init__.py" +reason = "Native bindings own configuration resolution; this package only declares the configuration models, constants and errors callers import." [[targets.python.rule_exceptions]] rule = "FFR309" -path = "src/sqlbuild/compiler/migrations/__init__.py" -reason = "Native bindings own migration planning and state rows; this package only declares the migration models, decisions and table names the executor imports." +path = "src/sqlbuild/integrations/dbt/__init__.py" +reason = "Native bindings own the dbt interop commands, the dbt runner, its JSON event stream and project initialisation; this package keeps the dbt models, types and errors and the PyYAML and Jinja profile glue they call." [[targets.python.rule_exceptions]] rule = "FFR309" @@ -115,11 +1216,6 @@ rule = "FFR309" path = "src/sqlbuild/errors/contracts/__init__.py" reason = "Native bindings read expected-error codes, messages and help; this package only declares the shared and executor input errors callers raise." -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/presentation/__init__.py" -reason = "Native bindings own CLI styling, terminal width and the transient line coordinator; this package only declares the display options and presentation types callers pass." - [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/dag/types.py" @@ -190,11 +1286,6 @@ rule = "FFR309" path = "src/sqlbuild/sql_values/exceptions.py" reason = "Typed SQL values are normalized and state literals rendered natively; the namespace package keeps the value models, types and the rendering error adapters raise." -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/compiler/lineage/__init__.py" -reason = "Column lineage is built natively; the package keeps only the lineage models, enums and errors the native graph and its callers share." - [[targets.python.rule_exceptions]] rule = "FFR601" path = "src/sqlbuild/adapter/contract/classes/base_adapter.py" @@ -448,6 +1539,19 @@ rule = "FPRSL105" path = "crates/sqlbuild-analysis/src/sql_tokens/main/builtin_function_names.rs" reason = "This test-support entry exists only for the SQL lint tests in sqlbuild-rules, which the analyzer does not count as importers." +[targets.rust.rule_options.FPRSR311] +allowed_entries = [ + "Cargo.toml", + "build.rs", + "src/", + "tests/", + "benches/", + "examples/", + "README*", + "LICENSE*", + "pyproject.toml", +] + [targets.rust.rule_options.FPRSL301] forbidden_packages = [ "sqlbuild-structure-checker", @@ -1138,6 +2242,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_progress.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs=15", @@ -1164,11 +2269,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs=16", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/inputs_view.rs=24", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs=20", @@ -1185,11 +2290,12 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_visibility.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs=27", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs=8", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs=13", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs=1", @@ -1214,6 +2320,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/log_records.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/discovery/faults.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs=1", @@ -1246,9 +2353,10 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs=16", + "crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs=4", @@ -1262,6 +2370,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs=2", @@ -1275,7 +2384,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs=66", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs=27", + "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs=41", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/models.rs=7", @@ -1291,10 +2400,12 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/functions.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lineage_facts.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/records/models.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs=2", @@ -1307,7 +2418,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_classes.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs=26", + "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs=33", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_facts.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_generated_references.rs=6", @@ -1329,12 +2440,13 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_resolution.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs=8", + "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_tolerant.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs=4", diff --git a/pyproject.toml b/pyproject.toml index f861ac0fc3..c26e3ebbf2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,7 @@ dev = [ "pytest-cov>=7.0.0", "pytest-xdist>=3.8.0", "ruff>=0.12.0", - "fensu>=0.23.0", + "fensu>=0.27.0", "testcontainers>=4.8.1", "ty>=0.0.1a19", ] diff --git a/scripts/cold_compile_performance/_helpers/random_dag_project.py b/scripts/cold_compile_performance/_helpers/random_dag_project.py deleted file mode 100644 index 7ecb629896..0000000000 --- a/scripts/cold_compile_performance/_helpers/random_dag_project.py +++ /dev/null @@ -1,153 +0,0 @@ -"""Seeded DuckDB projects whose models bind against shapes inferred from their producers.""" - -from __future__ import annotations - -import random -from collections.abc import Callable -from pathlib import Path - -from scripts.cold_compile_performance.models import RandomDagProject - -_RANDOM_DAG_ROOTS: int = 3 -_RANDOM_DAG_ROOT_SHARE: float = 0.1 -_RANDOM_DAG_FEATURE_SHARE: float = 0.1 -_RANDOM_DAG_WINDOW: int = 12 -_RANDOM_DAG_SOURCES: tuple[tuple[str, tuple[str, ...]], ...] = ( - ("raw_orders", ("id", "quantity", "status")), - ("raw_customers", ("id", "region")), -) - - -def write_random_dag_project(*, project_dir: Path, project: RandomDagProject) -> tuple[str, ...]: - """Write seeded models over inferred shapes; return model names in topological order.""" - - rng: random.Random = random.Random(project.seed) - project_dir.mkdir(parents=True, exist_ok=True) - rules: str = ", ".join(f'"{code}"' for code in project.rules) - (project_dir / "sqlbuild_project.toml").write_text( - f'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = [{rules}]\n', encoding="utf-8" - ) - (project_dir / "sources").mkdir(exist_ok=True) - (project_dir / "sources" / "raw.yml").write_text( - "sources:\n" - " - name: raw_orders\n description: Test source raw_orders.\n" - " expression: SELECT 1 AS id, 2 AS quantity, 'open' AS status\n" - " - name: raw_customers\n description: Test source raw_customers.\n" - " expression: SELECT 1 AS id, 'north' AS region\n", - encoding="utf-8", - ) - models: Path = project_dir / "models" - models.mkdir(exist_ok=True) - columns: dict[str, tuple[str, ...]] = {} - names: list[str] = [] - for index in range(project.model_count): - name: str = f"orders_{index:03d}" - options: str = "" - body: str - output: tuple[str, ...] - if index < _RANDOM_DAG_ROOTS or rng.random() < _RANDOM_DAG_ROOT_SHARE: - source, output = rng.choice(_RANDOM_DAG_SOURCES) - body = f'SELECT * FROM __source("{source}")' - else: - body, output = _random_dag_query( - rng=rng, index=index, names=names, columns=columns, errors=project.errors - ) - if project.run_ids and rng.random() < _RANDOM_DAG_FEATURE_SHARE: - options += ', tags ["${CTX:run.id}"]' - if project.analysis_opt_outs and rng.random() < _RANDOM_DAG_FEATURE_SHARE: - options += ", sql_analysis false" - (models / f"{name}.sql").write_text( - f'MODEL (description "Test model {name}.", materialized view{options});\n{body}\n', - encoding="utf-8", - ) - columns[name] = output - names.append(name) - if project.missing_reference: - (models / "orders_missing.sql").write_text( - 'MODEL (description "Test model orders_missing.", materialized view);\n' - 'SELECT id FROM __ref("orders_absent")\n', - encoding="utf-8", - ) - if project.cycle: - for name, upstream in ( - ("orders_loop_a", "orders_loop_b"), - ("orders_loop_b", "orders_loop_a"), - ): - (models / f"{name}.sql").write_text( - f'MODEL (description "Test model {name}.", materialized view);\n' - f'SELECT id FROM __ref("{upstream}")\n', - encoding="utf-8", - ) - return tuple(names) - - -def _random_dag_query( - *, - rng: random.Random, - index: int, - names: list[str], - columns: dict[str, tuple[str, ...]], - errors: bool, -) -> tuple[str, tuple[str, ...]]: - upstream: str = rng.choice(names[max(0, index - _RANDOM_DAG_WINDOW) :]) - other: str = rng.choice(names) - if errors and rng.random() < _RANDOM_DAG_FEATURE_SHARE: - return ( - f'SELECT id, missing_{index} FROM __ref("{upstream}")', - ("id", f"missing_{index}"), - ) - return rng.choice(_query_builders())( - index=index, upstream=upstream, other=other, columns=columns - ) - - -def _star_query( - *, index: int, upstream: str, other: str, columns: dict[str, tuple[str, ...]] -) -> tuple[str, tuple[str, ...]]: - del index, other - return f'SELECT * FROM __ref("{upstream}")', columns[upstream] - - -def _projection_query( - *, index: int, upstream: str, other: str, columns: dict[str, tuple[str, ...]] -) -> tuple[str, tuple[str, ...]]: - del other - return ( - f"SELECT id, {columns[upstream][-1]} AS kept_{index}, " - f'CAST(id AS BIGINT) * {index} AS value_{index} FROM __ref("{upstream}")', - ("id", f"kept_{index}", f"value_{index}"), - ) - - -def _join_query( - *, index: int, upstream: str, other: str, columns: dict[str, tuple[str, ...]] -) -> tuple[str, tuple[str, ...]]: - return ( - f"SELECT a.*, b.{columns[other][-1]} AS joined_{index} " - f'FROM __ref("{upstream}") AS a LEFT JOIN __ref("{other}") AS b ON a.id = b.id', - (*columns[upstream], f"joined_{index}"), - ) - - -def _union_query( - *, index: int, upstream: str, other: str, columns: dict[str, tuple[str, ...]] -) -> tuple[str, tuple[str, ...]]: - del index, columns - return ( - f'SELECT id FROM __ref("{upstream}") UNION ALL SELECT id FROM __ref("{other}")', - ("id",), - ) - - -def _aggregate_query( - *, index: int, upstream: str, other: str, columns: dict[str, tuple[str, ...]] -) -> tuple[str, tuple[str, ...]]: - del other, columns - return ( - f'SELECT id, COUNT(*) AS row_count_{index} FROM __ref("{upstream}") GROUP BY id', - ("id", f"row_count_{index}"), - ) - - -def _query_builders() -> tuple[Callable[..., tuple[str, tuple[str, ...]]], ...]: - return (_star_query, _projection_query, _join_query, _union_query, _aggregate_query) diff --git a/scripts/cold_compile_performance/models.py b/scripts/cold_compile_performance/models.py index 566a6ca7d8..9118eaf908 100644 --- a/scripts/cold_compile_performance/models.py +++ b/scripts/cold_compile_performance/models.py @@ -13,17 +13,3 @@ class VariedProjectStatistics: leaf_models: int array_models: int window_models: int - - -@dataclass(frozen=True) -class RandomDagProject: - """One seeded DuckDB project whose models bind against inferred upstream shapes.""" - - seed: int - model_count: int - errors: bool = False - run_ids: bool = False - analysis_opt_outs: bool = False - missing_reference: bool = False - cycle: bool = False - rules: tuple[str, ...] = () diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py b/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py index 63172895f0..8f95cc214a 100644 --- a/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py +++ b/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py @@ -54,9 +54,6 @@ _EXPRESSION_AUDIT: str = 'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE NOT (@expression)\n' _EXPRESSION_AUDIT_PATH: str = "models/marts/_sqlbuild/_audits/generic/expression_is_true.sql" _TEST_PATH: str = "tests/unit/test_stg_orders.sql" -_ORDER_ROW: str = ( - " SELECT 1 AS order_id, 10 AS customer_id, CAST(5 AS DOUBLE) AS amount, 'placed' AS status\n" -) def _mart(body: str) -> dict[str, str]: diff --git a/scripts/compiler_differential/_helpers/comparing/capture_text.py b/scripts/compiler_differential/_helpers/comparing/capture_text.py new file mode 100644 index 0000000000..a783261382 --- /dev/null +++ b/scripts/compiler_differential/_helpers/comparing/capture_text.py @@ -0,0 +1,43 @@ +"""Expand the shared nodes of a stage capture document.""" + +from __future__ import annotations + +import json + +from scripts.compiler_differential._helpers.comparing.compare import as_json_object, shared_digest +from scripts.compiler_differential.constants import ( + STAGE_CAPTURE_ROOT_KEY, + STAGE_CAPTURE_SHARED_NODES_KEY, +) + +_SHARED_DOCUMENT_KEYS: frozenset[str] = frozenset( + {STAGE_CAPTURE_SHARED_NODES_KEY, STAGE_CAPTURE_ROOT_KEY} +) + + +def expand_capture_text(text: str) -> object: + """Parse a whole capture document, replacing every shared reference by its node.""" + + document: object = json.loads(text) + mapping: dict[str, object] | None = as_json_object(document) + nodes: dict[str, object] | None = as_json_object( + None if mapping is None else mapping.get(STAGE_CAPTURE_SHARED_NODES_KEY) + ) + if mapping is None or nodes is None or set(mapping) != _SHARED_DOCUMENT_KEYS: + return document + expanded: dict[str, object] = {} + for digest, node in nodes.items(): + expanded[digest] = _expanded(value=node, nodes=expanded) + return _expanded(value=mapping[STAGE_CAPTURE_ROOT_KEY], nodes=expanded) + + +def _expanded(*, value: object, nodes: dict[str, object]) -> object: + digest: str | None = shared_digest(value) + if digest is not None: + return nodes[digest] + items: dict[str, object] | None = as_json_object(value) + if items is not None: + return {key: _expanded(value=item, nodes=nodes) for key, item in items.items()} + if isinstance(value, list): + return [_expanded(value=item, nodes=nodes) for item in value] + return value diff --git a/scripts/compiler_differential/_helpers/coverage/analysis.py b/scripts/compiler_differential/_helpers/coverage/analysis.py index 5116c308c5..d7a532970f 100644 --- a/scripts/compiler_differential/_helpers/coverage/analysis.py +++ b/scripts/compiler_differential/_helpers/coverage/analysis.py @@ -7,6 +7,7 @@ from collections.abc import Iterator, Sequence from pathlib import Path +from scripts.compiler_differential._helpers.comparing.capture_text import expand_capture_text from scripts.compiler_differential._helpers.comparing.compare import as_json_object from scripts.compiler_differential._helpers.coverage.capture_values import ( capture_problems, @@ -15,7 +16,6 @@ has_content, records, ) -from scripts.compiler_differential.classes.capture_file import expand_capture_text from scripts.compiler_differential.constants import ( ANALYSIS_CALLABLE_CAPTURE_FIELDS, ANALYSIS_DETAIL_KINDS, diff --git a/scripts/compiler_differential/_helpers/coverage/capture_values.py b/scripts/compiler_differential/_helpers/coverage/capture_values.py index 16d29d8956..12744bf938 100644 --- a/scripts/compiler_differential/_helpers/coverage/capture_values.py +++ b/scripts/compiler_differential/_helpers/coverage/capture_values.py @@ -5,8 +5,8 @@ import re from collections.abc import Iterator +from scripts.compiler_differential._helpers.comparing.capture_text import expand_capture_text from scripts.compiler_differential._helpers.comparing.compare import as_json_object -from scripts.compiler_differential.classes.capture_file import expand_capture_text from scripts.compiler_differential.constants import ( CALLABLE_MARKER, COLLECTION_CAPTURE_MARKERS, diff --git a/scripts/compiler_differential/_helpers/coverage/discovery.py b/scripts/compiler_differential/_helpers/coverage/discovery.py index da0e2810aa..b6823157d5 100644 --- a/scripts/compiler_differential/_helpers/coverage/discovery.py +++ b/scripts/compiler_differential/_helpers/coverage/discovery.py @@ -8,6 +8,7 @@ from collections.abc import Iterator from pathlib import Path +from scripts.compiler_differential._helpers.comparing.capture_text import expand_capture_text from scripts.compiler_differential._helpers.comparing.compare import as_json_object from scripts.compiler_differential._helpers.coverage.capture_values import ( capture_problems, @@ -17,7 +18,6 @@ named_values, records, ) -from scripts.compiler_differential.classes.capture_file import expand_capture_text from scripts.compiler_differential.constants import ( CALLABLE_CAPTURE_FIELDS, CRLF_BYTES, diff --git a/scripts/compiler_differential/_helpers/coverage/render.py b/scripts/compiler_differential/_helpers/coverage/render.py index 8f314e0bce..ee553ba958 100644 --- a/scripts/compiler_differential/_helpers/coverage/render.py +++ b/scripts/compiler_differential/_helpers/coverage/render.py @@ -8,6 +8,7 @@ from collections.abc import Iterator from pathlib import Path, PurePosixPath +from scripts.compiler_differential._helpers.comparing.capture_text import expand_capture_text from scripts.compiler_differential._helpers.comparing.compare import as_json_object from scripts.compiler_differential._helpers.coverage.capture_values import ( capture_problems, @@ -20,7 +21,6 @@ macro_function_source, takes_typed_reference, ) -from scripts.compiler_differential.classes.capture_file import expand_capture_text from scripts.compiler_differential.constants import ( BUILT_IN_AUDIT_NAMES, COMPILE_INPUTS_CAPTURE_TYPE, diff --git a/scripts/compiler_differential/classes/capture_file.py b/scripts/compiler_differential/classes/capture_file.py deleted file mode 100644 index 3017d571fd..0000000000 --- a/scripts/compiler_differential/classes/capture_file.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Read one stage capture from disk node by node instead of parsing it whole.""" - -from __future__ import annotations - -import json -from pathlib import Path - -from scripts.compiler_differential._helpers.comparing.compare import as_json_object, shared_digest -from scripts.compiler_differential._helpers.comparing.normalize import normalize_artifact_text -from scripts.compiler_differential.constants import ( - STAGE_CAPTURE_ROOT_KEY, - STAGE_CAPTURE_SHARED_NODES_KEY, -) - -_SHARED_HEADER: bytes = f"{{{json.dumps(STAGE_CAPTURE_SHARED_NODES_KEY)}: {{\n".encode() -_ROOT_PREFIX: bytes = f"{json.dumps(STAGE_CAPTURE_ROOT_KEY)}: ".encode() -_SHARED_DOCUMENT_KEYS: frozenset[str] = frozenset( - {STAGE_CAPTURE_SHARED_NODES_KEY, STAGE_CAPTURE_ROOT_KEY} -) -_DIGEST_CHARACTERS: int = 64 -_NODE_PREFIX_BYTES: int = _DIGEST_CHARACTERS + len(b'"": ') - - -class CaptureFile: - """A normalized stage capture whose shared nodes are parsed only when resolved.""" - - def __init__(self, path: Path) -> None: - self._path: Path = path - self._offsets: dict[str, int] = {} - self.root: object = self._index() - - def resolve(self, value: object) -> object: - """Return the shared node a reference names, or the value itself.""" - - digest: str | None = shared_digest(value) - return value if digest is None else self._node(digest) - - def _index(self) -> object: - with self._path.open("rb") as stream: - if stream.readline() != _SHARED_HEADER: - return _parse(self._path.read_bytes()) - root: object = None - offset: int = stream.tell() - for line in iter(stream.readline, b""): - if line.startswith(_ROOT_PREFIX): - root = _parse(line[len(_ROOT_PREFIX) :]) - elif line.startswith(b'"') and len(line) > _NODE_PREFIX_BYTES: - self._offsets[line[1 : _DIGEST_CHARACTERS + 1].decode("ascii")] = offset - offset += len(line) - return root - - def _node(self, digest: str) -> object: - offset: int | None = self._offsets.get(digest) - if offset is None: - raise json.JSONDecodeError(f"missing shared node {digest}", str(self._path), 0) - with self._path.open("rb") as stream: - _ = stream.seek(offset) - line: bytes = stream.readline() - return _parse(line[_NODE_PREFIX_BYTES:]) - - -def expand_capture_text(text: str) -> object: - """Parse a whole capture document, replacing every shared reference by its node.""" - - document: object = json.loads(text) - mapping: dict[str, object] | None = as_json_object(document) - nodes: dict[str, object] | None = as_json_object( - None if mapping is None else mapping.get(STAGE_CAPTURE_SHARED_NODES_KEY) - ) - if mapping is None or nodes is None or set(mapping) != _SHARED_DOCUMENT_KEYS: - return document - expanded: dict[str, object] = {} - for digest, node in nodes.items(): - expanded[digest] = _expanded(value=node, nodes=expanded) - return _expanded(value=mapping[STAGE_CAPTURE_ROOT_KEY], nodes=expanded) - - -def _expanded(*, value: object, nodes: dict[str, object]) -> object: - digest: str | None = shared_digest(value) - if digest is not None: - return nodes[digest] - items: dict[str, object] | None = as_json_object(value) - if items is not None: - return {key: _expanded(value=item, nodes=nodes) for key, item in items.items()} - if isinstance(value, list): - return [_expanded(value=item, nodes=nodes) for item in value] - return value - - -def _parse(data: bytes) -> object: - text: str = data.decode("utf-8").rstrip("\n") - return json.loads(normalize_artifact_text(text.removesuffix(","))) diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index f7d6084c08..d199a76cf2 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -125,15 +125,6 @@ ADAPTER_ROOT_PARTS: tuple[str, ...] = ("src", "sqlbuild", "adapter") ADAPTERS_ROOT_PARTS: tuple[str, ...] = ("src", "sqlbuild", "adapters") ADAPTER_PACKAGE_IMPORT_PREFIX: str = "sqlbuild.adapters." -ADAPTER_CONTRACT_CLASS_IMPORT_PREFIX: tuple[str, ...] = ( - "sqlbuild", - "adapter", - "contract", - "classes", -) -ADAPTER_CONTRACT_BASE_CLASS_NAMES: frozenset[str] = frozenset( - {"BaseAdapter", "DuckDbBackedAdapter", "StrictAdapter"} -) ADAPTER_CONTRACT_BASE_SYMBOLS: frozenset[tuple[str, ...]] = frozenset( { ( @@ -185,7 +176,6 @@ PLANNER_PATH_PREFIX: str = "src/sqlbuild/compiler/planner/" POLICY_EVALUATION_SCOPES: frozenset[str] = frozenset({"root", "tooling"}) POLICY_IMPLEMENTATION_PATH_PREFIX: str = "scripts/fensu_policy/" -PROVIDER_CLASS_NAME: str = "Provider" PROVIDER_MODULE_PARTS: tuple[str, ...] = ("src", "sqlbuild", "providers.py") PUBLIC_EXPORT_ASSIGNMENT_TARGETS: tuple[str, ...] = ("__all__",) ROOT_SCOPE_NAME: str = "root" @@ -280,10 +270,6 @@ EVENT_EXPORTER_DECORATOR_NAMES: frozenset[str] = frozenset( {"lifecycle_event_sink", "command_output_sink"} ) -OBSERVABILITY_MODULE_PREFIXES: tuple[tuple[str, ...], ...] = ( - ("sqlbuild", "observability"), - ("sqlbuild", "runtime", "observability"), -) OBSERVABILITY_LIFECYCLE_EVENT_MODULES: tuple[tuple[str, ...], ...] = ( ("sqlbuild", "observability"), ("sqlbuild", "runtime", "observability", "models"), diff --git a/scripts/native_module_paths/_helpers/host_roots.py b/scripts/native_module_paths/_helpers/host_roots.py new file mode 100644 index 0000000000..7c425af671 --- /dev/null +++ b/scripts/native_module_paths/_helpers/host_roots.py @@ -0,0 +1,54 @@ +"""Fensu dead-code roots that keep the native host table's Python targets alive.""" + +import ast +import importlib.util +import tomllib +from importlib.machinery import ModuleSpec +from pathlib import Path +from typing import Any + +import sqlbuild._native_host as native_host +from scripts.native_module_paths.constants import ( + HOST_ROOT_REASON, + MODULE_FILE_ATTRIBUTE, + PYTHON_SUFFIX, +) + + +def defining_target(*, target: tuple[str, str]) -> tuple[str, str]: + """Follow `from module import name as name` re-exports to the module that defines a target.""" + module, attribute = target + spec: ModuleSpec | None = importlib.util.find_spec(module) + origin: str | None = None if spec is None else spec.origin + if origin is None or not origin.endswith(PYTHON_SUFFIX): + return target + for node in ast.parse(Path(origin).read_text(encoding="utf-8")).body: + if not isinstance(node, ast.ImportFrom) or node.level != 0 or node.module is None: + continue + for alias in node.names: + if (alias.asname or alias.name) == attribute: + return defining_target(target=(node.module, alias.name)) + return target + + +def host_targets() -> set[tuple[str, str]]: + """The defining module and symbol of every host table entry that names a symbol.""" + targets: set[tuple[str, str]] = set() + for target in native_host.host_targets().values(): + if target[1] != MODULE_FILE_ATTRIBUTE: + targets.add(defining_target(target=target)) + return targets + + +def host_dead_code_roots(*, config: Path) -> set[tuple[str, str]]: + """Every module and symbol the Fensu dead-code roots in `config` keep for the host table.""" + parsed: dict[str, Any] = tomllib.loads(config.read_text(encoding="utf-8")) + roots: list[dict[str, Any]] = parsed["targets"]["python"]["dead_code"]["roots"] + found: set[tuple[str, str]] = set() + for root in roots: + if root["reason"] != HOST_ROOT_REASON: + continue + for module in root["modules"]: + for symbol in root["symbols"]: + found.add((module, symbol)) + return found diff --git a/scripts/native_module_paths/constants.py b/scripts/native_module_paths/constants.py index 192e7b5ecf..407caf0bd7 100644 --- a/scripts/native_module_paths/constants.py +++ b/scripts/native_module_paths/constants.py @@ -34,3 +34,6 @@ ) HOST_MODULE: str = "sqlbuild._native_host" HOST_ENTRY: re.Pattern[str] = re.compile(r"\bHostEntry\s*\{\s*name:\s*" + ATTRIBUTE_NAME) +HOST_ROOT_REASON: str = "Native code resolves these through the sqlbuild._native_host name table." +MODULE_FILE_ATTRIBUTE: str = "__file__" +PYTHON_SUFFIX: str = ".py" diff --git a/scripts/planner_differential/_helpers/capturing/canonical.py b/scripts/planner_differential/_helpers/capturing/canonical.py index 927fb20359..051a6d8013 100644 --- a/scripts/planner_differential/_helpers/capturing/canonical.py +++ b/scripts/planner_differential/_helpers/capturing/canonical.py @@ -6,7 +6,7 @@ import re from datetime import date, timedelta -from scripts.compiler_differential.classes.capture_file import expand_capture_text +from scripts.compiler_differential._helpers.comparing.capture_text import expand_capture_text from scripts.planner_differential.constants import ( COMPACT_TIMESTAMP_PATTERN, DATE_PATTERN, diff --git a/src/sqlbuild/adapter/contract/constants.py b/src/sqlbuild/adapter/contract/constants.py index 2f323717dd..0a895325e8 100644 --- a/src/sqlbuild/adapter/contract/constants.py +++ b/src/sqlbuild/adapter/contract/constants.py @@ -1,21 +1,5 @@ """Adapter constants.""" -INTEGER_TYPE_NAMES: frozenset[str] = frozenset( - {"TINYINT", "SMALLINT", "INT", "INTEGER", "BIGINT", "LONG", "INT64"} -) -DECIMAL_TYPE_NAMES: frozenset[str] = frozenset( - {"DECIMAL", "BIGDECIMAL", "NUMERIC", "NUMBER", "BIGNUMERIC"} -) -FLOAT_TYPE_NAMES: frozenset[str] = frozenset({"DOUBLE", "FLOAT", "FLOAT64", "REAL"}) -STRING_TYPE_NAMES: frozenset[str] = frozenset({"VARCHAR", "CHAR", "CHARACTER", "TEXT", "STRING"}) -BOOLEAN_TYPE_NAMES: frozenset[str] = frozenset({"BOOLEAN", "BOOL"}) -TIMESTAMP_TYPE_NAMES: frozenset[str] = frozenset( - {"TIMESTAMP", "TIMESTAMPTZ", "TIMESTAMPNTZ", "TIMESTAMPLTZ"} -) -DATE_TYPE_NAME: str = "DATE" -DATETIME_TYPE_NAME: str = "DATETIME" -POLYGLOT_CUSTOM_TYPE_NAME: str = "CUSTOM" -TIMESTAMP_TYPE_TOKEN: str = "TIMESTAMP" INTEGER_TYPE_TOKEN: str = "INT" TYPED_OBJECT_ENTRY_PART_COUNT: int = 2 TABLE_RELATION_TYPE_NAMES: frozenset[str] = frozenset( diff --git a/src/sqlbuild/adapters/bigquery/constants.py b/src/sqlbuild/adapters/bigquery/constants.py index d3024011fb..498d176b89 100644 --- a/src/sqlbuild/adapters/bigquery/constants.py +++ b/src/sqlbuild/adapters/bigquery/constants.py @@ -2,15 +2,12 @@ BOOLEAN_METADATA_TYPE_NAME: str = "BOOLEAN" BOOLEAN_WIRE_TYPE_NAME: str = "BOOL" -BIGNUMERIC_TYPE_NAME: str = "BIGNUMERIC" -CUSTOM_NORMALIZATION_TYPE_NAMES: frozenset[str] = frozenset({"INT64", "FLOAT64", "BIGNUMERIC"}) DATE_TYPE_NAME: str = "DATE" DATETIME_TYPE_NAME: str = "DATETIME" DECIMAL_METADATA_TYPE_NAMES: frozenset[str] = frozenset({"NUMERIC", "BIGNUMERIC"}) FLOAT_METADATA_TYPE_NAME: str = "FLOAT" FLOAT_WIRE_TYPE_NAME: str = "FLOAT64" INTEGER_METADATA_TYPE_NAME: str = "INTEGER" -INTEGER_PARSE_TYPE_NAMES: frozenset[str] = frozenset({"INT", "BIGINT"}) INTEGER_TYPE_TOKEN: str = "INT" INTEGER_WIRE_TYPE_NAME: str = "INT64" NOT_FOUND_ERROR_CLASS_NAME: str = "NotFound" diff --git a/src/sqlbuild/adapters/clickhouse/constants.py b/src/sqlbuild/adapters/clickhouse/constants.py index 7dce598a92..7b9047e237 100644 --- a/src/sqlbuild/adapters/clickhouse/constants.py +++ b/src/sqlbuild/adapters/clickhouse/constants.py @@ -40,7 +40,6 @@ ) VIEW_ENGINES: frozenset[str] = frozenset({"View", "MaterializedView", "LiveView", "WindowView"}) RELATION_NOT_FOUND_CODES: frozenset[int] = frozenset({60, 81}) -ATOMIC_DATABASE_ENGINES: frozenset[str] = frozenset({"Atomic", "Replicated"}) NULLABLE_WRAPPER: str = "Nullable" TYPE_WRAPPERS: tuple[str, ...] = ("Nullable", "LowCardinality") TYPE_ALIASES: dict[str, str] = { diff --git a/src/sqlbuild/adapters/duckdb/_helpers/__init__.py b/src/sqlbuild/adapters/duckdb/_helpers/__init__.py deleted file mode 100644 index d39eda62bb..0000000000 --- a/src/sqlbuild/adapters/duckdb/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""DuckDB adapter helpers.""" diff --git a/src/sqlbuild/adapters/snowflake/constants.py b/src/sqlbuild/adapters/snowflake/constants.py index 3816e406b6..c816a0575e 100644 --- a/src/sqlbuild/adapters/snowflake/constants.py +++ b/src/sqlbuild/adapters/snowflake/constants.py @@ -10,17 +10,9 @@ SECONDARY_ROLES_ALL: str = "ALL" SECONDARY_ROLES_NONE: str = "NONE" NUMBER_TYPE_NAME: str = "NUMBER" -DEFAULT_TEXT_LENGTH: int = 16_777_216 -INTEGER_PRECISION: int = 38 -INTEGER_SCALE: int = 0 -NORMALIZED_LTZ_INPUT_TYPE_NAME: str = "TIMESTAMPLTZ" -NORMALIZED_NTZ_INPUT_TYPE_NAMES: frozenset[str] = frozenset({"TIMESTAMP", "TIMESTAMPNTZ"}) -NORMALIZED_TZ_INPUT_TYPE_NAME: str = "TIMESTAMPTZ" STATUS_COLUMN_NAME: str = "status" SUCCESS_STATUS_TOKENS: tuple[str, ...] = ("success", "created", "dropped") TEXT_TYPE_NAMES: frozenset[str] = frozenset({"TEXT", "VARCHAR", "CHAR", "CHARACTER"}) -TEXT_TYPE_NAME: str = "TEXT" -UNBOUNDED_TEXT_TYPE_NAMES: frozenset[str] = frozenset({"STRING", "TEXT", "VARCHAR"}) TRUE_METADATA_VALUE: str = "YES" SHOW_LIKE_ESCAPE_CHARACTER: str = "\\" CURRENT_DATABASE_ATTRIBUTE: str = "_sqlbuild_current_database" diff --git a/src/sqlbuild/adapters/sqlite/constants.py b/src/sqlbuild/adapters/sqlite/constants.py index dcebc49096..8b35836965 100644 --- a/src/sqlbuild/adapters/sqlite/constants.py +++ b/src/sqlbuild/adapters/sqlite/constants.py @@ -15,7 +15,6 @@ LOCKED_ERROR_FRAGMENTS: tuple[str, ...] = ("database is locked", "database table is locked") MISSING_RELATION_ERROR_FRAGMENT: str = "no such table" READ_STATEMENT_KEYWORDS: frozenset[str] = frozenset({"SELECT", "PRAGMA", "EXPLAIN", "VALUES"}) -TRANSACTION_END_KEYWORDS: frozenset[str] = frozenset({"COMMIT", "END", "ROLLBACK"}) WRITE_GATE_POLL_SECONDS: float = 0.1 TYPEOF_TYPE_NAMES: dict[str, str] = { "integer": "INTEGER", diff --git a/src/sqlbuild/compiler/frontier/__init__.py b/src/sqlbuild/compiler/frontier/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/lineage/__init__.py b/src/sqlbuild/compiler/lineage/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/migrations/__init__.py b/src/sqlbuild/compiler/migrations/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/planner/__init__.py b/src/sqlbuild/compiler/planner/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/executor/clone/types.py b/src/sqlbuild/executor/clone/types.py index 9f30a97f63..2a9606126d 100644 --- a/src/sqlbuild/executor/clone/types.py +++ b/src/sqlbuild/executor/clone/types.py @@ -19,10 +19,6 @@ def __call__( ) -> None: ... -class CloneFingerprintProgressReporter(CloneFingerprintProgressCallback, Protocol): - def write_interrupted(self) -> None: ... - - class CloneStatus(StrEnum): SUCCESS = "success" WARNING = "warning" diff --git a/src/sqlbuild/executor/python_nodes/models.py b/src/sqlbuild/executor/python_nodes/models.py index 1ff25a8a67..4237e12f99 100644 --- a/src/sqlbuild/executor/python_nodes/models.py +++ b/src/sqlbuild/executor/python_nodes/models.py @@ -112,16 +112,6 @@ class PythonNodeFanInDecision: skip_mode: SkipMode | None = None -@dataclass(frozen=True) -class CursorWindow: - """Resolved cursor bounds for one Python execution run.""" - - start_cursor_ts: datetime | None = None - end_cursor_ts: datetime | None = None - start_cursor_int: int | None = None - end_cursor_int: int | None = None - - from sqlbuild.executor.python_nodes.classes.run_state import PythonNodeRunState # noqa: E402 diff --git a/src/sqlbuild/integrations/dagster/_helpers/translation.py b/src/sqlbuild/integrations/dagster/_helpers/translation.py index 4a1997d92e..a8dff56731 100644 --- a/src/sqlbuild/integrations/dagster/_helpers/translation.py +++ b/src/sqlbuild/integrations/dagster/_helpers/translation.py @@ -41,15 +41,3 @@ def translate_sqlbuild_dag( ) return {**dag, "nodes": translated_nodes, "checks": translated_checks} - - -def is_dagster_asset_enabled(node: Mapping[str, Any]) -> bool: - """Return whether a translated DAG node is owned by the SQLBuild asset definition.""" - - return bool(node.get(DAGSTER_ASSET_ENABLED_FIELD, True)) - - -def is_dagster_check_enabled(check: Mapping[str, Any]) -> bool: - """Return whether a translated DAG check is owned by the SQLBuild asset definition.""" - - return bool(check.get(DAGSTER_CHECK_ENABLED_FIELD, True)) diff --git a/src/sqlbuild/integrations/dagster/constants.py b/src/sqlbuild/integrations/dagster/constants.py index d02a1fee47..8afa16ddce 100644 --- a/src/sqlbuild/integrations/dagster/constants.py +++ b/src/sqlbuild/integrations/dagster/constants.py @@ -49,23 +49,9 @@ CLONE_COMMAND: str = "clone" CHECK_COMMAND: str = "check" STANDALONE_CHECK_COMMANDS: frozenset[str] = frozenset({"audit", CHECK_COMMAND, "test"}) -EVENT_OUTPUT_FLAG: str = "--event-output" LIVE_EVENT_COMMANDS: frozenset[str] = ASSET_SELECTION_COMMANDS EXPLICIT_SELECTION_FLAGS: frozenset[str] = frozenset({"--select", "-s", "--select-file"}) -JSON_OUTPUT_FLAGS: frozenset[str] = frozenset({"--json", "--json-output"}) -JSON_OUTPUT_FLAG: str = "--json-output" -SELECT_FILE_FLAG: str = "--select-file" -SCENARIO_TEST_COMMAND: tuple[str, str] = ("scenario", "test") SCENARIO_CHECK_KIND: str = NodeKind.SCENARIO.value -SCENARIO_VALUE_FLAGS: frozenset[str] = frozenset( - { - "--max-snapshot-rows", - "--max-snapshot-total-rows", - "--max-snapshot-bytes", - "--max-snapshot-total-bytes", - } -) -LOAD_COMMAND: str = "load" LOAD_SELECTABLE_NODE_KIND_MEMBERS: frozenset[NodeKind] = frozenset( {NodeKind.SOURCE, NodeKind.LOADER} ) @@ -146,7 +132,6 @@ ) SUCCESS_EXECUTION_STATUS: str = ExecutionStatus.SUCCESS.value FAILED_EXECUTION_STATUS: str = ExecutionStatus.FAILED.value -CHECK_NAME_SEPARATOR_CHARACTER: str = "_" CHECK_METADATA_EXCLUDED_KEYS: frozenset[str] = frozenset( {"passed", "steps", "expected_results", "assertion_results"} ) diff --git a/tests/unit/scripts/native_module_paths/_helpers/_test_types.py b/tests/unit/scripts/native_module_paths/_helpers/_test_types.py index 7040d2fcc8..89d2e334f2 100644 --- a/tests/unit/scripts/native_module_paths/_helpers/_test_types.py +++ b/tests/unit/scripts/native_module_paths/_helpers/_test_types.py @@ -40,3 +40,10 @@ class HostUnknownNameCase: description: str name: str expected_error: str + + +@dataclass(frozen=True) +class HostDeadCodeRootCase: + description: str + expected_missing_targets: tuple[tuple[str, str], ...] + expected_extra_targets: tuple[tuple[str, str], ...] diff --git a/tests/unit/scripts/native_module_paths/_helpers/helpers.py b/tests/unit/scripts/native_module_paths/_helpers/helpers.py index facfc3700e..5f2fe828c8 100644 --- a/tests/unit/scripts/native_module_paths/_helpers/helpers.py +++ b/tests/unit/scripts/native_module_paths/_helpers/helpers.py @@ -1,7 +1,10 @@ +from pathlib import Path + from scripts.native_module_paths._helpers.crates import native_module_path_errors from scripts.native_module_paths._helpers.scan import scan_native_source SOURCE_LABEL: str = "lib.rs" +FENSU_CONFIG: Path = Path(__file__).parents[5] / "fensu.toml" def native_source_errors(source: str) -> tuple[str, ...]: diff --git a/tests/unit/scripts/native_module_paths/_helpers/test_native_host.py b/tests/unit/scripts/native_module_paths/_helpers/test_native_host.py index 5ff77bc5ee..aa49f0ad8f 100644 --- a/tests/unit/scripts/native_module_paths/_helpers/test_native_host.py +++ b/tests/unit/scripts/native_module_paths/_helpers/test_native_host.py @@ -10,13 +10,15 @@ import sqlbuild._native as native import sqlbuild._native_host as native_host +from scripts.native_module_paths._helpers.host_roots import host_dead_code_roots, host_targets from tests.unit.scripts.native_module_paths._helpers._test_types import ( + HostDeadCodeRootCase, HostImportCase, HostKindCase, HostTableCase, HostUnknownNameCase, ) -from tests.unit.scripts.native_module_paths._helpers.helpers import resolved_kind +from tests.unit.scripts.native_module_paths._helpers.helpers import FENSU_CONFIG, resolved_kind @pytest.mark.parametrize( @@ -115,5 +117,26 @@ def test_given_unknown_name_when_reading_native_host_then_raises_attribute_error assert str(raised.value) == test_case.expected_error +@pytest.mark.parametrize( + "test_case", + [ + HostDeadCodeRootCase( + description="every host target is a dead-code root and every host root a target", + expected_missing_targets=(), + expected_extra_targets=(), + ) + ], + ids=lambda case: case.description, +) +def test_given_native_host_table_when_reading_fensu_roots_then_roots_match_targets( + test_case: HostDeadCodeRootCase, +) -> None: + targets: set[tuple[str, str]] = host_targets() + roots: set[tuple[str, str]] = host_dead_code_roots(config=FENSU_CONFIG) + + assert tuple(sorted(targets - roots)) == test_case.expected_missing_targets + assert tuple(sorted(roots - targets)) == test_case.expected_extra_targets + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py b/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py index 86d7bd65ee..616cc76a25 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py @@ -8,7 +8,7 @@ import pytest -from scripts.compiler_differential.classes.capture_file import expand_capture_text +from scripts.compiler_differential._helpers.comparing.capture_text import expand_capture_text from sqlbuild._native import ( STAGE_CAPTURE_DIR_ENV_VAR, CompilerStage, diff --git a/uv.lock b/uv.lock index 980908966c..10418364db 100644 --- a/uv.lock +++ b/uv.lock @@ -1345,31 +1345,31 @@ wheels = [ [[package]] name = "fensu" -version = "0.23.0" +version = "0.27.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "fensu-cli" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b4/f4/d037850c050343d2548b901a994a77783e7a02d352fe35aa2477644a185c/fensu-0.23.0.tar.gz", hash = "sha256:997c31c26e9b6ea88b2797898a5644a95ae543ba568fb5f3482efd2e17b54d61", size = 508009, upload-time = "2026-09-24T15:29:54.691Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/a3/d1eea20ab654f3c4b1b170cc8eeaf27d5092095cd74e6bc819bfff8d9312/fensu-0.27.0.tar.gz", hash = "sha256:023692a73c4ac2fdadbb167c59a58dc944c7a41b0d78a40fa260f6923a8b3e7b", size = 538895, upload-time = "2026-10-17T05:38:49.174Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1a/67/597b68a104ae906705e8fc27ec53f57eaad36d869a5323b71518aa77592d/fensu-0.23.0-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:2ee20effc98db0d4e682c3f3a55b72ffca81386228c9092cab82b4bd044d1ddc", size = 4026956, upload-time = "2026-09-24T15:29:34.609Z" }, - { url = "https://files.pythonhosted.org/packages/13/0e/1e4583e15a49bc066530ea88faa7257caa131a41bf084c89eb9b34a944a7/fensu-0.23.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:39f85e019e2a66a1cfa280efc02b47740fee8270a6e4befef4885f46f8e03753", size = 3892516, upload-time = "2026-09-24T15:29:36.742Z" }, - { url = "https://files.pythonhosted.org/packages/b4/d6/a10db3b1c7f5fb667bd6e634f494d527c29637b2fb9bc8f68d13f0300482/fensu-0.23.0-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3002c67ba494386e1c6e94f5f2abbb4b355a174dce0ecf8ef71756c203b7d26a", size = 4119441, upload-time = "2026-09-24T15:29:38.636Z" }, - { url = "https://files.pythonhosted.org/packages/3e/a6/0ac54e58f3d0aec31b6ad6fa46bb8d3454540e7f6b066d6832880cf81fbb/fensu-0.23.0-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ed8a57574977971b2f3c599b5beaac9bd905612aba89971b7b1697f790310e5", size = 4184904, upload-time = "2026-09-24T15:29:40.493Z" }, - { url = "https://files.pythonhosted.org/packages/99/57/7bb9c5c2b971c26bcbf1a1c10a54097925cbaff07f3c8bcb9aefad3d3865/fensu-0.23.0-cp312-abi3-win_amd64.whl", hash = "sha256:98d78a6582ea21fc456c18a31b7d01f3ff43b8ed687de9e51ef8497e54ee2514", size = 3864765, upload-time = "2026-09-24T15:29:42.311Z" }, + { url = "https://files.pythonhosted.org/packages/2c/b7/7b4053d22df1307dbaebb4ed48280bb9c3663ee649005eb6ab5e2fec28b8/fensu-0.27.0-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d982c4fec0d2210b78fcf67098c32ca9f8b8592f338b2a327fcd1747b59bb310", size = 4154209, upload-time = "2026-10-17T05:38:28.74Z" }, + { url = "https://files.pythonhosted.org/packages/7a/0d/b7058d187e9908ec18f245c1b168c24b746633a0b13666d69052157da3eb/fensu-0.27.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:8c3cca4a4391028d7b07bb9ead34340f6ba0662c36ea4ef31f69b44a46847289", size = 4004244, upload-time = "2026-10-17T05:38:31.356Z" }, + { url = "https://files.pythonhosted.org/packages/f8/bc/b1eee94e617e954a2188c1802d3770b27624a97f775cbed310f54389c100/fensu-0.27.0-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5640ffe3cf8916b1533dfe51ca27d65bcea71c9fbd62ae5cfaa2079e8007bdaf", size = 4230195, upload-time = "2026-10-17T05:38:33.057Z" }, + { url = "https://files.pythonhosted.org/packages/7c/01/9f895e6a4427c8ee80ce5d779aceab5a2b994a156a4c1f3bf664108ab489/fensu-0.27.0-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7d26d2713a69f2e05b07c6f778e1e2f86896e26b74247e60952018288381c04", size = 4307270, upload-time = "2026-10-17T05:38:34.827Z" }, + { url = "https://files.pythonhosted.org/packages/08/61/2a380a704a49eea91a3539935af7b2d4ad09aa1d87d787f87c1de847ae4b/fensu-0.27.0-cp312-abi3-win_amd64.whl", hash = "sha256:15a9aef82221f25812d9c4f74e782dd677c73d566980eaed60b762da13ee2b3d", size = 3977639, upload-time = "2026-10-17T05:38:36.612Z" }, ] [[package]] name = "fensu-cli" -version = "0.23.0" +version = "0.27.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4a/6f/48493231455bdaceeb637594765c15c46b59252b2526b5ee9478b0c0841c/fensu_cli-0.23.0.tar.gz", hash = "sha256:0a598d0013c662b4bc4e7785dc91f8d480665a06750c2114acbafe8de9ef3dba", size = 715137, upload-time = "2026-09-24T15:29:56.142Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/97/2540b6ea4bca959ac577502fe0c96b432c3b4e694daa1ca68df3a2a06b8a/fensu_cli-0.27.0.tar.gz", hash = "sha256:4276e7fcf7dddc4c36afc347536bf1fd7114fff246ad344544991a1edf7b4ee3", size = 760869, upload-time = "2026-10-17T05:38:50.61Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/34/ce/87ca84074f6ba2f278d2739505c6f6d1cde53b8b66085b373666873607a3/fensu_cli-0.23.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:4ccbf9ed3182344ef9e6fefe1d6d62f0770b825d18059bce1954347dc1d3615e", size = 7745194, upload-time = "2026-09-24T15:29:44.933Z" }, - { url = "https://files.pythonhosted.org/packages/c1/b1/21d23c39b2c8dd51e544b6546fada9bd317214a1e4a6f44b3351cc73d4be/fensu_cli-0.23.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:8f8d1851c1711359ad2688a3aaa1717af8e48a2c51700a54c647082c3fe51e15", size = 7271582, upload-time = "2026-09-24T15:29:46.978Z" }, - { url = "https://files.pythonhosted.org/packages/aa/56/c190cdedf6e2afa67e58bd6cd6188853983f160e14b73494a08906e84869/fensu_cli-0.23.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d21e87ba99a9c600783c7107e2d06cce15cdce379921a8c996884c16b4c8a407", size = 7362814, upload-time = "2026-09-24T15:29:49.078Z" }, - { url = "https://files.pythonhosted.org/packages/74/13/54147ab05067512127c99df415b73667f878693bf4bd50eb1e03fcb84f76/fensu_cli-0.23.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:07ab456f0f8eb90a611e842e3ad1ddf44053b5eb5a5c69d4499f248bd21d01b9", size = 7795295, upload-time = "2026-09-24T15:29:50.828Z" }, - { url = "https://files.pythonhosted.org/packages/70/53/2c0a86d35cb37123dffd5644020cb5284169f83bd0571281e341896b8946/fensu_cli-0.23.0-py3-none-win_amd64.whl", hash = "sha256:8fa9948fd8ee64319a105f70085ddb790b4131bb5ff88636ae6d27ef51b14f57", size = 7490965, upload-time = "2026-09-24T15:29:53.097Z" }, + { url = "https://files.pythonhosted.org/packages/0b/69/9491487602985731af4a7363754970bb35baece4e27cf2a7a0a95ebe4ae7/fensu_cli-0.27.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e90c8d298904d02c639e87541ba51a31ea3912af0bc354cdfea9d95f4428fd41", size = 7935880, upload-time = "2026-10-17T05:38:38.606Z" }, + { url = "https://files.pythonhosted.org/packages/de/4e/7e84f824b3902708ce4c8399f3402fb278cba9c48ed37877d67a4bad5245/fensu_cli-0.27.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1a9f3a2fb8ec8930af9b5cd59f3f1c027cd80a1ebae1a22c7d07133b80d7c65a", size = 7457913, upload-time = "2026-10-17T05:38:40.624Z" }, + { url = "https://files.pythonhosted.org/packages/05/9f/fc20897baafdac0039faef2cc3a324b059fdfe7f4a1522bac386f1e5f8ae/fensu_cli-0.27.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cf73108b3778d0d74e11857e743042f4241174ec8f16aeee7e5c8c4769e41e7b", size = 7537447, upload-time = "2026-10-17T05:38:42.834Z" }, + { url = "https://files.pythonhosted.org/packages/46/be/1fc7c87cbf320ff215dcb204df808c592fefec6a0892a1103564a1c69b76/fensu_cli-0.27.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:435dad914ec0d061d5fceb10d8b049f75cdb5c031a156efe2f8309647c744b8c", size = 8035160, upload-time = "2026-10-17T05:38:45.074Z" }, + { url = "https://files.pythonhosted.org/packages/ef/5b/53050980866370c7a41dc7e3e61de86cf67e5848d8560e6a9d166040679a/fensu_cli-0.27.0-py3-none-win_amd64.whl", hash = "sha256:68e91a38dab7e081c4ba504e22a7709169d8d6c470137834adcad116b52fa280", size = 7652792, upload-time = "2026-10-17T05:38:47.26Z" }, ] [[package]] @@ -4350,7 +4350,7 @@ dev = [ { name = "dbt-snowflake", specifier = "==1.12.0b1" }, { name = "dbt-sqlserver", specifier = ">=1.9.0,<1.11" }, { name = "dlt", extras = ["duckdb", "filesystem", "sql-database"], specifier = ">=1.28.0,<2" }, - { name = "fensu", specifier = ">=0.23.0" }, + { name = "fensu", specifier = ">=0.27.0" }, { name = "ingestr", specifier = ">=1.0.0" }, { name = "jsonschema", specifier = ">=4.23.0" }, { name = "maturin", specifier = ">=1.9,<2" },