From 2cd06dc49d583a43126fa6ac69d267146ee8d62d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 04:37:49 +0000 Subject: [PATCH 01/29] feat(analysis): give native dialects a parse and type contract The native dialect trait now has parse and normalize_type, with read_whole and type_match provided on top. read_whole refuses parser gaps, empty input, more than one statement and statements the parser kept unparsed. type_match reports NotCompared when either side has no normalized type, so the contract check makes no finding instead of guessing. Parser gaps carry the parser's message and whether the failure is a syntax error, and the P026 skip appends that message. AnalysisRoute gains an Unavailable arm, which skips a model with P026 instead of analysing it. The contracts type comparer goes through the route's type_match, and normalize_text_type is public so native dialects can reuse the shared text type rules. --- .../src/analysis_route/constants.rs | 8 ++ .../main/clickhouse_type_match.rs | 2 +- .../src/analysis_route/main/native_gap.rs | 2 + .../analysis_route/main/skipped_analysis.rs | 24 +++- .../src/analysis_route/mod.rs | 4 + .../src/analysis_route/models.rs | 23 +++- .../src/analysis_route/tests.rs | 2 + .../tests/test_clickhouse_type_match.rs | 14 +- .../tests/test_native_dialect.rs | 127 ++++++++++++++++++ .../analysis_route/tests/test_native_gap.rs | 4 + .../src/analysis_route/tests/test_types.rs | 54 ++++++-- .../src/analysis_route/types.rs | 65 ++++++++- .../src/contracts/_helpers/columns.rs | 2 +- .../src/contracts/_helpers/type_comparison.rs | 45 ++++--- .../src/contracts/constants.rs | 2 - .../src/type_system/main/mod.rs | 1 + .../type_system/main/normalize_text_type.rs | 10 ++ .../_helpers/input_loops/model_loop.rs | 11 +- .../input_loops/model_loop_opt_out.rs | 30 ++++- .../_helpers/lineage/project_lineage.rs | 4 +- 20 files changed, 381 insertions(+), 53 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs create mode 100644 crates/sqlbuild-analysis/src/type_system/main/normalize_text_type.rs diff --git a/crates/sqlbuild-analysis/src/analysis_route/constants.rs b/crates/sqlbuild-analysis/src/analysis_route/constants.rs index 1e31d63436..eeb7ce5e8c 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/constants.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/constants.rs @@ -4,6 +4,14 @@ use crate::analysis_route::types::NativeDialect; /// Each dialect, by its adapter's analysis dialect name, whose SQL a native module analyses. pub(crate) const NATIVE_DIALECTS: &[(&str, &dyn NativeDialect)] = &[]; +/// The dialect a project with no analysis dialect is named by. +pub(crate) const GENERIC_DIALECT: &str = "generic"; +/// The gap of SQL that holds more than one statement. +pub(crate) const MORE_THAN_ONE_STATEMENT: &str = "more than one statement"; +/// The gap of SQL that holds no statement. +pub(crate) const NO_STATEMENT: &str = "no statement"; +/// The gap of a statement the parser kept as raw text. +pub(crate) const UNPARSED_STATEMENT: &str = "unparsed statement"; /// The warning, or under `require_sql_analysis` the error, for a model analysis skips. pub(crate) const SKIPPED_ANALYSIS_CODE: &str = "P026"; /// ClickHouse's top-level nullability wrapper, which the native profile spells as a flag. diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/clickhouse_type_match.rs b/crates/sqlbuild-analysis/src/analysis_route/main/clickhouse_type_match.rs index 91223ddd45..a5ee54f86d 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/clickhouse_type_match.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/clickhouse_type_match.rs @@ -6,7 +6,7 @@ use crate::analysis_route::models::TypeMatch; pub fn clickhouse_type_match(declared: &str, inferred: &str, inferred_nullable: bool) -> TypeMatch { let (declared_base, declared_nullable) = unwrapped(declared); let (inferred_base, wrapped_nullable) = unwrapped(inferred); - TypeMatch { + TypeMatch::Compared { same_type: declared_base == inferred_base, nullable_into_non_null: !declared_nullable && (inferred_nullable || wrapped_nullable), } diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/native_gap.rs b/crates/sqlbuild-analysis/src/analysis_route/main/native_gap.rs index 2d2c02726f..e7c3f74397 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/native_gap.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/native_gap.rs @@ -27,7 +27,9 @@ fn located(sql: &str, gap: ParserGap) -> LocatedGap { let position: TextPosition = text_position(sql, before.chars().count()); LocatedGap { construct: gap.construct, + message: gap.message, line: position.line, column: position.column, + syntax: gap.syntax, } } diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/skipped_analysis.rs b/crates/sqlbuild-analysis/src/analysis_route/main/skipped_analysis.rs index 3fba7a616f..eacef989f7 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/skipped_analysis.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/skipped_analysis.rs @@ -1,15 +1,33 @@ -use crate::analysis_route::constants::SKIPPED_ANALYSIS_CODE; +use crate::analysis_route::constants::{GENERIC_DIALECT, SKIPPED_ANALYSIS_CODE}; use crate::analysis_route::models::LocatedGap; -/// P026's code and message for a model whose SQL the dialect's native parser cannot read. +/// P026's code and message for a model whose SQL the dialect's native parser cannot read, with +/// the parser's own message when it says more than the construct's name. #[must_use] pub fn skipped_analysis(model: &str, dialect: &str, gap: &LocatedGap) -> (&'static str, String) { + let detail: String = if gap.message.is_empty() || gap.message == gap.construct { + String::new() + } else { + format!(": {}", gap.message) + }; ( SKIPPED_ANALYSIS_CODE, format!( "SQL analysis skipped for model '{model}': the {dialect} parser cannot read {} at \ - line {}, column {}", + line {}, column {}{detail}", gap.construct, gap.line, gap.column ), ) } + +/// P026's code and message for a model whose dialect no native parser reads. +#[must_use] +pub fn unavailable_analysis(model: &str, dialect: Option<&str>) -> (&'static str, String) { + ( + SKIPPED_ANALYSIS_CODE, + format!( + "SQL analysis skipped for model '{model}': no native parser reads the {} dialect", + dialect.unwrap_or(GENERIC_DIALECT) + ), + ) +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/mod.rs b/crates/sqlbuild-analysis/src/analysis_route/mod.rs index 76ff5d49a0..2e8da84999 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/mod.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/mod.rs @@ -5,5 +5,9 @@ pub mod main; pub mod models; pub mod types; +pub use crate::analysis_route::main::analysis_route::analysis_route; +pub use crate::analysis_route::models::{AnalysisRoute, ParserGap, TypeMatch}; +pub use crate::analysis_route::types::NativeDialect; + #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-analysis/src/analysis_route/models.rs b/crates/sqlbuild-analysis/src/analysis_route/models.rs index a93a53d297..0d4e8f6f57 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/models.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/models.rs @@ -9,27 +9,40 @@ pub enum AnalysisRoute { Legacy, /// The dialect's native module; a model it cannot read whole is not analysed. Native(&'static dyn NativeDialect), + /// No native module reads the dialect: nothing is analysed, and no Polyglot path stands in. + Unavailable, } -/// The first construct a native parser cannot read, and its byte offset in the SQL it read. +/// The first construct or error that stops a native parser, at its byte offset in the SQL it read. #[derive(Clone, Debug, PartialEq, Eq)] pub struct ParserGap { + /// The construct, such as `PIVOT`, or the parser's error message. pub construct: String, + /// The parser's bare message, without the line and column an error's `Display` adds. + pub message: String, pub offset: usize, + /// Whether the parser reports invalid syntax; false for an unsupported construct or a limit. + pub syntax: bool, } /// A gap with the line and column, from 1, of its offset in the analysed SQL. #[derive(Clone, Debug, PartialEq, Eq)] pub struct LocatedGap { pub construct: String, + pub message: String, pub line: usize, pub column: usize, + pub syntax: bool, } /// How an inferred column type meets its declared type: the type itself, then its nullability. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct TypeMatch { - pub same_type: bool, - /// The inferred column may be null, but its declared type does not allow it. - pub nullable_into_non_null: bool, +pub enum TypeMatch { + /// One side is a type the dialect cannot spell or refuses: no finding and no match. + NotCompared, + Compared { + same_type: bool, + /// The inferred column may be null, but its declared type does not allow it. + nullable_into_non_null: bool, + }, } diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests.rs b/crates/sqlbuild-analysis/src/analysis_route/tests.rs index b8464bdb55..93f21c6de3 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/test_clickhouse_type_match.rs"] mod test_clickhouse_type_match; +#[path = "tests/test_native_dialect.rs"] +mod test_native_dialect; #[path = "tests/test_native_gap.rs"] mod test_native_gap; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_clickhouse_type_match.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_clickhouse_type_match.rs index e23bab65e4..4e55839f60 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_clickhouse_type_match.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_clickhouse_type_match.rs @@ -11,7 +11,7 @@ fn given_clickhouse_column_types_when_matching_then_wrappers_are_stripped_and_nu declared: "Nullable(String)", inferred: "String", inferred_nullable: true, - expected_match: TypeMatch { + expected_match: TypeMatch::Compared { same_type: true, nullable_into_non_null: false, }, @@ -21,7 +21,7 @@ fn given_clickhouse_column_types_when_matching_then_wrappers_are_stripped_and_nu declared: "LowCardinality(String)", inferred: "String", inferred_nullable: false, - expected_match: TypeMatch { + expected_match: TypeMatch::Compared { same_type: true, nullable_into_non_null: false, }, @@ -31,7 +31,7 @@ fn given_clickhouse_column_types_when_matching_then_wrappers_are_stripped_and_nu declared: "LowCardinality(Nullable(String))", inferred: "Nullable(String)", inferred_nullable: false, - expected_match: TypeMatch { + expected_match: TypeMatch::Compared { same_type: true, nullable_into_non_null: false, }, @@ -41,7 +41,7 @@ fn given_clickhouse_column_types_when_matching_then_wrappers_are_stripped_and_nu declared: "String", inferred: "String", inferred_nullable: true, - expected_match: TypeMatch { + expected_match: TypeMatch::Compared { same_type: true, nullable_into_non_null: true, }, @@ -51,7 +51,7 @@ fn given_clickhouse_column_types_when_matching_then_wrappers_are_stripped_and_nu declared: "Nullable(Int64)", inferred: "Int64", inferred_nullable: false, - expected_match: TypeMatch { + expected_match: TypeMatch::Compared { same_type: true, nullable_into_non_null: false, }, @@ -61,7 +61,7 @@ fn given_clickhouse_column_types_when_matching_then_wrappers_are_stripped_and_nu declared: "Nullable(Int32)", inferred: "Int64", inferred_nullable: true, - expected_match: TypeMatch { + expected_match: TypeMatch::Compared { same_type: false, nullable_into_non_null: false, }, @@ -71,7 +71,7 @@ fn given_clickhouse_column_types_when_matching_then_wrappers_are_stripped_and_nu declared: "Array(Nullable(String))", inferred: "Array(String)", inferred_nullable: false, - expected_match: TypeMatch { + expected_match: TypeMatch::Compared { same_type: false, nullable_into_non_null: false, }, diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs new file mode 100644 index 0000000000..18baa18a2a --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs @@ -0,0 +1,127 @@ +use crate::analysis_route::main::skipped_analysis::{skipped_analysis, unavailable_analysis}; +use crate::analysis_route::models::{LocatedGap, TypeMatch}; +use crate::analysis_route::tests::test_types::{ + NativeTypeMatchTestCase, PivotlessDialect, ReadWholeTestCase, +}; +use crate::analysis_route::types::NativeDialect; + +#[test] +fn given_sql_when_reading_it_whole_then_only_one_parsed_statement_without_a_gap_is_read() { + let test_cases = [ + ReadWholeTestCase { + description: "one query is read whole", + sql: "SELECT a FROM t", + expected_gap: None, + }, + ReadWholeTestCase { + description: "two statements are not one model", + sql: "SELECT 1; SELECT 2", + expected_gap: Some("more than one statement"), + }, + ReadWholeTestCase { + description: "no statement at all", + sql: " ", + expected_gap: Some("no statement"), + }, + ReadWholeTestCase { + description: "a statement kept as raw text is not read", + sql: "RAW VACUUM t", + expected_gap: Some("unparsed statement"), + }, + ReadWholeTestCase { + description: "a construct the parser refuses stops it first", + sql: "SELECT * FROM t PIVOT (sum(x) FOR y IN (1))", + expected_gap: Some("PIVOT"), + }, + ]; + for test_case in test_cases { + assert_eq!( + PivotlessDialect + .read_whole(test_case.sql) + .err() + .map(|gap| gap.construct), + test_case.expected_gap.map(str::to_owned), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_type_the_dialect_refuses_when_matching_types_then_nothing_is_compared() { + let test_cases = [ + NativeTypeMatchTestCase { + description: "a refused declared type is not compared", + declared: "NOT_A_TYPE", + inferred: "INTEGER", + expected_match: TypeMatch::NotCompared, + }, + NativeTypeMatchTestCase { + description: "a refused inferred type is not compared", + declared: "INTEGER", + inferred: "NOT_A_TYPE", + expected_match: TypeMatch::NotCompared, + }, + NativeTypeMatchTestCase { + description: "two known, different types still mismatch", + declared: "INTEGER", + inferred: "TEXT", + expected_match: TypeMatch::Compared { + same_type: false, + nullable_into_non_null: false, + }, + }, + NativeTypeMatchTestCase { + description: "the same type matches", + declared: "INTEGER", + inferred: "integer", + expected_match: TypeMatch::Compared { + same_type: true, + nullable_into_non_null: false, + }, + }, + ]; + for test_case in test_cases { + assert_eq!( + PivotlessDialect.type_match(test_case.declared, test_case.inferred, true), + test_case.expected_match, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_parser_message_or_no_parser_when_skipping_analysis_then_p026_says_why() { + let syntax: LocatedGap = LocatedGap { + construct: "syntax error".to_owned(), + message: "syntax error at or near \"FROM\"".to_owned(), + line: 1, + column: 15, + syntax: true, + }; + assert_eq!( + skipped_analysis("orders", "postgres", &syntax), + ( + "P026", + "SQL analysis skipped for model 'orders': the postgres parser cannot read syntax \ + error at line 1, column 15: syntax error at or near \"FROM\"" + .to_owned() + ), + "the parser's message follows the construct" + ); + assert_eq!( + unavailable_analysis("orders", Some("trino")), + ( + "P026", + "SQL analysis skipped for model 'orders': no native parser reads the trino dialect" + .to_owned() + ), + "an unknown dialect is named" + ); + assert_eq!( + unavailable_analysis("orders", None).1, + "SQL analysis skipped for model 'orders': no native parser reads the generic dialect", + "no dialect is the generic dialect" + ); +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_gap.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_gap.rs index 75a1600999..a69a2d3638 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_gap.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_gap.rs @@ -105,8 +105,10 @@ fn given_model_sql_when_probing_the_native_parser_then_its_gap_is_located_after_ placeholders: &[], expected_gap: Some(LocatedGap { construct: "PIVOT".to_owned(), + message: "PIVOT".to_owned(), line: 3, column: 3, + syntax: false, }), }, ]; @@ -136,8 +138,10 @@ fn given_a_located_gap_when_skipping_analysis_then_p026_names_the_model_and_the_ dialect: "postgres", gap: LocatedGap { construct: "PIVOT".to_owned(), + message: "PIVOT".to_owned(), line: 3, column: 3, + syntax: false, }, expected_code: "P026", expected_message: "SQL analysis skipped for model 'orders_pivot': the postgres parser \ diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs index 9587ebda22..69fd87107b 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs @@ -1,5 +1,10 @@ +use sqlquack_syntax::expressions::statements::models::Command; +use sqlquack_syntax::{Expression, Parsed}; + use crate::analysis_route::models::{LocatedGap, ParserGap, TypeMatch}; use crate::analysis_route::types::NativeDialect; +use crate::type_system::main::normalize_text_type::normalize_text_type; +use crate::type_system::models::NormalizedType; pub(super) struct NativeGapTestCase { pub(super) description: &'static str, @@ -23,7 +28,8 @@ pub(super) struct SkippedAnalysisTestCase { pub(super) expected_message: &'static str, } -/// A test dialect whose parser refuses PIVOT, unfilled placeholders and one-part stubs. +/// A test dialect over the PostgreSQL parser that also refuses PIVOT, unfilled placeholders and +/// one-part stubs, keeps SQL after `RAW ` as raw text, and refuses the type `NOT_A_TYPE`. pub(super) struct PivotlessDialect; impl NativeDialect for PivotlessDialect { @@ -31,25 +37,57 @@ impl NativeDialect for PivotlessDialect { "postgres" } - fn read_whole(&self, sql: &str) -> Result<(), ParserGap> { + fn stub_prefix(&self) -> &'static str { + "__sqlbuild." + } + + fn parse(&self, sql: &str) -> Result { let refused = ["PIVOT", "@@@", "__sqlbuild_udf_", "__cursor_", "@'"] .iter() .filter_map(|word| sql.find(word).map(|offset| (offset, *word))) .min(); - match refused { - Some((offset, word)) => Err(ParserGap { + if let Some(text) = sql.strip_prefix("RAW ") { + return Ok(Parsed { + statements: vec![Expression::Command(Box::new(Command { + this: text.to_owned(), + }))], + gaps: Vec::new(), + }); + } + if let Some((offset, word)) = refused { + return Err(ParserGap { construct: word.to_owned(), + message: word.to_owned(), offset, - }), - None => Ok(()), + syntax: false, + }); } + sqlquack_syntax_postgres::parse(sql).map_err(|error| ParserGap { + construct: error.message.clone(), + message: error.message, + offset: error.position, + syntax: error.syntax, + }) } - fn stub_prefix(&self) -> &'static str { - "__sqlbuild." + fn normalize_type(&self, spelling: &str) -> Option { + (!spelling.eq_ignore_ascii_case("NOT_A_TYPE")).then(|| normalize_text_type(spelling, None)) } } +pub(super) struct ReadWholeTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) expected_gap: Option<&'static str>, +} + +pub(super) struct NativeTypeMatchTestCase { + pub(super) description: &'static str, + pub(super) declared: &'static str, + pub(super) inferred: &'static str, + pub(super) expected_match: TypeMatch, +} + pub(super) struct TypeMatchTestCase { pub(super) description: &'static str, pub(super) declared: &'static str, diff --git a/crates/sqlbuild-analysis/src/analysis_route/types.rs b/crates/sqlbuild-analysis/src/analysis_route/types.rs index fedc80ad24..b358e77278 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/types.rs @@ -1,15 +1,72 @@ //! What a dialect's native analysis module provides. -use crate::analysis_route::models::ParserGap; +use sqlquack_syntax::{Expression, Parsed}; + +use crate::analysis_route::constants::{MORE_THAN_ONE_STATEMENT, NO_STATEMENT, UNPARSED_STATEMENT}; +use crate::analysis_route::models::{ParserGap, TypeMatch}; +use crate::type_system::models::NormalizedType; /// A dialect whose SQL SQLQuack analyses in place of Polyglot. pub trait NativeDialect: Sync { /// The adapter's analysis dialect name, such as `postgres`. fn name(&self) -> &'static str; - /// Whether the native parser reads `sql` whole: one statement, no gap and no unparsed bytes. - fn read_whole(&self, sql: &str) -> Result<(), ParserGap>; - /// The engine-accepted prefix of UDF and cursor stubs: `__sqlbuild_`, or `__sqlbuild.` for T-SQL. fn stub_prefix(&self) -> &'static str; + + /// Every statement of `sql` with the parser's named gaps, or the first error that stops it. + fn parse(&self, sql: &str) -> Result; + + /// A type as written in this dialect, as SQLBuild compares types; `None` when the dialect + /// cannot spell it or its engine refuses it, and nothing is compared against it. + fn normalize_type(&self, spelling: &str) -> Option; + + /// The one statement of `sql` when the parser reads it whole: no gap, and not raw text. + fn read_whole(&self, sql: &str) -> Result { + let parsed: Parsed = self.parse(sql)?; + if let Some(gap) = parsed.gaps.first() { + return Err(ParserGap { + construct: gap.construct.to_owned(), + message: gap.construct.to_owned(), + offset: gap.start, + syntax: false, + }); + } + let mut statements = parsed.statements.into_iter(); + let (Some(statement), None) = (statements.next(), statements.next()) else { + let construct: &str = if sql.trim().is_empty() { + NO_STATEMENT + } else { + MORE_THAN_ONE_STATEMENT + }; + return Err(whole_gap(construct, 0)); + }; + match statement { + Expression::Command(_) => Err(whole_gap(UNPARSED_STATEMENT, 0)), + statement => Ok(statement), + } + } + + /// Whether a declared and an inferred type match, nullability apart; `NotCompared` when + /// `normalize_type` gives `None` for either. An override must keep that rule. + fn type_match(&self, declared: &str, inferred: &str, inferred_nullable: bool) -> TypeMatch { + let _ = inferred_nullable; + match (self.normalize_type(declared), self.normalize_type(inferred)) { + (Some(declared), Some(inferred)) => TypeMatch::Compared { + same_type: declared == inferred, + nullable_into_non_null: false, + }, + _ => TypeMatch::NotCompared, + } + } +} + +/// A gap `read_whole` names itself, which no parser reports as invalid syntax. +fn whole_gap(construct: &str, offset: usize) -> ParserGap { + ParserGap { + construct: construct.to_owned(), + message: construct.to_owned(), + offset, + syntax: false, + } } diff --git a/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs b/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs index a5273a5217..811f5ebce4 100644 --- a/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs +++ b/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs @@ -310,7 +310,7 @@ fn column_type( }, })); }; - if check.types.types_equal(declared_type, inferred_type)? == TypeComparison::Equal { + if check.types.types_equal(declared_type, inferred_type)? != TypeComparison::Different { return Ok(None); } Ok(Some(ContractDiagnostic { diff --git a/crates/sqlbuild-analysis/src/contracts/_helpers/type_comparison.rs b/crates/sqlbuild-analysis/src/contracts/_helpers/type_comparison.rs index e439b2d73e..5b3513bcb7 100644 --- a/crates/sqlbuild-analysis/src/contracts/_helpers/type_comparison.rs +++ b/crates/sqlbuild-analysis/src/contracts/_helpers/type_comparison.rs @@ -4,9 +4,8 @@ use std::cell::RefCell; use std::collections::HashMap; use crate::analysis_route::main::analysis_route::analysis_route; -use crate::analysis_route::main::clickhouse_type_match::clickhouse_type_match; -use crate::analysis_route::models::AnalysisRoute; -use crate::contracts::constants::{CLICKHOUSE_DIALECT, UPPER_FOLDING_DIALECT}; +use crate::analysis_route::models::{AnalysisRoute, TypeMatch}; +use crate::contracts::constants::UPPER_FOLDING_DIALECT; use crate::contracts::models::{DeclaredColumn, InferredOutputColumn}; use crate::type_system::errors::TypeNormalizationError; use crate::type_system::main::normalize_type::normalize_type; @@ -17,13 +16,15 @@ use crate::type_system::models::{NormalizedType, TypeNormalization}; pub(crate) enum TypeComparison { Equal, Different, + /// A native route cannot spell one of the types, or has no parser: no finding either way. + NotCompared, } /// One request's type comparisons, normalizing each type string once as Python's cache does. pub(crate) struct TypeComparer<'a> { dialect: &'a str, - /// ClickHouse on its native route, whose types are compared without top-level wrappers. - native_clickhouse: bool, + /// The dialect's route: a native module compares types itself, and no route compares none. + route: AnalysisRoute, normalized: RefCell>>, } @@ -36,8 +37,7 @@ impl<'a> TypeComparer<'a> { pub(crate) fn new(dialect: &'a str) -> Self { Self { dialect, - native_clickhouse: dialect == CLICKHOUSE_DIALECT - && matches!(analysis_route(Some(dialect)), AnalysisRoute::Native(_)), + route: analysis_route(Some(dialect)), normalized: RefCell::new(HashMap::new()), } } @@ -48,12 +48,18 @@ impl<'a> TypeComparer<'a> { left: &str, right: &str, ) -> Result { - if self.native_clickhouse { - return Ok(if clickhouse_type_match(left, right, false).same_type { - TypeComparison::Equal - } else { - TypeComparison::Different - }); + match self.route { + AnalysisRoute::Legacy => {} + AnalysisRoute::Unavailable => return Ok(TypeComparison::NotCompared), + AnalysisRoute::Native(dialect) => { + return Ok(match dialect.type_match(left, right, false) { + TypeMatch::NotCompared => TypeComparison::NotCompared, + TypeMatch::Compared { + same_type: true, .. + } => TypeComparison::Equal, + TypeMatch::Compared { .. } => TypeComparison::Different, + }); + } } let left: NormalizedType = self.normalized(left)?; let right: NormalizedType = self.normalized(right)?; @@ -73,15 +79,20 @@ impl<'a> TypeComparer<'a> { if declared.not_null && inferred.nullable { return true; } - let (true, Some(declared_type), Some(inferred_type)) = ( - self.native_clickhouse, + let (AnalysisRoute::Native(dialect), Some(declared_type), Some(inferred_type)) = ( + self.route, declared.declared_type.as_deref(), inferred.inferred_type.as_deref(), ) else { return false; }; - clickhouse_type_match(declared_type, inferred_type, inferred.nullable) - .nullable_into_non_null + matches!( + dialect.type_match(declared_type, inferred_type, inferred.nullable), + TypeMatch::Compared { + nullable_into_non_null: true, + .. + } + ) } fn normalized(&self, type_sql: &str) -> Result { diff --git a/crates/sqlbuild-analysis/src/contracts/constants.rs b/crates/sqlbuild-analysis/src/contracts/constants.rs index 08cb6e0278..c1edad4679 100644 --- a/crates/sqlbuild-analysis/src/contracts/constants.rs +++ b/crates/sqlbuild-analysis/src/contracts/constants.rs @@ -42,7 +42,5 @@ pub(crate) const PROMOTION_REMOVE_CONTRACT_HELP: &str = "or remove the model's e pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; pub(crate) const ADDITIONAL_HELP_SEPARATOR: &str = "\n = help: "; -/// The analysis dialect name whose native route compares ClickHouse types without wrappers. -pub(crate) const CLICKHOUSE_DIALECT: &str = "clickhouse"; /// The dialect that folds unquoted identifiers to upper case. pub(crate) const UPPER_FOLDING_DIALECT: &str = "snowflake"; diff --git a/crates/sqlbuild-analysis/src/type_system/main/mod.rs b/crates/sqlbuild-analysis/src/type_system/main/mod.rs index 7434eded53..1ac950cb8f 100644 --- a/crates/sqlbuild-analysis/src/type_system/main/mod.rs +++ b/crates/sqlbuild-analysis/src/type_system/main/mod.rs @@ -1,3 +1,4 @@ +pub mod normalize_text_type; pub mod normalize_type; pub mod polyglot_type_args; pub mod same_type; diff --git a/crates/sqlbuild-analysis/src/type_system/main/normalize_text_type.rs b/crates/sqlbuild-analysis/src/type_system/main/normalize_text_type.rs new file mode 100644 index 0000000000..51e6458938 --- /dev/null +++ b/crates/sqlbuild-analysis/src/type_system/main/normalize_text_type.rs @@ -0,0 +1,10 @@ +//! `normalize_text_type`: a type's comparison shape read from its text alone, without Polyglot. + +use crate::type_system::_helpers::fallback::normalize_with_fallback; +use crate::type_system::models::{NormalizedType, TypeDialect}; + +/// `type_sql`'s name, family and parameters as the text fallback reads them under `dialect`. +#[must_use] +pub fn normalize_text_type(type_sql: &str, dialect: Option) -> NormalizedType { + normalize_with_fallback(type_sql, dialect) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs index c61261dd75..b75a525fa2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs @@ -73,7 +73,8 @@ use crate::bindings::_helpers::input_loops::model_loop_macros::{ }; use crate::bindings::_helpers::input_loops::model_loop_named::{SCHEMA_DECLARATION, named_usages}; use crate::bindings::_helpers::input_loops::model_loop_opt_out::{ - OptOut, OptOutModel, reject_unparseable_query, rejected_opt_out, skip_unreadable, + OptOut, OptOutModel, reject_unparseable_query, rejected_opt_out, skip_unavailable, + skip_unreadable, }; use crate::bindings::_helpers::input_loops::model_loop_prepass::{ NativeHeader, all_header_facts, native_header, @@ -204,6 +205,8 @@ pub(crate) struct NativeModelLoop { pub(crate) project_config_path: String, /// Whether the dialect's SQL is analysed by Polyglot or by its native module. pub(crate) route: AnalysisRoute, + /// The adapter's analysis dialect name, which P026 names when no native parser reads it. + pub(crate) dialect: Option, contract_classes: Py, loop_classes: Py, } @@ -265,6 +268,7 @@ impl NativeModelLoop { }, project_config_path, route: analysis_route(dialect.as_deref()), + dialect, contract_classes, loop_classes, }) @@ -837,6 +841,11 @@ impl NativeModelLoop { skipped = !read; (read, None) } + AnalysisRoute::Unavailable => { + skip_unavailable(self.settings, self.dialect.as_deref(), context, model)?; + skipped = true; + (false, None) + } }, AnalysisGate::Disabled => match rejected_opt_out(self, context, model)? { OptOut::Accepted => (false, None), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs index b392bc8aaf..234f5502b4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs @@ -5,7 +5,9 @@ use std::path::Path; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyBoolMethods, PyDict, PyDictMethods, PyList, PyString, PyTuple}; use sqlbuild_analysis::analysis_route::main::native_gap::native_gap; -use sqlbuild_analysis::analysis_route::main::skipped_analysis::skipped_analysis; +use sqlbuild_analysis::analysis_route::main::skipped_analysis::{ + skipped_analysis, unavailable_analysis, +}; use sqlbuild_analysis::analysis_route::models::{AnalysisRoute, LocatedGap}; use sqlbuild_analysis::analysis_route::types::NativeDialect; use sqlbuild_analysis::assembly::project::models::{SyntaxCheck, SyntaxMode}; @@ -77,6 +79,7 @@ pub(crate) fn rejected_opt_out<'py>( let readable: bool = match model_loop.route { AnalysisRoute::Legacy => syntax_valid(model.config.py(), GENERIC_DIALECT, &checks)?, AnalysisRoute::Native(dialect) => first_gap(dialect, context, &checks)?.is_none(), + AnalysisRoute::Unavailable => false, }; if readable { Ok(OptOut::Rejected(location(model_loop, context, model)?)) @@ -97,6 +100,29 @@ pub(crate) fn skip_unreadable( return Ok(true); }; let (code, message) = skipped_analysis(model.name, dialect.name(), &gap); + report_skip(settings, context, model, code, message)?; + Ok(false) +} + +/// P026 for a model whose dialect no native parser reads; it warns, or raises, and is not analysed. +pub(crate) fn skip_unavailable( + settings: ModelLoopSettings, + dialect: Option<&str>, + context: &LoopContext<'_, '_>, + model: &OptOutModel<'_, '_>, +) -> PyResult<()> { + let (code, message) = unavailable_analysis(model.name, dialect); + report_skip(settings, context, model, code, message) +} + +/// Reports P026 as a warning, or raises it under `require_sql_analysis`. +fn report_skip( + settings: ModelLoopSettings, + context: &LoopContext<'_, '_>, + model: &OptOutModel<'_, '_>, + code: &str, + message: String, +) -> PyResult<()> { let py: Python<'_> = model.config.py(); let help: &str = &context.classes.syntax_help; if settings.require_sql_analysis { @@ -134,7 +160,7 @@ pub(crate) fn skip_unreadable( .classes .report_diagnostic .call((), Some(&arguments))?; - Ok(false) + Ok(()) } /// The model's analysis SQL and each hook statement, as filled to run, with its placeholders. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs index 30e49894fc..7bc1ff9fd9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs @@ -170,9 +170,9 @@ fn fast_lineage<'py>( let mut work: Vec> = Vec::with_capacity(models.len()); let mut requests: Vec = Vec::new(); let dialect_name: Option = dialect.map(|name| name.extract()).transpose()?; - let native: bool = matches!( + let native: bool = !matches!( analysis_route(dialect_name.as_deref()), - AnalysisRoute::Native(_) + AnalysisRoute::Legacy ); for model in models { let (has_star, star_resolved): (bool, bool) = star_flags(model)?; From b320f8e305a9a5d373ea99bc131a8907e637ef25 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 07:34:27 +0000 Subject: [PATCH 02/29] feat(analysis): analyse native dialect queries with sqlquack-semantics A native dialect now names its sqlquack-semantics profile, and the route provides analyse and macro_body on top of read_whole. analyse turns the query's lineage and findings into the facts the analysis session keeps: output columns typed through the dialect's normalize_type, lineage rows over the model's referenced resources, the columns deciding rows, the star flags, and findings in SQLBuild's codes through the same sqlbuild_code the Polyglot path uses, so implicit-conversion findings stay blocking. Unknown functions a dialect declares as extensions are not reported. StandInSchema gives probe queries, such as a macro body's, one relation of known columns. read_whole now reports SQL with no statement, such as a lone comment, as no statement rather than more than one. --- Cargo.lock | 38 ++- Cargo.toml | 2 + crates/sqlbuild-analysis/Cargo.toml | 1 + .../src/analysis_route/_helpers/mod.rs | 1 + .../_helpers/native_analysis.rs | 169 +++++++++++++ .../src/analysis_route/constants.rs | 8 + .../src/analysis_route/mod.rs | 5 +- .../src/analysis_route/models.rs | 75 ++++++ .../tests/test_native_dialect.rs | 238 +++++++++++++++++- .../src/analysis_route/tests/test_types.rs | 60 ++++- .../src/analysis_route/types.rs | 54 +++- .../_helpers/diagnostics.rs | 2 +- .../semantic_validation/main/diagnostics.rs | 7 + 13 files changed, 640 insertions(+), 20 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs create mode 100644 crates/sqlbuild-analysis/src/analysis_route/_helpers/native_analysis.rs diff --git a/Cargo.lock b/Cargo.lock index 6269367f6c..7a4ed51429 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4091,7 +4091,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -4142,6 +4142,7 @@ dependencies = [ "sqlbuild-scopes", "sqlbuild-sqltext", "sqlclean", + "sqlquack-semantics", "sqlquack-syntax", "sqlquack-syntax-postgres", ] @@ -4496,6 +4497,37 @@ dependencies = [ "sha2 0.10.9", ] +[[package]] +name = "sqlquack-core" +version = "0.2.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=a9dde6a00b144c6306425cfa84516ad816d8c49c#a9dde6a00b144c6306425cfa84516ad816d8c49c" +dependencies = [ + "serde", + "serde_json", + "sqlquack-syntax", +] + +[[package]] +name = "sqlquack-semantics" +version = "0.2.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=a9dde6a00b144c6306425cfa84516ad816d8c49c#a9dde6a00b144c6306425cfa84516ad816d8c49c" +dependencies = [ + "serde", + "serde_json", + "sqlquack-core", + "sqlquack-snowflake", + "sqlquack-syntax", +] + +[[package]] +name = "sqlquack-snowflake" +version = "0.2.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=a9dde6a00b144c6306425cfa84516ad816d8c49c#a9dde6a00b144c6306425cfa84516ad816d8c49c" +dependencies = [ + "serde", + "sqlquack-core", +] + [[package]] name = "sqlquack-syntax" version = "0.2.0" @@ -4618,7 +4650,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -5054,7 +5086,7 @@ checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" dependencies = [ "memoffset", "tempfile", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 65a30d1f3a..e5e3ab99ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -176,6 +176,8 @@ sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" } sqlbuild-state = { path = "crates/sqlbuild-state" } # SQL lint Rules, quality Rules and the formatter. sqlclean = { git = "https://github.com/sqlstack-io/sqlclean.git", rev = "cf79656d247277a30bc81ef39b6dee420e721221" } +# Scopes, name resolution, types and lineage for the native analysis route. +sqlquack-semantics = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "a9dde6a00b144c6306425cfa84516ad816d8c49c" } # The lossless SQL tokenizer behind query fingerprints. sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "a9dde6a00b144c6306425cfa84516ad816d8c49c" } sqlquack-syntax-postgres = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "a9dde6a00b144c6306425cfa84516ad816d8c49c" } diff --git a/crates/sqlbuild-analysis/Cargo.toml b/crates/sqlbuild-analysis/Cargo.toml index 80a5ae2313..95c643ebd3 100644 --- a/crates/sqlbuild-analysis/Cargo.toml +++ b/crates/sqlbuild-analysis/Cargo.toml @@ -26,6 +26,7 @@ sqlbuild-render.workspace = true sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true sqlclean.workspace = true +sqlquack-semantics.workspace = true sqlquack-syntax.workspace = true sqlquack-syntax-postgres.workspace = true diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs new file mode 100644 index 0000000000..801bee7312 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod native_analysis; diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/native_analysis.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/native_analysis.rs new file mode 100644 index 0000000000..e03c775d96 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/native_analysis.rs @@ -0,0 +1,169 @@ +//! The provided native analysis: sqlquack-semantics lineage and findings as SQLBuild's facts. + +use sqlquack_semantics::lineage::models::{ + LineageContext, NameFinding, Nullability, OutputLineage, QueryLineage, TransformKind, +}; +use sqlquack_semantics::{statement_lineage, statement_name_findings}; +use sqlquack_syntax::Expression; + +use crate::analysis_route::constants::{ERROR_SEVERITY, UNKNOWN_FUNCTION_CODE}; +use crate::analysis_route::models::{AnalysisInput, NativeAnalysis}; +use crate::analysis_route::types::NativeDialect; +use crate::assembly::analysis_session::constants::{ + CONFIDENCE_HIGH, CONFIDENCE_MEDIUM, CONFIDENCE_UNKNOWN, NON_NULL_NULLABILITY, + NULLABLE_NULLABILITY, TRANSFORM_AGGREGATION, TRANSFORM_CAST, TRANSFORM_CONSTANT, + TRANSFORM_DIRECT, TRANSFORM_EXPRESSION, UNKNOWN_NULLABILITY, +}; +use crate::assembly::analysis_session::models::{ColumnFact, LineageRow}; +use crate::semantic_validation::main::diagnostics::sqlbuild_code; +use crate::semantic_validation::models::DiagnosticRow; + +/// The analysis of `statement`, the one statement of `input.sql`, under `dialect`'s profile. +pub(crate) fn native_analysis( + dialect: &D, + statement: &Expression, + input: &AnalysisInput<'_>, +) -> NativeAnalysis { + let context: LineageContext<'_> = LineageContext { + profile: dialect.profile(), + schema: input.schema, + sql: Some(input.sql), + }; + let lineage: QueryLineage = statement_lineage(statement, context); + NativeAnalysis { + columns: lineage + .outputs + .iter() + .map(|output| column_fact(dialect, output)) + .collect(), + lineage: lineage + .outputs + .iter() + .map(|output| lineage_row(output, input.references)) + .collect(), + row_sources: lineage + .row_sources + .into_iter() + .map(|source| (source.relation, source.column)) + .collect(), + has_star: lineage.projects_star, + star_resolved: !lineage.open_star, + diagnostics: statement_name_findings(statement, context) + .into_iter() + .filter_map(|finding| diagnostic_row(input, finding)) + .collect(), + } +} + +/// An output column typed through the dialect, so both routes spell its type alike. +fn column_fact(dialect: &D, output: &OutputLineage) -> ColumnFact { + ColumnFact { + name: output.name.clone(), + data_type: output + .data_type + .as_deref() + .and_then(|spelling| dialect.normalize_type(spelling)) + .map(|normalized| normalized.normalized_name), + nullability: match output.nullability { + Nullability::Unknown => UNKNOWN_NULLABILITY, + Nullability::Nullable => NULLABLE_NULLABILITY, + Nullability::NonNull => NON_NULL_NULLABILITY, + } + .to_owned(), + } +} + +/// An output column's lineage over the model's referenced resources, as the legacy rows give it: +/// columns of relations the model does not reference are left out, and a column reading none of +/// its references is of unknown confidence unless it is a constant. +fn lineage_row(output: &OutputLineage, references: &[(String, String, String)]) -> LineageRow { + let transform_code: u8 = match output.kind { + TransformKind::Constant => TRANSFORM_CONSTANT, + TransformKind::Direct => TRANSFORM_DIRECT, + TransformKind::Cast => TRANSFORM_CAST, + TransformKind::Expression => TRANSFORM_EXPRESSION, + TransformKind::Aggregation => TRANSFORM_AGGREGATION, + }; + let sources: Vec<(String, String, String)> = output + .sources + .iter() + .filter_map(|source| { + references + .iter() + .find(|(analysis_name, _, _)| *analysis_name == source.relation) + .map(|(_, resource_type, resource_name)| { + ( + resource_type.clone(), + resource_name.clone(), + source.column.clone(), + ) + }) + }) + .collect(); + let confidence_code: u8 = if sources.is_empty() && transform_code != TRANSFORM_CONSTANT { + CONFIDENCE_UNKNOWN + } else if output.resolved { + CONFIDENCE_HIGH + } else { + CONFIDENCE_MEDIUM + }; + LineageRow { + output_column: output.name.clone(), + transform_code, + confidence_code, + sources, + } +} + +/// A finding in SQLBuild's code at its position in `input.sql`; `None` for a code SQLBuild does +/// not report, and for an unknown function the dialect declares as an extension. +fn diagnostic_row(input: &AnalysisInput<'_>, finding: NameFinding) -> Option { + if finding.code == UNKNOWN_FUNCTION_CODE + && input + .extension_functions + .iter() + .any(|name| name.eq_ignore_ascii_case(function_name(&finding))) + { + return None; + } + let code: String = sqlbuild_code(finding.code)?; + let (line, column, start, end) = match finding.span { + Some(span) => { + let (line, column) = line_column(input.sql, span.start); + (Some(line), Some(column), Some(span.start), Some(span.end)) + } + None => (None, None, None, None), + }; + Some(DiagnosticRow { + code, + message: finding.message, + line, + column, + start, + end, + severity: ERROR_SEVERITY.to_owned(), + }) +} + +/// The function an unknown-function finding names: the name as written when the finding keeps +/// it, else the one its message quotes, as in `Unknown function 'NAME' for dialect DuckDB`. +fn function_name(finding: &NameFinding) -> &str { + if finding.written.is_empty() { + finding.message.split('\'').nth(1).unwrap_or_default() + } else { + &finding.written + } +} + +/// The line and column, from 1, of character `offset` in `sql`. +fn line_column(sql: &str, offset: usize) -> (usize, usize) { + sql.chars() + .take(offset) + .fold((1, 1), |(line, column), character| { + if character == '\n' { + (line + 1, 1) + } else { + (line, column + 1) + } + }) +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/constants.rs b/crates/sqlbuild-analysis/src/analysis_route/constants.rs index eeb7ce5e8c..9e622059ae 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/constants.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/constants.rs @@ -18,3 +18,11 @@ pub(crate) const SKIPPED_ANALYSIS_CODE: &str = "P026"; pub(crate) const NULLABLE_WRAPPER: &str = "Nullable"; /// ClickHouse's top-level dictionary-encoding wrapper, which leaves the column's type as it is. pub(crate) const LOW_CARDINALITY_WRAPPER: &str = "LowCardinality"; +/// The finding of a function the dialect's catalogue does not know. +pub(crate) const UNKNOWN_FUNCTION_CODE: &str = "E202"; +/// The severity of every finding the native analysis reports, as the Polyglot path blocks them. +pub(crate) const ERROR_SEVERITY: &str = "error"; +/// What `macro_body` puts before a body to read it as the one projection of a query. +pub(crate) const MACRO_BODY_PREFIX: &str = "SELECT "; +/// The gap of a macro body that is not one expression. +pub(crate) const NOT_ONE_EXPRESSION: &str = "not one expression"; diff --git a/crates/sqlbuild-analysis/src/analysis_route/mod.rs b/crates/sqlbuild-analysis/src/analysis_route/mod.rs index 2e8da84999..2a9f4bce70 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/mod.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/mod.rs @@ -1,12 +1,15 @@ //! Which engine analyses a dialect's SQL: the Polyglot path, or a native module over SQLQuack. +pub(crate) mod _helpers; pub(crate) mod constants; pub mod main; pub mod models; pub mod types; pub use crate::analysis_route::main::analysis_route::analysis_route; -pub use crate::analysis_route::models::{AnalysisRoute, ParserGap, TypeMatch}; +pub use crate::analysis_route::models::{ + AnalysisInput, AnalysisRoute, NativeAnalysis, ParserGap, StandInSchema, TypeMatch, +}; pub use crate::analysis_route::types::NativeDialect; #[cfg(test)] diff --git a/crates/sqlbuild-analysis/src/analysis_route/models.rs b/crates/sqlbuild-analysis/src/analysis_route/models.rs index 0d4e8f6f57..0c487b2f09 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/models.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/models.rs @@ -1,6 +1,11 @@ //! The route a dialect's analysis takes, and what stops a native parser reading SQL whole. +use sqlquack_semantics::lineage::models::{Nullability, SchemaColumn}; +use sqlquack_semantics::lineage::types::LineageSchema; + use crate::analysis_route::types::NativeDialect; +use crate::assembly::analysis_session::models::{ColumnFact, LineageRow}; +use crate::semantic_validation::models::DiagnosticRow; /// The engine that analyses a dialect's SQL. #[derive(Clone, Copy)] @@ -46,3 +51,73 @@ pub enum TypeMatch { nullable_into_non_null: bool, }, } + +/// What a native module analyses: one query, the relations it may read, and how to name them. +#[derive(Clone, Copy, Debug)] +pub struct AnalysisInput<'a> { + /// The analysis SQL, normalised with the dialect's stubs and its placeholders filled. + pub sql: &'a str, + /// The columns, types and nullability of the relations the query may read. + pub schema: &'a dyn LineageSchema, + /// `(analysis name, resource type, resource name)` of each relation the model references. + pub references: &'a [(String, String, String)], + /// Functions the dialect's catalogue cannot know, such as MotherDuck's `md_*`, matched without + /// case: no unknown-function finding names them, and they stay untyped. + pub extension_functions: &'a [String], +} + +/// One query's analysis, in the facts the analysis session keeps for a model. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct NativeAnalysis { + /// The output columns in order, each typed through the dialect's `normalize_type`. + pub columns: Vec, + /// One row per output column: how it is derived and the referenced columns it reads. + pub lineage: Vec, + /// `(relation, column)` of the columns that decide which rows the query returns. + pub row_sources: Vec<(String, String)>, + /// The outermost query level projects a star. + pub has_star: bool, + /// Every column a star projects is known. + pub star_resolved: bool, + /// The findings SQLBuild reports, in its codes and at character positions in `sql`. + pub diagnostics: Vec, +} + +/// One relation of known columns, standing in for what a probe query reads, such as a macro +/// body's arguments as the columns of `SELECT () AS b FROM `. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StandInSchema { + relation: String, + columns: Vec, +} + +impl StandInSchema { + /// `relation` with each `(column, type as the dialect spells it, nullable)` in order. + #[must_use] + pub fn new(relation: &str, columns: &[(&str, &str, bool)]) -> Self { + Self { + relation: relation.to_owned(), + columns: columns + .iter() + .map(|&(name, data_type, nullable)| SchemaColumn { + name: name.to_owned(), + quoted: false, + data_type: Some(data_type.to_owned()), + nullability: if nullable { + Nullability::Nullable + } else { + Nullability::NonNull + }, + }) + .collect(), + } + } +} + +impl LineageSchema for StandInSchema { + fn columns(&self, relation: &str) -> Option<&[SchemaColumn]> { + relation + .eq_ignore_ascii_case(&self.relation) + .then_some(self.columns.as_slice()) + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs index 18baa18a2a..93e1955581 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs @@ -1,9 +1,47 @@ use crate::analysis_route::main::skipped_analysis::{skipped_analysis, unavailable_analysis}; -use crate::analysis_route::models::{LocatedGap, TypeMatch}; +use crate::analysis_route::models::{ + AnalysisInput, LocatedGap, NativeAnalysis, StandInSchema, TypeMatch, +}; use crate::analysis_route::tests::test_types::{ - NativeTypeMatchTestCase, PivotlessDialect, ReadWholeTestCase, + CataloguedDialect, FindingTestCase, MacroBodyTestCase, NativeTypeMatchTestCase, + PivotlessDialect, ReadWholeTestCase, }; use crate::analysis_route::types::NativeDialect; +use crate::assembly::analysis_session::models::LineageRow; + +/// `orders` with a non-null INTEGER id and a nullable VARCHAR status. +fn orders() -> StandInSchema { + StandInSchema::new( + "orders", + &[("id", "INTEGER", false), ("status", "VARCHAR", true)], + ) +} + +/// The analysis of `sql` over `orders`, which the model references as the model `shop.orders`. +fn analysed( + dialect: &dyn NativeDialect, + sql: &str, + extension_functions: &[&str], +) -> NativeAnalysis { + let schema: StandInSchema = orders(); + let references: [(String, String, String); 1] = [( + "orders".to_owned(), + "model".to_owned(), + "shop.orders".to_owned(), + )]; + let extension_functions: Vec = extension_functions + .iter() + .map(|name| (*name).to_owned()) + .collect(); + dialect + .analyse(&AnalysisInput { + sql, + schema: &schema, + references: &references, + extension_functions: &extension_functions, + }) + .expect("the test query is read whole") +} #[test] fn given_sql_when_reading_it_whole_then_only_one_parsed_statement_without_a_gap_is_read() { @@ -23,6 +61,11 @@ fn given_sql_when_reading_it_whole_then_only_one_parsed_statement_without_a_gap_ sql: " ", expected_gap: Some("no statement"), }, + ReadWholeTestCase { + description: "a comment alone holds no statement, not several", + sql: "-- nothing to read", + expected_gap: Some("no statement"), + }, ReadWholeTestCase { description: "a statement kept as raw text is not read", sql: "RAW VACUUM t", @@ -125,3 +168,194 @@ fn given_a_parser_message_or_no_parser_when_skipping_analysis_then_p026_says_why "no dialect is the generic dialect" ); } + +#[test] +fn given_findings_when_analysing_then_they_take_sqlbuild_codes_and_positions() { + let test_cases = [ + FindingTestCase { + description: "a VARCHAR column as a condition is an implicit conversion, blocked as B211", + dialect: &PivotlessDialect, + sql: "SELECT id FROM orders WHERE status", + extension_functions: &[], + expected_findings: &[("B211", Some((1, 29)))], + }, + FindingTestCase { + description: "a cast to a type the engine lacks is blocked as B218, at the value cast", + dialect: &PivotlessDialect, + sql: "SELECT CAST(id AS NOT_A_TYPE) AS x FROM orders", + extension_functions: &[], + expected_findings: &[("B218", Some((1, 13)))], + }, + FindingTestCase { + description: "an unknown column on its own line keeps its line and column", + dialect: &PivotlessDialect, + sql: "SELECT id,\n nope\nFROM orders", + extension_functions: &[], + expected_findings: &[("B002", Some((2, 3)))], + }, + FindingTestCase { + description: "a function the catalogue does not know is B101", + dialect: &CataloguedDialect, + sql: "SELECT CURRENT_TIMESTAMP() AS created_at FROM orders", + extension_functions: &[], + expected_findings: &[("B101", None)], + }, + FindingTestCase { + description: "a declared extension function is known, matched without case", + dialect: &CataloguedDialect, + sql: "SELECT CURRENT_TIMESTAMP() AS created_at FROM orders", + extension_functions: &["current_timestamp"], + expected_findings: &[], + }, + FindingTestCase { + description: "an undeclared function is still unknown beside declared extensions", + dialect: &CataloguedDialect, + sql: "SELECT CURRENT_TIMESTAMP() AS created_at FROM orders", + extension_functions: &["md_user_info"], + expected_findings: &[("B101", None)], + }, + ]; + for test_case in test_cases { + let findings: Vec<(String, Option, Option)> = analysed( + test_case.dialect, + test_case.sql, + test_case.extension_functions, + ) + .diagnostics + .into_iter() + .map(|row| (row.code, row.line, row.column)) + .collect(); + let expected: Vec<(String, Option, Option)> = test_case + .expected_findings + .iter() + .map(|&(code, position)| { + ( + code.to_owned(), + position.map(|(line, _)| line), + position.map(|(_, column)| column), + ) + }) + .collect(); + assert_eq!(findings, expected, "{}", test_case.description); + } +} + +#[test] +fn given_a_query_over_referenced_columns_when_analysing_then_lineage_reads_the_resources() { + let analysis: NativeAnalysis = analysed( + &PivotlessDialect, + "SELECT id, CAST(id AS TEXT) AS id_text, 1 AS one, nope FROM orders WHERE status = 'paid'", + &[], + ); + let source = |column: &str| { + ( + "model".to_owned(), + "shop.orders".to_owned(), + column.to_owned(), + ) + }; + assert_eq!( + analysis.lineage, + vec![ + LineageRow { + output_column: "id".to_owned(), + transform_code: 0, + confidence_code: 1, + sources: vec![source("id")], + }, + LineageRow { + output_column: "id_text".to_owned(), + transform_code: 1, + confidence_code: 1, + sources: vec![source("id")], + }, + LineageRow { + output_column: "one".to_owned(), + transform_code: 5, + confidence_code: 1, + sources: Vec::new(), + }, + LineageRow { + output_column: "nope".to_owned(), + transform_code: 0, + confidence_code: 2, + sources: vec![source("nope")], + }, + ], + "direct, cast and constant columns read the referenced model; an unresolved column is \ + read from the one relation at medium confidence" + ); + assert_eq!( + analysis.row_sources, + vec![("orders".to_owned(), "status".to_owned())], + "the filter's column decides rows" + ); + assert_eq!( + ( + analysis.columns[0].data_type.clone(), + analysis.columns[0].nullability.as_str() + ), + ( + PivotlessDialect + .normalize_type("INTEGER") + .map(|normalized| normalized.normalized_name), + "non_null" + ), + "a column is typed through the dialect and keeps the schema's nullability" + ); +} + +#[test] +fn given_a_star_when_analysing_then_it_is_resolved_only_over_known_columns() { + let known: NativeAnalysis = analysed(&PivotlessDialect, "SELECT * FROM orders", &[]); + assert_eq!( + (known.has_star, known.star_resolved, known.columns.len()), + (true, true, 2), + "a star over a known relation projects its columns" + ); + let unknown: NativeAnalysis = analysed(&PivotlessDialect, "SELECT * FROM elsewhere", &[]); + assert_eq!( + (unknown.has_star, unknown.star_resolved), + (true, false), + "a star over an unknown relation leaves columns out" + ); +} + +#[test] +fn given_a_macro_body_when_reading_it_then_only_one_expression_is_read() { + let test_cases = [ + MacroBodyTestCase { + description: "one expression is the body", + body: "a + 1", + expected_gap: None, + }, + MacroBodyTestCase { + description: "two expressions are not one body", + body: "a, b", + expected_gap: Some(("not one expression", 0)), + }, + MacroBodyTestCase { + description: "a query is not a body", + body: "1 FROM t", + expected_gap: Some(("not one expression", 0)), + }, + MacroBodyTestCase { + description: "a refused construct is placed in the body, not the query around it", + body: "x PIVOT", + expected_gap: Some(("PIVOT", 2)), + }, + ]; + for test_case in test_cases { + assert_eq!( + PivotlessDialect + .macro_body(test_case.body) + .err() + .map(|gap| (gap.construct, gap.offset)), + test_case + .expected_gap + .map(|(construct, offset)| (construct.to_owned(), offset)), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs index 69fd87107b..7ba1c2226f 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs @@ -1,5 +1,7 @@ +use sqlquack_semantics::dialects::constants::{DUCKDB, POSTGRES}; +use sqlquack_semantics::dialects::types::DialectProfile; use sqlquack_syntax::expressions::statements::models::Command; -use sqlquack_syntax::{Expression, Parsed}; +use sqlquack_syntax::{Dialect, Expression, ParseOptions, Parsed, parse_strict}; use crate::analysis_route::models::{LocatedGap, ParserGap, TypeMatch}; use crate::analysis_route::types::NativeDialect; @@ -70,11 +72,51 @@ impl NativeDialect for PivotlessDialect { }) } + fn profile(&self) -> &'static dyn DialectProfile { + &POSTGRES + } + fn normalize_type(&self, spelling: &str) -> Option { (!spelling.eq_ignore_ascii_case("NOT_A_TYPE")).then(|| normalize_text_type(spelling, None)) } } +/// DuckDB's shared parser under the DuckDB profile, which knows DuckDB's keyword-only functions, as +/// MotherDuck projects are analysed. +pub(super) struct CataloguedDialect; + +impl NativeDialect for CataloguedDialect { + fn name(&self) -> &'static str { + "duckdb" + } + + fn stub_prefix(&self) -> &'static str { + "__sqlbuild_" + } + + fn parse(&self, sql: &str) -> Result { + parse_strict(sql, Dialect::DuckDB, &ParseOptions::default()) + .map(|statements| Parsed { + statements, + gaps: Vec::new(), + }) + .map_err(|failure| ParserGap { + construct: failure.to_string(), + message: failure.to_string(), + offset: 0, + syntax: false, + }) + } + + fn profile(&self) -> &'static dyn DialectProfile { + &DUCKDB + } + + fn normalize_type(&self, spelling: &str) -> Option { + PivotlessDialect.normalize_type(spelling) + } +} + pub(super) struct ReadWholeTestCase { pub(super) description: &'static str, pub(super) sql: &'static str, @@ -95,3 +137,19 @@ pub(super) struct TypeMatchTestCase { pub(super) inferred_nullable: bool, pub(super) expected_match: TypeMatch, } + +pub(super) struct FindingTestCase { + pub(super) description: &'static str, + pub(super) dialect: &'static dyn NativeDialect, + pub(super) sql: &'static str, + pub(super) extension_functions: &'static [&'static str], + /// The code of each finding in order, with its line and column when the parse placed it. + pub(super) expected_findings: &'static [(&'static str, Option<(usize, usize)>)], +} + +pub(super) struct MacroBodyTestCase { + pub(super) description: &'static str, + pub(super) body: &'static str, + /// `(construct, offset in the body)` of the gap, when the body is not read. + pub(super) expected_gap: Option<(&'static str, usize)>, +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/types.rs b/crates/sqlbuild-analysis/src/analysis_route/types.rs index b358e77278..35ecde38c0 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/types.rs @@ -1,9 +1,14 @@ //! What a dialect's native analysis module provides. +use sqlquack_semantics::dialects::types::DialectProfile; use sqlquack_syntax::{Expression, Parsed}; -use crate::analysis_route::constants::{MORE_THAN_ONE_STATEMENT, NO_STATEMENT, UNPARSED_STATEMENT}; -use crate::analysis_route::models::{ParserGap, TypeMatch}; +use crate::analysis_route::_helpers::native_analysis::native_analysis; +use crate::analysis_route::constants::{ + MACRO_BODY_PREFIX, MORE_THAN_ONE_STATEMENT, NO_STATEMENT, NOT_ONE_EXPRESSION, + UNPARSED_STATEMENT, +}; +use crate::analysis_route::models::{AnalysisInput, NativeAnalysis, ParserGap, TypeMatch}; use crate::type_system::models::NormalizedType; /// A dialect whose SQL SQLQuack analyses in place of Polyglot. @@ -17,6 +22,9 @@ pub trait NativeDialect: Sync { /// Every statement of `sql` with the parser's named gaps, or the first error that stops it. fn parse(&self, sql: &str) -> Result; + /// The dialect's sqlquack-semantics profile: how its names resolve, compare and type. + fn profile(&self) -> &'static dyn DialectProfile; + /// A type as written in this dialect, as SQLBuild compares types; `None` when the dialect /// cannot spell it or its engine refuses it, and nothing is compared against it. fn normalize_type(&self, spelling: &str) -> Option; @@ -33,17 +41,39 @@ pub trait NativeDialect: Sync { }); } let mut statements = parsed.statements.into_iter(); - let (Some(statement), None) = (statements.next(), statements.next()) else { - let construct: &str = if sql.trim().is_empty() { - NO_STATEMENT - } else { - MORE_THAN_ONE_STATEMENT - }; - return Err(whole_gap(construct, 0)); + match (statements.next(), statements.next()) { + (None, _) => Err(whole_gap(NO_STATEMENT, 0)), + (Some(_), Some(_)) => Err(whole_gap(MORE_THAN_ONE_STATEMENT, 0)), + (Some(Expression::Command(_)), None) => Err(whole_gap(UNPARSED_STATEMENT, 0)), + (Some(statement), None) => Ok(statement), + } + } + + /// The outputs, lineage and findings of the query `input.sql`, read whole, against + /// `input.schema`; findings are in SQLBuild's codes, as the Polyglot path reports them. + fn analyse(&self, input: &AnalysisInput<'_>) -> Result { + let statement: Expression = self.read_whole(input.sql)?; + Ok(native_analysis(self, &statement, input)) + } + + /// A macro body's one expression, read as the projection of `SELECT `; a gap's offset + /// is in `body`. + fn macro_body(&self, body: &str) -> Result { + let statement: Expression = self + .read_whole(&format!("{MACRO_BODY_PREFIX}{body}")) + .map_err(|gap| ParserGap { + offset: gap.offset.saturating_sub(MACRO_BODY_PREFIX.len()), + ..gap + })?; + let Expression::Select(select) = statement else { + return Err(whole_gap(NOT_ONE_EXPRESSION, 0)); }; - match statement { - Expression::Command(_) => Err(whole_gap(UNPARSED_STATEMENT, 0)), - statement => Ok(statement), + if select.from.is_some() { + return Err(whole_gap(NOT_ONE_EXPRESSION, 0)); + } + match <[Expression; 1]>::try_from(select.expressions) { + Ok([expression]) => Ok(expression), + Err(_) => Err(whole_gap(NOT_ONE_EXPRESSION, 0)), } } diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs index b198ae8a54..4440d80a4e 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs @@ -177,7 +177,7 @@ pub(crate) fn map_diagnostics( Ok(ValidationResult { valid, errors }) } -fn sqlbuild_code(code: &str) -> Option { +pub(crate) fn sqlbuild_code(code: &str) -> Option { let direct = match code { "E001" | "E002" | "E003" | "E004" | "E200" => Some("B000"), "E201" => Some("B002"), diff --git a/crates/sqlbuild-analysis/src/semantic_validation/main/diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_validation/main/diagnostics.rs index ae7f192632..0f271d481a 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/main/diagnostics.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/main/diagnostics.rs @@ -7,3 +7,10 @@ pub fn binding_diagnostics( ) -> Result, String> { crate::semantic_validation::_helpers::diagnostics::binding_diagnostics(sql, dialect, rows) } + +/// SQLBuild's code for an analysis finding: E codes directly, implicit-conversion W codes through +/// their blocking B codes; `None` for a code SQLBuild does not report. +#[must_use] +pub fn sqlbuild_code(code: &str) -> Option { + crate::semantic_validation::_helpers::diagnostics::sqlbuild_code(code) +} From eed3343a74d4670bafa8c902cc94d3d3abe58824 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 07:36:48 +0000 Subject: [PATCH 03/29] fix(types): take a fallback type's family from its top-level type The text fallback gave family Timestamp to any type whose text held TIMESTAMP, so an ARRAY, MAP or STRUCT holding a timestamp read as a timestamp where Polyglot reads it as Other. The family now comes from the leading type name, so TIMESTAMP(3), TIMESTAMP WITH TIME ZONE and TIMESTAMP_NTZ stay timestamps and complex types do not. --- .../src/type_system/_helpers/fallback.rs | 7 ++- .../type_system/tests/test_normalize_type.rs | 50 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/crates/sqlbuild-analysis/src/type_system/_helpers/fallback.rs b/crates/sqlbuild-analysis/src/type_system/_helpers/fallback.rs index 6ee0ba4bf1..83970ed723 100644 --- a/crates/sqlbuild-analysis/src/type_system/_helpers/fallback.rs +++ b/crates/sqlbuild-analysis/src/type_system/_helpers/fallback.rs @@ -97,7 +97,12 @@ pub(crate) fn normalize_with_fallback( let name: &str = if bigquery { "BOOL" } else { base_type }; return simple(name, TypeFamily::Boolean); } - if base_type.contains(TIMESTAMP_TYPE_TOKEN) { + // The top-level type decides the family, so a MAP or STRUCT holding a timestamp is not one. + let top_level: &str = base_type + .split(|character: char| !(character.is_ascii_alphanumeric() || character == '_')) + .next() + .unwrap_or(base_type); + if top_level.contains(TIMESTAMP_TYPE_TOKEN) { return simple( ×tamp_normalized_name(base_type, dialect), TypeFamily::Timestamp, diff --git a/crates/sqlbuild-analysis/src/type_system/tests/test_normalize_type.rs b/crates/sqlbuild-analysis/src/type_system/tests/test_normalize_type.rs index afee3fc479..ba70a0280d 100644 --- a/crates/sqlbuild-analysis/src/type_system/tests/test_normalize_type.rs +++ b/crates/sqlbuild-analysis/src/type_system/tests/test_normalize_type.rs @@ -1,5 +1,7 @@ use crate::type_system::errors::TypeNormalizationError; +use crate::type_system::main::normalize_text_type::normalize_text_type; use crate::type_system::main::normalize_type::normalize_type; +use crate::type_system::models::TypeFamily; use crate::type_system::tests::helpers::normalization_text; use crate::type_system::tests::test_types::{BracketDepthTestCase, NormalizeTypeTestCase}; @@ -179,3 +181,51 @@ fn given_array_suffixes_when_normalizing_on_a_worker_stack_then_python_supported ); } } + +#[test] +fn given_complex_types_holding_a_timestamp_when_normalizing_text_then_the_top_level_type_decides() { + let test_cases = [ + ( + "an array of timestamps is an array", + "ARRAY", + TypeFamily::Other, + ), + ( + "a map of structs holding a timestamp is a map", + "MAP>", + TypeFamily::Other, + ), + ( + "a struct holding a timestamp is a struct", + "STRUCT", + TypeFamily::Other, + ), + ( + "a bare timestamp is a timestamp", + "TIMESTAMP", + TypeFamily::Timestamp, + ), + ( + "a timestamp with precision is a timestamp", + "TIMESTAMP(3)", + TypeFamily::Timestamp, + ), + ( + "a zoned timestamp is a timestamp", + "TIMESTAMP WITH TIME ZONE", + TypeFamily::Timestamp, + ), + ( + "Snowflake's NTZ alias is a timestamp", + "TIMESTAMP_NTZ", + TypeFamily::Timestamp, + ), + ]; + for (description, type_sql, expected_family) in test_cases { + assert_eq!( + normalize_text_type(type_sql, None).family, + expected_family, + "{description}" + ); + } +} From 73af3475df2956002993a71ca59ed95b99acc0b7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 07:41:45 +0000 Subject: [PATCH 04/29] fix(analysis): compare ClickHouse nullability without a type, and keep stubs in code type_match now takes the inferred type as optional, and NotCompared keeps whether the declared type alone forbids null. A ClickHouse column declared without Nullable that receives a nullable value of unknown type is flagged again, while no type finding is made. ClickHouse types are compared without whitespace outside quoted strings, so Decimal(10, 2) is Decimal(10,2). Respelling cursor stubs for a native parser now leaves text inside strings and comments, and longer names that end in a stub's text, as written. --- .../main/clickhouse_type_match.rs | 32 +++++++++- .../src/analysis_route/models.rs | 8 ++- .../tests/test_clickhouse_type_match.rs | 61 ++++++++++++++++--- .../tests/test_native_dialect.rs | 24 ++++++-- .../src/analysis_route/tests/test_types.rs | 4 +- .../src/analysis_route/types.rs | 21 +++++-- .../src/contracts/_helpers/type_comparison.rs | 20 +++--- .../_helpers/normalization.rs | 55 ++++++++++++++--- .../tests/test_normalize_batch.rs | 45 +++++++++++++- .../semantic_validation/tests/test_types.rs | 6 ++ 10 files changed, 232 insertions(+), 44 deletions(-) diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/clickhouse_type_match.rs b/crates/sqlbuild-analysis/src/analysis_route/main/clickhouse_type_match.rs index a5ee54f86d..b380ebf65f 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/clickhouse_type_match.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/clickhouse_type_match.rs @@ -1,17 +1,43 @@ use crate::analysis_route::constants::{LOW_CARDINALITY_WRAPPER, NULLABLE_WRAPPER}; use crate::analysis_route::models::TypeMatch; -/// A declared and an inferred ClickHouse type compared without top-level wrappers, nullability apart. +/// A declared and an inferred ClickHouse type compared without top-level wrappers or spacing, +/// nullability apart. A declared type without `Nullable` forbids null even when the inferred type +/// is unknown. An arm compares spellings it has respelt through its aliases. #[must_use] -pub fn clickhouse_type_match(declared: &str, inferred: &str, inferred_nullable: bool) -> TypeMatch { +pub fn clickhouse_type_match( + declared: &str, + inferred: Option<&str>, + inferred_nullable: bool, +) -> TypeMatch { let (declared_base, declared_nullable) = unwrapped(declared); + let Some(inferred) = inferred else { + return TypeMatch::NotCompared { + nullable_into_non_null: !declared_nullable && inferred_nullable, + }; + }; let (inferred_base, wrapped_nullable) = unwrapped(inferred); TypeMatch::Compared { - same_type: declared_base == inferred_base, + same_type: unspaced(declared_base) == unspaced(inferred_base), nullable_into_non_null: !declared_nullable && (inferred_nullable || wrapped_nullable), } } +/// `type_sql` without whitespace outside its quoted strings, so `Decimal(10, 2)` is `Decimal(10,2)` +/// and an `Enum8('a b' = 1)` value keeps its space. +fn unspaced(type_sql: &str) -> String { + let mut quoted: bool = false; + type_sql + .chars() + .filter(|character| { + if *character == '\'' { + quoted = !quoted; + } + quoted || !character.is_whitespace() + }) + .collect() +} + /// The type inside its top-level `Nullable` and `LowCardinality` wrappers, and whether it is nullable. fn unwrapped(type_sql: &str) -> (&str, bool) { let mut current: &str = type_sql.trim(); diff --git a/crates/sqlbuild-analysis/src/analysis_route/models.rs b/crates/sqlbuild-analysis/src/analysis_route/models.rs index 0c487b2f09..1a90169536 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/models.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/models.rs @@ -43,8 +43,12 @@ pub struct LocatedGap { /// How an inferred column type meets its declared type: the type itself, then its nullability. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum TypeMatch { - /// One side is a type the dialect cannot spell or refuses: no finding and no match. - NotCompared, + /// One side is a type the dialect cannot spell, refuses or does not know: no type finding and + /// no match, though the declared type alone may still forbid null. + NotCompared { + /// The inferred column may be null, but its declared type does not allow it. + nullable_into_non_null: bool, + }, Compared { same_type: bool, /// The inferred column may be null, but its declared type does not allow it. diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_clickhouse_type_match.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_clickhouse_type_match.rs index 4e55839f60..c6cab99ac9 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_clickhouse_type_match.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_clickhouse_type_match.rs @@ -9,7 +9,7 @@ fn given_clickhouse_column_types_when_matching_then_wrappers_are_stripped_and_nu TypeMatchTestCase { description: "a nullable String flag matches DESCRIBE's Nullable(String)", declared: "Nullable(String)", - inferred: "String", + inferred: Some("String"), inferred_nullable: true, expected_match: TypeMatch::Compared { same_type: true, @@ -19,7 +19,7 @@ fn given_clickhouse_column_types_when_matching_then_wrappers_are_stripped_and_nu TypeMatchTestCase { description: "a declared LowCardinality(String) is a String", declared: "LowCardinality(String)", - inferred: "String", + inferred: Some("String"), inferred_nullable: false, expected_match: TypeMatch::Compared { same_type: true, @@ -29,7 +29,7 @@ fn given_clickhouse_column_types_when_matching_then_wrappers_are_stripped_and_nu TypeMatchTestCase { description: "LowCardinality around Nullable allows null", declared: "LowCardinality(Nullable(String))", - inferred: "Nullable(String)", + inferred: Some("Nullable(String)"), inferred_nullable: false, expected_match: TypeMatch::Compared { same_type: true, @@ -39,7 +39,7 @@ fn given_clickhouse_column_types_when_matching_then_wrappers_are_stripped_and_nu TypeMatchTestCase { description: "a nullable inference into a plain String declaration is a nullability conflict", declared: "String", - inferred: "String", + inferred: Some("String"), inferred_nullable: true, expected_match: TypeMatch::Compared { same_type: true, @@ -49,7 +49,7 @@ fn given_clickhouse_column_types_when_matching_then_wrappers_are_stripped_and_nu TypeMatchTestCase { description: "a non-null inference into a Nullable declaration is no conflict", declared: "Nullable(Int64)", - inferred: "Int64", + inferred: Some("Int64"), inferred_nullable: false, expected_match: TypeMatch::Compared { same_type: true, @@ -59,7 +59,7 @@ fn given_clickhouse_column_types_when_matching_then_wrappers_are_stripped_and_nu TypeMatchTestCase { description: "different base types differ whatever their wrappers", declared: "Nullable(Int32)", - inferred: "Int64", + inferred: Some("Int64"), inferred_nullable: true, expected_match: TypeMatch::Compared { same_type: false, @@ -69,13 +69,60 @@ fn given_clickhouse_column_types_when_matching_then_wrappers_are_stripped_and_nu TypeMatchTestCase { description: "a Nullable nested inside Array is part of the type, not stripped", declared: "Array(Nullable(String))", - inferred: "Array(String)", + inferred: Some("Array(String)"), inferred_nullable: false, expected_match: TypeMatch::Compared { same_type: false, nullable_into_non_null: false, }, }, + TypeMatchTestCase { + description: "spacing inside a type's arguments is not a difference", + declared: "Decimal(10, 2)", + inferred: Some("Decimal(10,2)"), + inferred_nullable: false, + expected_match: TypeMatch::Compared { + same_type: true, + nullable_into_non_null: false, + }, + }, + TypeMatchTestCase { + description: "spacing inside a quoted enum value is part of the type", + declared: "Enum8('a b' = 1)", + inferred: Some("Enum8('ab' = 1)"), + inferred_nullable: false, + expected_match: TypeMatch::Compared { + same_type: false, + nullable_into_non_null: false, + }, + }, + TypeMatchTestCase { + description: "a nullable value of unknown type into a plain String is a nullability conflict", + declared: "String", + inferred: None, + inferred_nullable: true, + expected_match: TypeMatch::NotCompared { + nullable_into_non_null: true, + }, + }, + TypeMatchTestCase { + description: "a nullable value of unknown type into a Nullable column is no conflict", + declared: "Nullable(String)", + inferred: None, + inferred_nullable: true, + expected_match: TypeMatch::NotCompared { + nullable_into_non_null: false, + }, + }, + TypeMatchTestCase { + description: "a non-null value of unknown type is no conflict", + declared: "String", + inferred: None, + inferred_nullable: false, + expected_match: TypeMatch::NotCompared { + nullable_into_non_null: false, + }, + }, ]; for test_case in test_cases { assert_eq!( diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs index 93e1955581..1e9b9c7ae5 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs @@ -96,19 +96,31 @@ fn given_a_type_the_dialect_refuses_when_matching_types_then_nothing_is_compared NativeTypeMatchTestCase { description: "a refused declared type is not compared", declared: "NOT_A_TYPE", - inferred: "INTEGER", - expected_match: TypeMatch::NotCompared, + inferred: Some("INTEGER"), + expected_match: TypeMatch::NotCompared { + nullable_into_non_null: false, + }, }, NativeTypeMatchTestCase { description: "a refused inferred type is not compared", declared: "INTEGER", - inferred: "NOT_A_TYPE", - expected_match: TypeMatch::NotCompared, + inferred: Some("NOT_A_TYPE"), + expected_match: TypeMatch::NotCompared { + nullable_into_non_null: false, + }, + }, + NativeTypeMatchTestCase { + description: "an unknown inferred type is not compared", + declared: "INTEGER", + inferred: None, + expected_match: TypeMatch::NotCompared { + nullable_into_non_null: false, + }, }, NativeTypeMatchTestCase { description: "two known, different types still mismatch", declared: "INTEGER", - inferred: "TEXT", + inferred: Some("TEXT"), expected_match: TypeMatch::Compared { same_type: false, nullable_into_non_null: false, @@ -117,7 +129,7 @@ fn given_a_type_the_dialect_refuses_when_matching_types_then_nothing_is_compared NativeTypeMatchTestCase { description: "the same type matches", declared: "INTEGER", - inferred: "integer", + inferred: Some("integer"), expected_match: TypeMatch::Compared { same_type: true, nullable_into_non_null: false, diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs index 7ba1c2226f..545760014f 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs @@ -126,14 +126,14 @@ pub(super) struct ReadWholeTestCase { pub(super) struct NativeTypeMatchTestCase { pub(super) description: &'static str, pub(super) declared: &'static str, - pub(super) inferred: &'static str, + pub(super) inferred: Option<&'static str>, pub(super) expected_match: TypeMatch, } pub(super) struct TypeMatchTestCase { pub(super) description: &'static str, pub(super) declared: &'static str, - pub(super) inferred: &'static str, + pub(super) inferred: Option<&'static str>, pub(super) inferred_nullable: bool, pub(super) expected_match: TypeMatch, } diff --git a/crates/sqlbuild-analysis/src/analysis_route/types.rs b/crates/sqlbuild-analysis/src/analysis_route/types.rs index 35ecde38c0..2860e079ff 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/types.rs @@ -77,16 +77,27 @@ pub trait NativeDialect: Sync { } } - /// Whether a declared and an inferred type match, nullability apart; `NotCompared` when - /// `normalize_type` gives `None` for either. An override must keep that rule. - fn type_match(&self, declared: &str, inferred: &str, inferred_nullable: bool) -> TypeMatch { + /// Whether a declared and an inferred type match, nullability apart; `NotCompared` when the + /// inferred type is unknown or `normalize_type` gives `None` for either side. An override must + /// keep that rule, and may still say whether the declared type alone forbids null. + fn type_match( + &self, + declared: &str, + inferred: Option<&str>, + inferred_nullable: bool, + ) -> TypeMatch { let _ = inferred_nullable; - match (self.normalize_type(declared), self.normalize_type(inferred)) { + match ( + self.normalize_type(declared), + inferred.and_then(|inferred| self.normalize_type(inferred)), + ) { (Some(declared), Some(inferred)) => TypeMatch::Compared { same_type: declared == inferred, nullable_into_non_null: false, }, - _ => TypeMatch::NotCompared, + _ => TypeMatch::NotCompared { + nullable_into_non_null: false, + }, } } } diff --git a/crates/sqlbuild-analysis/src/contracts/_helpers/type_comparison.rs b/crates/sqlbuild-analysis/src/contracts/_helpers/type_comparison.rs index 5b3513bcb7..fba3ccc4aa 100644 --- a/crates/sqlbuild-analysis/src/contracts/_helpers/type_comparison.rs +++ b/crates/sqlbuild-analysis/src/contracts/_helpers/type_comparison.rs @@ -52,8 +52,8 @@ impl<'a> TypeComparer<'a> { AnalysisRoute::Legacy => {} AnalysisRoute::Unavailable => return Ok(TypeComparison::NotCompared), AnalysisRoute::Native(dialect) => { - return Ok(match dialect.type_match(left, right, false) { - TypeMatch::NotCompared => TypeComparison::NotCompared, + return Ok(match dialect.type_match(left, Some(right), false) { + TypeMatch::NotCompared { .. } => TypeComparison::NotCompared, TypeMatch::Compared { same_type: true, .. } => TypeComparison::Equal, @@ -79,18 +79,22 @@ impl<'a> TypeComparer<'a> { if declared.not_null && inferred.nullable { return true; } - let (AnalysisRoute::Native(dialect), Some(declared_type), Some(inferred_type)) = ( - self.route, - declared.declared_type.as_deref(), - inferred.inferred_type.as_deref(), - ) else { + let (AnalysisRoute::Native(dialect), Some(declared_type)) = + (self.route, declared.declared_type.as_deref()) + else { return false; }; matches!( - dialect.type_match(declared_type, inferred_type, inferred.nullable), + dialect.type_match( + declared_type, + inferred.inferred_type.as_deref(), + inferred.nullable + ), TypeMatch::Compared { nullable_into_non_null: true, .. + } | TypeMatch::NotCompared { + nullable_into_non_null: true } ) } diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs index daaeed17d4..cea9883053 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs @@ -194,17 +194,52 @@ pub(crate) fn normalize_native_sql( let udf_prefix: String = format!("{stub_prefix}udf_"); let normalized: String = normalize(sql, dialect, (&none, &none), &udf_prefix)?.sql; let stubs: &Regex = NATIVE_STUBS.as_ref()?; - Ok(stubs - .replace_all(&normalized, |captures: ®ex::Captures<'_>| { - match (captures.get(1), captures.get(2)) { - (Some(bound), _) => format!("{stub_prefix}cursor_{}()", bound.as_str()), - (None, Some(parameter)) => { - format!("'{HOOK_PARAMETER_PREFIX}{}'", parameter.as_str()) - } - (None, None) => captures[0].to_owned(), + let non_code: Vec<(usize, usize)> = non_code_ranges(&normalized)?; + let mut native: String = String::with_capacity(normalized.len()); + let mut copied_until: usize = 0; + for captures in stubs.captures_iter(&normalized) { + let whole = captures.get(0).ok_or("missing stub match")?; + let quoted: bool = non_code + .iter() + .any(|&(start, end)| start <= whole.start() && whole.start() < end); + // A cursor stub is a whole name: `x__cursor_start()` is another function. + let glued: bool = captures.get(1).is_some() + && normalized[..whole.start()] + .chars() + .next_back() + .is_some_and(|character| character.is_alphanumeric() || character == '_'); + if quoted || glued { + continue; + } + native.push_str(&normalized[copied_until..whole.start()]); + match (captures.get(1), captures.get(2)) { + (Some(bound), _) => { + native.push_str(&format!("{stub_prefix}cursor_{}()", bound.as_str())); + } + (None, Some(parameter)) => { + native.push_str(&format!("'{HOOK_PARAMETER_PREFIX}{}'", parameter.as_str())); } - }) - .into_owned()) + (None, None) => native.push_str(whole.as_str()), + } + copied_until = whole.end(); + } + native.push_str(&normalized[copied_until..]); + Ok(native) +} + +/// The byte ranges of `sql`'s strings, quoted identifiers and comments, in order. +fn non_code_ranges(sql: &str) -> Result, String> { + let mut ranges: Vec<(usize, usize)> = Vec::new(); + let mut index: usize = 0; + while index < sql.len() { + if let Some(end) = skip_non_code(sql, index)? { + ranges.push((index, end)); + index = end; + continue; + } + index += sql[index..].chars().next().map_or(1, char::len_utf8); + } + Ok(ranges) } pub(crate) fn normalize_dialect_sql_mapped( diff --git a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs index 112492fef7..bbc5a4ea36 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs @@ -1,7 +1,10 @@ use std::collections::HashMap; use crate::semantic_validation::main::normalize_batch::normalize_analysis_sqls; -use crate::semantic_validation::tests::test_types::NormalizationBatchTestCase; +use crate::semantic_validation::main::normalize_native::normalize_native_sql; +use crate::semantic_validation::tests::test_types::{ + NativeStubTestCase, NormalizationBatchTestCase, +}; use crate::semantic_validation::types::NormalizationRequest; #[test] @@ -100,3 +103,43 @@ fn given_batch_when_normalizing_on_pool_or_in_turn_then_matches_single_calls_in_ ); } } + +#[test] +fn given_cursor_stubs_when_normalizing_for_a_native_parser_then_only_whole_names_in_code_are_respelt() + { + let test_cases = [ + NativeStubTestCase { + description: "a cursor stub in code takes the dialect's prefix", + sql: "SELECT * FROM t WHERE ts > __cursor_start()", + expected_sql: "SELECT * FROM t WHERE ts > __sqlbuild.cursor_start()", + }, + NativeStubTestCase { + description: "a cursor stub's text inside a string literal is left as written", + sql: "SELECT '__cursor_start()' AS s FROM t", + expected_sql: "SELECT '__cursor_start()' AS s FROM t", + }, + NativeStubTestCase { + description: "a cursor stub's text inside a comment is left as written", + sql: "SELECT 1 FROM t -- __cursor_end()\n", + expected_sql: "SELECT 1 FROM t -- __cursor_end()\n", + }, + NativeStubTestCase { + description: "a longer name ending in a cursor stub's text is another function", + sql: "SELECT x__cursor_end() FROM t", + expected_sql: "SELECT x__cursor_end() FROM t", + }, + NativeStubTestCase { + description: "a quoted stub beside a stub in code leaves only the code one respelt", + sql: "SELECT '__cursor_end()', __cursor_end() FROM t", + expected_sql: "SELECT '__cursor_end()', __sqlbuild.cursor_end() FROM t", + }, + ]; + for test_case in test_cases { + assert_eq!( + normalize_native_sql(test_case.sql, "postgres", "__sqlbuild.").as_deref(), + Ok(test_case.expected_sql), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_types.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_types.rs index 201a5dc6fc..27fa2325ea 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_types.rs @@ -53,6 +53,12 @@ pub(crate) struct NormalizationBatchTestCase { pub(crate) expected_failures: &'static [usize], } +pub(crate) struct NativeStubTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_sql: &'static str, +} + pub(crate) struct TruthTestScanTestCase { pub(crate) description: &'static str, pub(crate) sql: &'static str, From 6e514a2d46f1820b89b8148e4e5aefced5fd5fb3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 07:43:08 +0000 Subject: [PATCH 05/29] fix(analysis): skip the Polyglot pivot proof for natively analysed dialects The dynamic pivot proof reads Polyglot's parse of the model. A dialect on a native or unavailable analysis route now gets a proof failure naming why, instead of a Polyglot parse. Every dialect is still on the Polyglot route, so no proof changes yet. --- .../assembly/analysis_session/_helpers/dynamic_pivot.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs index e49ac7c85c..455457cc61 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs @@ -8,6 +8,8 @@ use serde_json::{Map, Value}; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use sqlbuild_core::panics::main::native_failure::native_failure; +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; use crate::assembly::analysis_session::_helpers::dict_walk::{ casefold, column_name, dict_list, identifier_name, is_single_wildcard, nested, node_key, parse_one_or_error, payload, relation_name, render_type, to_dict, truthy, upper, @@ -101,6 +103,13 @@ pub(crate) fn pivot_outcome( facts.dialect ))); } + // The proof reads Polyglot's tree; a dialect a native module analyses is never parsed by it. + if !matches!(analysis_route(Some(facts.dialect)), AnalysisRoute::Legacy) { + return Ok(failure(format!( + "dynamic pivot proofs do not read the {} native parser's tree", + facts.dialect + ))); + } let parse_dialect: &str = if folded_dialect == MOTHERDUCK_DIALECT { DUCKDB_DIALECT } else { From b2d0feeefdf7d489d6a9718267d694fcd3c0a8d3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 07:44:02 +0000 Subject: [PATCH 06/29] Revert "fix(analysis): skip the Polyglot pivot proof for natively analysed dialects" The pivot proof moves onto SQLQuack's parser for every dialect in its own change, so no route needs to skip it. --- .../assembly/analysis_session/_helpers/dynamic_pivot.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs index 455457cc61..e49ac7c85c 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs @@ -8,8 +8,6 @@ use serde_json::{Map, Value}; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use sqlbuild_core::panics::main::native_failure::native_failure; -use crate::analysis_route::main::analysis_route::analysis_route; -use crate::analysis_route::models::AnalysisRoute; use crate::assembly::analysis_session::_helpers::dict_walk::{ casefold, column_name, dict_list, identifier_name, is_single_wildcard, nested, node_key, parse_one_or_error, payload, relation_name, render_type, to_dict, truthy, upper, @@ -103,13 +101,6 @@ pub(crate) fn pivot_outcome( facts.dialect ))); } - // The proof reads Polyglot's tree; a dialect a native module analyses is never parsed by it. - if !matches!(analysis_route(Some(facts.dialect)), AnalysisRoute::Legacy) { - return Ok(failure(format!( - "dynamic pivot proofs do not read the {} native parser's tree", - facts.dialect - ))); - } let parse_dialect: &str = if folded_dialect == MOTHERDUCK_DIALECT { DUCKDB_DIALECT } else { From 583e3ea7f79f4822d3e0a91e3c9d4da5de4fcea4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 08:10:26 +0000 Subject: [PATCH 07/29] refactor(analysis): shape the native analysis route as fensu's Rust rules require Each analysis_route and semantic_validation entry module now exposes one function, with ClickHouse's spelling helpers and the read_whole gap in _helpers. Doc comments are one line, the route's tests run every case from a named test case table, and report_skip takes the P026 code and message together. The analysis_route module no longer re-exports its entries. normalize_text_type and clickhouse_type_match keep an exception for having no importer yet: their native dialect modules land in their own changes. --- .../_helpers/clickhouse_types.rs | 53 +++ .../src/analysis_route/_helpers/mod.rs | 2 + .../_helpers/native_analysis.rs | 43 +-- .../src/analysis_route/_helpers/whole_gap.rs | 11 + .../main/clickhouse_type_match.rs | 59 +--- .../src/analysis_route/main/mod.rs | 1 + .../analysis_route/main/skipped_analysis.rs | 17 +- .../main/unavailable_analysis.rs | 13 + .../src/analysis_route/mod.rs | 6 - .../src/analysis_route/models.rs | 9 +- .../src/analysis_route/tests.rs | 2 + .../src/analysis_route/tests/helpers.rs | 53 +++ .../tests/test_native_dialect.rs | 313 +++++++++--------- .../analysis_route/tests/test_native_gap.rs | 46 ++- .../src/analysis_route/tests/test_types.rs | 36 +- .../src/analysis_route/types.rs | 24 +- .../_helpers/normalization.rs | 1 - .../src/semantic_validation/main.rs | 1 + .../semantic_validation/main/diagnostics.rs | 7 - .../semantic_validation/main/sqlbuild_code.rs | 5 + .../src/type_system/_helpers/fallback.rs | 1 - .../type_system/tests/test_normalize_type.rs | 83 ++--- .../src/type_system/tests/test_types.rs | 6 + .../input_loops/model_loop_opt_out.rs | 24 +- fensu.toml | 10 + 25 files changed, 466 insertions(+), 360 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/analysis_route/_helpers/clickhouse_types.rs create mode 100644 crates/sqlbuild-analysis/src/analysis_route/_helpers/whole_gap.rs create mode 100644 crates/sqlbuild-analysis/src/analysis_route/main/unavailable_analysis.rs create mode 100644 crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs create mode 100644 crates/sqlbuild-analysis/src/semantic_validation/main/sqlbuild_code.rs diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/clickhouse_types.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/clickhouse_types.rs new file mode 100644 index 0000000000..9018bfc67b --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/clickhouse_types.rs @@ -0,0 +1,53 @@ +use crate::analysis_route::constants::{LOW_CARDINALITY_WRAPPER, NULLABLE_WRAPPER}; + +/// `type_sql` without whitespace outside its quoted strings. +pub(crate) fn unspaced(type_sql: &str) -> String { + let mut quoted: bool = false; + type_sql + .chars() + .filter(|character| { + if *character == '\'' { + quoted = !quoted; + } + quoted || !character.is_whitespace() + }) + .collect() +} + +/// The type inside its top-level `Nullable` and `LowCardinality` wrappers, and whether it is nullable. +pub(crate) fn unwrapped(type_sql: &str) -> (&str, bool) { + let mut current: &str = type_sql.trim(); + let mut nullable: bool = false; + loop { + if let Some(inner) = wrapped(current, NULLABLE_WRAPPER) { + nullable = true; + current = inner; + } else if let Some(inner) = wrapped(current, LOW_CARDINALITY_WRAPPER) { + current = inner; + } else { + return (current, nullable); + } + } +} + +/// The argument of `wrapper(...)` when the whole of `type_sql` is that one call. +fn wrapped<'a>(type_sql: &'a str, wrapper: &str) -> Option<&'a str> { + let head: &str = type_sql.get(..wrapper.len())?; + if !head.eq_ignore_ascii_case(wrapper) { + return None; + } + let inner: &str = type_sql[wrapper.len()..] + .trim_start() + .strip_prefix('(')? + .strip_suffix(')')?; + let mut depth: usize = 0; + for character in inner.chars() { + match character { + '(' => depth += 1, + ')' if depth == 0 => return None, + ')' => depth -= 1, + _ => {} + } + } + Some(inner.trim()) +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs index 801bee7312..715122942d 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs @@ -1 +1,3 @@ +pub(crate) mod clickhouse_types; pub(crate) mod native_analysis; +pub(crate) mod whole_gap; diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/native_analysis.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/native_analysis.rs index e03c775d96..ffe1bca27b 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/_helpers/native_analysis.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/native_analysis.rs @@ -1,7 +1,8 @@ //! The provided native analysis: sqlquack-semantics lineage and findings as SQLBuild's facts. use sqlquack_semantics::lineage::models::{ - LineageContext, NameFinding, Nullability, OutputLineage, QueryLineage, TransformKind, + LineageContext, NameFinding, Nullability, OutputLineage, QueryLineage, SourceColumn, + TransformKind, }; use sqlquack_semantics::{statement_lineage, statement_name_findings}; use sqlquack_syntax::Expression; @@ -15,7 +16,7 @@ use crate::assembly::analysis_session::constants::{ TRANSFORM_DIRECT, TRANSFORM_EXPRESSION, UNKNOWN_NULLABILITY, }; use crate::assembly::analysis_session::models::{ColumnFact, LineageRow}; -use crate::semantic_validation::main::diagnostics::sqlbuild_code; +use crate::semantic_validation::main::sqlbuild_code::sqlbuild_code; use crate::semantic_validation::models::DiagnosticRow; /// The analysis of `statement`, the one statement of `input.sql`, under `dialect`'s profile. @@ -73,9 +74,7 @@ fn column_fact(dialect: &D, output: &OutputLineage) - } } -/// An output column's lineage over the model's referenced resources, as the legacy rows give it: -/// columns of relations the model does not reference are left out, and a column reading none of -/// its references is of unknown confidence unless it is a constant. +/// An output column's lineage over the model's referenced resources, as the legacy rows give it. fn lineage_row(output: &OutputLineage, references: &[(String, String, String)]) -> LineageRow { let transform_code: u8 = match output.kind { TransformKind::Constant => TRANSFORM_CONSTANT, @@ -87,18 +86,7 @@ fn lineage_row(output: &OutputLineage, references: &[(String, String, String)]) let sources: Vec<(String, String, String)> = output .sources .iter() - .filter_map(|source| { - references - .iter() - .find(|(analysis_name, _, _)| *analysis_name == source.relation) - .map(|(_, resource_type, resource_name)| { - ( - resource_type.clone(), - resource_name.clone(), - source.column.clone(), - ) - }) - }) + .filter_map(|source| referenced_source(source, references)) .collect(); let confidence_code: u8 = if sources.is_empty() && transform_code != TRANSFORM_CONSTANT { CONFIDENCE_UNKNOWN @@ -115,8 +103,22 @@ fn lineage_row(output: &OutputLineage, references: &[(String, String, String)]) } } -/// A finding in SQLBuild's code at its position in `input.sql`; `None` for a code SQLBuild does -/// not report, and for an unknown function the dialect declares as an extension. +/// `(resource type, resource name, column)` of a source column the model references. +fn referenced_source( + source: &SourceColumn, + references: &[(String, String, String)], +) -> Option<(String, String, String)> { + let (_, resource_type, resource_name) = references + .iter() + .find(|(analysis_name, _, _)| *analysis_name == source.relation)?; + Some(( + resource_type.clone(), + resource_name.clone(), + source.column.clone(), + )) +} + +/// A finding in SQLBuild's code at its position in `input.sql`, unless SQLBuild does not report it. fn diagnostic_row(input: &AnalysisInput<'_>, finding: NameFinding) -> Option { if finding.code == UNKNOWN_FUNCTION_CODE && input @@ -145,8 +147,7 @@ fn diagnostic_row(input: &AnalysisInput<'_>, finding: NameFinding) -> Option &str { if finding.written.is_empty() { finding.message.split('\'').nth(1).unwrap_or_default() diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/whole_gap.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/whole_gap.rs new file mode 100644 index 0000000000..7b4fda1d47 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/whole_gap.rs @@ -0,0 +1,11 @@ +use crate::analysis_route::models::ParserGap; + +/// A gap `read_whole` names itself, which no parser reports as invalid syntax. +pub(crate) fn whole_gap(construct: &str, offset: usize) -> ParserGap { + ParserGap { + construct: construct.to_owned(), + message: construct.to_owned(), + offset, + syntax: false, + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/clickhouse_type_match.rs b/crates/sqlbuild-analysis/src/analysis_route/main/clickhouse_type_match.rs index b380ebf65f..2a497aa248 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/clickhouse_type_match.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/clickhouse_type_match.rs @@ -1,9 +1,7 @@ -use crate::analysis_route::constants::{LOW_CARDINALITY_WRAPPER, NULLABLE_WRAPPER}; +use crate::analysis_route::_helpers::clickhouse_types::{unspaced, unwrapped}; use crate::analysis_route::models::TypeMatch; -/// A declared and an inferred ClickHouse type compared without top-level wrappers or spacing, -/// nullability apart. A declared type without `Nullable` forbids null even when the inferred type -/// is unknown. An arm compares spellings it has respelt through its aliases. +/// A declared and an inferred ClickHouse type compared without top-level wrappers or spacing. #[must_use] pub fn clickhouse_type_match( declared: &str, @@ -22,56 +20,3 @@ pub fn clickhouse_type_match( nullable_into_non_null: !declared_nullable && (inferred_nullable || wrapped_nullable), } } - -/// `type_sql` without whitespace outside its quoted strings, so `Decimal(10, 2)` is `Decimal(10,2)` -/// and an `Enum8('a b' = 1)` value keeps its space. -fn unspaced(type_sql: &str) -> String { - let mut quoted: bool = false; - type_sql - .chars() - .filter(|character| { - if *character == '\'' { - quoted = !quoted; - } - quoted || !character.is_whitespace() - }) - .collect() -} - -/// The type inside its top-level `Nullable` and `LowCardinality` wrappers, and whether it is nullable. -fn unwrapped(type_sql: &str) -> (&str, bool) { - let mut current: &str = type_sql.trim(); - let mut nullable: bool = false; - loop { - if let Some(inner) = wrapped(current, NULLABLE_WRAPPER) { - nullable = true; - current = inner; - } else if let Some(inner) = wrapped(current, LOW_CARDINALITY_WRAPPER) { - current = inner; - } else { - return (current, nullable); - } - } -} - -/// The argument of `wrapper(...)` when the whole of `type_sql` is that one call. -fn wrapped<'a>(type_sql: &'a str, wrapper: &str) -> Option<&'a str> { - let head: &str = type_sql.get(..wrapper.len())?; - if !head.eq_ignore_ascii_case(wrapper) { - return None; - } - let inner: &str = type_sql[wrapper.len()..] - .trim_start() - .strip_prefix('(')? - .strip_suffix(')')?; - let mut depth: usize = 0; - for character in inner.chars() { - match character { - '(' => depth += 1, - ')' if depth == 0 => return None, - ')' => depth -= 1, - _ => {} - } - } - Some(inner.trim()) -} diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs b/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs index 87c5c0cfd3..4991f95634 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs @@ -2,3 +2,4 @@ pub mod analysis_route; pub mod clickhouse_type_match; pub mod native_gap; pub mod skipped_analysis; +pub mod unavailable_analysis; diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/skipped_analysis.rs b/crates/sqlbuild-analysis/src/analysis_route/main/skipped_analysis.rs index eacef989f7..6e0d6abaa2 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/skipped_analysis.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/skipped_analysis.rs @@ -1,8 +1,7 @@ -use crate::analysis_route::constants::{GENERIC_DIALECT, SKIPPED_ANALYSIS_CODE}; +use crate::analysis_route::constants::SKIPPED_ANALYSIS_CODE; use crate::analysis_route::models::LocatedGap; -/// P026's code and message for a model whose SQL the dialect's native parser cannot read, with -/// the parser's own message when it says more than the construct's name. +/// P026's code and message for a model whose SQL the dialect's native parser cannot read. #[must_use] pub fn skipped_analysis(model: &str, dialect: &str, gap: &LocatedGap) -> (&'static str, String) { let detail: String = if gap.message.is_empty() || gap.message == gap.construct { @@ -19,15 +18,3 @@ pub fn skipped_analysis(model: &str, dialect: &str, gap: &LocatedGap) -> (&'stat ), ) } - -/// P026's code and message for a model whose dialect no native parser reads. -#[must_use] -pub fn unavailable_analysis(model: &str, dialect: Option<&str>) -> (&'static str, String) { - ( - SKIPPED_ANALYSIS_CODE, - format!( - "SQL analysis skipped for model '{model}': no native parser reads the {} dialect", - dialect.unwrap_or(GENERIC_DIALECT) - ), - ) -} diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/unavailable_analysis.rs b/crates/sqlbuild-analysis/src/analysis_route/main/unavailable_analysis.rs new file mode 100644 index 0000000000..97257e54f6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/main/unavailable_analysis.rs @@ -0,0 +1,13 @@ +use crate::analysis_route::constants::{GENERIC_DIALECT, SKIPPED_ANALYSIS_CODE}; + +/// P026's code and message for a model whose dialect no native parser reads. +#[must_use] +pub fn unavailable_analysis(model: &str, dialect: Option<&str>) -> (&'static str, String) { + ( + SKIPPED_ANALYSIS_CODE, + format!( + "SQL analysis skipped for model '{model}': no native parser reads the {} dialect", + dialect.unwrap_or(GENERIC_DIALECT) + ), + ) +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/mod.rs b/crates/sqlbuild-analysis/src/analysis_route/mod.rs index 2a9f4bce70..4d7376c049 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/mod.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/mod.rs @@ -6,11 +6,5 @@ pub mod main; pub mod models; pub mod types; -pub use crate::analysis_route::main::analysis_route::analysis_route; -pub use crate::analysis_route::models::{ - AnalysisInput, AnalysisRoute, NativeAnalysis, ParserGap, StandInSchema, TypeMatch, -}; -pub use crate::analysis_route::types::NativeDialect; - #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-analysis/src/analysis_route/models.rs b/crates/sqlbuild-analysis/src/analysis_route/models.rs index 1a90169536..7330e008d6 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/models.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/models.rs @@ -43,8 +43,7 @@ pub struct LocatedGap { /// How an inferred column type meets its declared type: the type itself, then its nullability. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum TypeMatch { - /// One side is a type the dialect cannot spell, refuses or does not know: no type finding and - /// no match, though the declared type alone may still forbid null. + /// One side's type is unknown or refused: no type finding, though null may still be forbidden. NotCompared { /// The inferred column may be null, but its declared type does not allow it. nullable_into_non_null: bool, @@ -65,8 +64,7 @@ pub struct AnalysisInput<'a> { pub schema: &'a dyn LineageSchema, /// `(analysis name, resource type, resource name)` of each relation the model references. pub references: &'a [(String, String, String)], - /// Functions the dialect's catalogue cannot know, such as MotherDuck's `md_*`, matched without - /// case: no unknown-function finding names them, and they stay untyped. + /// Functions the catalogue cannot know, such as MotherDuck's `md_*`; never reported unknown. pub extension_functions: &'a [String], } @@ -87,8 +85,7 @@ pub struct NativeAnalysis { pub diagnostics: Vec, } -/// One relation of known columns, standing in for what a probe query reads, such as a macro -/// body's arguments as the columns of `SELECT () AS b FROM `. +/// One relation of known columns, standing in for what a probe query such as a macro's reads. #[derive(Clone, Debug, PartialEq, Eq)] pub struct StandInSchema { relation: String, diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests.rs b/crates/sqlbuild-analysis/src/analysis_route/tests.rs index 93f21c6de3..add990df01 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/helpers.rs"] +mod helpers; #[path = "tests/test_clickhouse_type_match.rs"] mod test_clickhouse_type_match; #[path = "tests/test_native_dialect.rs"] diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs new file mode 100644 index 0000000000..27e20d3ec9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs @@ -0,0 +1,53 @@ +use crate::analysis_route::models::{AnalysisInput, NativeAnalysis, StandInSchema}; +use crate::analysis_route::types::NativeDialect; +use crate::assembly::analysis_session::models::LineageRow; + +/// `orders` with a non-null INTEGER id and a nullable VARCHAR status. +pub(super) fn orders() -> StandInSchema { + StandInSchema::new( + "orders", + &[("id", "INTEGER", false), ("status", "VARCHAR", true)], + ) +} + +/// The analysis of `sql` over `orders`, which the model references as the model `shop.orders`. +pub(super) fn analysed( + dialect: &dyn NativeDialect, + sql: &str, + extension_functions: &[&str], +) -> NativeAnalysis { + let schema: StandInSchema = orders(); + let references: [(String, String, String); 1] = [( + "orders".to_owned(), + "model".to_owned(), + "shop.orders".to_owned(), + )]; + let extension_functions: Vec = extension_functions + .iter() + .map(|name| (*name).to_owned()) + .collect(); + dialect + .analyse(&AnalysisInput { + sql, + schema: &schema, + references: &references, + extension_functions: &extension_functions, + }) + .expect("the test query is read whole") +} + +/// The columns a lineage row reads, each of which must come from the model `shop.orders`. +pub(super) fn resource_columns(row: &LineageRow) -> Vec<&str> { + row.sources + .iter() + .map(|(resource_type, resource_name, column)| { + assert_eq!( + (resource_type.as_str(), resource_name.as_str()), + ("model", "shop.orders"), + "a source of {}", + row.output_column + ); + column.as_str() + }) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs index 1e9b9c7ae5..9f8c470052 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs @@ -1,47 +1,12 @@ -use crate::analysis_route::main::skipped_analysis::{skipped_analysis, unavailable_analysis}; -use crate::analysis_route::models::{ - AnalysisInput, LocatedGap, NativeAnalysis, StandInSchema, TypeMatch, -}; +use crate::analysis_route::main::unavailable_analysis::unavailable_analysis; +use crate::analysis_route::models::{NativeAnalysis, TypeMatch}; +use crate::analysis_route::tests::helpers::{analysed, resource_columns}; use crate::analysis_route::tests::test_types::{ - CataloguedDialect, FindingTestCase, MacroBodyTestCase, NativeTypeMatchTestCase, - PivotlessDialect, ReadWholeTestCase, + CataloguedDialect, ColumnTestCase, FindingTestCase, LineageTestCase, MacroBodyTestCase, + NativeTypeMatchTestCase, PivotlessDialect, ReadWholeTestCase, StarTestCase, + UnavailableAnalysisTestCase, }; use crate::analysis_route::types::NativeDialect; -use crate::assembly::analysis_session::models::LineageRow; - -/// `orders` with a non-null INTEGER id and a nullable VARCHAR status. -fn orders() -> StandInSchema { - StandInSchema::new( - "orders", - &[("id", "INTEGER", false), ("status", "VARCHAR", true)], - ) -} - -/// The analysis of `sql` over `orders`, which the model references as the model `shop.orders`. -fn analysed( - dialect: &dyn NativeDialect, - sql: &str, - extension_functions: &[&str], -) -> NativeAnalysis { - let schema: StandInSchema = orders(); - let references: [(String, String, String); 1] = [( - "orders".to_owned(), - "model".to_owned(), - "shop.orders".to_owned(), - )]; - let extension_functions: Vec = extension_functions - .iter() - .map(|name| (*name).to_owned()) - .collect(); - dialect - .analyse(&AnalysisInput { - sql, - schema: &schema, - references: &references, - extension_functions: &extension_functions, - }) - .expect("the test query is read whole") -} #[test] fn given_sql_when_reading_it_whole_then_only_one_parsed_statement_without_a_gap_is_read() { @@ -147,38 +112,29 @@ fn given_a_type_the_dialect_refuses_when_matching_types_then_nothing_is_compared } #[test] -fn given_a_parser_message_or_no_parser_when_skipping_analysis_then_p026_says_why() { - let syntax: LocatedGap = LocatedGap { - construct: "syntax error".to_owned(), - message: "syntax error at or near \"FROM\"".to_owned(), - line: 1, - column: 15, - syntax: true, - }; - assert_eq!( - skipped_analysis("orders", "postgres", &syntax), - ( - "P026", - "SQL analysis skipped for model 'orders': the postgres parser cannot read syntax \ - error at line 1, column 15: syntax error at or near \"FROM\"" - .to_owned() - ), - "the parser's message follows the construct" - ); - assert_eq!( - unavailable_analysis("orders", Some("trino")), - ( - "P026", - "SQL analysis skipped for model 'orders': no native parser reads the trino dialect" - .to_owned() - ), - "an unknown dialect is named" - ); - assert_eq!( - unavailable_analysis("orders", None).1, - "SQL analysis skipped for model 'orders': no native parser reads the generic dialect", - "no dialect is the generic dialect" - ); +fn given_no_native_parser_when_skipping_analysis_then_p026_names_the_dialect() { + let test_cases = [ + UnavailableAnalysisTestCase { + description: "an unknown dialect is named", + dialect: Some("trino"), + expected_message: "SQL analysis skipped for model 'orders': no native parser reads \ + the trino dialect", + }, + UnavailableAnalysisTestCase { + description: "no dialect is the generic dialect", + dialect: None, + expected_message: "SQL analysis skipped for model 'orders': no native parser reads \ + the generic dialect", + }, + ]; + for test_case in test_cases { + assert_eq!( + unavailable_analysis("orders", test_case.dialect), + ("P026", test_case.expected_message.to_owned()), + "{}", + test_case.description + ); + } } #[test] @@ -228,109 +184,146 @@ fn given_findings_when_analysing_then_they_take_sqlbuild_codes_and_positions() { }, ]; for test_case in test_cases { - let findings: Vec<(String, Option, Option)> = analysed( + let analysis: NativeAnalysis = analysed( test_case.dialect, test_case.sql, test_case.extension_functions, - ) - .diagnostics - .into_iter() - .map(|row| (row.code, row.line, row.column)) - .collect(); - let expected: Vec<(String, Option, Option)> = test_case - .expected_findings + ); + let findings: Vec<(&str, Option<(usize, usize)>)> = analysis + .diagnostics .iter() - .map(|&(code, position)| { + .map(|row| (row.code.as_str(), row.line.zip(row.column))) + .collect(); + assert_eq!( + findings, test_case.expected_findings, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_query_over_referenced_columns_when_analysing_then_lineage_reads_the_resources() { + let test_cases = [ + LineageTestCase { + description: "direct, cast and constant columns read the model; an unresolved one is \ + read from the one relation at medium confidence", + sql: "SELECT id, CAST(id AS TEXT) AS id_text, 1 AS one, nope FROM orders \ + WHERE status = 'paid'", + expected_lineage: &[ + ("id", 0, 1, &["id"]), + ("id_text", 1, 1, &["id"]), + ("one", 5, 1, &[]), + ("nope", 0, 2, &["nope"]), + ], + expected_row_sources: &[("orders", "status")], + }, + LineageTestCase { + description: "a query without a filter has no columns deciding rows", + sql: "SELECT status FROM orders", + expected_lineage: &[("status", 0, 1, &["status"])], + expected_row_sources: &[], + }, + ]; + for test_case in test_cases { + let analysis: NativeAnalysis = analysed(&PivotlessDialect, test_case.sql, &[]); + let lineage: Vec<(&str, u8, u8, Vec<&str>)> = analysis + .lineage + .iter() + .map(|row| { ( - code.to_owned(), - position.map(|(line, _)| line), - position.map(|(_, column)| column), + row.output_column.as_str(), + row.transform_code, + row.confidence_code, + resource_columns(row), ) }) .collect(); - assert_eq!(findings, expected, "{}", test_case.description); + let row_sources: Vec<(&str, &str)> = analysis + .row_sources + .iter() + .map(|(relation, column)| (relation.as_str(), column.as_str())) + .collect(); + assert_eq!( + (lineage, row_sources), + ( + test_case + .expected_lineage + .iter() + .map(|&(name, transform, confidence, columns)| { + (name, transform, confidence, columns.to_vec()) + }) + .collect::>(), + test_case.expected_row_sources.to_vec() + ), + "{}", + test_case.description + ); } } #[test] -fn given_a_query_over_referenced_columns_when_analysing_then_lineage_reads_the_resources() { - let analysis: NativeAnalysis = analysed( - &PivotlessDialect, - "SELECT id, CAST(id AS TEXT) AS id_text, 1 AS one, nope FROM orders WHERE status = 'paid'", - &[], - ); - let source = |column: &str| { - ( - "model".to_owned(), - "shop.orders".to_owned(), - column.to_owned(), - ) - }; - assert_eq!( - analysis.lineage, - vec![ - LineageRow { - output_column: "id".to_owned(), - transform_code: 0, - confidence_code: 1, - sources: vec![source("id")], - }, - LineageRow { - output_column: "id_text".to_owned(), - transform_code: 1, - confidence_code: 1, - sources: vec![source("id")], - }, - LineageRow { - output_column: "one".to_owned(), - transform_code: 5, - confidence_code: 1, - sources: Vec::new(), - }, - LineageRow { - output_column: "nope".to_owned(), - transform_code: 0, - confidence_code: 2, - sources: vec![source("nope")], - }, - ], - "direct, cast and constant columns read the referenced model; an unresolved column is \ - read from the one relation at medium confidence" - ); - assert_eq!( - analysis.row_sources, - vec![("orders".to_owned(), "status".to_owned())], - "the filter's column decides rows" - ); - assert_eq!( - ( - analysis.columns[0].data_type.clone(), - analysis.columns[0].nullability.as_str() - ), - ( - PivotlessDialect - .normalize_type("INTEGER") - .map(|normalized| normalized.normalized_name), - "non_null" - ), - "a column is typed through the dialect and keeps the schema's nullability" - ); +fn given_schema_columns_when_analysing_then_they_keep_their_types_and_nullability() { + let test_cases = [ + ColumnTestCase { + description: "a non-null INTEGER column", + sql: "SELECT id FROM orders", + expected_columns: &[("id", Some("INTEGER"), "non_null")], + }, + ColumnTestCase { + description: "a nullable VARCHAR column", + sql: "SELECT status FROM orders", + expected_columns: &[("status", Some("VARCHAR"), "nullable")], + }, + ]; + for test_case in test_cases { + let analysis: NativeAnalysis = analysed(&PivotlessDialect, test_case.sql, &[]); + let columns: Vec<(&str, Option<&str>, &str)> = analysis + .columns + .iter() + .map(|column| { + ( + column.name.as_str(), + column.data_type.as_deref(), + column.nullability.as_str(), + ) + }) + .collect(); + assert_eq!( + columns, test_case.expected_columns, + "{}", + test_case.description + ); + } } #[test] fn given_a_star_when_analysing_then_it_is_resolved_only_over_known_columns() { - let known: NativeAnalysis = analysed(&PivotlessDialect, "SELECT * FROM orders", &[]); - assert_eq!( - (known.has_star, known.star_resolved, known.columns.len()), - (true, true, 2), - "a star over a known relation projects its columns" - ); - let unknown: NativeAnalysis = analysed(&PivotlessDialect, "SELECT * FROM elsewhere", &[]); - assert_eq!( - (unknown.has_star, unknown.star_resolved), - (true, false), - "a star over an unknown relation leaves columns out" - ); + let test_cases = [ + StarTestCase { + description: "a star over a known relation projects its columns", + sql: "SELECT * FROM orders", + expected_star: (true, true, 2), + }, + StarTestCase { + description: "a star over an unknown relation leaves columns out", + sql: "SELECT * FROM elsewhere", + expected_star: (true, false, 0), + }, + ]; + for test_case in test_cases { + let analysis: NativeAnalysis = analysed(&PivotlessDialect, test_case.sql, &[]); + assert_eq!( + ( + analysis.has_star, + analysis.star_resolved, + analysis.columns.len() + ), + test_case.expected_star, + "{}", + test_case.description + ); + } } #[test] diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_gap.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_gap.rs index a69a2d3638..5c395c6d1f 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_gap.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_gap.rs @@ -132,21 +132,39 @@ fn given_model_sql_when_probing_the_native_parser_then_its_gap_is_located_after_ #[test] fn given_a_located_gap_when_skipping_analysis_then_p026_names_the_model_and_the_gap() { - let test_cases = [SkippedAnalysisTestCase { - description: "a PIVOT gap on line 3", - model: "orders_pivot", - dialect: "postgres", - gap: LocatedGap { - construct: "PIVOT".to_owned(), - message: "PIVOT".to_owned(), - line: 3, - column: 3, - syntax: false, - }, - expected_code: "P026", - expected_message: "SQL analysis skipped for model 'orders_pivot': the postgres parser \ + let test_cases = [ + SkippedAnalysisTestCase { + description: "the parser's message follows the construct", + model: "orders", + dialect: "postgres", + gap: LocatedGap { + construct: "syntax error".to_owned(), + message: "syntax error at or near \"FROM\"".to_owned(), + line: 1, + column: 15, + syntax: true, + }, + expected_code: "P026", + expected_message: "SQL analysis skipped for model 'orders': the postgres parser \ + cannot read syntax error at line 1, column 15: syntax error at \ + or near \"FROM\"", + }, + SkippedAnalysisTestCase { + description: "a PIVOT gap on line 3", + model: "orders_pivot", + dialect: "postgres", + gap: LocatedGap { + construct: "PIVOT".to_owned(), + message: "PIVOT".to_owned(), + line: 3, + column: 3, + syntax: false, + }, + expected_code: "P026", + expected_message: "SQL analysis skipped for model 'orders_pivot': the postgres parser \ cannot read PIVOT at line 3, column 3", - }]; + }, + ]; for test_case in test_cases { assert_eq!( skipped_analysis(test_case.model, test_case.dialect, &test_case.gap), diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs index 545760014f..95d2188b32 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs @@ -30,8 +30,7 @@ pub(super) struct SkippedAnalysisTestCase { pub(super) expected_message: &'static str, } -/// A test dialect over the PostgreSQL parser that also refuses PIVOT, unfilled placeholders and -/// one-part stubs, keeps SQL after `RAW ` as raw text, and refuses the type `NOT_A_TYPE`. +/// The PostgreSQL parser, refusing PIVOT, placeholders, one-part stubs and the type `NOT_A_TYPE`. pub(super) struct PivotlessDialect; impl NativeDialect for PivotlessDialect { @@ -81,8 +80,7 @@ impl NativeDialect for PivotlessDialect { } } -/// DuckDB's shared parser under the DuckDB profile, which knows DuckDB's keyword-only functions, as -/// MotherDuck projects are analysed. +/// DuckDB's shared parser under the DuckDB profile, as MotherDuck projects are analysed. pub(super) struct CataloguedDialect; impl NativeDialect for CataloguedDialect { @@ -153,3 +151,33 @@ pub(super) struct MacroBodyTestCase { /// `(construct, offset in the body)` of the gap, when the body is not read. pub(super) expected_gap: Option<(&'static str, usize)>, } + +pub(super) struct UnavailableAnalysisTestCase { + pub(super) description: &'static str, + pub(super) dialect: Option<&'static str>, + pub(super) expected_message: &'static str, +} + +/// Each output's name, transform code, confidence code and the columns it reads. +type ExpectedLineage = &'static [(&'static str, u8, u8, &'static [&'static str])]; + +pub(super) struct LineageTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) expected_lineage: ExpectedLineage, + pub(super) expected_row_sources: &'static [(&'static str, &'static str)], +} + +pub(super) struct ColumnTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + /// Each output's name, type and nullability. + pub(super) expected_columns: &'static [(&'static str, Option<&'static str>, &'static str)], +} + +pub(super) struct StarTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + /// Whether it projects a star, whether every star column is known, and the column count. + pub(super) expected_star: (bool, bool, usize), +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/types.rs b/crates/sqlbuild-analysis/src/analysis_route/types.rs index 2860e079ff..d28a0b24fc 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/types.rs @@ -4,6 +4,7 @@ use sqlquack_semantics::dialects::types::DialectProfile; use sqlquack_syntax::{Expression, Parsed}; use crate::analysis_route::_helpers::native_analysis::native_analysis; +use crate::analysis_route::_helpers::whole_gap::whole_gap; use crate::analysis_route::constants::{ MACRO_BODY_PREFIX, MORE_THAN_ONE_STATEMENT, NO_STATEMENT, NOT_ONE_EXPRESSION, UNPARSED_STATEMENT, @@ -25,8 +26,7 @@ pub trait NativeDialect: Sync { /// The dialect's sqlquack-semantics profile: how its names resolve, compare and type. fn profile(&self) -> &'static dyn DialectProfile; - /// A type as written in this dialect, as SQLBuild compares types; `None` when the dialect - /// cannot spell it or its engine refuses it, and nothing is compared against it. + /// A type as written in this dialect, as SQLBuild compares it; `None` when unknown or refused. fn normalize_type(&self, spelling: &str) -> Option; /// The one statement of `sql` when the parser reads it whole: no gap, and not raw text. @@ -49,15 +49,13 @@ pub trait NativeDialect: Sync { } } - /// The outputs, lineage and findings of the query `input.sql`, read whole, against - /// `input.schema`; findings are in SQLBuild's codes, as the Polyglot path reports them. + /// The outputs, lineage and findings of the query `input.sql` in SQLBuild's facts and codes. fn analyse(&self, input: &AnalysisInput<'_>) -> Result { let statement: Expression = self.read_whole(input.sql)?; Ok(native_analysis(self, &statement, input)) } - /// A macro body's one expression, read as the projection of `SELECT `; a gap's offset - /// is in `body`. + /// A macro body's one expression, read as the projection of `SELECT `. fn macro_body(&self, body: &str) -> Result { let statement: Expression = self .read_whole(&format!("{MACRO_BODY_PREFIX}{body}")) @@ -77,9 +75,7 @@ pub trait NativeDialect: Sync { } } - /// Whether a declared and an inferred type match, nullability apart; `NotCompared` when the - /// inferred type is unknown or `normalize_type` gives `None` for either side. An override must - /// keep that rule, and may still say whether the declared type alone forbids null. + /// Whether a declared and an inferred type match; an override keeps `NotCompared` for unknowns. fn type_match( &self, declared: &str, @@ -101,13 +97,3 @@ pub trait NativeDialect: Sync { } } } - -/// A gap `read_whole` names itself, which no parser reports as invalid syntax. -fn whole_gap(construct: &str, offset: usize) -> ParserGap { - ParserGap { - construct: construct.to_owned(), - message: construct.to_owned(), - offset, - syntax: false, - } -} diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs index cea9883053..febdc7140a 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs @@ -202,7 +202,6 @@ pub(crate) fn normalize_native_sql( let quoted: bool = non_code .iter() .any(|&(start, end)| start <= whole.start() && whole.start() < end); - // A cursor stub is a whole name: `x__cursor_start()` is another function. let glued: bool = captures.get(1).is_some() && normalized[..whole.start()] .chars() diff --git a/crates/sqlbuild-analysis/src/semantic_validation/main.rs b/crates/sqlbuild-analysis/src/semantic_validation/main.rs index 930655bff7..2db687f9e5 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/main.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/main.rs @@ -8,6 +8,7 @@ pub mod normalize_batch; pub mod normalize_dialect; pub mod normalize_mapped; pub mod normalize_native; +pub mod sqlbuild_code; pub mod truth_test_spans; use crate::semantic_validation::models::ProjectCatalog; diff --git a/crates/sqlbuild-analysis/src/semantic_validation/main/diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_validation/main/diagnostics.rs index 0f271d481a..ae7f192632 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/main/diagnostics.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/main/diagnostics.rs @@ -7,10 +7,3 @@ pub fn binding_diagnostics( ) -> Result, String> { crate::semantic_validation::_helpers::diagnostics::binding_diagnostics(sql, dialect, rows) } - -/// SQLBuild's code for an analysis finding: E codes directly, implicit-conversion W codes through -/// their blocking B codes; `None` for a code SQLBuild does not report. -#[must_use] -pub fn sqlbuild_code(code: &str) -> Option { - crate::semantic_validation::_helpers::diagnostics::sqlbuild_code(code) -} diff --git a/crates/sqlbuild-analysis/src/semantic_validation/main/sqlbuild_code.rs b/crates/sqlbuild-analysis/src/semantic_validation/main/sqlbuild_code.rs new file mode 100644 index 0000000000..0187252e45 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_validation/main/sqlbuild_code.rs @@ -0,0 +1,5 @@ +/// SQLBuild's code for an analysis finding; `None` for a code SQLBuild does not report. +#[must_use] +pub fn sqlbuild_code(code: &str) -> Option { + crate::semantic_validation::_helpers::diagnostics::sqlbuild_code(code) +} diff --git a/crates/sqlbuild-analysis/src/type_system/_helpers/fallback.rs b/crates/sqlbuild-analysis/src/type_system/_helpers/fallback.rs index 83970ed723..6cc429a7f6 100644 --- a/crates/sqlbuild-analysis/src/type_system/_helpers/fallback.rs +++ b/crates/sqlbuild-analysis/src/type_system/_helpers/fallback.rs @@ -97,7 +97,6 @@ pub(crate) fn normalize_with_fallback( let name: &str = if bigquery { "BOOL" } else { base_type }; return simple(name, TypeFamily::Boolean); } - // The top-level type decides the family, so a MAP or STRUCT holding a timestamp is not one. let top_level: &str = base_type .split(|character: char| !(character.is_ascii_alphanumeric() || character == '_')) .next() diff --git a/crates/sqlbuild-analysis/src/type_system/tests/test_normalize_type.rs b/crates/sqlbuild-analysis/src/type_system/tests/test_normalize_type.rs index ba70a0280d..73a83a8cba 100644 --- a/crates/sqlbuild-analysis/src/type_system/tests/test_normalize_type.rs +++ b/crates/sqlbuild-analysis/src/type_system/tests/test_normalize_type.rs @@ -3,7 +3,9 @@ use crate::type_system::main::normalize_text_type::normalize_text_type; use crate::type_system::main::normalize_type::normalize_type; use crate::type_system::models::TypeFamily; use crate::type_system::tests::helpers::normalization_text; -use crate::type_system::tests::test_types::{BracketDepthTestCase, NormalizeTypeTestCase}; +use crate::type_system::tests::test_types::{ + BracketDepthTestCase, NormalizeTypeTestCase, TextFamilyTestCase, +}; #[test] fn given_type_strings_when_normalizing_then_python_normalization_is_returned() { @@ -185,47 +187,48 @@ fn given_array_suffixes_when_normalizing_on_a_worker_stack_then_python_supported #[test] fn given_complex_types_holding_a_timestamp_when_normalizing_text_then_the_top_level_type_decides() { let test_cases = [ - ( - "an array of timestamps is an array", - "ARRAY", - TypeFamily::Other, - ), - ( - "a map of structs holding a timestamp is a map", - "MAP>", - TypeFamily::Other, - ), - ( - "a struct holding a timestamp is a struct", - "STRUCT", - TypeFamily::Other, - ), - ( - "a bare timestamp is a timestamp", - "TIMESTAMP", - TypeFamily::Timestamp, - ), - ( - "a timestamp with precision is a timestamp", - "TIMESTAMP(3)", - TypeFamily::Timestamp, - ), - ( - "a zoned timestamp is a timestamp", - "TIMESTAMP WITH TIME ZONE", - TypeFamily::Timestamp, - ), - ( - "Snowflake's NTZ alias is a timestamp", - "TIMESTAMP_NTZ", - TypeFamily::Timestamp, - ), + TextFamilyTestCase { + description: "an array of timestamps is an array", + type_sql: "ARRAY", + expected_family: TypeFamily::Other, + }, + TextFamilyTestCase { + description: "a map of structs holding a timestamp is a map", + type_sql: "MAP>", + expected_family: TypeFamily::Other, + }, + TextFamilyTestCase { + description: "a struct holding a timestamp is a struct", + type_sql: "STRUCT", + expected_family: TypeFamily::Other, + }, + TextFamilyTestCase { + description: "a bare timestamp is a timestamp", + type_sql: "TIMESTAMP", + expected_family: TypeFamily::Timestamp, + }, + TextFamilyTestCase { + description: "a timestamp with precision is a timestamp", + type_sql: "TIMESTAMP(3)", + expected_family: TypeFamily::Timestamp, + }, + TextFamilyTestCase { + description: "a zoned timestamp is a timestamp", + type_sql: "TIMESTAMP WITH TIME ZONE", + expected_family: TypeFamily::Timestamp, + }, + TextFamilyTestCase { + description: "Snowflake's NTZ alias is a timestamp", + type_sql: "TIMESTAMP_NTZ", + expected_family: TypeFamily::Timestamp, + }, ]; - for (description, type_sql, expected_family) in test_cases { + for test_case in test_cases { assert_eq!( - normalize_text_type(type_sql, None).family, - expected_family, - "{description}" + normalize_text_type(test_case.type_sql, None).family, + test_case.expected_family, + "{}", + test_case.description ); } } diff --git a/crates/sqlbuild-analysis/src/type_system/tests/test_types.rs b/crates/sqlbuild-analysis/src/type_system/tests/test_types.rs index 2b9aeec95d..46c3592c52 100644 --- a/crates/sqlbuild-analysis/src/type_system/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/type_system/tests/test_types.rs @@ -30,3 +30,9 @@ pub(super) struct SameTypeTestCase { pub(super) types: (&'static str, &'static str), pub(super) expected_same: bool, } + +pub(super) struct TextFamilyTestCase { + pub(super) description: &'static str, + pub(super) type_sql: &'static str, + pub(super) expected_family: crate::type_system::models::TypeFamily, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs index 234f5502b4..4020b6e5f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs @@ -5,9 +5,8 @@ use std::path::Path; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyBoolMethods, PyDict, PyDictMethods, PyList, PyString, PyTuple}; use sqlbuild_analysis::analysis_route::main::native_gap::native_gap; -use sqlbuild_analysis::analysis_route::main::skipped_analysis::{ - skipped_analysis, unavailable_analysis, -}; +use sqlbuild_analysis::analysis_route::main::skipped_analysis::skipped_analysis; +use sqlbuild_analysis::analysis_route::main::unavailable_analysis::unavailable_analysis; use sqlbuild_analysis::analysis_route::models::{AnalysisRoute, LocatedGap}; use sqlbuild_analysis::analysis_route::types::NativeDialect; use sqlbuild_analysis::assembly::project::models::{SyntaxCheck, SyntaxMode}; @@ -99,8 +98,12 @@ pub(crate) fn skip_unreadable( let Some(gap) = first_gap(dialect, context, &checks)? else { return Ok(true); }; - let (code, message) = skipped_analysis(model.name, dialect.name(), &gap); - report_skip(settings, context, model, code, message)?; + report_skip( + settings, + context, + model, + skipped_analysis(model.name, dialect.name(), &gap), + )?; Ok(false) } @@ -111,8 +114,12 @@ pub(crate) fn skip_unavailable( context: &LoopContext<'_, '_>, model: &OptOutModel<'_, '_>, ) -> PyResult<()> { - let (code, message) = unavailable_analysis(model.name, dialect); - report_skip(settings, context, model, code, message) + report_skip( + settings, + context, + model, + unavailable_analysis(model.name, dialect), + ) } /// Reports P026 as a warning, or raises it under `require_sql_analysis`. @@ -120,8 +127,7 @@ fn report_skip( settings: ModelLoopSettings, context: &LoopContext<'_, '_>, model: &OptOutModel<'_, '_>, - code: &str, - message: String, + (code, message): (&str, String), ) -> PyResult<()> { let py: Python<'_> = model.config.py(); let help: &str = &context.classes.syntax_help; diff --git a/fensu.toml b/fensu.toml index bee7f18a71..3fee255b7a 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2932,3 +2932,13 @@ reason = "The override keeps queue.Queue.put's positional protocol, which every rule = "FFS201" path = "src/sqlbuild/orchestration_declarations/models/sensor_context.py" reason = "Only SensorContext lives here: invocation-local ctx.cursor assignment proposes progress, committed solely by native fenced completion. The package regression guards this declaration-only scope; class-symbol exceptions are unsupported." + +[[targets.rust.rule_exceptions]] +rule = "FPRSL105" +path = "crates/sqlbuild-analysis/src/type_system/main/normalize_text_type.rs" +reason = "Native dialect modules in analysis_route type their spellings through this entry; each dialect's module lands in its own change." + +[[targets.rust.rule_exceptions]] +rule = "FPRSL105" +path = "crates/sqlbuild-analysis/src/analysis_route/main/clickhouse_type_match.rs" +reason = "The ClickHouse native dialect module's type_match calls this entry; that module lands in its own change." From 80039dcd539e1e1d78f97dbe9e6f658abd399799 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:10:41 +0000 Subject: [PATCH 08/29] chore(deps): pin sqlquack-semantics to 19c51fa2 with the other sqlquack crates --- Cargo.lock | 12 ++++++------ Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 21077bdb23..7d467327d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4091,7 +4091,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -4500,7 +4500,7 @@ dependencies = [ [[package]] name = "sqlquack-core" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=a9dde6a00b144c6306425cfa84516ad816d8c49c#a9dde6a00b144c6306425cfa84516ad816d8c49c" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5#19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" dependencies = [ "serde", "serde_json", @@ -4510,7 +4510,7 @@ dependencies = [ [[package]] name = "sqlquack-semantics" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=a9dde6a00b144c6306425cfa84516ad816d8c49c#a9dde6a00b144c6306425cfa84516ad816d8c49c" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5#19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" dependencies = [ "serde", "serde_json", @@ -4522,7 +4522,7 @@ dependencies = [ [[package]] name = "sqlquack-snowflake" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=a9dde6a00b144c6306425cfa84516ad816d8c49c#a9dde6a00b144c6306425cfa84516ad816d8c49c" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5#19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" dependencies = [ "serde", "sqlquack-core", @@ -4651,7 +4651,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -5087,7 +5087,7 @@ checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" dependencies = [ "memoffset", "tempfile", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index bbe26153f4..c4475e8387 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -177,7 +177,7 @@ sqlbuild-state = { path = "crates/sqlbuild-state" } # SQL lint Rules, quality Rules and the formatter. sqlclean = { git = "https://github.com/sqlstack-io/sqlclean.git", rev = "cf79656d247277a30bc81ef39b6dee420e721221" } # Scopes, name resolution, types and lineage for the native analysis route. -sqlquack-semantics = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "a9dde6a00b144c6306425cfa84516ad816d8c49c" } +sqlquack-semantics = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" } # The lossless SQL tokenizer behind query fingerprints. sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" } sqlquack-syntax-postgres = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" } From a003a5cf07741bd409180e238aed944234ee7dc5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:55:20 +0000 Subject: [PATCH 09/29] fix(analysis): let a native route's parser decide syntax, after the reference scan On a native analysis route the gap gate reads the model SQL and hooks whole, so assembly no longer reruns Polyglot's syntax validation there: a model the native parser reads gets no Polyglot P001. The gate now runs after reference scanning, so a located scanner error wins over a raw normalisation failure, and a model whose reference calls already failed P012 gets no P026. The Polyglot route keeps its order and validation. --- .../_helpers/compile_driver/assembly_models.rs | 10 +++++++++- .../bindings/_helpers/input_loops/model_loop.rs | 16 +++++++++++++--- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs index af5593c480..99db37ef08 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs @@ -4,6 +4,8 @@ use pyo3::IntoPyObject; use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSetMethods, PyList, PyString, PyTuple}; +use sqlbuild_analysis::analysis_route::main::analysis_route::analysis_route; +use sqlbuild_analysis::analysis_route::models::AnalysisRoute; use crate::bindings::_helpers::analysis_session::model_analyses::ModelAnalyses; use crate::bindings::_helpers::analysis_session::profile_inputs::profile_dialect; @@ -116,7 +118,13 @@ fn compiled_model<'py>( syntax()?, ) }; - let validates: bool = input_validated && !syntax_validated; + let analysis_dialect: Option = + native_field(&context.profile, "sql_analysis_dialect")?.extract()?; + let legacy_route: bool = matches!( + analysis_route(analysis_dialect.as_deref()), + AnalysisRoute::Legacy + ); + let validates: bool = input_validated && !syntax_validated && legacy_route; if request.sql_validation_enabled && validates { for hook_name in HOOK_NAMES { validate_hook_sql_syntax( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs index b75a525fa2..731e152bd8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs @@ -717,8 +717,10 @@ impl NativeModelLoop { relative_path, contents, }; - let (analysed, rejected_opt_out, analysis_skipped) = - self.analysis(context, &opt_out_model)?; + let legacy_gate = match self.route { + AnalysisRoute::Legacy => Some(self.analysis(context, &opt_out_model, true)?), + AnalysisRoute::Native(_) | AnalysisRoute::Unavailable => None, + }; let scanned = match extract_sql_references(&query_sql, &self.syntax) { ReferenceExtraction::Extracted(scanned) => scanned, ReferenceExtraction::Failed(failure) => { @@ -751,6 +753,10 @@ impl NativeModelLoop { contents, }, )?; + let (analysed, rejected_opt_out, analysis_skipped) = match legacy_gate { + Some(gate) => gate, + None => self.analysis(context, &opt_out_model, scanned.invalid_calls.is_empty())?, + }; let analysis: (bool, Option>) = ( analysed && scanned.invalid_calls.is_empty(), rejected_opt_out, @@ -818,11 +824,12 @@ impl NativeModelLoop { }) } - /// Whether the model's SQL is analysed, its rejected opt-out, and whether its parser skipped it. + /// Analysed, rejected opt-out, skipped; a native route reads no SQL whose calls P012 failed. fn analysis<'py>( &self, context: &LoopContext<'_, 'py>, model: &OptOutModel<'_, 'py>, + calls_valid: bool, ) -> PyResult<(bool, Option>, bool)> { let mut skipped: bool = false; let (analysed, rejected) = @@ -836,6 +843,9 @@ impl NativeModelLoop { } AnalysisGate::Enabled => match self.route { AnalysisRoute::Legacy => (true, None), + AnalysisRoute::Native(_) | AnalysisRoute::Unavailable if !calls_valid => { + (false, None) + } AnalysisRoute::Native(dialect) => { let read: bool = skip_unreadable(self.settings, dialect, context, model)?; skipped = !read; From 382d3cf7ba1f5264409282377c2102fb4afbb66c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 10:38:07 +0000 Subject: [PATCH 10/29] feat(analysis): analyse models on a native route with the dialect's own parser A wave whose dialect routes natively now analyses each model in parallel on the analysis pool through the dialect's analyse, with the session's schema and nullability, instead of Polyglot. Placeholder defaults are filled before the parse, SQL macro checks still run on the outcome, and the Polyglot completion and cache attachment steps are skipped for native routes. --- .../src/analysis_route/tests.rs | 2 + .../tests/test_native_outcome.rs | 82 +++++++++++++ .../src/analysis_route/tests/test_types.rs | 9 ++ .../_helpers/analysis_cache.rs | 4 + .../assembly/analysis_session/_helpers/mod.rs | 1 + .../_helpers/native_outcome.rs | 112 ++++++++++++++++++ .../analysis_session/_helpers/session.rs | 46 ++++++- .../src/assembly/analysis_session/models.rs | 23 ++++ 8 files changed, 277 insertions(+), 2 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/native_outcome.rs diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests.rs b/crates/sqlbuild-analysis/src/analysis_route/tests.rs index add990df01..0b05ee5ffb 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests.rs @@ -6,5 +6,7 @@ mod test_clickhouse_type_match; mod test_native_dialect; #[path = "tests/test_native_gap.rs"] mod test_native_gap; +#[path = "tests/test_native_outcome.rs"] +mod test_native_outcome; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs new file mode 100644 index 0000000000..59cb6fdaf1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs @@ -0,0 +1,82 @@ +use crate::analysis_route::tests::test_types::{CataloguedDialect, NativeOutcomeTestCase}; +use crate::assembly::analysis_session::_helpers::mappings::ShapeTable; +use crate::assembly::analysis_session::_helpers::native_outcome::native_outcome; +use crate::assembly::analysis_session::models::{ModelOutcome, ModelRequest}; +use crate::assembly::analysis_session::types::Shapes; + +#[test] +fn given_session_shapes_when_a_native_route_analyses_a_model_then_its_facts_follow_them() { + let test_cases = [ + NativeOutcomeTestCase { + description: "a closed source expands a star with its types and known nullability", + sql: "SELECT * FROM raw_orders", + shape: &[("order_id", "INTEGER"), ("status", "VARCHAR")], + expected: &[ + "order_id INTEGER non_null", + "status VARCHAR nullable", + "succeeded=true star=true/true", + ], + }, + NativeOutcomeTestCase { + description: "a relation the session has no shape for is unknown, not empty", + sql: "SELECT * FROM raw_orders", + shape: &[], + expected: &["succeeded=true star=true/false"], + }, + NativeOutcomeTestCase { + description: "SQL the dialect cannot read fails the analysis", + sql: "SELECT FROM WHERE", + shape: &[("order_id", "INTEGER")], + expected: &["succeeded=false star=false/false"], + }, + ]; + + for test_case in test_cases { + let shape: Vec<(String, String)> = test_case + .shape + .iter() + .map(|(name, data_type)| ((*name).to_owned(), (*data_type).to_owned())) + .collect(); + let schema: Shapes = vec![("raw_orders".to_owned(), shape)]; + let nullability: ShapeTable = ShapeTable::from_shapes(&vec![( + "raw_orders".to_owned(), + vec![ + ("order_id".to_owned(), "non_null".to_owned()), + ("status".to_owned(), "nullable".to_owned()), + ], + )]); + let model: ModelRequest = ModelRequest { + name: "orders".to_owned(), + query_sql: test_case.sql.to_owned(), + required_names: vec!["raw_orders".to_owned()], + ..ModelRequest::default() + }; + let (outcome, failure): (ModelOutcome, Option) = + native_outcome(&CataloguedDialect, &model, (schema, &nullability)); + let analysis = &outcome.analysis; + let mut lines: Vec = analysis + .columns + .iter() + .flatten() + .map(|column| { + format!( + "{} {} {}", + column.name, + column.data_type.as_deref().unwrap_or("-"), + column.nullability + ) + }) + .collect(); + lines.push(format!( + "succeeded={} star={}/{}", + analysis.analysis_succeeded, analysis.has_star, analysis.star_resolved + )); + assert_eq!(lines, test_case.expected, "{}", test_case.description); + assert_eq!( + failure.is_some(), + !analysis.analysis_succeeded, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs index 95d2188b32..a6a5fdbb1e 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs @@ -181,3 +181,12 @@ pub(super) struct StarTestCase { /// Whether it projects a star, whether every star column is known, and the column count. pub(super) expected_star: (bool, bool, usize), } + +pub(super) struct NativeOutcomeTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + /// `raw_orders`' shape as the session knows it; empty when it does not. + pub(super) shape: &'static [(&'static str, &'static str)], + /// Each output column as `name type nullability`, then `succeeded star=/`. + pub(super) expected: &'static [&'static str], +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs index 757a894a90..bd20efd783 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs @@ -7,6 +7,7 @@ use sqlbuild_cache::digest::types::ContentDigest; use sqlbuild_cache::store::errors::StoreDecodeError; use sqlbuild_cache::store::models::NativeStore; +use crate::analysis_route::models::AnalysisRoute; use crate::assembly::analysis_session::constants::{ ANALYSIS_CACHE_FORMAT, VARINT_CONTINUATION, VARINT_MAX_BYTES, VARINT_PAYLOAD_BITS, VARINT_PAYLOAD_MASK, @@ -27,6 +28,9 @@ pub(crate) type CachedOutcome = (ModelOutcome, Option); impl AnalysisSession { /// Read and fill `store`; its environment already names the build and Python's settings. pub(crate) fn attach_cache(&mut self, store: NativeStore) { + if matches!(self.route(), AnalysisRoute::Native(_)) { + return; + } let models: usize = self.request.models.len(); self.cache = Some(AnalysisCache { store, diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mod.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mod.rs index 61615bd519..b9e6bfc973 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mod.rs @@ -6,5 +6,6 @@ pub(crate) mod dict_walk; pub(crate) mod dynamic_pivot; pub(crate) mod enrichment; pub(crate) mod mappings; +pub(crate) mod native_outcome; pub(crate) mod query_columns; pub(crate) mod session; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/native_outcome.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/native_outcome.rs new file mode 100644 index 0000000000..c10454aa27 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/native_outcome.rs @@ -0,0 +1,112 @@ +//! One model's outcome from its dialect's native analysis module, in the session's facts. + +use std::collections::HashMap; + +use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; +use sqlquack_semantics::lineage::models::{Nullability, SchemaColumn}; + +use crate::analysis_route::models::{AnalysisInput, NativeAnalysis, ParserGap}; +use crate::analysis_route::types::NativeDialect; +use crate::assembly::analysis_session::_helpers::mappings::ShapeTable; +use crate::assembly::analysis_session::constants::UNKNOWN_TYPE; +use crate::assembly::analysis_session::models::{ + LineageFacts, ModelAnalysis, ModelOutcome, ModelRequest, SessionSchema, +}; +use crate::assembly::analysis_session::types::{Pairs, Shapes}; +use crate::semantic_validation::main::normalize_native::normalize_native_sql; + +/// The model's outcome read against `schema`, its complete input shapes, and why it failed if it did. +pub(crate) fn native_outcome( + dialect: &dyn NativeDialect, + model: &ModelRequest, + (schema, nullability): (Shapes, &ShapeTable), +) -> (ModelOutcome, Option) { + let normalised: Result = + normalize_native_sql(&model.query_sql, dialect.name(), dialect.stub_prefix()); + let (cleaned_sql, analysed): (String, Result) = match normalised { + Ok(normalised) => { + let sql: String = placeholder_defaults(&normalised, &model.placeholders); + let relations: SessionSchema = session_schema(&schema, nullability); + let analysed = dialect + .analyse(&AnalysisInput { + sql: &sql, + schema: &relations, + references: &model.lineage_references, + extension_functions: &[], + }) + .map_err(|gap: ParserGap| gap.message); + (sql, analysed) + } + Err(reason) => (String::new(), Err(reason)), + }; + let failure: Option = analysed + .as_ref() + .err() + .map(|reason| format!("{}: {reason}", model.name)); + let analysis: ModelAnalysis = match analysed { + Ok(native) => ModelAnalysis { + analysis_succeeded: true, + columns: Some(native.columns), + lineage: LineageFacts::Native(native.lineage), + has_star: native.has_star, + star_resolved: native.star_resolved, + binding_diagnostics: native.diagnostics, + binding_validated: true, + }, + Err(_) => ModelAnalysis { + analysis_succeeded: false, + columns: None, + lineage: LineageFacts::Native(Vec::new()), + has_star: false, + star_resolved: false, + binding_diagnostics: Vec::new(), + binding_validated: true, + }, + }; + let outcome: ModelOutcome = ModelOutcome { + analysis, + cleaned_sql, + deferred_cleaning: None, + validated_schema: schema, + fused_binding_validated: true, + sql_macro_findings: Vec::new(), + }; + (outcome, failure) +} + +/// Each relation with a known shape, its columns typed and with the nullability the session knows. +fn session_schema(schema: &Shapes, nullability: &ShapeTable) -> SessionSchema { + let relations: HashMap> = schema + .iter() + .filter(|(_, columns)| !columns.is_empty()) + .map(|(relation, columns)| { + let known: Option<&Pairs> = nullability.get(relation); + let typed: Vec = columns + .iter() + .map(|(name, data_type)| SchemaColumn { + name: name.clone(), + quoted: false, + data_type: (!data_type.is_empty() && data_type != UNKNOWN_TYPE) + .then(|| data_type.clone()), + nullability: column_nullability(known, name), + }) + .collect(); + (relation.clone(), typed) + }) + .collect(); + SessionSchema { relations } +} + +fn column_nullability(known: Option<&Pairs>, column: &str) -> Nullability { + let spelled: Option<&str> = known.and_then(|pairs| { + pairs + .iter() + .find(|(name, _)| name == column) + .map(|(_, value)| value.as_str()) + }); + match spelled { + Some("non_null") => Nullability::NonNull, + Some("nullable") => Nullability::Nullable, + _ => Nullability::Unknown, + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs index 19327a8c92..80ccfc6d07 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs @@ -4,12 +4,17 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; use rayon::ThreadPool; -use rayon::iter::{IndexedParallelIterator, IntoParallelRefIterator, ParallelIterator}; +use rayon::iter::{ + IndexedParallelIterator, IntoParallelIterator, IntoParallelRefIterator, ParallelIterator, +}; use sqlbuild_cache::digest::types::ContentDigest; use sqlbuild_cache::store::errors::StoreDecodeError; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use sqlbuild_core::panics::main::native_failure::native_failure; +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; +use crate::analysis_route::types::NativeDialect; use crate::assembly::analysis_session::_helpers::analysis_cache::{ CachedOutcome, decode_outcome, encode_outcome, }; @@ -29,6 +34,7 @@ use crate::assembly::analysis_session::_helpers::mappings::{ ShapeTable, dict_from_pairs, same_keys, same_relations, }; use crate::assembly::analysis_session::_helpers::mappings::{producers, waves}; +use crate::assembly::analysis_session::_helpers::native_outcome::native_outcome; use crate::assembly::analysis_session::constants::{UNKNOWN_NULLABILITY, UNKNOWN_TYPE}; use crate::assembly::analysis_session::main::resolved_cleaned_sql::resolved_cleaned_sql; use crate::assembly::analysis_session::models::{ @@ -226,6 +232,9 @@ impl AnalysisSession { /// Python's `_analyze_model_sql_requests` for one wave, less the models the cache answers. fn analyze_wave(&mut self, models: &[usize]) -> Result<(), String> { + if let AnalysisRoute::Native(dialect) = self.route() { + return self.analyze_native_wave(dialect, models); + } let schemas: Vec = models .iter() .map(|model| self.binding_schema(&self.request.models[*model])) @@ -294,6 +303,39 @@ impl AnalysisSession { self.check_sql_macros(models) } + /// Each model of the wave analysed by its dialect's native module, in parallel. + fn analyze_native_wave( + &mut self, + dialect: &'static dyn NativeDialect, + models: &[usize], + ) -> Result<(), String> { + let inputs: Vec<(usize, Shapes)> = models + .iter() + .map(|model| (*model, self.binding_schema(&self.request.models[*model]))) + .collect(); + let pool = self.catalog.native.analysis_pool()?; + let request: &SessionRequest = &self.request; + let nullability: &ShapeTable = &self.available_nullability; + let outcomes: Vec<(ModelOutcome, Option)> = pool.install(|| { + inputs + .into_par_iter() + .map(|(model, schema)| { + native_outcome(dialect, &request.models[model], (schema, nullability)) + }) + .collect() + }); + for (model, (outcome, failure)) in models.iter().zip(outcomes) { + self.failures.extend(failure); + self.outcomes[*model] = Some(outcome); + } + self.check_sql_macros(models) + } + + /// The route of the session's dialect: a native module, or Polyglot. + pub(crate) fn route(&self) -> AnalysisRoute { + analysis_route(Some(&self.request.dialect)) + } + /// Each model's SQL macro call arguments checked against its analysed types, cache hits too. fn check_sql_macros(&mut self, models: &[usize]) -> Result<(), String> { let checked: Vec = models @@ -376,7 +418,7 @@ impl AnalysisSession { /// Python's `_complete_inferred_bindings` star and type pass for one wave. fn complete_wave(&mut self, wave: usize) -> Result<(), String> { - if self.cyclic { + if self.cyclic || matches!(self.route(), AnalysisRoute::Native(_)) { return Ok(()); } let mut candidates: Vec<(usize, Shapes, bool)> = Vec::new(); diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs index 594038d064..b3f7ff410c 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs @@ -7,6 +7,8 @@ use std::sync::Arc; use rayon::ThreadPool; use sqlbuild_cache::digest::types::ContentDigest; use sqlbuild_cache::store::models::NativeStore; +use sqlquack_semantics::lineage::models::SchemaColumn; +use sqlquack_semantics::lineage::types::LineageSchema; use crate::assembly::analysis_session::_helpers::catalog_state::SessionCatalog; use crate::assembly::analysis_session::_helpers::mappings::ShapeTable; @@ -336,6 +338,27 @@ pub struct AnalysisCache { pub(crate) stats: AnalysisCacheStats, } +/// The relations one model reads, as the session knows them, for a native route's analysis. +#[derive(Debug, Default)] +pub(crate) struct SessionSchema { + /// Columns of each relation with a known shape, by the analysis name the query reads. + pub(crate) relations: HashMap>, +} + +impl LineageSchema for SessionSchema { + fn columns(&self, relation: &str) -> Option<&[SchemaColumn]> { + self.relations + .get(relation) + .or_else(|| { + self.relations + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(relation)) + .map(|(_, columns)| columns) + }) + .map(Vec::as_slice) + } +} + /// One compile's model analysis, advanced phase by phase. #[derive(Debug)] pub struct AnalysisSession { From 183734280b142622e565f38477c1693139590e46 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 10:41:01 +0000 Subject: [PATCH 11/29] fix(analysis): find a native route's stubs with the dialect's own quoting rules Finding which cursor stubs and hook parameters sit in code used one quoting policy for every dialect, with backslash escapes in single-quoted strings. A DuckDB or Postgres string ending in a literal backslash, such as 'C:\', then read as unclosed and the model failed with a P001. The scan now uses the dialect's lexical syntax, so E'' escape strings, Snowflake backslash escapes and // comments still hide stubs. The dialect syntax mapping is the same one the rules crate moves to sqlbuild-sqltext. --- .../_helpers/normalization.rs | 20 ++------- .../tests/test_normalize_batch.rs | 37 +++++++++++++++- .../semantic_validation/tests/test_types.rs | 1 + .../sql_scan/main/dialect_lexical_syntax.rs | 43 +++++++++++++++++++ .../sqlbuild-sqltext/src/sql_scan/main/mod.rs | 1 + 5 files changed, 85 insertions(+), 17 deletions(-) create mode 100644 crates/sqlbuild-sqltext/src/sql_scan/main/dialect_lexical_syntax.rs diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs index febdc7140a..5dd9a55e9a 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs @@ -2,6 +2,8 @@ use crate::semantic_checks::constants::UDF_NAME_PREFIX; use regex::Regex; +use sqlbuild_sqltext::sql_scan::main::dialect_lexical_syntax::dialect_lexical_syntax; +use sqlbuild_sqltext::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges; use sqlbuild_sqltext::sql_scan::main::matching_paren::matching_paren; use sqlbuild_sqltext::sql_scan::main::non_code_end::non_code_end; use sqlbuild_sqltext::sql_scan::models::QuotePolicy; @@ -194,7 +196,8 @@ pub(crate) fn normalize_native_sql( let udf_prefix: String = format!("{stub_prefix}udf_"); let normalized: String = normalize(sql, dialect, (&none, &none), &udf_prefix)?.sql; let stubs: &Regex = NATIVE_STUBS.as_ref()?; - let non_code: Vec<(usize, usize)> = non_code_ranges(&normalized)?; + let non_code: Vec<(usize, usize)> = + dialect_non_code_ranges(&normalized, &dialect_lexical_syntax(dialect)); let mut native: String = String::with_capacity(normalized.len()); let mut copied_until: usize = 0; for captures in stubs.captures_iter(&normalized) { @@ -226,21 +229,6 @@ pub(crate) fn normalize_native_sql( Ok(native) } -/// The byte ranges of `sql`'s strings, quoted identifiers and comments, in order. -fn non_code_ranges(sql: &str) -> Result, String> { - let mut ranges: Vec<(usize, usize)> = Vec::new(); - let mut index: usize = 0; - while index < sql.len() { - if let Some(end) = skip_non_code(sql, index)? { - ranges.push((index, end)); - index = end; - continue; - } - index += sql[index..].chars().next().map_or(1, char::len_utf8); - } - Ok(ranges) -} - pub(crate) fn normalize_dialect_sql_mapped( sql: &str, dialect: &str, diff --git a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs index bbc5a4ea36..05db7af22e 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs @@ -109,34 +109,69 @@ fn given_cursor_stubs_when_normalizing_for_a_native_parser_then_only_whole_names { let test_cases = [ NativeStubTestCase { + dialect: "postgres", description: "a cursor stub in code takes the dialect's prefix", sql: "SELECT * FROM t WHERE ts > __cursor_start()", expected_sql: "SELECT * FROM t WHERE ts > __sqlbuild.cursor_start()", }, NativeStubTestCase { + dialect: "postgres", description: "a cursor stub's text inside a string literal is left as written", sql: "SELECT '__cursor_start()' AS s FROM t", expected_sql: "SELECT '__cursor_start()' AS s FROM t", }, NativeStubTestCase { + dialect: "postgres", description: "a cursor stub's text inside a comment is left as written", sql: "SELECT 1 FROM t -- __cursor_end()\n", expected_sql: "SELECT 1 FROM t -- __cursor_end()\n", }, NativeStubTestCase { + dialect: "postgres", description: "a longer name ending in a cursor stub's text is another function", sql: "SELECT x__cursor_end() FROM t", expected_sql: "SELECT x__cursor_end() FROM t", }, NativeStubTestCase { + dialect: "postgres", description: "a quoted stub beside a stub in code leaves only the code one respelt", sql: "SELECT '__cursor_end()', __cursor_end() FROM t", expected_sql: "SELECT '__cursor_end()', __sqlbuild.cursor_end() FROM t", }, + NativeStubTestCase { + dialect: "duckdb", + description: "a DuckDB backslash before a closing quote is literal, so the stub after it is code", + sql: r"SELECT 'C:\' AS folder, __cursor_end() FROM t", + expected_sql: r"SELECT 'C:\' AS folder, __sqlbuild.cursor_end() FROM t", + }, + NativeStubTestCase { + dialect: "postgres", + description: "a Postgres escape string's backslash-quoted quote keeps the stub after it quoted", + sql: r"SELECT E'O\'Brien __cursor_end()', __cursor_end() FROM t", + expected_sql: r"SELECT E'O\'Brien __cursor_end()', __sqlbuild.cursor_end() FROM t", + }, + NativeStubTestCase { + dialect: "postgres", + description: "a plain Postgres string's backslash is literal, so the stub after it is code", + sql: r"SELECT 'C:\', __cursor_end() FROM t", + expected_sql: r"SELECT 'C:\', __sqlbuild.cursor_end() FROM t", + }, + NativeStubTestCase { + dialect: "snowflake", + description: "a Snowflake backslash-quoted quote keeps the stub after it quoted", + sql: r"SELECT 'it\'s __cursor_end()', __cursor_end() FROM t", + expected_sql: r"SELECT 'it\'s __cursor_end()', __sqlbuild.cursor_end() FROM t", + }, + NativeStubTestCase { + dialect: "snowflake", + description: "a cursor stub's text inside a Snowflake // comment is left as written", + sql: "SELECT 1 FROM t // __cursor_end()\n", + expected_sql: "SELECT 1 FROM t // __cursor_end()\n", + }, ]; for test_case in test_cases { assert_eq!( - normalize_native_sql(test_case.sql, "postgres", "__sqlbuild.").as_deref(), + normalize_native_sql(test_case.sql, test_case.dialect, "__sqlbuild.").as_deref(), Ok(test_case.expected_sql), "{}", test_case.description diff --git a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_types.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_types.rs index 27fa2325ea..8a51815a13 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_types.rs @@ -55,6 +55,7 @@ pub(crate) struct NormalizationBatchTestCase { pub(crate) struct NativeStubTestCase { pub(crate) description: &'static str, + pub(crate) dialect: &'static str, pub(crate) sql: &'static str, pub(crate) expected_sql: &'static str, } diff --git a/crates/sqlbuild-sqltext/src/sql_scan/main/dialect_lexical_syntax.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/dialect_lexical_syntax.rs new file mode 100644 index 0000000000..ff94e94b32 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/dialect_lexical_syntax.rs @@ -0,0 +1,43 @@ +//! `dialect_lexical_syntax`: the comment and quoted-text syntax of a dialect's SQL, by dialect name. + +use crate::sql_scan::models::LexicalSyntax; + +/// The comment and quoted-text syntax that SQLQuack tokenizes for `dialect`. +pub fn dialect_lexical_syntax(dialect: &str) -> LexicalSyntax { + let strings = |quotes: &[&str]| quotes.iter().map(|quote| (*quote).to_owned()).collect(); + let lines = |prefixes: &[&str]| prefixes.iter().map(|prefix| (*prefix).to_owned()).collect(); + match dialect.to_ascii_lowercase().as_str() { + "bigquery" => LexicalSyntax { + backslash_escape_quotes: strings(&["'", "\""]), + raw_string_prefix: true, + triple_quoted_strings: true, + line_comment_prefixes: lines(&["--", "#"]), + ..LexicalSyntax::default() + }, + "snowflake" => LexicalSyntax { + backslash_escape_quotes: strings(&["'"]), + line_comment_prefixes: lines(&["--", "//"]), + ..LexicalSyntax::default() + }, + "clickhouse" => LexicalSyntax { + backslash_escape_quotes: strings(&["'"]), + line_comment_prefixes: lines(&["--", "#", "//"]), + ..LexicalSyntax::default() + }, + "databricks" | "duckdb" => LexicalSyntax { + line_comment_prefixes: lines(&["--"]), + ..LexicalSyntax::default() + }, + "mssql" | "sqlserver" | "tsql" => LexicalSyntax { + nested_block_comments: true, + line_comment_prefixes: lines(&["--"]), + ..LexicalSyntax::default() + }, + _ => LexicalSyntax { + escape_string_prefix: true, + nested_block_comments: true, + line_comment_prefixes: lines(&["--"]), + ..LexicalSyntax::default() + }, + } +} diff --git a/crates/sqlbuild-sqltext/src/sql_scan/main/mod.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/mod.rs index 4735e4b172..3cc4afefde 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/main/mod.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/mod.rs @@ -1,4 +1,5 @@ pub mod comment_end; +pub mod dialect_lexical_syntax; pub mod dialect_matching_paren; pub mod dialect_non_code_end; pub mod dialect_non_code_ranges; From aac5217e0827691918c53f4ea321f54333e1ab3c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 10:56:52 +0000 Subject: [PATCH 12/29] fix(analysis): keep the analysis cache on a native route A native route skipped the analysis cache, so every compile re-analysed every model and a warm run could not answer from stored outcomes. The cache now attaches on every route. A native wave analyses only the models the cache misses, a hit's cleaned SQL is the dialect's normalised SQL, and the Polyglot catalog is only prepared for the Polyglot route. --- .../_helpers/analysis_cache.rs | 4 - .../_helpers/native_outcome.rs | 43 ++++++---- .../analysis_session/_helpers/session.rs | 85 +++++++++++-------- 3 files changed, 75 insertions(+), 57 deletions(-) diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs index bd20efd783..757a894a90 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs @@ -7,7 +7,6 @@ use sqlbuild_cache::digest::types::ContentDigest; use sqlbuild_cache::store::errors::StoreDecodeError; use sqlbuild_cache::store::models::NativeStore; -use crate::analysis_route::models::AnalysisRoute; use crate::assembly::analysis_session::constants::{ ANALYSIS_CACHE_FORMAT, VARINT_CONTINUATION, VARINT_MAX_BYTES, VARINT_PAYLOAD_BITS, VARINT_PAYLOAD_MASK, @@ -28,9 +27,6 @@ pub(crate) type CachedOutcome = (ModelOutcome, Option); impl AnalysisSession { /// Read and fill `store`; its environment already names the build and Python's settings. pub(crate) fn attach_cache(&mut self, store: NativeStore) { - if matches!(self.route(), AnalysisRoute::Native(_)) { - return; - } let models: usize = self.request.models.len(); self.cache = Some(AnalysisCache { store, diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/native_outcome.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/native_outcome.rs index c10454aa27..9211877862 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/native_outcome.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/native_outcome.rs @@ -21,24 +21,22 @@ pub(crate) fn native_outcome( model: &ModelRequest, (schema, nullability): (Shapes, &ShapeTable), ) -> (ModelOutcome, Option) { - let normalised: Result = - normalize_native_sql(&model.query_sql, dialect.name(), dialect.stub_prefix()); - let (cleaned_sql, analysed): (String, Result) = match normalised { - Ok(normalised) => { - let sql: String = placeholder_defaults(&normalised, &model.placeholders); - let relations: SessionSchema = session_schema(&schema, nullability); - let analysed = dialect - .analyse(&AnalysisInput { - sql: &sql, - schema: &relations, - references: &model.lineage_references, - extension_functions: &[], - }) - .map_err(|gap: ParserGap| gap.message); - (sql, analysed) - } - Err(reason) => (String::new(), Err(reason)), - }; + let (cleaned_sql, analysed): (String, Result) = + match native_cleaned_sql(dialect, model) { + Ok(sql) => { + let relations: SessionSchema = session_schema(&schema, nullability); + let analysed = dialect + .analyse(&AnalysisInput { + sql: &sql, + schema: &relations, + references: &model.lineage_references, + extension_functions: &[], + }) + .map_err(|gap: ParserGap| gap.message); + (sql, analysed) + } + Err(reason) => (String::new(), Err(reason)), + }; let failure: Option = analysed .as_ref() .err() @@ -74,6 +72,15 @@ pub(crate) fn native_outcome( (outcome, failure) } +/// The model's SQL as its dialect's parser reads it: stubs respelt and placeholders filled. +pub(crate) fn native_cleaned_sql( + dialect: &dyn NativeDialect, + model: &ModelRequest, +) -> Result { + normalize_native_sql(&model.query_sql, dialect.name(), dialect.stub_prefix()) + .map(|normalised| placeholder_defaults(&normalised, &model.placeholders)) +} + /// Each relation with a known shape, its columns typed and with the nullability the session knows. fn session_schema(schema: &Shapes, nullability: &ShapeTable) -> SessionSchema { let relations: HashMap> = schema diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs index 80ccfc6d07..4e5371158f 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs @@ -4,9 +4,7 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; use rayon::ThreadPool; -use rayon::iter::{ - IndexedParallelIterator, IntoParallelIterator, IntoParallelRefIterator, ParallelIterator, -}; +use rayon::iter::{IndexedParallelIterator, IntoParallelRefIterator, ParallelIterator}; use sqlbuild_cache::digest::types::ContentDigest; use sqlbuild_cache::store::errors::StoreDecodeError; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; @@ -34,7 +32,9 @@ use crate::assembly::analysis_session::_helpers::mappings::{ ShapeTable, dict_from_pairs, same_keys, same_relations, }; use crate::assembly::analysis_session::_helpers::mappings::{producers, waves}; -use crate::assembly::analysis_session::_helpers::native_outcome::native_outcome; +use crate::assembly::analysis_session::_helpers::native_outcome::{ + native_cleaned_sql, native_outcome, +}; use crate::assembly::analysis_session::constants::{UNKNOWN_NULLABILITY, UNKNOWN_TYPE}; use crate::assembly::analysis_session::main::resolved_cleaned_sql::resolved_cleaned_sql; use crate::assembly::analysis_session::models::{ @@ -232,14 +232,14 @@ impl AnalysisSession { /// Python's `_analyze_model_sql_requests` for one wave, less the models the cache answers. fn analyze_wave(&mut self, models: &[usize]) -> Result<(), String> { - if let AnalysisRoute::Native(dialect) = self.route() { - return self.analyze_native_wave(dialect, models); - } let schemas: Vec = models .iter() .map(|model| self.binding_schema(&self.request.models[*model])) .collect(); let misses: Vec = self.read_cache(models, &schemas)?; + if let AnalysisRoute::Native(dialect) = self.route() { + return self.analyze_native_wave(dialect, models, &misses, &schemas); + } let batch = Batch { dialect: &self.request.dialect, function_return_types: &self.request.function_return_types, @@ -303,30 +303,29 @@ impl AnalysisSession { self.check_sql_macros(models) } - /// Each model of the wave analysed by its dialect's native module, in parallel. + /// Each model of the wave the cache did not answer, analysed by its dialect's native module. fn analyze_native_wave( &mut self, dialect: &'static dyn NativeDialect, models: &[usize], + misses: &[usize], + schemas: &[Shapes], ) -> Result<(), String> { - let inputs: Vec<(usize, Shapes)> = models - .iter() - .map(|model| (*model, self.binding_schema(&self.request.models[*model]))) - .collect(); let pool = self.catalog.native.analysis_pool()?; let request: &SessionRequest = &self.request; let nullability: &ShapeTable = &self.available_nullability; let outcomes: Vec<(ModelOutcome, Option)> = pool.install(|| { - inputs - .into_par_iter() - .map(|(model, schema)| { - native_outcome(dialect, &request.models[model], (schema, nullability)) + misses + .par_iter() + .map(|position| { + let model: &ModelRequest = &request.models[models[*position]]; + native_outcome(dialect, model, (schemas[*position].clone(), nullability)) }) .collect() }); - for (model, (outcome, failure)) in models.iter().zip(outcomes) { + for (position, (outcome, failure)) in misses.iter().zip(outcomes) { self.failures.extend(failure); - self.outcomes[*model] = Some(outcome); + self.outcomes[models[*position]] = Some(outcome); } self.check_sql_macros(models) } @@ -584,22 +583,8 @@ impl AnalysisSession { let Some(mut cache) = self.cache.take() else { return Ok((0..models.len()).collect()); }; - let mut required: Vec = Vec::new(); - let mut seen: HashSet<&str> = HashSet::new(); - for model in models { - for reference in &self.request.models[*model].references { - if seen.insert(reference.analysis_name.as_str()) { - required.push(reference.analysis_name.clone()); - } - } - } - self.catalog.prepare_analysis( - &required, - &self.available_types, - &self.available_nullability, - ); - for schema in schemas { - let _ = self.catalog.prepare(&[("", schema)]); + if matches!(self.route(), AnalysisRoute::Legacy) { + self.prepare_catalog(models, schemas); } let pool: Arc = self.catalog.native.analysis_pool()?; let session_digest: ContentDigest = cache.session_digest; @@ -663,7 +648,28 @@ impl AnalysisSession { Ok(misses) } - /// Normalize the cleaned SQL of the cache hits among `models` that still lack it. + /// Load the relations and schemas a Polyglot analysis of `models` reads into its catalog. + fn prepare_catalog(&mut self, models: &[usize], schemas: &[Shapes]) { + let mut required: Vec = Vec::new(); + let mut seen: HashSet<&str> = HashSet::new(); + for model in models { + for reference in &self.request.models[*model].references { + if seen.insert(reference.analysis_name.as_str()) { + required.push(reference.analysis_name.clone()); + } + } + } + self.catalog.prepare_analysis( + &required, + &self.available_types, + &self.available_nullability, + ); + for schema in schemas { + let _ = self.catalog.prepare(&[("", schema)]); + } + } + + /// Normalize the cleaned SQL of the cache hits among `models` that still lack it. fn ensure_cleaned(&mut self, models: &[usize]) -> Result<(), String> { let pending: Vec = models .iter() @@ -673,6 +679,15 @@ impl AnalysisSession { if pending.is_empty() { return Ok(()); } + if let AnalysisRoute::Native(dialect) = self.route() { + for model in pending { + let sql: String = + native_cleaned_sql(dialect, &self.request.models[model]).unwrap_or_default(); + self.outcome_mut(model)?.cleaned_sql = sql; + self.pending_cleaned[model] = false; + } + return Ok(()); + } let schemas: Vec = pending .iter() .map(|model| self.binding_schema(&self.request.models[*model])) From 06bad7a71817697065cdb958cdb44b1d92107e36 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 10:56:52 +0000 Subject: [PATCH 13/29] fix(lineage): build fast lineage natively for models without facts On a native route a model without lineage facts, such as one with SQL analysis turned off, got no fast lineage, where Polyglot used to parse it. Its lineage is now read by the dialect's own parser and analysis over the fast lineage schema, with its placeholder defaults filled. SQL the parser cannot read is still left out, and the debug event gives P026's reason with the construct, line and column. --- .../src/analysis_route/main/native_gap.rs | 2 +- .../analysis_route/main/skipped_analysis.rs | 21 ++- .../tests/test_native_outcome.rs | 96 +++++++++++++- .../src/analysis_route/tests/test_types.rs | 7 + .../src/lineage/_helpers/mod.rs | 1 + .../src/lineage/_helpers/native_lineage.rs | 120 ++++++++++++++++++ .../src/lineage/main/build_fast_lineage.rs | 47 +++++-- .../sqlbuild-analysis/src/lineage/models.rs | 2 + .../src/lineage/tests/helpers.rs | 1 + .../lineage/tests/test_build_fast_lineage.rs | 1 + .../_helpers/lineage/project_lineage.rs | 35 +++-- 11 files changed, 302 insertions(+), 31 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/lineage/_helpers/native_lineage.rs diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/native_gap.rs b/crates/sqlbuild-analysis/src/analysis_route/main/native_gap.rs index e7c3f74397..58168f4625 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/native_gap.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/native_gap.rs @@ -22,7 +22,7 @@ pub fn native_gap( } /// The gap with the line and column of its offset; an offset past the end is the end. -fn located(sql: &str, gap: ParserGap) -> LocatedGap { +pub(crate) fn located(sql: &str, gap: ParserGap) -> LocatedGap { let before: &str = sql.get(..gap.offset).unwrap_or(sql); let position: TextPosition = text_position(sql, before.chars().count()); LocatedGap { diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/skipped_analysis.rs b/crates/sqlbuild-analysis/src/analysis_route/main/skipped_analysis.rs index 6e0d6abaa2..3a09a72095 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/skipped_analysis.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/skipped_analysis.rs @@ -4,17 +4,24 @@ use crate::analysis_route::models::LocatedGap; /// P026's code and message for a model whose SQL the dialect's native parser cannot read. #[must_use] pub fn skipped_analysis(model: &str, dialect: &str, gap: &LocatedGap) -> (&'static str, String) { + ( + SKIPPED_ANALYSIS_CODE, + format!( + "SQL analysis skipped for model '{model}': {}", + unreadable_reason(dialect, gap) + ), + ) +} + +/// Why the dialect's native parser cannot read SQL, as P026 words it. +pub(crate) fn unreadable_reason(dialect: &str, gap: &LocatedGap) -> String { let detail: String = if gap.message.is_empty() || gap.message == gap.construct { String::new() } else { format!(": {}", gap.message) }; - ( - SKIPPED_ANALYSIS_CODE, - format!( - "SQL analysis skipped for model '{model}': the {dialect} parser cannot read {} at \ - line {}, column {}{detail}", - gap.construct, gap.line, gap.column - ), + format!( + "the {dialect} parser cannot read {} at line {}, column {}{detail}", + gap.construct, gap.line, gap.column ) } diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs index 59cb6fdaf1..bbb1531713 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs @@ -1,8 +1,12 @@ -use crate::analysis_route::tests::test_types::{CataloguedDialect, NativeOutcomeTestCase}; +use crate::analysis_route::tests::test_types::{ + CataloguedDialect, NativeLineageTestCase, NativeOutcomeTestCase, PivotlessDialect, +}; use crate::assembly::analysis_session::_helpers::mappings::ShapeTable; use crate::assembly::analysis_session::_helpers::native_outcome::native_outcome; use crate::assembly::analysis_session::models::{ModelOutcome, ModelRequest}; use crate::assembly::analysis_session::types::Shapes; +use crate::lineage::_helpers::native_lineage::native_model_lineage; +use crate::lineage::models::{FastLineageOutcome, LineageResourceType, LineageSchemaResource}; #[test] fn given_session_shapes_when_a_native_route_analyses_a_model_then_its_facts_follow_them() { @@ -80,3 +84,93 @@ fn given_session_shapes_when_a_native_route_analyses_a_model_then_its_facts_foll ); } } + +#[test] +fn given_a_model_without_facts_when_fast_lineage_reads_it_natively_then_it_has_lineage_or_a_reason() +{ + let schema: Vec = vec![ + LineageSchemaResource { + resource_type: LineageResourceType::Model, + name: "orders".to_owned(), + columns: vec!["order_id".to_owned(), "amount".to_owned()], + }, + LineageSchemaResource { + resource_type: LineageResourceType::Source, + name: "raw_payments".to_owned(), + columns: vec!["payment_id".to_owned()], + }, + ]; + let test_cases = [ + NativeLineageTestCase { + description: "a referenced model's columns are the upstreams of direct and computed outputs", + sql: r#"SELECT o.order_id, o.amount * 2 AS doubled FROM __ref("orders") AS o"#, + placeholders: &[], + expected: &[ + "order_id direct high [model:orders.order_id]", + "doubled expression high [model:orders.amount]", + "star=false", + ], + }, + NativeLineageTestCase { + description: "a placeholder takes its default before the parse, and a source is a source", + sql: r#"SELECT payment_id FROM __source("raw_payments") WHERE payment_id > @@@min_id"#, + placeholders: &[("min_id", "0")], + expected: &[ + "payment_id direct high [source:raw_payments.payment_id]", + "star=false", + ], + }, + NativeLineageTestCase { + description: "a star over a relation of unknown columns is a star without outputs", + sql: r#"SELECT * FROM __ref("unknown_model")"#, + placeholders: &[], + expected: &["star=true"], + }, + NativeLineageTestCase { + description: "SQL the parser cannot read gives P026's reason, located", + sql: "SELECT 1\nFROM t PIVOT (sum(x) FOR y IN (1))", + placeholders: &[], + expected: &["unparsed: the postgres parser cannot read PIVOT at line 2, column 8"], + }, + ]; + + for test_case in test_cases { + let placeholders: Vec<(String, String)> = test_case + .placeholders + .iter() + .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) + .collect(); + let outcome: FastLineageOutcome = + native_model_lineage(&PivotlessDialect, test_case.sql, &placeholders, &schema); + let lines: Vec = match outcome { + FastLineageOutcome::Built { columns, has_star } => columns + .iter() + .map(|column| { + let sources: Vec = column + .upstream_columns + .iter() + .map(|source| { + format!( + "{}:{}.{}", + source.resource_type.as_str(), + source.resource_name, + source.column_name + ) + }) + .collect(); + format!( + "{} {} {} [{}]", + column.output_column, + column.transform_kind.as_str(), + column.confidence.as_str(), + sources.join(", ") + ) + }) + .chain([format!("star={has_star}")]) + .collect(), + FastLineageOutcome::Unparsed(reason) => vec![format!("unparsed: {reason}")], + other => vec![format!("{other:?}")], + }; + assert_eq!(lines, test_case.expected, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs index a6a5fdbb1e..6e6be06911 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs @@ -190,3 +190,10 @@ pub(super) struct NativeOutcomeTestCase { /// Each output column as `name type nullability`, then `succeeded star=/`. pub(super) expected: &'static [&'static str], } + +pub(super) struct NativeLineageTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) placeholders: &'static [(&'static str, &'static str)], + pub(super) expected: &'static [&'static str], +} diff --git a/crates/sqlbuild-analysis/src/lineage/_helpers/mod.rs b/crates/sqlbuild-analysis/src/lineage/_helpers/mod.rs index 139d0d6c87..31d6889d68 100644 --- a/crates/sqlbuild-analysis/src/lineage/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/lineage/_helpers/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod authored_files; pub(crate) mod environment_markers; pub(crate) mod fingerprint_digest; +pub(crate) mod native_lineage; pub(crate) mod parsed_lineage; pub(crate) mod projections; pub(crate) mod python_payload; diff --git a/crates/sqlbuild-analysis/src/lineage/_helpers/native_lineage.rs b/crates/sqlbuild-analysis/src/lineage/_helpers/native_lineage.rs new file mode 100644 index 0000000000..328085b21c --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage/_helpers/native_lineage.rs @@ -0,0 +1,120 @@ +//! Fast lineage of a model without facts, read by its dialect's native analysis module. + +use std::collections::HashMap; + +use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; +use sqlquack_semantics::lineage::models::{Nullability, SchemaColumn}; + +use crate::analysis_route::main::native_gap::located; +use crate::analysis_route::main::skipped_analysis::unreadable_reason; +use crate::analysis_route::models::{AnalysisInput, NativeAnalysis, ParserGap}; +use crate::analysis_route::types::NativeDialect; +use crate::assembly::analysis_session::constants::{ + CONFIDENCE_HIGH, CONFIDENCE_MEDIUM, TRANSFORM_AGGREGATION, TRANSFORM_CAST, TRANSFORM_CONSTANT, + TRANSFORM_DIRECT, TRANSFORM_STAR, +}; +use crate::assembly::analysis_session::models::{LineageRow, SessionSchema}; +use crate::lineage::_helpers::references::physical_resources; +use crate::lineage::models::{ + FastLineageOutcome, LineageColumn, LineageConfidence, LineageResourceType, + LineageSchemaResource, LineageSource, LineageTransformKind, +}; +use crate::semantic_validation::main::normalize_native::normalize_native_sql; + +/// The model's lineage over `schema`'s column names, or why the dialect's parser cannot read it. +pub(crate) fn native_model_lineage( + dialect: &dyn NativeDialect, + query_sql: &str, + placeholders: &[(String, String)], + schema: &[LineageSchemaResource], +) -> FastLineageOutcome { + let normalised: String = + match normalize_native_sql(query_sql, dialect.name(), dialect.stub_prefix()) { + Ok(normalised) => normalised, + Err(reason) => return FastLineageOutcome::Unparsed(reason), + }; + let sql: String = placeholder_defaults(&normalised, placeholders); + let mut references: Vec<(String, String, String)> = Vec::new(); + for resource in physical_resources(query_sql) { + if !references + .iter() + .any(|(name, _, _)| *name == resource.resource_name) + { + references.push(( + resource.resource_name.clone(), + resource.resource_type.as_str().to_owned(), + resource.resource_name, + )); + } + } + let relations: SessionSchema = names_schema(schema); + let analysed: Result = dialect.analyse(&AnalysisInput { + sql: &sql, + schema: &relations, + references: &references, + extension_functions: &[], + }); + match analysed { + Ok(native) => FastLineageOutcome::Built { + columns: native.lineage.iter().map(lineage_column).collect(), + has_star: native.has_star, + }, + Err(gap) => { + FastLineageOutcome::Unparsed(unreadable_reason(dialect.name(), &located(&sql, gap))) + } + } +} + +/// Each resource with known columns, untyped and of unknown nullability; the first of a name wins. +fn names_schema(schema: &[LineageSchemaResource]) -> SessionSchema { + let mut relations: HashMap> = HashMap::new(); + for resource in schema + .iter() + .filter(|resource| !resource.columns.is_empty()) + { + let _ = relations.entry(resource.name.clone()).or_insert_with(|| { + resource + .columns + .iter() + .map(|name| SchemaColumn { + name: name.clone(), + quoted: false, + data_type: None, + nullability: Nullability::Unknown, + }) + .collect() + }); + } + SessionSchema { relations } +} + +/// A session lineage row as a fast lineage column; sources of other resource kinds are dropped. +fn lineage_column(row: &LineageRow) -> LineageColumn { + LineageColumn { + output_column: row.output_column.clone(), + transform_kind: match row.transform_code { + TRANSFORM_DIRECT => LineageTransformKind::Direct, + TRANSFORM_CAST => LineageTransformKind::Cast, + TRANSFORM_AGGREGATION => LineageTransformKind::Aggregation, + TRANSFORM_STAR => LineageTransformKind::Star, + TRANSFORM_CONSTANT => LineageTransformKind::Constant, + _ => LineageTransformKind::Expression, + }, + confidence: match row.confidence_code { + CONFIDENCE_HIGH => LineageConfidence::High, + CONFIDENCE_MEDIUM => LineageConfidence::Medium, + _ => LineageConfidence::Unknown, + }, + upstream_columns: row + .sources + .iter() + .filter_map(|(resource_type, resource_name, column_name)| { + Some(LineageSource { + resource_type: LineageResourceType::from_value(resource_type)?, + resource_name: resource_name.clone(), + column_name: column_name.clone(), + }) + }) + .collect(), + } +} diff --git a/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs b/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs index 94dcca4ac8..0d74ba11a7 100644 --- a/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs +++ b/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs @@ -8,6 +8,9 @@ use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use sqlbuild_core::panics::main::native_failure::native_failure; +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; +use crate::lineage::_helpers::native_lineage::native_model_lineage; use crate::lineage::_helpers::parsed_lineage::{ ParsedModel, parsed_model_lineage, proxy_parse_options, }; @@ -52,23 +55,49 @@ fn model_outcomes(request: &FastLineageRequest) -> Vec = proxy_parse_options(); request .models .par_iter() - .map(|model| match model { - FastLineageModel::StarExpansion { - query_sql, - existing_columns, - } => Ok(FastLineageOutcome::StarColumns(star_lineage( + .map(|model| match (model, route) { + ( + FastLineageModel::StarExpansion { + query_sql, + existing_columns, + }, + _, + ) => Ok(FastLineageOutcome::StarColumns(star_lineage( &schema, &physical_resources(query_sql), existing_columns.iter().map(String::as_str), ))), - FastLineageModel::Parse { - query_sql, - inferred_columns, - } => { + ( + FastLineageModel::Parse { + query_sql, + placeholders, + .. + }, + AnalysisRoute::Native(module), + ) => catch_compiler_panic(|| { + Ok(native_model_lineage( + module, + query_sql, + placeholders, + &request.schema, + )) + }), + (FastLineageModel::Parse { .. }, AnalysisRoute::Unavailable) => { + Ok(FastLineageOutcome::Omitted) + } + ( + FastLineageModel::Parse { + query_sql, + inferred_columns, + .. + }, + AnalysisRoute::Legacy, + ) => { let model = ParsedModel { query_sql, inferred_names: inferred_columns, diff --git a/crates/sqlbuild-analysis/src/lineage/models.rs b/crates/sqlbuild-analysis/src/lineage/models.rs index d8d4a2657b..2632824ed2 100644 --- a/crates/sqlbuild-analysis/src/lineage/models.rs +++ b/crates/sqlbuild-analysis/src/lineage/models.rs @@ -106,6 +106,8 @@ pub enum FastLineageModel { Parse { query_sql: String, inferred_columns: Vec, + /// `@@@name` defaults a native parser reads in place of each placeholder. + placeholders: Vec<(String, String)>, }, } diff --git a/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs b/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs index bf332a89da..0553e0e346 100644 --- a/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs @@ -45,6 +45,7 @@ pub(crate) fn deep_union_outcome_lines( FastLineageModel::Parse { query_sql: sql, inferred_columns: Vec::new(), + placeholders: Vec::new(), }, ) }) diff --git a/crates/sqlbuild-analysis/src/lineage/tests/test_build_fast_lineage.rs b/crates/sqlbuild-analysis/src/lineage/tests/test_build_fast_lineage.rs index 9998ac560d..9e9f1a77de 100644 --- a/crates/sqlbuild-analysis/src/lineage/tests/test_build_fast_lineage.rs +++ b/crates/sqlbuild-analysis/src/lineage/tests/test_build_fast_lineage.rs @@ -234,6 +234,7 @@ fn given_models_without_compact_facts_when_building_then_matches_python_lineage( FastLineageModel::Parse { query_sql: test_case.sql.to_owned(), inferred_columns: strings(test_case.inferred_columns), + placeholders: Vec::new(), }, ); assert_eq!( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs index 7bc1ff9fd9..ad9c50e3f5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs @@ -30,6 +30,7 @@ use crate::bindings::_helpers::lineage::lineage_objects::{LineageClasses, log_li use crate::bindings::_helpers::lineage::project_schema::{ fast_schema, inferred_names, rich_schema, }; +use crate::bindings::_helpers::project_assembly::syntax_checks::model_placeholders; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::ErrorId; @@ -154,8 +155,6 @@ struct FastModel<'py> { /// A model's lineage facts: none, Python facts, or a compiled model's native facts and edge count. enum FastFacts<'py> { Missing, - /// No facts, on a native route, where Polyglot never parses the model: no lineage. - Skipped, Python(Bound<'py, PyAny>), Native(Bound<'py, PyAny>, usize), } @@ -191,19 +190,15 @@ fn fast_lineage<'py>( continue; } let facts: Bound<'py, PyAny> = native_field(model, "fast_lineage_columns")?; - if facts.is_none() && native { - work.push(FastModel { - name: native_field(model, "name")?.extract()?, - facts: FastFacts::Skipped, - has_star, - expands_star, - }); - continue; - } if facts.is_none() { requests.push(FastLineageModel::Parse { query_sql: native_field(model, "query_sql")?.extract()?, inferred_columns: inferred_names(model)?, + placeholders: if native { + placeholder_pairs(model)? + } else { + Vec::new() + }, }); } else if expands_star { requests.push(FastLineageModel::StarExpansion { @@ -266,7 +261,6 @@ fn fast_lineage<'py>( } CompactColumns::Facts(columns.unbind()) } - FastFacts::Skipped => continue, FastFacts::Missing => { match outcomes.next() { Some(FastLineageOutcome::Built { columns, has_star }) => { @@ -282,7 +276,11 @@ fn fast_lineage<'py>( } Some(FastLineageOutcome::Unparsed(detail)) => log_lineage_event( py, - "fast column lineage parse failed; falling back", + if native { + "fast column lineage not built: the native parser cannot read the model" + } else { + "fast column lineage parse failed; falling back" + }, &model.name, Some(&detail), )?, @@ -310,6 +308,17 @@ fn fast_lineage<'py>( .map(Bound::into_any) } +/// The model's `@@@name` placeholder defaults, which a native parser reads in their place. +fn placeholder_pairs(model: &Bound<'_, PyAny>) -> PyResult> { + let Some(placeholders) = model_placeholders(model)? else { + return Ok(Vec::new()); + }; + placeholders + .iter() + .map(|(name, value)| Ok((name.extract()?, value.extract()?))) + .collect() +} + /// The output column names of compact facts or plain column facts. fn output_columns(classes: &LineageClasses<'_>, facts: &Bound<'_, PyAny>) -> PyResult> { if facts.is_instance(&classes.compact_facts)? { From f07c13aefa720e0e1d0b58e202579d43bced499e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 11:06:26 +0000 Subject: [PATCH 14/29] test(types): re-record the generated type golden for nested timestamps The text fallback now takes a type's family from its top-level type, so a STRUCT, ARRAY or MAP holding a timestamp reads as other, not timestamp. Only those 666 dialect entries move; every top-level timestamp is unchanged. --- ...rized_nested_padded_and_invalid_types.json | 1332 ++++++++--------- 1 file changed, 666 insertions(+), 666 deletions(-) diff --git a/tests/goldens/python_oracle/types_generated_seeded_scalar_parameterized_nested_padded_and_invalid_types.json b/tests/goldens/python_oracle/types_generated_seeded_scalar_parameterized_nested_padded_and_invalid_types.json index 1729e43aef..04b9dd9f09 100644 --- a/tests/goldens/python_oracle/types_generated_seeded_scalar_parameterized_nested_padded_and_invalid_types.json +++ b/tests/goldens/python_oracle/types_generated_seeded_scalar_parameterized_nested_padded_and_invalid_types.json @@ -478,18 +478,18 @@ [ "type", { -"DuckDB": "98a80237e4e0a12f35bbf17e", -"None": "98a80237e4e0a12f35bbf17e", -"bigquery": "98a80237e4e0a12f35bbf17e", -"databricks": "98a80237e4e0a12f35bbf17e", -"duckdb": "98a80237e4e0a12f35bbf17e", -"generic": "98a80237e4e0a12f35bbf17e", +"DuckDB": "f1dcf3349867744866bb1299", +"None": "f1dcf3349867744866bb1299", +"bigquery": "f1dcf3349867744866bb1299", +"databricks": "f1dcf3349867744866bb1299", +"duckdb": "f1dcf3349867744866bb1299", +"generic": "f1dcf3349867744866bb1299", "motherduck": "901c5133ed47d4769286e860", -"mysql": "98a80237e4e0a12f35bbf17e", -"postgres": "98a80237e4e0a12f35bbf17e", -"postgresql": "98a80237e4e0a12f35bbf17e", -"snowflake": "98a80237e4e0a12f35bbf17e", -"tsql": "98a80237e4e0a12f35bbf17e" +"mysql": "f1dcf3349867744866bb1299", +"postgres": "f1dcf3349867744866bb1299", +"postgresql": "f1dcf3349867744866bb1299", +"snowflake": "f1dcf3349867744866bb1299", +"tsql": "f1dcf3349867744866bb1299" } ], [ @@ -529,18 +529,18 @@ [ "type", { -"DuckDB": "9a79cb58680bebe49009a72d", -"None": "9a79cb58680bebe49009a72d", -"bigquery": "9a79cb58680bebe49009a72d", -"databricks": "9a79cb58680bebe49009a72d", -"duckdb": "9a79cb58680bebe49009a72d", -"generic": "9a79cb58680bebe49009a72d", +"DuckDB": "7c70b66e45566b521b449adc", +"None": "7c70b66e45566b521b449adc", +"bigquery": "7c70b66e45566b521b449adc", +"databricks": "7c70b66e45566b521b449adc", +"duckdb": "7c70b66e45566b521b449adc", +"generic": "7c70b66e45566b521b449adc", "motherduck": "901c5133ed47d4769286e860", -"mysql": "9a79cb58680bebe49009a72d", -"postgres": "9a79cb58680bebe49009a72d", -"postgresql": "9a79cb58680bebe49009a72d", -"snowflake": "9a79cb58680bebe49009a72d", -"tsql": "9a79cb58680bebe49009a72d" +"mysql": "7c70b66e45566b521b449adc", +"postgres": "7c70b66e45566b521b449adc", +"postgresql": "7c70b66e45566b521b449adc", +"snowflake": "7c70b66e45566b521b449adc", +"tsql": "7c70b66e45566b521b449adc" } ], [ @@ -954,18 +954,18 @@ [ "type", { -"DuckDB": "1950e32420d1d40d41ffae64", -"None": "1950e32420d1d40d41ffae64", -"bigquery": "1950e32420d1d40d41ffae64", -"databricks": "1950e32420d1d40d41ffae64", -"duckdb": "1950e32420d1d40d41ffae64", -"generic": "1950e32420d1d40d41ffae64", +"DuckDB": "8f2569067b02a6a41c0caf62", +"None": "8f2569067b02a6a41c0caf62", +"bigquery": "8f2569067b02a6a41c0caf62", +"databricks": "8f2569067b02a6a41c0caf62", +"duckdb": "8f2569067b02a6a41c0caf62", +"generic": "8f2569067b02a6a41c0caf62", "motherduck": "901c5133ed47d4769286e860", -"mysql": "1950e32420d1d40d41ffae64", -"postgres": "1950e32420d1d40d41ffae64", -"postgresql": "1950e32420d1d40d41ffae64", -"snowflake": "1950e32420d1d40d41ffae64", -"tsql": "1950e32420d1d40d41ffae64" +"mysql": "8f2569067b02a6a41c0caf62", +"postgres": "8f2569067b02a6a41c0caf62", +"postgresql": "8f2569067b02a6a41c0caf62", +"snowflake": "8f2569067b02a6a41c0caf62", +"tsql": "8f2569067b02a6a41c0caf62" } ], [ @@ -1311,18 +1311,18 @@ [ "type", { -"DuckDB": "06aec7cca4b30489405665e2", -"None": "06aec7cca4b30489405665e2", -"bigquery": "06aec7cca4b30489405665e2", -"databricks": "06aec7cca4b30489405665e2", -"duckdb": "06aec7cca4b30489405665e2", -"generic": "06aec7cca4b30489405665e2", +"DuckDB": "da06e77421c17a0eee87e6cc", +"None": "da06e77421c17a0eee87e6cc", +"bigquery": "da06e77421c17a0eee87e6cc", +"databricks": "da06e77421c17a0eee87e6cc", +"duckdb": "da06e77421c17a0eee87e6cc", +"generic": "da06e77421c17a0eee87e6cc", "motherduck": "901c5133ed47d4769286e860", -"mysql": "06aec7cca4b30489405665e2", -"postgres": "06aec7cca4b30489405665e2", -"postgresql": "06aec7cca4b30489405665e2", -"snowflake": "06aec7cca4b30489405665e2", -"tsql": "06aec7cca4b30489405665e2" +"mysql": "da06e77421c17a0eee87e6cc", +"postgres": "da06e77421c17a0eee87e6cc", +"postgresql": "da06e77421c17a0eee87e6cc", +"snowflake": "da06e77421c17a0eee87e6cc", +"tsql": "da06e77421c17a0eee87e6cc" } ], [ @@ -1396,35 +1396,35 @@ [ "type", { -"DuckDB": "7aa59eed1bba74a9dee48e59", -"None": "7aa59eed1bba74a9dee48e59", -"bigquery": "7aa59eed1bba74a9dee48e59", -"databricks": "7aa59eed1bba74a9dee48e59", -"duckdb": "7aa59eed1bba74a9dee48e59", -"generic": "7aa59eed1bba74a9dee48e59", +"DuckDB": "85679d39a40088722a25077b", +"None": "85679d39a40088722a25077b", +"bigquery": "85679d39a40088722a25077b", +"databricks": "85679d39a40088722a25077b", +"duckdb": "85679d39a40088722a25077b", +"generic": "85679d39a40088722a25077b", "motherduck": "901c5133ed47d4769286e860", -"mysql": "7aa59eed1bba74a9dee48e59", -"postgres": "7aa59eed1bba74a9dee48e59", -"postgresql": "7aa59eed1bba74a9dee48e59", -"snowflake": "7aa59eed1bba74a9dee48e59", -"tsql": "7aa59eed1bba74a9dee48e59" +"mysql": "85679d39a40088722a25077b", +"postgres": "85679d39a40088722a25077b", +"postgresql": "85679d39a40088722a25077b", +"snowflake": "85679d39a40088722a25077b", +"tsql": "85679d39a40088722a25077b" } ], [ "type", { -"DuckDB": "66e7166890ccedeaad6c241a", -"None": "66e7166890ccedeaad6c241a", -"bigquery": "66e7166890ccedeaad6c241a", -"databricks": "66e7166890ccedeaad6c241a", -"duckdb": "66e7166890ccedeaad6c241a", -"generic": "66e7166890ccedeaad6c241a", +"DuckDB": "af83290b9888e9ea215c9c2a", +"None": "af83290b9888e9ea215c9c2a", +"bigquery": "af83290b9888e9ea215c9c2a", +"databricks": "af83290b9888e9ea215c9c2a", +"duckdb": "af83290b9888e9ea215c9c2a", +"generic": "af83290b9888e9ea215c9c2a", "motherduck": "901c5133ed47d4769286e860", -"mysql": "66e7166890ccedeaad6c241a", -"postgres": "66e7166890ccedeaad6c241a", -"postgresql": "66e7166890ccedeaad6c241a", -"snowflake": "66e7166890ccedeaad6c241a", -"tsql": "66e7166890ccedeaad6c241a" +"mysql": "af83290b9888e9ea215c9c2a", +"postgres": "af83290b9888e9ea215c9c2a", +"postgresql": "af83290b9888e9ea215c9c2a", +"snowflake": "af83290b9888e9ea215c9c2a", +"tsql": "af83290b9888e9ea215c9c2a" } ], [ @@ -1889,18 +1889,18 @@ [ "type", { -"DuckDB": "caf173733defe796376d4efe", -"None": "caf173733defe796376d4efe", -"bigquery": "caf173733defe796376d4efe", -"databricks": "caf173733defe796376d4efe", -"duckdb": "caf173733defe796376d4efe", -"generic": "caf173733defe796376d4efe", +"DuckDB": "a6458f2eeceb4012953d4be0", +"None": "a6458f2eeceb4012953d4be0", +"bigquery": "a6458f2eeceb4012953d4be0", +"databricks": "a6458f2eeceb4012953d4be0", +"duckdb": "a6458f2eeceb4012953d4be0", +"generic": "a6458f2eeceb4012953d4be0", "motherduck": "901c5133ed47d4769286e860", -"mysql": "caf173733defe796376d4efe", -"postgres": "caf173733defe796376d4efe", -"postgresql": "caf173733defe796376d4efe", -"snowflake": "caf173733defe796376d4efe", -"tsql": "caf173733defe796376d4efe" +"mysql": "a6458f2eeceb4012953d4be0", +"postgres": "a6458f2eeceb4012953d4be0", +"postgresql": "a6458f2eeceb4012953d4be0", +"snowflake": "a6458f2eeceb4012953d4be0", +"tsql": "a6458f2eeceb4012953d4be0" } ], [ @@ -1974,18 +1974,18 @@ [ "type", { -"DuckDB": "4f8af4650ce9c18ed0cb0bca", -"None": "4f8af4650ce9c18ed0cb0bca", -"bigquery": "4f8af4650ce9c18ed0cb0bca", -"databricks": "4f8af4650ce9c18ed0cb0bca", -"duckdb": "4f8af4650ce9c18ed0cb0bca", -"generic": "4f8af4650ce9c18ed0cb0bca", +"DuckDB": "a5e56c4a2e3c796d3dca7042", +"None": "a5e56c4a2e3c796d3dca7042", +"bigquery": "a5e56c4a2e3c796d3dca7042", +"databricks": "a5e56c4a2e3c796d3dca7042", +"duckdb": "a5e56c4a2e3c796d3dca7042", +"generic": "a5e56c4a2e3c796d3dca7042", "motherduck": "901c5133ed47d4769286e860", -"mysql": "4f8af4650ce9c18ed0cb0bca", -"postgres": "4f8af4650ce9c18ed0cb0bca", -"postgresql": "4f8af4650ce9c18ed0cb0bca", -"snowflake": "4f8af4650ce9c18ed0cb0bca", -"tsql": "4f8af4650ce9c18ed0cb0bca" +"mysql": "a5e56c4a2e3c796d3dca7042", +"postgres": "a5e56c4a2e3c796d3dca7042", +"postgresql": "a5e56c4a2e3c796d3dca7042", +"snowflake": "a5e56c4a2e3c796d3dca7042", +"tsql": "a5e56c4a2e3c796d3dca7042" } ], [ @@ -2603,18 +2603,18 @@ [ "type", { -"DuckDB": "ad35c4a2523ce8c2f57acf51", -"None": "ad35c4a2523ce8c2f57acf51", -"bigquery": "ad35c4a2523ce8c2f57acf51", -"databricks": "ad35c4a2523ce8c2f57acf51", -"duckdb": "ad35c4a2523ce8c2f57acf51", -"generic": "ad35c4a2523ce8c2f57acf51", +"DuckDB": "e7bc9109e1c25d865732885c", +"None": "e7bc9109e1c25d865732885c", +"bigquery": "e7bc9109e1c25d865732885c", +"databricks": "e7bc9109e1c25d865732885c", +"duckdb": "e7bc9109e1c25d865732885c", +"generic": "e7bc9109e1c25d865732885c", "motherduck": "901c5133ed47d4769286e860", -"mysql": "ad35c4a2523ce8c2f57acf51", -"postgres": "ad35c4a2523ce8c2f57acf51", -"postgresql": "ad35c4a2523ce8c2f57acf51", -"snowflake": "ad35c4a2523ce8c2f57acf51", -"tsql": "ad35c4a2523ce8c2f57acf51" +"mysql": "e7bc9109e1c25d865732885c", +"postgres": "e7bc9109e1c25d865732885c", +"postgresql": "e7bc9109e1c25d865732885c", +"snowflake": "e7bc9109e1c25d865732885c", +"tsql": "e7bc9109e1c25d865732885c" } ], [ @@ -4252,18 +4252,18 @@ [ "type", { -"DuckDB": "96ecb0bd0c27167bf4e31a08", -"None": "96ecb0bd0c27167bf4e31a08", -"bigquery": "96ecb0bd0c27167bf4e31a08", -"databricks": "96ecb0bd0c27167bf4e31a08", -"duckdb": "96ecb0bd0c27167bf4e31a08", -"generic": "96ecb0bd0c27167bf4e31a08", +"DuckDB": "a1ccf5c0db81a578bab8cda9", +"None": "a1ccf5c0db81a578bab8cda9", +"bigquery": "a1ccf5c0db81a578bab8cda9", +"databricks": "a1ccf5c0db81a578bab8cda9", +"duckdb": "a1ccf5c0db81a578bab8cda9", +"generic": "a1ccf5c0db81a578bab8cda9", "motherduck": "901c5133ed47d4769286e860", -"mysql": "96ecb0bd0c27167bf4e31a08", -"postgres": "96ecb0bd0c27167bf4e31a08", -"postgresql": "96ecb0bd0c27167bf4e31a08", -"snowflake": "96ecb0bd0c27167bf4e31a08", -"tsql": "96ecb0bd0c27167bf4e31a08" +"mysql": "a1ccf5c0db81a578bab8cda9", +"postgres": "a1ccf5c0db81a578bab8cda9", +"postgresql": "a1ccf5c0db81a578bab8cda9", +"snowflake": "a1ccf5c0db81a578bab8cda9", +"tsql": "a1ccf5c0db81a578bab8cda9" } ], [ @@ -4423,17 +4423,17 @@ "type", { "DuckDB": "c344d36a6751b773df74ee3c", -"None": "87fed6659d6324c5ef0c6ee2", -"bigquery": "87fed6659d6324c5ef0c6ee2", -"databricks": "87fed6659d6324c5ef0c6ee2", +"None": "2d031f8bbd88fca9a9492b54", +"bigquery": "2d031f8bbd88fca9a9492b54", +"databricks": "2d031f8bbd88fca9a9492b54", "duckdb": "c344d36a6751b773df74ee3c", -"generic": "87fed6659d6324c5ef0c6ee2", +"generic": "2d031f8bbd88fca9a9492b54", "motherduck": "901c5133ed47d4769286e860", -"mysql": "87fed6659d6324c5ef0c6ee2", -"postgres": "87fed6659d6324c5ef0c6ee2", -"postgresql": "87fed6659d6324c5ef0c6ee2", -"snowflake": "87fed6659d6324c5ef0c6ee2", -"tsql": "87fed6659d6324c5ef0c6ee2" +"mysql": "2d031f8bbd88fca9a9492b54", +"postgres": "2d031f8bbd88fca9a9492b54", +"postgresql": "2d031f8bbd88fca9a9492b54", +"snowflake": "2d031f8bbd88fca9a9492b54", +"tsql": "2d031f8bbd88fca9a9492b54" } ], [ @@ -4779,18 +4779,18 @@ [ "type", { -"DuckDB": "2795ccedbfd3fa0bed5fdfcd", -"None": "2795ccedbfd3fa0bed5fdfcd", -"bigquery": "2795ccedbfd3fa0bed5fdfcd", -"databricks": "2795ccedbfd3fa0bed5fdfcd", -"duckdb": "2795ccedbfd3fa0bed5fdfcd", -"generic": "2795ccedbfd3fa0bed5fdfcd", +"DuckDB": "ebd0935007c5cca9d0c14021", +"None": "ebd0935007c5cca9d0c14021", +"bigquery": "ebd0935007c5cca9d0c14021", +"databricks": "ebd0935007c5cca9d0c14021", +"duckdb": "ebd0935007c5cca9d0c14021", +"generic": "ebd0935007c5cca9d0c14021", "motherduck": "901c5133ed47d4769286e860", -"mysql": "2795ccedbfd3fa0bed5fdfcd", -"postgres": "2795ccedbfd3fa0bed5fdfcd", -"postgresql": "2795ccedbfd3fa0bed5fdfcd", -"snowflake": "2795ccedbfd3fa0bed5fdfcd", -"tsql": "2795ccedbfd3fa0bed5fdfcd" +"mysql": "ebd0935007c5cca9d0c14021", +"postgres": "ebd0935007c5cca9d0c14021", +"postgresql": "ebd0935007c5cca9d0c14021", +"snowflake": "ebd0935007c5cca9d0c14021", +"tsql": "ebd0935007c5cca9d0c14021" } ], [ @@ -5357,18 +5357,18 @@ [ "type", { -"DuckDB": "c4860fb394ecca5e4515f1c9", -"None": "c4860fb394ecca5e4515f1c9", -"bigquery": "c4860fb394ecca5e4515f1c9", -"databricks": "c4860fb394ecca5e4515f1c9", -"duckdb": "c4860fb394ecca5e4515f1c9", -"generic": "c4860fb394ecca5e4515f1c9", +"DuckDB": "8e8e75c867befbf0dd5b522f", +"None": "8e8e75c867befbf0dd5b522f", +"bigquery": "8e8e75c867befbf0dd5b522f", +"databricks": "8e8e75c867befbf0dd5b522f", +"duckdb": "8e8e75c867befbf0dd5b522f", +"generic": "8e8e75c867befbf0dd5b522f", "motherduck": "901c5133ed47d4769286e860", -"mysql": "c4860fb394ecca5e4515f1c9", -"postgres": "c4860fb394ecca5e4515f1c9", -"postgresql": "c4860fb394ecca5e4515f1c9", -"snowflake": "c4860fb394ecca5e4515f1c9", -"tsql": "c4860fb394ecca5e4515f1c9" +"mysql": "8e8e75c867befbf0dd5b522f", +"postgres": "8e8e75c867befbf0dd5b522f", +"postgresql": "8e8e75c867befbf0dd5b522f", +"snowflake": "8e8e75c867befbf0dd5b522f", +"tsql": "8e8e75c867befbf0dd5b522f" } ], [ @@ -5697,18 +5697,18 @@ [ "type", { -"DuckDB": "445eba49a8d5ccbd6775e1b6", -"None": "445eba49a8d5ccbd6775e1b6", -"bigquery": "445eba49a8d5ccbd6775e1b6", -"databricks": "445eba49a8d5ccbd6775e1b6", -"duckdb": "445eba49a8d5ccbd6775e1b6", -"generic": "445eba49a8d5ccbd6775e1b6", +"DuckDB": "a8cae774dd1c333f42d6a630", +"None": "a8cae774dd1c333f42d6a630", +"bigquery": "a8cae774dd1c333f42d6a630", +"databricks": "a8cae774dd1c333f42d6a630", +"duckdb": "a8cae774dd1c333f42d6a630", +"generic": "a8cae774dd1c333f42d6a630", "motherduck": "901c5133ed47d4769286e860", -"mysql": "445eba49a8d5ccbd6775e1b6", -"postgres": "445eba49a8d5ccbd6775e1b6", -"postgresql": "445eba49a8d5ccbd6775e1b6", -"snowflake": "445eba49a8d5ccbd6775e1b6", -"tsql": "445eba49a8d5ccbd6775e1b6" +"mysql": "a8cae774dd1c333f42d6a630", +"postgres": "a8cae774dd1c333f42d6a630", +"postgresql": "a8cae774dd1c333f42d6a630", +"snowflake": "a8cae774dd1c333f42d6a630", +"tsql": "a8cae774dd1c333f42d6a630" } ], [ @@ -6054,18 +6054,18 @@ [ "type", { -"DuckDB": "07aa3fe366bb8a3f270ea3d3", -"None": "07aa3fe366bb8a3f270ea3d3", -"bigquery": "07aa3fe366bb8a3f270ea3d3", -"databricks": "07aa3fe366bb8a3f270ea3d3", -"duckdb": "07aa3fe366bb8a3f270ea3d3", -"generic": "07aa3fe366bb8a3f270ea3d3", +"DuckDB": "7a9cb09cc9d8e6470f5c18fb", +"None": "7a9cb09cc9d8e6470f5c18fb", +"bigquery": "7a9cb09cc9d8e6470f5c18fb", +"databricks": "7a9cb09cc9d8e6470f5c18fb", +"duckdb": "7a9cb09cc9d8e6470f5c18fb", +"generic": "7a9cb09cc9d8e6470f5c18fb", "motherduck": "901c5133ed47d4769286e860", -"mysql": "07aa3fe366bb8a3f270ea3d3", -"postgres": "07aa3fe366bb8a3f270ea3d3", -"postgresql": "07aa3fe366bb8a3f270ea3d3", -"snowflake": "07aa3fe366bb8a3f270ea3d3", -"tsql": "07aa3fe366bb8a3f270ea3d3" +"mysql": "7a9cb09cc9d8e6470f5c18fb", +"postgres": "7a9cb09cc9d8e6470f5c18fb", +"postgresql": "7a9cb09cc9d8e6470f5c18fb", +"snowflake": "7a9cb09cc9d8e6470f5c18fb", +"tsql": "7a9cb09cc9d8e6470f5c18fb" } ], [ @@ -6139,18 +6139,18 @@ [ "type", { -"DuckDB": "c426d93641829fa3d8f49b8d", -"None": "c426d93641829fa3d8f49b8d", -"bigquery": "c426d93641829fa3d8f49b8d", -"databricks": "c426d93641829fa3d8f49b8d", -"duckdb": "c426d93641829fa3d8f49b8d", -"generic": "c426d93641829fa3d8f49b8d", +"DuckDB": "efb799417b64225e2ee93447", +"None": "efb799417b64225e2ee93447", +"bigquery": "efb799417b64225e2ee93447", +"databricks": "efb799417b64225e2ee93447", +"duckdb": "efb799417b64225e2ee93447", +"generic": "efb799417b64225e2ee93447", "motherduck": "901c5133ed47d4769286e860", -"mysql": "c426d93641829fa3d8f49b8d", -"postgres": "c426d93641829fa3d8f49b8d", -"postgresql": "c426d93641829fa3d8f49b8d", -"snowflake": "c426d93641829fa3d8f49b8d", -"tsql": "c426d93641829fa3d8f49b8d" +"mysql": "efb799417b64225e2ee93447", +"postgres": "efb799417b64225e2ee93447", +"postgresql": "efb799417b64225e2ee93447", +"snowflake": "efb799417b64225e2ee93447", +"tsql": "efb799417b64225e2ee93447" } ], [ @@ -6224,18 +6224,18 @@ [ "type", { -"DuckDB": "be15dde1d6e0b4bb3f6da66f", -"None": "be15dde1d6e0b4bb3f6da66f", -"bigquery": "be15dde1d6e0b4bb3f6da66f", -"databricks": "be15dde1d6e0b4bb3f6da66f", -"duckdb": "be15dde1d6e0b4bb3f6da66f", -"generic": "be15dde1d6e0b4bb3f6da66f", +"DuckDB": "400acbf184a17c6b468aab31", +"None": "400acbf184a17c6b468aab31", +"bigquery": "400acbf184a17c6b468aab31", +"databricks": "400acbf184a17c6b468aab31", +"duckdb": "400acbf184a17c6b468aab31", +"generic": "400acbf184a17c6b468aab31", "motherduck": "901c5133ed47d4769286e860", -"mysql": "be15dde1d6e0b4bb3f6da66f", -"postgres": "be15dde1d6e0b4bb3f6da66f", -"postgresql": "be15dde1d6e0b4bb3f6da66f", -"snowflake": "be15dde1d6e0b4bb3f6da66f", -"tsql": "be15dde1d6e0b4bb3f6da66f" +"mysql": "400acbf184a17c6b468aab31", +"postgres": "400acbf184a17c6b468aab31", +"postgresql": "400acbf184a17c6b468aab31", +"snowflake": "400acbf184a17c6b468aab31", +"tsql": "400acbf184a17c6b468aab31" } ], [ @@ -6751,18 +6751,18 @@ [ "type", { -"DuckDB": "182a094bae8a9a33f67845ad", -"None": "182a094bae8a9a33f67845ad", -"bigquery": "182a094bae8a9a33f67845ad", -"databricks": "182a094bae8a9a33f67845ad", -"duckdb": "182a094bae8a9a33f67845ad", -"generic": "182a094bae8a9a33f67845ad", +"DuckDB": "fa13cacc15beaca279323bfb", +"None": "fa13cacc15beaca279323bfb", +"bigquery": "fa13cacc15beaca279323bfb", +"databricks": "fa13cacc15beaca279323bfb", +"duckdb": "fa13cacc15beaca279323bfb", +"generic": "fa13cacc15beaca279323bfb", "motherduck": "901c5133ed47d4769286e860", -"mysql": "182a094bae8a9a33f67845ad", -"postgres": "182a094bae8a9a33f67845ad", -"postgresql": "182a094bae8a9a33f67845ad", -"snowflake": "182a094bae8a9a33f67845ad", -"tsql": "182a094bae8a9a33f67845ad" +"mysql": "fa13cacc15beaca279323bfb", +"postgres": "fa13cacc15beaca279323bfb", +"postgresql": "fa13cacc15beaca279323bfb", +"snowflake": "fa13cacc15beaca279323bfb", +"tsql": "fa13cacc15beaca279323bfb" } ], [ @@ -6955,18 +6955,18 @@ [ "type", { -"DuckDB": "7032aa8254308ba815883957", -"None": "7032aa8254308ba815883957", -"bigquery": "7032aa8254308ba815883957", -"databricks": "7032aa8254308ba815883957", -"duckdb": "7032aa8254308ba815883957", -"generic": "7032aa8254308ba815883957", +"DuckDB": "8e263d7157f195e571a4862b", +"None": "8e263d7157f195e571a4862b", +"bigquery": "8e263d7157f195e571a4862b", +"databricks": "8e263d7157f195e571a4862b", +"duckdb": "8e263d7157f195e571a4862b", +"generic": "8e263d7157f195e571a4862b", "motherduck": "901c5133ed47d4769286e860", -"mysql": "7032aa8254308ba815883957", -"postgres": "7032aa8254308ba815883957", -"postgresql": "7032aa8254308ba815883957", -"snowflake": "7032aa8254308ba815883957", -"tsql": "7032aa8254308ba815883957" +"mysql": "8e263d7157f195e571a4862b", +"postgres": "8e263d7157f195e571a4862b", +"postgresql": "8e263d7157f195e571a4862b", +"snowflake": "8e263d7157f195e571a4862b", +"tsql": "8e263d7157f195e571a4862b" } ], [ @@ -7210,18 +7210,18 @@ [ "type", { -"DuckDB": "8789309b90fc585ddbbe90f9", -"None": "8789309b90fc585ddbbe90f9", -"bigquery": "8789309b90fc585ddbbe90f9", -"databricks": "8789309b90fc585ddbbe90f9", -"duckdb": "8789309b90fc585ddbbe90f9", -"generic": "8789309b90fc585ddbbe90f9", +"DuckDB": "6b4faa1df4db47a526d4ae10", +"None": "6b4faa1df4db47a526d4ae10", +"bigquery": "6b4faa1df4db47a526d4ae10", +"databricks": "6b4faa1df4db47a526d4ae10", +"duckdb": "6b4faa1df4db47a526d4ae10", +"generic": "6b4faa1df4db47a526d4ae10", "motherduck": "901c5133ed47d4769286e860", -"mysql": "8789309b90fc585ddbbe90f9", -"postgres": "8789309b90fc585ddbbe90f9", -"postgresql": "8789309b90fc585ddbbe90f9", -"snowflake": "8789309b90fc585ddbbe90f9", -"tsql": "8789309b90fc585ddbbe90f9" +"mysql": "6b4faa1df4db47a526d4ae10", +"postgres": "6b4faa1df4db47a526d4ae10", +"postgresql": "6b4faa1df4db47a526d4ae10", +"snowflake": "6b4faa1df4db47a526d4ae10", +"tsql": "6b4faa1df4db47a526d4ae10" } ], [ @@ -7330,17 +7330,17 @@ "type", { "DuckDB": "c729c38d2df6efedc6288247", -"None": "eea1002d88cc68fad7424aca", -"bigquery": "eea1002d88cc68fad7424aca", -"databricks": "eea1002d88cc68fad7424aca", +"None": "75f1ffee735aaf2dc58efd48", +"bigquery": "75f1ffee735aaf2dc58efd48", +"databricks": "75f1ffee735aaf2dc58efd48", "duckdb": "c729c38d2df6efedc6288247", -"generic": "eea1002d88cc68fad7424aca", +"generic": "75f1ffee735aaf2dc58efd48", "motherduck": "901c5133ed47d4769286e860", -"mysql": "eea1002d88cc68fad7424aca", -"postgres": "eea1002d88cc68fad7424aca", -"postgresql": "eea1002d88cc68fad7424aca", -"snowflake": "eea1002d88cc68fad7424aca", -"tsql": "eea1002d88cc68fad7424aca" +"mysql": "75f1ffee735aaf2dc58efd48", +"postgres": "75f1ffee735aaf2dc58efd48", +"postgresql": "75f1ffee735aaf2dc58efd48", +"snowflake": "75f1ffee735aaf2dc58efd48", +"tsql": "75f1ffee735aaf2dc58efd48" } ], [ @@ -7465,18 +7465,18 @@ [ "type", { -"DuckDB": "733b04338829c5844270d25e", -"None": "733b04338829c5844270d25e", -"bigquery": "733b04338829c5844270d25e", -"databricks": "733b04338829c5844270d25e", -"duckdb": "733b04338829c5844270d25e", -"generic": "733b04338829c5844270d25e", +"DuckDB": "1bbf30e0166bb61bed8be4d6", +"None": "1bbf30e0166bb61bed8be4d6", +"bigquery": "1bbf30e0166bb61bed8be4d6", +"databricks": "1bbf30e0166bb61bed8be4d6", +"duckdb": "1bbf30e0166bb61bed8be4d6", +"generic": "1bbf30e0166bb61bed8be4d6", "motherduck": "901c5133ed47d4769286e860", -"mysql": "733b04338829c5844270d25e", -"postgres": "733b04338829c5844270d25e", -"postgresql": "733b04338829c5844270d25e", -"snowflake": "733b04338829c5844270d25e", -"tsql": "733b04338829c5844270d25e" +"mysql": "1bbf30e0166bb61bed8be4d6", +"postgres": "1bbf30e0166bb61bed8be4d6", +"postgresql": "1bbf30e0166bb61bed8be4d6", +"snowflake": "1bbf30e0166bb61bed8be4d6", +"tsql": "1bbf30e0166bb61bed8be4d6" } ], [ @@ -8264,18 +8264,18 @@ [ "type", { -"DuckDB": "cc33965a669608ec39e8c246", -"None": "cc33965a669608ec39e8c246", -"bigquery": "cc33965a669608ec39e8c246", -"databricks": "cc33965a669608ec39e8c246", -"duckdb": "cc33965a669608ec39e8c246", -"generic": "cc33965a669608ec39e8c246", +"DuckDB": "30a870a18e4d367086a7d302", +"None": "30a870a18e4d367086a7d302", +"bigquery": "30a870a18e4d367086a7d302", +"databricks": "30a870a18e4d367086a7d302", +"duckdb": "30a870a18e4d367086a7d302", +"generic": "30a870a18e4d367086a7d302", "motherduck": "901c5133ed47d4769286e860", -"mysql": "cc33965a669608ec39e8c246", -"postgres": "cc33965a669608ec39e8c246", -"postgresql": "cc33965a669608ec39e8c246", -"snowflake": "cc33965a669608ec39e8c246", -"tsql": "cc33965a669608ec39e8c246" +"mysql": "30a870a18e4d367086a7d302", +"postgres": "30a870a18e4d367086a7d302", +"postgresql": "30a870a18e4d367086a7d302", +"snowflake": "30a870a18e4d367086a7d302", +"tsql": "30a870a18e4d367086a7d302" } ], [ @@ -8740,18 +8740,18 @@ [ "type", { -"DuckDB": "9626d393c137a37008bb4b89", -"None": "9626d393c137a37008bb4b89", -"bigquery": "9626d393c137a37008bb4b89", -"databricks": "9626d393c137a37008bb4b89", -"duckdb": "9626d393c137a37008bb4b89", -"generic": "9626d393c137a37008bb4b89", +"DuckDB": "386957bbe33b0277f99cac09", +"None": "386957bbe33b0277f99cac09", +"bigquery": "386957bbe33b0277f99cac09", +"databricks": "386957bbe33b0277f99cac09", +"duckdb": "386957bbe33b0277f99cac09", +"generic": "386957bbe33b0277f99cac09", "motherduck": "901c5133ed47d4769286e860", -"mysql": "9626d393c137a37008bb4b89", -"postgres": "9626d393c137a37008bb4b89", -"postgresql": "9626d393c137a37008bb4b89", -"snowflake": "9626d393c137a37008bb4b89", -"tsql": "9626d393c137a37008bb4b89" +"mysql": "386957bbe33b0277f99cac09", +"postgres": "386957bbe33b0277f99cac09", +"postgresql": "386957bbe33b0277f99cac09", +"snowflake": "386957bbe33b0277f99cac09", +"tsql": "386957bbe33b0277f99cac09" } ], [ @@ -8791,18 +8791,18 @@ [ "type", { -"DuckDB": "8fe65454f387031aa6231faf", -"None": "8fe65454f387031aa6231faf", -"bigquery": "8fe65454f387031aa6231faf", -"databricks": "8fe65454f387031aa6231faf", -"duckdb": "8fe65454f387031aa6231faf", -"generic": "8fe65454f387031aa6231faf", +"DuckDB": "9304d412ef114a51a170158d", +"None": "9304d412ef114a51a170158d", +"bigquery": "9304d412ef114a51a170158d", +"databricks": "9304d412ef114a51a170158d", +"duckdb": "9304d412ef114a51a170158d", +"generic": "9304d412ef114a51a170158d", "motherduck": "901c5133ed47d4769286e860", -"mysql": "8fe65454f387031aa6231faf", -"postgres": "8fe65454f387031aa6231faf", -"postgresql": "8fe65454f387031aa6231faf", -"snowflake": "8fe65454f387031aa6231faf", -"tsql": "8fe65454f387031aa6231faf" +"mysql": "9304d412ef114a51a170158d", +"postgres": "9304d412ef114a51a170158d", +"postgresql": "9304d412ef114a51a170158d", +"snowflake": "9304d412ef114a51a170158d", +"tsql": "9304d412ef114a51a170158d" } ], [ @@ -9607,18 +9607,18 @@ [ "type", { -"DuckDB": "c4d601bdc1b1f357b3e9bbdf", -"None": "c4d601bdc1b1f357b3e9bbdf", -"bigquery": "c4d601bdc1b1f357b3e9bbdf", -"databricks": "c4d601bdc1b1f357b3e9bbdf", -"duckdb": "c4d601bdc1b1f357b3e9bbdf", -"generic": "c4d601bdc1b1f357b3e9bbdf", +"DuckDB": "5551318f9b0e622b4b928ca3", +"None": "5551318f9b0e622b4b928ca3", +"bigquery": "5551318f9b0e622b4b928ca3", +"databricks": "5551318f9b0e622b4b928ca3", +"duckdb": "5551318f9b0e622b4b928ca3", +"generic": "5551318f9b0e622b4b928ca3", "motherduck": "901c5133ed47d4769286e860", -"mysql": "c4d601bdc1b1f357b3e9bbdf", -"postgres": "c4d601bdc1b1f357b3e9bbdf", -"postgresql": "c4d601bdc1b1f357b3e9bbdf", -"snowflake": "c4d601bdc1b1f357b3e9bbdf", -"tsql": "c4d601bdc1b1f357b3e9bbdf" +"mysql": "5551318f9b0e622b4b928ca3", +"postgres": "5551318f9b0e622b4b928ca3", +"postgresql": "5551318f9b0e622b4b928ca3", +"snowflake": "5551318f9b0e622b4b928ca3", +"tsql": "5551318f9b0e622b4b928ca3" } ], [ @@ -9828,18 +9828,18 @@ [ "type", { -"DuckDB": "b6b82efd03def9465e4f97b2", -"None": "b6b82efd03def9465e4f97b2", -"bigquery": "b6b82efd03def9465e4f97b2", -"databricks": "b6b82efd03def9465e4f97b2", -"duckdb": "b6b82efd03def9465e4f97b2", -"generic": "b6b82efd03def9465e4f97b2", +"DuckDB": "a8face56f690e623ad76d780", +"None": "a8face56f690e623ad76d780", +"bigquery": "a8face56f690e623ad76d780", +"databricks": "a8face56f690e623ad76d780", +"duckdb": "a8face56f690e623ad76d780", +"generic": "a8face56f690e623ad76d780", "motherduck": "901c5133ed47d4769286e860", -"mysql": "b6b82efd03def9465e4f97b2", -"postgres": "b6b82efd03def9465e4f97b2", -"postgresql": "b6b82efd03def9465e4f97b2", -"snowflake": "b6b82efd03def9465e4f97b2", -"tsql": "b6b82efd03def9465e4f97b2" +"mysql": "a8face56f690e623ad76d780", +"postgres": "a8face56f690e623ad76d780", +"postgresql": "a8face56f690e623ad76d780", +"snowflake": "a8face56f690e623ad76d780", +"tsql": "a8face56f690e623ad76d780" } ], [ @@ -10117,18 +10117,18 @@ [ "type", { -"DuckDB": "4b27937f44f966ff9f3a7c76", -"None": "4b27937f44f966ff9f3a7c76", -"bigquery": "4b27937f44f966ff9f3a7c76", -"databricks": "4b27937f44f966ff9f3a7c76", -"duckdb": "4b27937f44f966ff9f3a7c76", -"generic": "4b27937f44f966ff9f3a7c76", +"DuckDB": "c641dec96086f9f69b467459", +"None": "c641dec96086f9f69b467459", +"bigquery": "c641dec96086f9f69b467459", +"databricks": "c641dec96086f9f69b467459", +"duckdb": "c641dec96086f9f69b467459", +"generic": "c641dec96086f9f69b467459", "motherduck": "901c5133ed47d4769286e860", -"mysql": "4b27937f44f966ff9f3a7c76", -"postgres": "4b27937f44f966ff9f3a7c76", -"postgresql": "4b27937f44f966ff9f3a7c76", -"snowflake": "4b27937f44f966ff9f3a7c76", -"tsql": "4b27937f44f966ff9f3a7c76" +"mysql": "c641dec96086f9f69b467459", +"postgres": "c641dec96086f9f69b467459", +"postgresql": "c641dec96086f9f69b467459", +"snowflake": "c641dec96086f9f69b467459", +"tsql": "c641dec96086f9f69b467459" } ], [ @@ -10236,18 +10236,18 @@ [ "type", { -"DuckDB": "14c32cfb5e5135338d015f42", -"None": "14c32cfb5e5135338d015f42", -"bigquery": "14c32cfb5e5135338d015f42", -"databricks": "14c32cfb5e5135338d015f42", -"duckdb": "14c32cfb5e5135338d015f42", -"generic": "14c32cfb5e5135338d015f42", +"DuckDB": "f8e63510f9a31c4fc8f60cb4", +"None": "f8e63510f9a31c4fc8f60cb4", +"bigquery": "f8e63510f9a31c4fc8f60cb4", +"databricks": "f8e63510f9a31c4fc8f60cb4", +"duckdb": "f8e63510f9a31c4fc8f60cb4", +"generic": "f8e63510f9a31c4fc8f60cb4", "motherduck": "901c5133ed47d4769286e860", -"mysql": "14c32cfb5e5135338d015f42", -"postgres": "14c32cfb5e5135338d015f42", -"postgresql": "14c32cfb5e5135338d015f42", -"snowflake": "14c32cfb5e5135338d015f42", -"tsql": "14c32cfb5e5135338d015f42" +"mysql": "f8e63510f9a31c4fc8f60cb4", +"postgres": "f8e63510f9a31c4fc8f60cb4", +"postgresql": "f8e63510f9a31c4fc8f60cb4", +"snowflake": "f8e63510f9a31c4fc8f60cb4", +"tsql": "f8e63510f9a31c4fc8f60cb4" } ], [ @@ -10865,18 +10865,18 @@ [ "type", { -"DuckDB": "c703e456945a8f06d005c98b", -"None": "c703e456945a8f06d005c98b", -"bigquery": "c703e456945a8f06d005c98b", -"databricks": "c703e456945a8f06d005c98b", -"duckdb": "c703e456945a8f06d005c98b", -"generic": "c703e456945a8f06d005c98b", +"DuckDB": "d1ea13fb2c3d459fff6bf7cd", +"None": "d1ea13fb2c3d459fff6bf7cd", +"bigquery": "d1ea13fb2c3d459fff6bf7cd", +"databricks": "d1ea13fb2c3d459fff6bf7cd", +"duckdb": "d1ea13fb2c3d459fff6bf7cd", +"generic": "d1ea13fb2c3d459fff6bf7cd", "motherduck": "901c5133ed47d4769286e860", -"mysql": "c703e456945a8f06d005c98b", -"postgres": "c703e456945a8f06d005c98b", -"postgresql": "c703e456945a8f06d005c98b", -"snowflake": "c703e456945a8f06d005c98b", -"tsql": "c703e456945a8f06d005c98b" +"mysql": "d1ea13fb2c3d459fff6bf7cd", +"postgres": "d1ea13fb2c3d459fff6bf7cd", +"postgresql": "d1ea13fb2c3d459fff6bf7cd", +"snowflake": "d1ea13fb2c3d459fff6bf7cd", +"tsql": "d1ea13fb2c3d459fff6bf7cd" } ], [ @@ -11307,18 +11307,18 @@ [ "type", { -"DuckDB": "aea8a680fa18770e56bf6220", -"None": "aea8a680fa18770e56bf6220", -"bigquery": "aea8a680fa18770e56bf6220", -"databricks": "aea8a680fa18770e56bf6220", -"duckdb": "aea8a680fa18770e56bf6220", -"generic": "aea8a680fa18770e56bf6220", +"DuckDB": "6fa5d7254a7ed25d37629163", +"None": "6fa5d7254a7ed25d37629163", +"bigquery": "6fa5d7254a7ed25d37629163", +"databricks": "6fa5d7254a7ed25d37629163", +"duckdb": "6fa5d7254a7ed25d37629163", +"generic": "6fa5d7254a7ed25d37629163", "motherduck": "901c5133ed47d4769286e860", -"mysql": "aea8a680fa18770e56bf6220", -"postgres": "aea8a680fa18770e56bf6220", -"postgresql": "aea8a680fa18770e56bf6220", -"snowflake": "aea8a680fa18770e56bf6220", -"tsql": "aea8a680fa18770e56bf6220" +"mysql": "6fa5d7254a7ed25d37629163", +"postgres": "6fa5d7254a7ed25d37629163", +"postgresql": "6fa5d7254a7ed25d37629163", +"snowflake": "6fa5d7254a7ed25d37629163", +"tsql": "6fa5d7254a7ed25d37629163" } ], [ @@ -13534,18 +13534,18 @@ [ "type", { -"DuckDB": "7c11e0be7630905d93bd9ce1", -"None": "7c11e0be7630905d93bd9ce1", -"bigquery": "7c11e0be7630905d93bd9ce1", -"databricks": "7c11e0be7630905d93bd9ce1", -"duckdb": "7c11e0be7630905d93bd9ce1", -"generic": "7c11e0be7630905d93bd9ce1", +"DuckDB": "86825c198b081bf6e208eaa7", +"None": "86825c198b081bf6e208eaa7", +"bigquery": "86825c198b081bf6e208eaa7", +"databricks": "86825c198b081bf6e208eaa7", +"duckdb": "86825c198b081bf6e208eaa7", +"generic": "86825c198b081bf6e208eaa7", "motherduck": "901c5133ed47d4769286e860", -"mysql": "7c11e0be7630905d93bd9ce1", -"postgres": "7c11e0be7630905d93bd9ce1", -"postgresql": "7c11e0be7630905d93bd9ce1", -"snowflake": "7c11e0be7630905d93bd9ce1", -"tsql": "7c11e0be7630905d93bd9ce1" +"mysql": "86825c198b081bf6e208eaa7", +"postgres": "86825c198b081bf6e208eaa7", +"postgresql": "86825c198b081bf6e208eaa7", +"snowflake": "86825c198b081bf6e208eaa7", +"tsql": "86825c198b081bf6e208eaa7" } ], [ @@ -14112,18 +14112,18 @@ [ "type", { -"DuckDB": "b55b02912e30e27dcb40ad93", -"None": "b55b02912e30e27dcb40ad93", -"bigquery": "b55b02912e30e27dcb40ad93", -"databricks": "b55b02912e30e27dcb40ad93", -"duckdb": "b55b02912e30e27dcb40ad93", -"generic": "b55b02912e30e27dcb40ad93", +"DuckDB": "f923673fc4041d12a88250a4", +"None": "f923673fc4041d12a88250a4", +"bigquery": "f923673fc4041d12a88250a4", +"databricks": "f923673fc4041d12a88250a4", +"duckdb": "f923673fc4041d12a88250a4", +"generic": "f923673fc4041d12a88250a4", "motherduck": "901c5133ed47d4769286e860", -"mysql": "b55b02912e30e27dcb40ad93", -"postgres": "b55b02912e30e27dcb40ad93", -"postgresql": "b55b02912e30e27dcb40ad93", -"snowflake": "b55b02912e30e27dcb40ad93", -"tsql": "b55b02912e30e27dcb40ad93" +"mysql": "f923673fc4041d12a88250a4", +"postgres": "f923673fc4041d12a88250a4", +"postgresql": "f923673fc4041d12a88250a4", +"snowflake": "f923673fc4041d12a88250a4", +"tsql": "f923673fc4041d12a88250a4" } ], [ @@ -15234,35 +15234,35 @@ [ "type", { -"DuckDB": "8917b70db141a02ac2e2bb1e", -"None": "8917b70db141a02ac2e2bb1e", -"bigquery": "8917b70db141a02ac2e2bb1e", -"databricks": "8917b70db141a02ac2e2bb1e", -"duckdb": "8917b70db141a02ac2e2bb1e", -"generic": "8917b70db141a02ac2e2bb1e", +"DuckDB": "9122e0d12e93512217da87fa", +"None": "9122e0d12e93512217da87fa", +"bigquery": "9122e0d12e93512217da87fa", +"databricks": "9122e0d12e93512217da87fa", +"duckdb": "9122e0d12e93512217da87fa", +"generic": "9122e0d12e93512217da87fa", "motherduck": "901c5133ed47d4769286e860", -"mysql": "8917b70db141a02ac2e2bb1e", -"postgres": "8917b70db141a02ac2e2bb1e", -"postgresql": "8917b70db141a02ac2e2bb1e", -"snowflake": "8917b70db141a02ac2e2bb1e", -"tsql": "8917b70db141a02ac2e2bb1e" +"mysql": "9122e0d12e93512217da87fa", +"postgres": "9122e0d12e93512217da87fa", +"postgresql": "9122e0d12e93512217da87fa", +"snowflake": "9122e0d12e93512217da87fa", +"tsql": "9122e0d12e93512217da87fa" } ], [ "type", { -"DuckDB": "ef480cafe4d260b70ce5214c", -"None": "ef480cafe4d260b70ce5214c", -"bigquery": "ef480cafe4d260b70ce5214c", -"databricks": "ef480cafe4d260b70ce5214c", -"duckdb": "ef480cafe4d260b70ce5214c", -"generic": "ef480cafe4d260b70ce5214c", +"DuckDB": "b89338179329ef976f911a46", +"None": "b89338179329ef976f911a46", +"bigquery": "b89338179329ef976f911a46", +"databricks": "b89338179329ef976f911a46", +"duckdb": "b89338179329ef976f911a46", +"generic": "b89338179329ef976f911a46", "motherduck": "901c5133ed47d4769286e860", -"mysql": "ef480cafe4d260b70ce5214c", -"postgres": "ef480cafe4d260b70ce5214c", -"postgresql": "ef480cafe4d260b70ce5214c", -"snowflake": "ef480cafe4d260b70ce5214c", -"tsql": "ef480cafe4d260b70ce5214c" +"mysql": "b89338179329ef976f911a46", +"postgres": "b89338179329ef976f911a46", +"postgresql": "b89338179329ef976f911a46", +"snowflake": "b89338179329ef976f911a46", +"tsql": "b89338179329ef976f911a46" } ], [ @@ -15846,18 +15846,18 @@ [ "type", { -"DuckDB": "d44c023c720f385769f2ab81", -"None": "d44c023c720f385769f2ab81", -"bigquery": "d44c023c720f385769f2ab81", -"databricks": "d44c023c720f385769f2ab81", -"duckdb": "d44c023c720f385769f2ab81", -"generic": "d44c023c720f385769f2ab81", +"DuckDB": "63904421f84671303411fe93", +"None": "63904421f84671303411fe93", +"bigquery": "63904421f84671303411fe93", +"databricks": "63904421f84671303411fe93", +"duckdb": "63904421f84671303411fe93", +"generic": "63904421f84671303411fe93", "motherduck": "901c5133ed47d4769286e860", -"mysql": "d44c023c720f385769f2ab81", -"postgres": "d44c023c720f385769f2ab81", -"postgresql": "d44c023c720f385769f2ab81", -"snowflake": "d44c023c720f385769f2ab81", -"tsql": "d44c023c720f385769f2ab81" +"mysql": "63904421f84671303411fe93", +"postgres": "63904421f84671303411fe93", +"postgresql": "63904421f84671303411fe93", +"snowflake": "63904421f84671303411fe93", +"tsql": "63904421f84671303411fe93" } ], [ @@ -16288,35 +16288,35 @@ [ "type", { -"DuckDB": "24b677b214ecdacdb007642e", -"None": "24b677b214ecdacdb007642e", -"bigquery": "24b677b214ecdacdb007642e", -"databricks": "24b677b214ecdacdb007642e", -"duckdb": "24b677b214ecdacdb007642e", -"generic": "24b677b214ecdacdb007642e", +"DuckDB": "15897ece5a8fa1f12e32bed8", +"None": "15897ece5a8fa1f12e32bed8", +"bigquery": "15897ece5a8fa1f12e32bed8", +"databricks": "15897ece5a8fa1f12e32bed8", +"duckdb": "15897ece5a8fa1f12e32bed8", +"generic": "15897ece5a8fa1f12e32bed8", "motherduck": "901c5133ed47d4769286e860", -"mysql": "24b677b214ecdacdb007642e", -"postgres": "24b677b214ecdacdb007642e", -"postgresql": "24b677b214ecdacdb007642e", -"snowflake": "24b677b214ecdacdb007642e", -"tsql": "24b677b214ecdacdb007642e" +"mysql": "15897ece5a8fa1f12e32bed8", +"postgres": "15897ece5a8fa1f12e32bed8", +"postgresql": "15897ece5a8fa1f12e32bed8", +"snowflake": "15897ece5a8fa1f12e32bed8", +"tsql": "15897ece5a8fa1f12e32bed8" } ], [ "type", { -"DuckDB": "4c2128e6078118fd0d3856fa", -"None": "4c2128e6078118fd0d3856fa", -"bigquery": "4c2128e6078118fd0d3856fa", -"databricks": "4c2128e6078118fd0d3856fa", -"duckdb": "4c2128e6078118fd0d3856fa", -"generic": "4c2128e6078118fd0d3856fa", +"DuckDB": "2377bd0302f3eba296b44fee", +"None": "2377bd0302f3eba296b44fee", +"bigquery": "2377bd0302f3eba296b44fee", +"databricks": "2377bd0302f3eba296b44fee", +"duckdb": "2377bd0302f3eba296b44fee", +"generic": "2377bd0302f3eba296b44fee", "motherduck": "901c5133ed47d4769286e860", -"mysql": "4c2128e6078118fd0d3856fa", -"postgres": "4c2128e6078118fd0d3856fa", -"postgresql": "4c2128e6078118fd0d3856fa", -"snowflake": "4c2128e6078118fd0d3856fa", -"tsql": "4c2128e6078118fd0d3856fa" +"mysql": "2377bd0302f3eba296b44fee", +"postgres": "2377bd0302f3eba296b44fee", +"postgresql": "2377bd0302f3eba296b44fee", +"snowflake": "2377bd0302f3eba296b44fee", +"tsql": "2377bd0302f3eba296b44fee" } ], [ @@ -16469,7 +16469,7 @@ "postgres": "6f851103102e7b986535376f", "postgresql": "6f851103102e7b986535376f", "snowflake": "87800133cac38d9119180c0a", -"tsql": "0e9ef5373e5d52970e8e1656" +"tsql": "6ec12741f1435c1a4011e935" } ], [ @@ -16543,18 +16543,18 @@ [ "type", { -"DuckDB": "3009be83c84cdfc92bbb075e", -"None": "3009be83c84cdfc92bbb075e", -"bigquery": "3009be83c84cdfc92bbb075e", -"databricks": "3009be83c84cdfc92bbb075e", -"duckdb": "3009be83c84cdfc92bbb075e", -"generic": "3009be83c84cdfc92bbb075e", +"DuckDB": "f9afa4d07d3af890563ba291", +"None": "f9afa4d07d3af890563ba291", +"bigquery": "f9afa4d07d3af890563ba291", +"databricks": "f9afa4d07d3af890563ba291", +"duckdb": "f9afa4d07d3af890563ba291", +"generic": "f9afa4d07d3af890563ba291", "motherduck": "901c5133ed47d4769286e860", -"mysql": "3009be83c84cdfc92bbb075e", -"postgres": "3009be83c84cdfc92bbb075e", -"postgresql": "3009be83c84cdfc92bbb075e", -"snowflake": "3009be83c84cdfc92bbb075e", -"tsql": "3009be83c84cdfc92bbb075e" +"mysql": "f9afa4d07d3af890563ba291", +"postgres": "f9afa4d07d3af890563ba291", +"postgresql": "f9afa4d07d3af890563ba291", +"snowflake": "f9afa4d07d3af890563ba291", +"tsql": "f9afa4d07d3af890563ba291" } ], [ @@ -16747,18 +16747,18 @@ [ "type", { -"DuckDB": "07f8200cee37aa0f6990da3d", -"None": "07f8200cee37aa0f6990da3d", -"bigquery": "07f8200cee37aa0f6990da3d", -"databricks": "07f8200cee37aa0f6990da3d", -"duckdb": "07f8200cee37aa0f6990da3d", -"generic": "07f8200cee37aa0f6990da3d", +"DuckDB": "293b92524b77d21b6865addc", +"None": "293b92524b77d21b6865addc", +"bigquery": "293b92524b77d21b6865addc", +"databricks": "293b92524b77d21b6865addc", +"duckdb": "293b92524b77d21b6865addc", +"generic": "293b92524b77d21b6865addc", "motherduck": "901c5133ed47d4769286e860", -"mysql": "07f8200cee37aa0f6990da3d", -"postgres": "07f8200cee37aa0f6990da3d", -"postgresql": "07f8200cee37aa0f6990da3d", -"snowflake": "07f8200cee37aa0f6990da3d", -"tsql": "07f8200cee37aa0f6990da3d" +"mysql": "293b92524b77d21b6865addc", +"postgres": "293b92524b77d21b6865addc", +"postgresql": "293b92524b77d21b6865addc", +"snowflake": "293b92524b77d21b6865addc", +"tsql": "293b92524b77d21b6865addc" } ], [ @@ -17342,18 +17342,18 @@ [ "type", { -"DuckDB": "d03c5e5797a8cbc0c6a81112", -"None": "d03c5e5797a8cbc0c6a81112", -"bigquery": "d03c5e5797a8cbc0c6a81112", -"databricks": "d03c5e5797a8cbc0c6a81112", -"duckdb": "d03c5e5797a8cbc0c6a81112", -"generic": "d03c5e5797a8cbc0c6a81112", +"DuckDB": "02a19edb16071249930beb5e", +"None": "02a19edb16071249930beb5e", +"bigquery": "02a19edb16071249930beb5e", +"databricks": "02a19edb16071249930beb5e", +"duckdb": "02a19edb16071249930beb5e", +"generic": "02a19edb16071249930beb5e", "motherduck": "901c5133ed47d4769286e860", -"mysql": "d03c5e5797a8cbc0c6a81112", -"postgres": "d03c5e5797a8cbc0c6a81112", -"postgresql": "d03c5e5797a8cbc0c6a81112", -"snowflake": "d03c5e5797a8cbc0c6a81112", -"tsql": "d03c5e5797a8cbc0c6a81112" +"mysql": "02a19edb16071249930beb5e", +"postgres": "02a19edb16071249930beb5e", +"postgresql": "02a19edb16071249930beb5e", +"snowflake": "02a19edb16071249930beb5e", +"tsql": "02a19edb16071249930beb5e" } ], [ @@ -18651,18 +18651,18 @@ [ "type", { -"DuckDB": "d043825e95555167dc033b5f", -"None": "d043825e95555167dc033b5f", -"bigquery": "d043825e95555167dc033b5f", -"databricks": "d043825e95555167dc033b5f", -"duckdb": "d043825e95555167dc033b5f", -"generic": "d043825e95555167dc033b5f", +"DuckDB": "2726169407c1e410edb2f532", +"None": "2726169407c1e410edb2f532", +"bigquery": "2726169407c1e410edb2f532", +"databricks": "2726169407c1e410edb2f532", +"duckdb": "2726169407c1e410edb2f532", +"generic": "2726169407c1e410edb2f532", "motherduck": "901c5133ed47d4769286e860", -"mysql": "d043825e95555167dc033b5f", -"postgres": "d043825e95555167dc033b5f", -"postgresql": "d043825e95555167dc033b5f", -"snowflake": "d043825e95555167dc033b5f", -"tsql": "d043825e95555167dc033b5f" +"mysql": "2726169407c1e410edb2f532", +"postgres": "2726169407c1e410edb2f532", +"postgresql": "2726169407c1e410edb2f532", +"snowflake": "2726169407c1e410edb2f532", +"tsql": "2726169407c1e410edb2f532" } ], [ @@ -18906,18 +18906,18 @@ [ "type", { -"DuckDB": "200c00626c910f6fd6177bb5", -"None": "200c00626c910f6fd6177bb5", -"bigquery": "200c00626c910f6fd6177bb5", -"databricks": "200c00626c910f6fd6177bb5", -"duckdb": "200c00626c910f6fd6177bb5", -"generic": "200c00626c910f6fd6177bb5", +"DuckDB": "e8e44701a29d5f80cd954213", +"None": "e8e44701a29d5f80cd954213", +"bigquery": "e8e44701a29d5f80cd954213", +"databricks": "e8e44701a29d5f80cd954213", +"duckdb": "e8e44701a29d5f80cd954213", +"generic": "e8e44701a29d5f80cd954213", "motherduck": "901c5133ed47d4769286e860", -"mysql": "200c00626c910f6fd6177bb5", -"postgres": "200c00626c910f6fd6177bb5", -"postgresql": "200c00626c910f6fd6177bb5", -"snowflake": "200c00626c910f6fd6177bb5", -"tsql": "200c00626c910f6fd6177bb5" +"mysql": "e8e44701a29d5f80cd954213", +"postgres": "e8e44701a29d5f80cd954213", +"postgresql": "e8e44701a29d5f80cd954213", +"snowflake": "e8e44701a29d5f80cd954213", +"tsql": "e8e44701a29d5f80cd954213" } ], [ @@ -19008,18 +19008,18 @@ [ "type", { -"DuckDB": "55a086bc8305ad303017ae10", -"None": "55a086bc8305ad303017ae10", -"bigquery": "55a086bc8305ad303017ae10", -"databricks": "55a086bc8305ad303017ae10", -"duckdb": "55a086bc8305ad303017ae10", -"generic": "55a086bc8305ad303017ae10", +"DuckDB": "d077d7ce7054943437bc495a", +"None": "d077d7ce7054943437bc495a", +"bigquery": "d077d7ce7054943437bc495a", +"databricks": "d077d7ce7054943437bc495a", +"duckdb": "d077d7ce7054943437bc495a", +"generic": "d077d7ce7054943437bc495a", "motherduck": "901c5133ed47d4769286e860", -"mysql": "55a086bc8305ad303017ae10", -"postgres": "55a086bc8305ad303017ae10", -"postgresql": "55a086bc8305ad303017ae10", -"snowflake": "55a086bc8305ad303017ae10", -"tsql": "55a086bc8305ad303017ae10" +"mysql": "d077d7ce7054943437bc495a", +"postgres": "d077d7ce7054943437bc495a", +"postgresql": "d077d7ce7054943437bc495a", +"snowflake": "d077d7ce7054943437bc495a", +"tsql": "d077d7ce7054943437bc495a" } ], [ @@ -19042,18 +19042,18 @@ [ "type", { -"DuckDB": "d3b6a6835220f9cedfa363c7", -"None": "d3b6a6835220f9cedfa363c7", -"bigquery": "d3b6a6835220f9cedfa363c7", -"databricks": "d3b6a6835220f9cedfa363c7", -"duckdb": "d3b6a6835220f9cedfa363c7", -"generic": "d3b6a6835220f9cedfa363c7", +"DuckDB": "9d9ab720ece92a330f5bf049", +"None": "9d9ab720ece92a330f5bf049", +"bigquery": "9d9ab720ece92a330f5bf049", +"databricks": "9d9ab720ece92a330f5bf049", +"duckdb": "9d9ab720ece92a330f5bf049", +"generic": "9d9ab720ece92a330f5bf049", "motherduck": "901c5133ed47d4769286e860", -"mysql": "d3b6a6835220f9cedfa363c7", -"postgres": "d3b6a6835220f9cedfa363c7", -"postgresql": "d3b6a6835220f9cedfa363c7", -"snowflake": "d3b6a6835220f9cedfa363c7", -"tsql": "d3b6a6835220f9cedfa363c7" +"mysql": "9d9ab720ece92a330f5bf049", +"postgres": "9d9ab720ece92a330f5bf049", +"postgresql": "9d9ab720ece92a330f5bf049", +"snowflake": "9d9ab720ece92a330f5bf049", +"tsql": "9d9ab720ece92a330f5bf049" } ], [ @@ -19246,18 +19246,18 @@ [ "type", { -"DuckDB": "e5199d22ebbd1383c1e821df", -"None": "e5199d22ebbd1383c1e821df", -"bigquery": "e5199d22ebbd1383c1e821df", -"databricks": "e5199d22ebbd1383c1e821df", -"duckdb": "e5199d22ebbd1383c1e821df", -"generic": "e5199d22ebbd1383c1e821df", +"DuckDB": "5a46370a4920d34e81123c9a", +"None": "5a46370a4920d34e81123c9a", +"bigquery": "5a46370a4920d34e81123c9a", +"databricks": "5a46370a4920d34e81123c9a", +"duckdb": "5a46370a4920d34e81123c9a", +"generic": "5a46370a4920d34e81123c9a", "motherduck": "901c5133ed47d4769286e860", -"mysql": "e5199d22ebbd1383c1e821df", -"postgres": "e5199d22ebbd1383c1e821df", -"postgresql": "e5199d22ebbd1383c1e821df", -"snowflake": "e5199d22ebbd1383c1e821df", -"tsql": "e5199d22ebbd1383c1e821df" +"mysql": "5a46370a4920d34e81123c9a", +"postgres": "5a46370a4920d34e81123c9a", +"postgresql": "5a46370a4920d34e81123c9a", +"snowflake": "5a46370a4920d34e81123c9a", +"tsql": "5a46370a4920d34e81123c9a" } ], [ @@ -19314,18 +19314,18 @@ [ "type", { -"DuckDB": "d91c8f17eb347629690400c7", -"None": "d91c8f17eb347629690400c7", -"bigquery": "d91c8f17eb347629690400c7", -"databricks": "d91c8f17eb347629690400c7", -"duckdb": "d91c8f17eb347629690400c7", -"generic": "d91c8f17eb347629690400c7", +"DuckDB": "a908355fb520168da033bacc", +"None": "a908355fb520168da033bacc", +"bigquery": "a908355fb520168da033bacc", +"databricks": "a908355fb520168da033bacc", +"duckdb": "a908355fb520168da033bacc", +"generic": "a908355fb520168da033bacc", "motherduck": "901c5133ed47d4769286e860", -"mysql": "d91c8f17eb347629690400c7", -"postgres": "d91c8f17eb347629690400c7", -"postgresql": "d91c8f17eb347629690400c7", -"snowflake": "d91c8f17eb347629690400c7", -"tsql": "d91c8f17eb347629690400c7" +"mysql": "a908355fb520168da033bacc", +"postgres": "a908355fb520168da033bacc", +"postgresql": "a908355fb520168da033bacc", +"snowflake": "a908355fb520168da033bacc", +"tsql": "a908355fb520168da033bacc" } ], [ @@ -19467,18 +19467,18 @@ [ "type", { -"DuckDB": "fd31c8e468772027804119ec", -"None": "fd31c8e468772027804119ec", -"bigquery": "fd31c8e468772027804119ec", -"databricks": "fd31c8e468772027804119ec", -"duckdb": "fd31c8e468772027804119ec", -"generic": "fd31c8e468772027804119ec", +"DuckDB": "7fe23b4f83ff4afac4eae8da", +"None": "7fe23b4f83ff4afac4eae8da", +"bigquery": "7fe23b4f83ff4afac4eae8da", +"databricks": "7fe23b4f83ff4afac4eae8da", +"duckdb": "7fe23b4f83ff4afac4eae8da", +"generic": "7fe23b4f83ff4afac4eae8da", "motherduck": "901c5133ed47d4769286e860", -"mysql": "fd31c8e468772027804119ec", -"postgres": "fd31c8e468772027804119ec", -"postgresql": "fd31c8e468772027804119ec", -"snowflake": "fd31c8e468772027804119ec", -"tsql": "fd31c8e468772027804119ec" +"mysql": "7fe23b4f83ff4afac4eae8da", +"postgres": "7fe23b4f83ff4afac4eae8da", +"postgresql": "7fe23b4f83ff4afac4eae8da", +"snowflake": "7fe23b4f83ff4afac4eae8da", +"tsql": "7fe23b4f83ff4afac4eae8da" } ], [ @@ -19790,18 +19790,18 @@ [ "type", { -"DuckDB": "4d90ccac34ef123bb0b2a15c", -"None": "4d90ccac34ef123bb0b2a15c", -"bigquery": "4d90ccac34ef123bb0b2a15c", -"databricks": "4d90ccac34ef123bb0b2a15c", -"duckdb": "4d90ccac34ef123bb0b2a15c", -"generic": "4d90ccac34ef123bb0b2a15c", +"DuckDB": "f1fdef11fa8bdd4185ea7944", +"None": "f1fdef11fa8bdd4185ea7944", +"bigquery": "f1fdef11fa8bdd4185ea7944", +"databricks": "f1fdef11fa8bdd4185ea7944", +"duckdb": "f1fdef11fa8bdd4185ea7944", +"generic": "f1fdef11fa8bdd4185ea7944", "motherduck": "901c5133ed47d4769286e860", -"mysql": "4d90ccac34ef123bb0b2a15c", -"postgres": "4d90ccac34ef123bb0b2a15c", -"postgresql": "4d90ccac34ef123bb0b2a15c", -"snowflake": "4d90ccac34ef123bb0b2a15c", -"tsql": "4d90ccac34ef123bb0b2a15c" +"mysql": "f1fdef11fa8bdd4185ea7944", +"postgres": "f1fdef11fa8bdd4185ea7944", +"postgresql": "f1fdef11fa8bdd4185ea7944", +"snowflake": "f1fdef11fa8bdd4185ea7944", +"tsql": "f1fdef11fa8bdd4185ea7944" } ], [ @@ -19960,18 +19960,18 @@ [ "type", { -"DuckDB": "2e5b1ffe62d65d4d8c389ecd", -"None": "2e5b1ffe62d65d4d8c389ecd", -"bigquery": "2e5b1ffe62d65d4d8c389ecd", -"databricks": "2e5b1ffe62d65d4d8c389ecd", -"duckdb": "2e5b1ffe62d65d4d8c389ecd", -"generic": "2e5b1ffe62d65d4d8c389ecd", +"DuckDB": "769fe9b23909b49818dba3fc", +"None": "769fe9b23909b49818dba3fc", +"bigquery": "769fe9b23909b49818dba3fc", +"databricks": "769fe9b23909b49818dba3fc", +"duckdb": "769fe9b23909b49818dba3fc", +"generic": "769fe9b23909b49818dba3fc", "motherduck": "901c5133ed47d4769286e860", -"mysql": "2e5b1ffe62d65d4d8c389ecd", -"postgres": "2e5b1ffe62d65d4d8c389ecd", -"postgresql": "2e5b1ffe62d65d4d8c389ecd", -"snowflake": "2e5b1ffe62d65d4d8c389ecd", -"tsql": "2e5b1ffe62d65d4d8c389ecd" +"mysql": "769fe9b23909b49818dba3fc", +"postgres": "769fe9b23909b49818dba3fc", +"postgresql": "769fe9b23909b49818dba3fc", +"snowflake": "769fe9b23909b49818dba3fc", +"tsql": "769fe9b23909b49818dba3fc" } ], [ @@ -20776,18 +20776,18 @@ [ "type", { -"DuckDB": "78709b3317abd96cc20340f7", -"None": "78709b3317abd96cc20340f7", -"bigquery": "78709b3317abd96cc20340f7", -"databricks": "78709b3317abd96cc20340f7", -"duckdb": "78709b3317abd96cc20340f7", -"generic": "78709b3317abd96cc20340f7", +"DuckDB": "7bd88d7cc8f9a92e789a9f44", +"None": "7bd88d7cc8f9a92e789a9f44", +"bigquery": "7bd88d7cc8f9a92e789a9f44", +"databricks": "7bd88d7cc8f9a92e789a9f44", +"duckdb": "7bd88d7cc8f9a92e789a9f44", +"generic": "7bd88d7cc8f9a92e789a9f44", "motherduck": "901c5133ed47d4769286e860", -"mysql": "78709b3317abd96cc20340f7", -"postgres": "78709b3317abd96cc20340f7", -"postgresql": "78709b3317abd96cc20340f7", -"snowflake": "78709b3317abd96cc20340f7", -"tsql": "78709b3317abd96cc20340f7" +"mysql": "7bd88d7cc8f9a92e789a9f44", +"postgres": "7bd88d7cc8f9a92e789a9f44", +"postgresql": "7bd88d7cc8f9a92e789a9f44", +"snowflake": "7bd88d7cc8f9a92e789a9f44", +"tsql": "7bd88d7cc8f9a92e789a9f44" } ], [ @@ -20929,18 +20929,18 @@ [ "type", { -"DuckDB": "dcbfb83ead5b0e32e500635b", -"None": "dcbfb83ead5b0e32e500635b", -"bigquery": "dcbfb83ead5b0e32e500635b", -"databricks": "dcbfb83ead5b0e32e500635b", -"duckdb": "dcbfb83ead5b0e32e500635b", -"generic": "dcbfb83ead5b0e32e500635b", +"DuckDB": "6672095306335323ebbfa61b", +"None": "6672095306335323ebbfa61b", +"bigquery": "6672095306335323ebbfa61b", +"databricks": "6672095306335323ebbfa61b", +"duckdb": "6672095306335323ebbfa61b", +"generic": "6672095306335323ebbfa61b", "motherduck": "901c5133ed47d4769286e860", -"mysql": "dcbfb83ead5b0e32e500635b", -"postgres": "dcbfb83ead5b0e32e500635b", -"postgresql": "dcbfb83ead5b0e32e500635b", -"snowflake": "dcbfb83ead5b0e32e500635b", -"tsql": "dcbfb83ead5b0e32e500635b" +"mysql": "6672095306335323ebbfa61b", +"postgres": "6672095306335323ebbfa61b", +"postgresql": "6672095306335323ebbfa61b", +"snowflake": "6672095306335323ebbfa61b", +"tsql": "6672095306335323ebbfa61b" } ], [ @@ -21490,18 +21490,18 @@ [ "type", { -"DuckDB": "277693b7c19b2696f01403ba", -"None": "277693b7c19b2696f01403ba", -"bigquery": "277693b7c19b2696f01403ba", -"databricks": "277693b7c19b2696f01403ba", -"duckdb": "277693b7c19b2696f01403ba", -"generic": "277693b7c19b2696f01403ba", +"DuckDB": "1a590fb9ec69f70166bed628", +"None": "1a590fb9ec69f70166bed628", +"bigquery": "1a590fb9ec69f70166bed628", +"databricks": "1a590fb9ec69f70166bed628", +"duckdb": "1a590fb9ec69f70166bed628", +"generic": "1a590fb9ec69f70166bed628", "motherduck": "901c5133ed47d4769286e860", -"mysql": "277693b7c19b2696f01403ba", -"postgres": "277693b7c19b2696f01403ba", -"postgresql": "277693b7c19b2696f01403ba", -"snowflake": "277693b7c19b2696f01403ba", -"tsql": "277693b7c19b2696f01403ba" +"mysql": "1a590fb9ec69f70166bed628", +"postgres": "1a590fb9ec69f70166bed628", +"postgresql": "1a590fb9ec69f70166bed628", +"snowflake": "1a590fb9ec69f70166bed628", +"tsql": "1a590fb9ec69f70166bed628" } ], [ @@ -22085,18 +22085,18 @@ [ "type", { -"DuckDB": "126765943545b2608ebe7128", -"None": "126765943545b2608ebe7128", -"bigquery": "126765943545b2608ebe7128", -"databricks": "126765943545b2608ebe7128", -"duckdb": "126765943545b2608ebe7128", -"generic": "126765943545b2608ebe7128", +"DuckDB": "fd0efa34f8363aad442b368c", +"None": "fd0efa34f8363aad442b368c", +"bigquery": "fd0efa34f8363aad442b368c", +"databricks": "fd0efa34f8363aad442b368c", +"duckdb": "fd0efa34f8363aad442b368c", +"generic": "fd0efa34f8363aad442b368c", "motherduck": "901c5133ed47d4769286e860", -"mysql": "126765943545b2608ebe7128", -"postgres": "126765943545b2608ebe7128", -"postgresql": "126765943545b2608ebe7128", -"snowflake": "126765943545b2608ebe7128", -"tsql": "126765943545b2608ebe7128" +"mysql": "fd0efa34f8363aad442b368c", +"postgres": "fd0efa34f8363aad442b368c", +"postgresql": "fd0efa34f8363aad442b368c", +"snowflake": "fd0efa34f8363aad442b368c", +"tsql": "fd0efa34f8363aad442b368c" } ], [ @@ -22442,18 +22442,18 @@ [ "type", { -"DuckDB": "cb06a5553de70a4924ed6600", -"None": "cb06a5553de70a4924ed6600", -"bigquery": "cb06a5553de70a4924ed6600", -"databricks": "cb06a5553de70a4924ed6600", -"duckdb": "cb06a5553de70a4924ed6600", -"generic": "cb06a5553de70a4924ed6600", +"DuckDB": "5aa2b9544add6f15c3e51f3e", +"None": "5aa2b9544add6f15c3e51f3e", +"bigquery": "5aa2b9544add6f15c3e51f3e", +"databricks": "5aa2b9544add6f15c3e51f3e", +"duckdb": "5aa2b9544add6f15c3e51f3e", +"generic": "5aa2b9544add6f15c3e51f3e", "motherduck": "901c5133ed47d4769286e860", -"mysql": "cb06a5553de70a4924ed6600", -"postgres": "cb06a5553de70a4924ed6600", -"postgresql": "cb06a5553de70a4924ed6600", -"snowflake": "cb06a5553de70a4924ed6600", -"tsql": "cb06a5553de70a4924ed6600" +"mysql": "5aa2b9544add6f15c3e51f3e", +"postgres": "5aa2b9544add6f15c3e51f3e", +"postgresql": "5aa2b9544add6f15c3e51f3e", +"snowflake": "5aa2b9544add6f15c3e51f3e", +"tsql": "5aa2b9544add6f15c3e51f3e" } ], [ @@ -22731,18 +22731,18 @@ [ "type", { -"DuckDB": "e2eee356d05f550b28bf85ad", -"None": "e2eee356d05f550b28bf85ad", -"bigquery": "e2eee356d05f550b28bf85ad", -"databricks": "e2eee356d05f550b28bf85ad", -"duckdb": "e2eee356d05f550b28bf85ad", -"generic": "e2eee356d05f550b28bf85ad", +"DuckDB": "6f1f2ad566432a6140ba8ac3", +"None": "6f1f2ad566432a6140ba8ac3", +"bigquery": "6f1f2ad566432a6140ba8ac3", +"databricks": "6f1f2ad566432a6140ba8ac3", +"duckdb": "6f1f2ad566432a6140ba8ac3", +"generic": "6f1f2ad566432a6140ba8ac3", "motherduck": "901c5133ed47d4769286e860", -"mysql": "e2eee356d05f550b28bf85ad", -"postgres": "e2eee356d05f550b28bf85ad", -"postgresql": "e2eee356d05f550b28bf85ad", -"snowflake": "e2eee356d05f550b28bf85ad", -"tsql": "e2eee356d05f550b28bf85ad" +"mysql": "6f1f2ad566432a6140ba8ac3", +"postgres": "6f1f2ad566432a6140ba8ac3", +"postgresql": "6f1f2ad566432a6140ba8ac3", +"snowflake": "6f1f2ad566432a6140ba8ac3", +"tsql": "6f1f2ad566432a6140ba8ac3" } ], [ @@ -22816,18 +22816,18 @@ [ "type", { -"DuckDB": "cd9c38fc2cf84c8551e29278", -"None": "cd9c38fc2cf84c8551e29278", -"bigquery": "cd9c38fc2cf84c8551e29278", -"databricks": "cd9c38fc2cf84c8551e29278", -"duckdb": "cd9c38fc2cf84c8551e29278", -"generic": "cd9c38fc2cf84c8551e29278", +"DuckDB": "17c9ac03f8051f8cd6cc398f", +"None": "17c9ac03f8051f8cd6cc398f", +"bigquery": "17c9ac03f8051f8cd6cc398f", +"databricks": "17c9ac03f8051f8cd6cc398f", +"duckdb": "17c9ac03f8051f8cd6cc398f", +"generic": "17c9ac03f8051f8cd6cc398f", "motherduck": "901c5133ed47d4769286e860", -"mysql": "cd9c38fc2cf84c8551e29278", -"postgres": "cd9c38fc2cf84c8551e29278", -"postgresql": "cd9c38fc2cf84c8551e29278", -"snowflake": "cd9c38fc2cf84c8551e29278", -"tsql": "cd9c38fc2cf84c8551e29278" +"mysql": "17c9ac03f8051f8cd6cc398f", +"postgres": "17c9ac03f8051f8cd6cc398f", +"postgresql": "17c9ac03f8051f8cd6cc398f", +"snowflake": "17c9ac03f8051f8cd6cc398f", +"tsql": "17c9ac03f8051f8cd6cc398f" } ], [ @@ -22918,18 +22918,18 @@ [ "type", { -"DuckDB": "1fdc0b9f28b115c175e88d46", -"None": "1fdc0b9f28b115c175e88d46", -"bigquery": "1fdc0b9f28b115c175e88d46", -"databricks": "1fdc0b9f28b115c175e88d46", -"duckdb": "1fdc0b9f28b115c175e88d46", -"generic": "1fdc0b9f28b115c175e88d46", +"DuckDB": "c748aee20c9eae68e05429a3", +"None": "c748aee20c9eae68e05429a3", +"bigquery": "c748aee20c9eae68e05429a3", +"databricks": "c748aee20c9eae68e05429a3", +"duckdb": "c748aee20c9eae68e05429a3", +"generic": "c748aee20c9eae68e05429a3", "motherduck": "901c5133ed47d4769286e860", -"mysql": "1fdc0b9f28b115c175e88d46", -"postgres": "1fdc0b9f28b115c175e88d46", -"postgresql": "1fdc0b9f28b115c175e88d46", -"snowflake": "1fdc0b9f28b115c175e88d46", -"tsql": "1fdc0b9f28b115c175e88d46" +"mysql": "c748aee20c9eae68e05429a3", +"postgres": "c748aee20c9eae68e05429a3", +"postgresql": "c748aee20c9eae68e05429a3", +"snowflake": "c748aee20c9eae68e05429a3", +"tsql": "c748aee20c9eae68e05429a3" } ], [ @@ -22969,18 +22969,18 @@ [ "type", { -"DuckDB": "c0d59e65280108f563dbb04b", -"None": "c0d59e65280108f563dbb04b", -"bigquery": "c0d59e65280108f563dbb04b", -"databricks": "c0d59e65280108f563dbb04b", -"duckdb": "c0d59e65280108f563dbb04b", -"generic": "c0d59e65280108f563dbb04b", +"DuckDB": "ac2d267e58647dba68e3f231", +"None": "ac2d267e58647dba68e3f231", +"bigquery": "ac2d267e58647dba68e3f231", +"databricks": "ac2d267e58647dba68e3f231", +"duckdb": "ac2d267e58647dba68e3f231", +"generic": "ac2d267e58647dba68e3f231", "motherduck": "901c5133ed47d4769286e860", -"mysql": "c0d59e65280108f563dbb04b", -"postgres": "c0d59e65280108f563dbb04b", -"postgresql": "c0d59e65280108f563dbb04b", -"snowflake": "c0d59e65280108f563dbb04b", -"tsql": "c0d59e65280108f563dbb04b" +"mysql": "ac2d267e58647dba68e3f231", +"postgres": "ac2d267e58647dba68e3f231", +"postgresql": "ac2d267e58647dba68e3f231", +"snowflake": "ac2d267e58647dba68e3f231", +"tsql": "ac2d267e58647dba68e3f231" } ], [ @@ -23904,18 +23904,18 @@ [ "type", { -"DuckDB": "7a00586f9fdb1d97256bfff8", -"None": "7a00586f9fdb1d97256bfff8", -"bigquery": "7a00586f9fdb1d97256bfff8", -"databricks": "7a00586f9fdb1d97256bfff8", -"duckdb": "7a00586f9fdb1d97256bfff8", -"generic": "7a00586f9fdb1d97256bfff8", +"DuckDB": "c2ad40bc8b050424142cf287", +"None": "c2ad40bc8b050424142cf287", +"bigquery": "c2ad40bc8b050424142cf287", +"databricks": "c2ad40bc8b050424142cf287", +"duckdb": "c2ad40bc8b050424142cf287", +"generic": "c2ad40bc8b050424142cf287", "motherduck": "901c5133ed47d4769286e860", -"mysql": "7a00586f9fdb1d97256bfff8", -"postgres": "7a00586f9fdb1d97256bfff8", -"postgresql": "7a00586f9fdb1d97256bfff8", -"snowflake": "7a00586f9fdb1d97256bfff8", -"tsql": "7a00586f9fdb1d97256bfff8" +"mysql": "c2ad40bc8b050424142cf287", +"postgres": "c2ad40bc8b050424142cf287", +"postgresql": "c2ad40bc8b050424142cf287", +"snowflake": "c2ad40bc8b050424142cf287", +"tsql": "c2ad40bc8b050424142cf287" } ], [ @@ -24057,18 +24057,18 @@ [ "type", { -"DuckDB": "ef2689646fd9e772e27d2727", -"None": "ef2689646fd9e772e27d2727", -"bigquery": "ef2689646fd9e772e27d2727", +"DuckDB": "da9c22183f7aed4dcccc3e86", +"None": "da9c22183f7aed4dcccc3e86", +"bigquery": "da9c22183f7aed4dcccc3e86", "databricks": "e63b3aab09c82955e2560928", -"duckdb": "ef2689646fd9e772e27d2727", -"generic": "ef2689646fd9e772e27d2727", +"duckdb": "da9c22183f7aed4dcccc3e86", +"generic": "da9c22183f7aed4dcccc3e86", "motherduck": "901c5133ed47d4769286e860", "mysql": "6c03a3450ea4742ee7396e3c", -"postgres": "ef2689646fd9e772e27d2727", -"postgresql": "ef2689646fd9e772e27d2727", -"snowflake": "ef2689646fd9e772e27d2727", -"tsql": "ef2689646fd9e772e27d2727" +"postgres": "da9c22183f7aed4dcccc3e86", +"postgresql": "da9c22183f7aed4dcccc3e86", +"snowflake": "da9c22183f7aed4dcccc3e86", +"tsql": "da9c22183f7aed4dcccc3e86" } ], [ From a6f1efd945b0ab58ac8bcb6460db7b03839f8b9b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 11:06:26 +0000 Subject: [PATCH 15/29] test(lineage): trace fast lineage after a warm restore The same project traced after a cold compile, from the analysis cache alone and from compile reuse, gives the compile report and traces of a run without any cache, so a restore keeps every analysed model's lineage facts. --- .../main/compile/fast_lineage/helpers.py | 28 +++++++++++++++---- .../fast_lineage/test_native_fast_lineage.py | 21 ++++++++++++++ 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py index a3222d6af4..18ab542280 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py @@ -20,6 +20,7 @@ class LineageRun(NamedTuple): compile_report: str compile_returncode: int traces: list[tuple[int, str, str]] + analysis_cache_hits: int = 0 def lineage_run( @@ -28,21 +29,34 @@ def lineage_run( files: dict[str, str], targets: tuple[str, ...], mode: str = "fast", + warm: str | None = None, ) -> LineageRun: - """Write the project, then compile and trace every target both ways in one lineage mode.""" + """Write the project, then compile and trace every target both ways in one lineage mode. + + `warm` names the store a cold compile fills first, so the measured compile and the traces + restore from it: `analysis cache` for the analysis store alone, `compile reuse` for both. + """ for relative_path, contents in files.items(): path: Path = project_dir / "project" / relative_path path.parent.mkdir(parents=True, exist_ok=True) _ = path.write_text(contents, encoding="utf-8") - environment: dict[str, str] = {REUSE_DISABLE_ENV_VAR: "1"} + environment: dict[str, str] = {REUSE_DISABLE_ENV_VAR: "0" if warm == "compile reuse" else "1"} + compile_args: tuple[str, ...] = ( + ("compile", "--json", "--no-cache", "--lineage-mode", mode) + if warm is None + else ("compile", "--json", "--lineage-mode", mode) + ) + if warm is not None: + cold: subprocess.CompletedProcess[str] = run_installed_sqb( + project_dir=project_dir / "project", args=compile_args, env=environment + ) + assert cold.returncode == 0, cold.stdout + cold.stderr compiled: subprocess.CompletedProcess[str] = run_installed_sqb( - project_dir=project_dir / "project", - args=("compile", "--json", "--no-cache", "--lineage-mode", mode), - env=environment, + project_dir=project_dir / "project", args=compile_args, env=environment ) report: dict[str, object] = json.loads(compiled.stdout.replace(str(project_dir), "")) - _ = report.pop("compile_timings", None) + timings: dict[str, int] = cast(dict[str, int], report.pop("compile_timings", {})) traces: list[tuple[int, str, str]] = [] for target, direction in product(targets, _DIRECTIONS): result: subprocess.CompletedProcess[str] = run_installed_sqb( @@ -56,6 +70,8 @@ def lineage_run( compile_report=json.dumps(report, indent=2), compile_returncode=compiled.returncode, traces=traces, + analysis_cache_hits=timings.get("analysis_batch_cache_hits", 0) + + timings.get("analysis_entry_cache_hits", 0), ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py index 930da556bc..39d6aea537 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py @@ -180,6 +180,27 @@ def test_given_project_when_compiling_then_fast_lineage_is_traced( ) +@pytest.mark.parametrize("warm", ["analysis cache", "compile reuse"]) +def test_given_a_warm_store_when_tracing_fast_lineage_then_it_matches_the_uncached_run( + warm: str, tmp_path: Path +) -> None: + targets: tuple[str, ...] = ( + "customer_totals.Total_Amount", + "order_copies.rounded", + "stg_orders.amount", + ) + uncached: LineageRun = lineage_run( + project_dir=tmp_path / "uncached", files=_PROJECT_FILES, targets=targets + ) + restored: LineageRun = lineage_run( + project_dir=tmp_path / "warm", files=_PROJECT_FILES, targets=targets, warm=warm + ) + + assert restored.compile_returncode == 0, restored.compile_report + assert (restored.compile_report, restored.traces) == (uncached.compile_report, uncached.traces) + assert warm != "analysis cache" or restored.analysis_cache_hits > 0 + + @pytest.mark.parametrize( "test_case", [ From 5bb708f6d9b863f3b0ecb40398e06a8cd7409aee Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 11:19:19 +0000 Subject: [PATCH 16/29] fix(analysis): gate a native route on a model's named hooks too A native route's analysis gate ran before hook expansion, when a named SQL hook is still only its name, so it read inline hooks but never named ones. A named hook the parser cannot read then reached the resource SQL check, whose Polyglot parse reported an unlocated P001, while an inline hook got P026. The gate now runs after hook expansion and reads both kinds, and on a native route the resource SQL check no longer turns a Polyglot rejection of hook SQL into a P001: the gate has given P026 for SQL the dialect's parser cannot read. --- .../_helpers/resource_sql/diagnostics.rs | 17 +++++++++--- .../_helpers/input_loops/model_loop.rs | 27 +++++++++++++------ 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs index 33ced0fba8..18aa73d84c 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs @@ -6,6 +6,8 @@ use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics::replace_intrinsics; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; use crate::semantic_checks::_helpers::explanation::opt_outs::unneeded_opt_out; use crate::semantic_checks::_helpers::metadata_checks::positions::{text_position, word_start}; use crate::semantic_checks::_helpers::resource_sql::help::{HelpContext, resource_sql_help}; @@ -98,11 +100,16 @@ fn binding_request( } /// Python's `_reported_diagnostic`: drop open-relation lookups, make parser rejections P001. -fn reported(resource: &ResourceSql, finding: &Finding) -> Option { +/// +/// On a native route the analysis gate reads model hooks with the dialect's parser and gives one +/// it cannot read P026, so Polyglot's rejection of hook SQL is never a P001 there. +fn reported(resource: &ResourceSql, finding: &Finding, native_hooks: bool) -> Option { if finding.code != UNKNOWN_TABLE_CODE { return Some(finding.clone()); } - if finding.message.starts_with(UNKNOWN_TABLE_PREFIX) { + if finding.message.starts_with(UNKNOWN_TABLE_PREFIX) + || (native_hooks && resource.kind == ResourceSqlKind::Model) + { return None; } Some(Finding { @@ -134,8 +141,12 @@ fn resource_diagnostics( checking: &Checking<'_>, ) -> Result, ResourceSqlFailure> { let mut found: Vec<(Key, ResourceSqlDiagnostic)> = Vec::new(); + let native_hooks: bool = matches!( + analysis_route(checking.request.dialect.as_deref()), + AnalysisRoute::Native(_) + ); for (index, native) in findings.iter().enumerate() { - let Some(finding) = reported(resource, native) else { + let Some(finding) = reported(resource, native, native_hooks) else { continue; }; let span = location(resource, checking.cleaned_sql, &finding)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs index 731e152bd8..4d9a371152 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs @@ -753,14 +753,6 @@ impl NativeModelLoop { contents, }, )?; - let (analysed, rejected_opt_out, analysis_skipped) = match legacy_gate { - Some(gate) => gate, - None => self.analysis(context, &opt_out_model, scanned.invalid_calls.is_empty())?, - }; - let analysis: (bool, Option>) = ( - analysed && scanned.invalid_calls.is_empty(), - rejected_opt_out, - ); let references: Vec = merge_call_site_references( scanned.references.iter().map(reference_row).collect(), argument_references, @@ -794,6 +786,25 @@ impl NativeModelLoop { } else { (values, Vec::new()) }; + let (analysed, rejected_opt_out, analysis_skipped) = match legacy_gate { + Some(gate) => gate, + None => { + let expanded_model: OptOutModel<'_, 'py> = OptOutModel { + name, + file_path, + config: &config, + values: &values, + query_sql: &query_sql, + relative_path, + contents, + }; + self.analysis(context, &expanded_model, scanned.invalid_calls.is_empty())? + } + }; + let analysis: (bool, Option>) = ( + analysed && scanned.invalid_calls.is_empty(), + rejected_opt_out, + ); let source: HeaderSource<'_, 'py> = HeaderSource { name, values: &values, From 7cf9e4752b1af86e1b85bb34615c4e2a7a01bb93 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 11:19:19 +0000 Subject: [PATCH 17/29] fix(analysis): know the adapter's declared functions on a native route A native route analysed every model with no extension functions, so a function the adapter declares a return type for was reported unknown. The session now passes those function names to the dialect's analysis. --- .../tests/test_native_outcome.rs | 49 ++++++++++++++++--- .../src/analysis_route/tests/test_types.rs | 4 +- .../_helpers/native_outcome.rs | 5 +- .../analysis_session/_helpers/session.rs | 10 +++- 4 files changed, 58 insertions(+), 10 deletions(-) diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs index bbb1531713..7a9aa872dc 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs @@ -15,23 +15,46 @@ fn given_session_shapes_when_a_native_route_analyses_a_model_then_its_facts_foll description: "a closed source expands a star with its types and known nullability", sql: "SELECT * FROM raw_orders", shape: &[("order_id", "INTEGER"), ("status", "VARCHAR")], + declared_functions: &[], expected: &[ "order_id INTEGER non_null", "status VARCHAR nullable", - "succeeded=true star=true/true", + "succeeded=true star=true/true findings=[]", ], }, NativeOutcomeTestCase { description: "a relation the session has no shape for is unknown, not empty", sql: "SELECT * FROM raw_orders", shape: &[], - expected: &["succeeded=true star=true/false"], + declared_functions: &[], + expected: &["succeeded=true star=true/false findings=[]"], }, NativeOutcomeTestCase { description: "SQL the dialect cannot read fails the analysis", sql: "SELECT FROM WHERE", shape: &[("order_id", "INTEGER")], - expected: &["succeeded=false star=false/false"], + declared_functions: &[], + expected: &["succeeded=false star=false/false findings=[]"], + }, + NativeOutcomeTestCase { + description: "a function the adapter declares is not unknown", + sql: "SELECT CURRENT_TIMESTAMP() AS at FROM raw_orders", + shape: &[("order_id", "INTEGER")], + declared_functions: &[("CURRENT_TIMESTAMP", "TIMESTAMPTZ")], + expected: &[ + "at TIMESTAMPTZ unknown", + "succeeded=true star=false/true findings=[]", + ], + }, + NativeOutcomeTestCase { + description: "the same function without the declaration is unknown", + sql: "SELECT CURRENT_TIMESTAMP() AS at FROM raw_orders", + shape: &[("order_id", "INTEGER")], + declared_functions: &[], + expected: &[ + "at TIMESTAMPTZ unknown", + "succeeded=true star=false/true findings=[\"B101\"]", + ], }, ]; @@ -55,8 +78,17 @@ fn given_session_shapes_when_a_native_route_analyses_a_model_then_its_facts_foll required_names: vec!["raw_orders".to_owned()], ..ModelRequest::default() }; - let (outcome, failure): (ModelOutcome, Option) = - native_outcome(&CataloguedDialect, &model, (schema, &nullability)); + let declared: Vec = test_case + .declared_functions + .iter() + .map(|(name, _)| (*name).to_owned()) + .collect(); + let (outcome, failure): (ModelOutcome, Option) = native_outcome( + &CataloguedDialect, + &model, + (schema, &nullability), + &declared, + ); let analysis = &outcome.analysis; let mut lines: Vec = analysis .columns @@ -71,8 +103,13 @@ fn given_session_shapes_when_a_native_route_analyses_a_model_then_its_facts_foll ) }) .collect(); + let findings: Vec<&str> = analysis + .binding_diagnostics + .iter() + .map(|row| row.code.as_str()) + .collect(); lines.push(format!( - "succeeded={} star={}/{}", + "succeeded={} star={}/{} findings={findings:?}", analysis.analysis_succeeded, analysis.has_star, analysis.star_resolved )); assert_eq!(lines, test_case.expected, "{}", test_case.description); diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs index 6e6be06911..d12c9f1af6 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs @@ -187,7 +187,9 @@ pub(super) struct NativeOutcomeTestCase { pub(super) sql: &'static str, /// `raw_orders`' shape as the session knows it; empty when it does not. pub(super) shape: &'static [(&'static str, &'static str)], - /// Each output column as `name type nullability`, then `succeeded star=/`. + /// The adapter's declared functions as `(name, return type)`. + pub(super) declared_functions: &'static [(&'static str, &'static str)], + /// Each output column as `name type nullability`, then success, star flags and finding codes. pub(super) expected: &'static [&'static str], } diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/native_outcome.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/native_outcome.rs index 9211877862..e3c64caa45 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/native_outcome.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/native_outcome.rs @@ -16,10 +16,13 @@ use crate::assembly::analysis_session::types::{Pairs, Shapes}; use crate::semantic_validation::main::normalize_native::normalize_native_sql; /// The model's outcome read against `schema`, its complete input shapes, and why it failed if it did. +/// +/// `extension_functions` are the functions the adapter declares, which are never unknown. pub(crate) fn native_outcome( dialect: &dyn NativeDialect, model: &ModelRequest, (schema, nullability): (Shapes, &ShapeTable), + extension_functions: &[String], ) -> (ModelOutcome, Option) { let (cleaned_sql, analysed): (String, Result) = match native_cleaned_sql(dialect, model) { @@ -30,7 +33,7 @@ pub(crate) fn native_outcome( sql: &sql, schema: &relations, references: &model.lineage_references, - extension_functions: &[], + extension_functions, }) .map_err(|gap: ParserGap| gap.message); (sql, analysed) diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs index 4e5371158f..eaa71c0be5 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs @@ -314,12 +314,18 @@ impl AnalysisSession { let pool = self.catalog.native.analysis_pool()?; let request: &SessionRequest = &self.request; let nullability: &ShapeTable = &self.available_nullability; + let declared: Vec = request + .function_return_types + .iter() + .map(|(name, _)| name.clone()) + .collect(); let outcomes: Vec<(ModelOutcome, Option)> = pool.install(|| { misses .par_iter() .map(|position| { let model: &ModelRequest = &request.models[models[*position]]; - native_outcome(dialect, model, (schemas[*position].clone(), nullability)) + let schema: Shapes = schemas[*position].clone(); + native_outcome(dialect, model, (schema, nullability), &declared) }) .collect() }); @@ -669,7 +675,7 @@ impl AnalysisSession { } } - /// Normalize the cleaned SQL of the cache hits among `models` that still lack it. + /// Normalize the cleaned SQL of the cache hits among `models` that still lack it. fn ensure_cleaned(&mut self, models: &[usize]) -> Result<(), String> { let pending: Vec = models .iter() From 9c886dd8146038628d209c600ab34309f013761c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 11:34:27 +0000 Subject: [PATCH 18/29] refactor(analysis): shape the native route's model and lineage entries as fensu requires A native model outcome and a native fast lineage are now entry modules of the analysis route, analysis_route::main::model_outcome and ::fast_lineage, each with one visible function. The session builds its own schema for the outcome, fast lineage passes in the references it scanned, and where a parser gap is and why it is unreadable live in one route helper. The warm-restore lineage test takes its stores from local test cases. Behaviour is unchanged. --- .../analysis_route/_helpers/located_gap.rs | 31 +++++ .../src/analysis_route/_helpers/mod.rs | 1 + .../main/fast_lineage.rs} | 27 +--- .../src/analysis_route/main/mod.rs | 2 + .../src/analysis_route/main/model_outcome.rs | 69 ++++++++++ .../src/analysis_route/main/native_gap.rs | 18 +-- .../analysis_route/main/skipped_analysis.rs | 14 +- .../src/analysis_route/tests/helpers.rs | 55 +++++++- .../tests/test_native_outcome.rs | 124 ++++++++---------- .../src/analysis_route/tests/test_types.rs | 9 +- .../analysis_session/_helpers/mappings.rs | 46 ++++++- .../assembly/analysis_session/_helpers/mod.rs | 1 - .../_helpers/native_outcome.rs | 122 ----------------- .../analysis_session/_helpers/session.rs | 18 ++- .../src/lineage/_helpers/mod.rs | 1 - .../src/lineage/main/build_fast_lineage.rs | 26 +++- .../_helpers/resource_sql/diagnostics.rs | 5 +- .../main/compile/fast_lineage/_test_types.py | 9 ++ .../main/compile/fast_lineage/helpers.py | 23 ++-- .../fast_lineage/test_native_fast_lineage.py | 43 ++++-- 20 files changed, 353 insertions(+), 291 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/analysis_route/_helpers/located_gap.rs rename crates/sqlbuild-analysis/src/{lineage/_helpers/native_lineage.rs => analysis_route/main/fast_lineage.rs} (82%) create mode 100644 crates/sqlbuild-analysis/src/analysis_route/main/model_outcome.rs delete mode 100644 crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/native_outcome.rs diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/located_gap.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/located_gap.rs new file mode 100644 index 0000000000..40d950161c --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/located_gap.rs @@ -0,0 +1,31 @@ +//! Where a native parser's gap is, and why the parser cannot read the SQL there. + +use crate::analysis_route::models::{LocatedGap, ParserGap}; +use crate::compiled_project::main::text_position::text_position; +use crate::compiled_project::models::TextPosition; + +/// The gap with the line and column of its offset; an offset past the end is the end. +pub(crate) fn located(sql: &str, gap: ParserGap) -> LocatedGap { + let before: &str = sql.get(..gap.offset).unwrap_or(sql); + let position: TextPosition = text_position(sql, before.chars().count()); + LocatedGap { + construct: gap.construct, + message: gap.message, + line: position.line, + column: position.column, + syntax: gap.syntax, + } +} + +/// Why the dialect's native parser cannot read SQL, as P026 words it. +pub(crate) fn unreadable_reason(dialect: &str, gap: &LocatedGap) -> String { + let detail: String = if gap.message.is_empty() || gap.message == gap.construct { + String::new() + } else { + format!(": {}", gap.message) + }; + format!( + "the {dialect} parser cannot read {} at line {}, column {}{detail}", + gap.construct, gap.line, gap.column + ) +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs index 715122942d..5c472fed40 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod clickhouse_types; +pub(crate) mod located_gap; pub(crate) mod native_analysis; pub(crate) mod whole_gap; diff --git a/crates/sqlbuild-analysis/src/lineage/_helpers/native_lineage.rs b/crates/sqlbuild-analysis/src/analysis_route/main/fast_lineage.rs similarity index 82% rename from crates/sqlbuild-analysis/src/lineage/_helpers/native_lineage.rs rename to crates/sqlbuild-analysis/src/analysis_route/main/fast_lineage.rs index 328085b21c..ee96e88baa 100644 --- a/crates/sqlbuild-analysis/src/lineage/_helpers/native_lineage.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/fast_lineage.rs @@ -5,8 +5,7 @@ use std::collections::HashMap; use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; use sqlquack_semantics::lineage::models::{Nullability, SchemaColumn}; -use crate::analysis_route::main::native_gap::located; -use crate::analysis_route::main::skipped_analysis::unreadable_reason; +use crate::analysis_route::_helpers::located_gap::{located, unreadable_reason}; use crate::analysis_route::models::{AnalysisInput, NativeAnalysis, ParserGap}; use crate::analysis_route::types::NativeDialect; use crate::assembly::analysis_session::constants::{ @@ -14,18 +13,17 @@ use crate::assembly::analysis_session::constants::{ TRANSFORM_DIRECT, TRANSFORM_STAR, }; use crate::assembly::analysis_session::models::{LineageRow, SessionSchema}; -use crate::lineage::_helpers::references::physical_resources; use crate::lineage::models::{ FastLineageOutcome, LineageColumn, LineageConfidence, LineageResourceType, LineageSchemaResource, LineageSource, LineageTransformKind, }; use crate::semantic_validation::main::normalize_native::normalize_native_sql; -/// The model's lineage over `schema`'s column names, or why the dialect's parser cannot read it. -pub(crate) fn native_model_lineage( +/// Lineage over `schema` and the `(name, type, resource)` references, or why it is unreadable. +pub fn fast_lineage( dialect: &dyn NativeDialect, - query_sql: &str, - placeholders: &[(String, String)], + (query_sql, placeholders): (&str, &[(String, String)]), + references: &[(String, String, String)], schema: &[LineageSchemaResource], ) -> FastLineageOutcome { let normalised: String = @@ -34,24 +32,11 @@ pub(crate) fn native_model_lineage( Err(reason) => return FastLineageOutcome::Unparsed(reason), }; let sql: String = placeholder_defaults(&normalised, placeholders); - let mut references: Vec<(String, String, String)> = Vec::new(); - for resource in physical_resources(query_sql) { - if !references - .iter() - .any(|(name, _, _)| *name == resource.resource_name) - { - references.push(( - resource.resource_name.clone(), - resource.resource_type.as_str().to_owned(), - resource.resource_name, - )); - } - } let relations: SessionSchema = names_schema(schema); let analysed: Result = dialect.analyse(&AnalysisInput { sql: &sql, schema: &relations, - references: &references, + references, extension_functions: &[], }); match analysed { diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs b/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs index 4991f95634..66e1c9dae3 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs @@ -1,5 +1,7 @@ pub mod analysis_route; pub mod clickhouse_type_match; +pub mod fast_lineage; +pub(crate) mod model_outcome; pub mod native_gap; pub mod skipped_analysis; pub mod unavailable_analysis; diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/model_outcome.rs b/crates/sqlbuild-analysis/src/analysis_route/main/model_outcome.rs new file mode 100644 index 0000000000..1395539f72 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/main/model_outcome.rs @@ -0,0 +1,69 @@ +//! One model's outcome from its dialect's native analysis module, in the session's facts. + +use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; + +use crate::analysis_route::models::{AnalysisInput, NativeAnalysis, ParserGap}; +use crate::analysis_route::types::NativeDialect; +use crate::assembly::analysis_session::models::{ + LineageFacts, ModelAnalysis, ModelOutcome, ModelRequest, SessionSchema, +}; +use crate::assembly::analysis_session::types::Shapes; +use crate::semantic_validation::main::normalize_native::normalize_native_sql; + +/// The model's outcome over `schema` with the adapter's declared functions, and why it failed. +pub(crate) fn model_outcome( + dialect: &dyn NativeDialect, + model: &ModelRequest, + (schema, relations): (Shapes, &SessionSchema), + extension_functions: &[String], +) -> (ModelOutcome, Option) { + let (cleaned_sql, analysed): (String, Result) = + match normalize_native_sql(&model.query_sql, dialect.name(), dialect.stub_prefix()) { + Ok(normalised) => { + let sql: String = placeholder_defaults(&normalised, &model.placeholders); + let analysed = dialect + .analyse(&AnalysisInput { + sql: &sql, + schema: relations, + references: &model.lineage_references, + extension_functions, + }) + .map_err(|gap: ParserGap| gap.message); + (sql, analysed) + } + Err(reason) => (String::new(), Err(reason)), + }; + let failure: Option = analysed + .as_ref() + .err() + .map(|reason| format!("{}: {reason}", model.name)); + let analysis: ModelAnalysis = match analysed { + Ok(native) => ModelAnalysis { + analysis_succeeded: true, + columns: Some(native.columns), + lineage: LineageFacts::Native(native.lineage), + has_star: native.has_star, + star_resolved: native.star_resolved, + binding_diagnostics: native.diagnostics, + binding_validated: true, + }, + Err(_) => ModelAnalysis { + analysis_succeeded: false, + columns: None, + lineage: LineageFacts::Native(Vec::new()), + has_star: false, + star_resolved: false, + binding_diagnostics: Vec::new(), + binding_validated: true, + }, + }; + let outcome: ModelOutcome = ModelOutcome { + analysis, + cleaned_sql, + deferred_cleaning: None, + validated_schema: schema, + fused_binding_validated: true, + sql_macro_findings: Vec::new(), + }; + (outcome, failure) +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/native_gap.rs b/crates/sqlbuild-analysis/src/analysis_route/main/native_gap.rs index 58168f4625..e484e7074a 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/native_gap.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/native_gap.rs @@ -1,10 +1,9 @@ use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; -use crate::analysis_route::models::{LocatedGap, ParserGap}; +use crate::analysis_route::_helpers::located_gap::located; +use crate::analysis_route::models::LocatedGap; use crate::analysis_route::types::NativeDialect; use crate::assembly::project::models::SyntaxCheck; -use crate::compiled_project::main::text_position::text_position; -use crate::compiled_project::models::TextPosition; use crate::semantic_validation::main::normalize_native::normalize_native_sql; /// The native parser's first gap in `check`'s analysis SQL, with the dialect's stub spelling. @@ -20,16 +19,3 @@ pub fn native_gap( .err() .map(|gap| located(&analysed, gap))) } - -/// The gap with the line and column of its offset; an offset past the end is the end. -pub(crate) fn located(sql: &str, gap: ParserGap) -> LocatedGap { - let before: &str = sql.get(..gap.offset).unwrap_or(sql); - let position: TextPosition = text_position(sql, before.chars().count()); - LocatedGap { - construct: gap.construct, - message: gap.message, - line: position.line, - column: position.column, - syntax: gap.syntax, - } -} diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/skipped_analysis.rs b/crates/sqlbuild-analysis/src/analysis_route/main/skipped_analysis.rs index 3a09a72095..d170a54955 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/skipped_analysis.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/skipped_analysis.rs @@ -1,3 +1,4 @@ +use crate::analysis_route::_helpers::located_gap::unreadable_reason; use crate::analysis_route::constants::SKIPPED_ANALYSIS_CODE; use crate::analysis_route::models::LocatedGap; @@ -12,16 +13,3 @@ pub fn skipped_analysis(model: &str, dialect: &str, gap: &LocatedGap) -> (&'stat ), ) } - -/// Why the dialect's native parser cannot read SQL, as P026 words it. -pub(crate) fn unreadable_reason(dialect: &str, gap: &LocatedGap) -> String { - let detail: String = if gap.message.is_empty() || gap.message == gap.construct { - String::new() - } else { - format!(": {}", gap.message) - }; - format!( - "the {dialect} parser cannot read {} at line {}, column {}{detail}", - gap.construct, gap.line, gap.column - ) -} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs index 27e20d3ec9..f3a99b4bbd 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs @@ -1,6 +1,12 @@ +use std::collections::HashMap; + +use sqlquack_semantics::lineage::models::{Nullability, SchemaColumn}; + use crate::analysis_route::models::{AnalysisInput, NativeAnalysis, StandInSchema}; use crate::analysis_route::types::NativeDialect; -use crate::assembly::analysis_session::models::LineageRow; +use crate::assembly::analysis_session::models::{LineageRow, SessionSchema}; +use crate::assembly::analysis_session::types::Shapes; +use crate::lineage::models::{LineageColumn, LineageSource}; /// `orders` with a non-null INTEGER id and a nullable VARCHAR status. pub(super) fn orders() -> StandInSchema { @@ -51,3 +57,50 @@ pub(super) fn resource_columns(row: &LineageRow) -> Vec<&str> { }) .collect() } + +/// `output transform confidence [type:resource.column, ...]`. +pub(super) fn column_line(column: &LineageColumn) -> String { + let sources: Vec = column.upstream_columns.iter().map(source_text).collect(); + format!( + "{} {} {} [{}]", + column.output_column, + column.transform_kind.as_str(), + column.confidence.as_str(), + sources.join(", ") + ) +} + +fn source_text(source: &LineageSource) -> String { + format!( + "{}:{}.{}", + source.resource_type.as_str(), + source.resource_name, + source.column_name + ) +} + +/// `raw_orders` with `shape` as the session's input shapes and its typed schema; none when empty. +pub(super) fn session_relations(shape: &[(&str, &str, Nullability)]) -> (Shapes, SessionSchema) { + let pairs: Vec<(String, String)> = shape + .iter() + .map(|(name, data_type, _)| ((*name).to_owned(), (*data_type).to_owned())) + .collect(); + let columns: Vec = shape.iter().map(schema_column).collect(); + let relations: HashMap> = std::iter::once(("raw_orders", columns)) + .filter(|(_, columns)| !columns.is_empty()) + .map(|(name, columns)| (name.to_owned(), columns)) + .collect(); + ( + vec![("raw_orders".to_owned(), pairs)], + SessionSchema { relations }, + ) +} + +fn schema_column(&(name, data_type, nullability): &(&str, &str, Nullability)) -> SchemaColumn { + SchemaColumn { + name: name.to_owned(), + quoted: false, + data_type: Some(data_type.to_owned()), + nullability, + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs index 7a9aa872dc..a1d77c535b 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs @@ -1,12 +1,15 @@ +use crate::analysis_route::main::fast_lineage::fast_lineage; +use crate::analysis_route::main::model_outcome::model_outcome; +use crate::analysis_route::tests::helpers::{column_line, session_relations}; use crate::analysis_route::tests::test_types::{ CataloguedDialect, NativeLineageTestCase, NativeOutcomeTestCase, PivotlessDialect, }; -use crate::assembly::analysis_session::_helpers::mappings::ShapeTable; -use crate::assembly::analysis_session::_helpers::native_outcome::native_outcome; -use crate::assembly::analysis_session::models::{ModelOutcome, ModelRequest}; +use crate::assembly::analysis_session::models::{ModelOutcome, ModelRequest, SessionSchema}; use crate::assembly::analysis_session::types::Shapes; -use crate::lineage::_helpers::native_lineage::native_model_lineage; -use crate::lineage::models::{FastLineageOutcome, LineageResourceType, LineageSchemaResource}; +use crate::lineage::models::{ + LineageColumn, LineageOutcomeParts, LineageResourceType, LineageSchemaResource, +}; +use sqlquack_semantics::lineage::models::Nullability; #[test] fn given_session_shapes_when_a_native_route_analyses_a_model_then_its_facts_follow_them() { @@ -14,9 +17,12 @@ fn given_session_shapes_when_a_native_route_analyses_a_model_then_its_facts_foll NativeOutcomeTestCase { description: "a closed source expands a star with its types and known nullability", sql: "SELECT * FROM raw_orders", - shape: &[("order_id", "INTEGER"), ("status", "VARCHAR")], + shape: &[ + ("order_id", "INTEGER", Nullability::NonNull), + ("status", "VARCHAR", Nullability::Nullable), + ], declared_functions: &[], - expected: &[ + expected_lines: &[ "order_id INTEGER non_null", "status VARCHAR nullable", "succeeded=true star=true/true findings=[]", @@ -27,21 +33,21 @@ fn given_session_shapes_when_a_native_route_analyses_a_model_then_its_facts_foll sql: "SELECT * FROM raw_orders", shape: &[], declared_functions: &[], - expected: &["succeeded=true star=true/false findings=[]"], + expected_lines: &["succeeded=true star=true/false findings=[]"], }, NativeOutcomeTestCase { description: "SQL the dialect cannot read fails the analysis", sql: "SELECT FROM WHERE", - shape: &[("order_id", "INTEGER")], + shape: &[("order_id", "INTEGER", Nullability::NonNull)], declared_functions: &[], - expected: &["succeeded=false star=false/false findings=[]"], + expected_lines: &["succeeded=false star=false/false findings=[]"], }, NativeOutcomeTestCase { description: "a function the adapter declares is not unknown", sql: "SELECT CURRENT_TIMESTAMP() AS at FROM raw_orders", - shape: &[("order_id", "INTEGER")], + shape: &[("order_id", "INTEGER", Nullability::NonNull)], declared_functions: &[("CURRENT_TIMESTAMP", "TIMESTAMPTZ")], - expected: &[ + expected_lines: &[ "at TIMESTAMPTZ unknown", "succeeded=true star=false/true findings=[]", ], @@ -49,9 +55,9 @@ fn given_session_shapes_when_a_native_route_analyses_a_model_then_its_facts_foll NativeOutcomeTestCase { description: "the same function without the declaration is unknown", sql: "SELECT CURRENT_TIMESTAMP() AS at FROM raw_orders", - shape: &[("order_id", "INTEGER")], + shape: &[("order_id", "INTEGER", Nullability::NonNull)], declared_functions: &[], - expected: &[ + expected_lines: &[ "at TIMESTAMPTZ unknown", "succeeded=true star=false/true findings=[\"B101\"]", ], @@ -59,19 +65,7 @@ fn given_session_shapes_when_a_native_route_analyses_a_model_then_its_facts_foll ]; for test_case in test_cases { - let shape: Vec<(String, String)> = test_case - .shape - .iter() - .map(|(name, data_type)| ((*name).to_owned(), (*data_type).to_owned())) - .collect(); - let schema: Shapes = vec![("raw_orders".to_owned(), shape)]; - let nullability: ShapeTable = ShapeTable::from_shapes(&vec![( - "raw_orders".to_owned(), - vec![ - ("order_id".to_owned(), "non_null".to_owned()), - ("status".to_owned(), "nullable".to_owned()), - ], - )]); + let (schema, relations): (Shapes, SessionSchema) = session_relations(test_case.shape); let model: ModelRequest = ModelRequest { name: "orders".to_owned(), query_sql: test_case.sql.to_owned(), @@ -83,12 +77,8 @@ fn given_session_shapes_when_a_native_route_analyses_a_model_then_its_facts_foll .iter() .map(|(name, _)| (*name).to_owned()) .collect(); - let (outcome, failure): (ModelOutcome, Option) = native_outcome( - &CataloguedDialect, - &model, - (schema, &nullability), - &declared, - ); + let (outcome, failure): (ModelOutcome, Option) = + model_outcome(&CataloguedDialect, &model, (schema, &relations), &declared); let analysis = &outcome.analysis; let mut lines: Vec = analysis .columns @@ -112,7 +102,7 @@ fn given_session_shapes_when_a_native_route_analyses_a_model_then_its_facts_foll "succeeded={} star={}/{} findings={findings:?}", analysis.analysis_succeeded, analysis.has_star, analysis.star_resolved )); - assert_eq!(lines, test_case.expected, "{}", test_case.description); + assert_eq!(lines, test_case.expected_lines, "{}", test_case.description); assert_eq!( failure.is_some(), !analysis.analysis_succeeded, @@ -137,37 +127,48 @@ fn given_a_model_without_facts_when_fast_lineage_reads_it_natively_then_it_has_l columns: vec!["payment_id".to_owned()], }, ]; + let references: Vec<(String, String, String)> = [ + ("orders", "model"), + ("unknown_model", "model"), + ("raw_payments", "source"), + ] + .iter() + .map(|(name, kind)| ((*name).to_owned(), (*kind).to_owned(), (*name).to_owned())) + .collect(); let test_cases = [ NativeLineageTestCase { description: "a referenced model's columns are the upstreams of direct and computed outputs", sql: r#"SELECT o.order_id, o.amount * 2 AS doubled FROM __ref("orders") AS o"#, placeholders: &[], - expected: &[ + expected_lines: &[ "order_id direct high [model:orders.order_id]", "doubled expression high [model:orders.amount]", - "star=false", + "built star=false", ], }, NativeLineageTestCase { description: "a placeholder takes its default before the parse, and a source is a source", sql: r#"SELECT payment_id FROM __source("raw_payments") WHERE payment_id > @@@min_id"#, placeholders: &[("min_id", "0")], - expected: &[ + expected_lines: &[ "payment_id direct high [source:raw_payments.payment_id]", - "star=false", + "built star=false", ], }, NativeLineageTestCase { description: "a star over a relation of unknown columns is a star without outputs", sql: r#"SELECT * FROM __ref("unknown_model")"#, placeholders: &[], - expected: &["star=true"], + expected_lines: &["built star=true"], }, NativeLineageTestCase { description: "SQL the parser cannot read gives P026's reason, located", sql: "SELECT 1\nFROM t PIVOT (sum(x) FOR y IN (1))", placeholders: &[], - expected: &["unparsed: the postgres parser cannot read PIVOT at line 2, column 8"], + expected_lines: &[ + "unparsed star=false", + "the postgres parser cannot read PIVOT at line 2, column 8", + ], }, ]; @@ -177,37 +178,16 @@ fn given_a_model_without_facts_when_fast_lineage_reads_it_natively_then_it_has_l .iter() .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) .collect(); - let outcome: FastLineageOutcome = - native_model_lineage(&PivotlessDialect, test_case.sql, &placeholders, &schema); - let lines: Vec = match outcome { - FastLineageOutcome::Built { columns, has_star } => columns - .iter() - .map(|column| { - let sources: Vec = column - .upstream_columns - .iter() - .map(|source| { - format!( - "{}:{}.{}", - source.resource_type.as_str(), - source.resource_name, - source.column_name - ) - }) - .collect(); - format!( - "{} {} {} [{}]", - column.output_column, - column.transform_kind.as_str(), - column.confidence.as_str(), - sources.join(", ") - ) - }) - .chain([format!("star={has_star}")]) - .collect(), - FastLineageOutcome::Unparsed(reason) => vec![format!("unparsed: {reason}")], - other => vec![format!("{other:?}")], - }; - assert_eq!(lines, test_case.expected, "{}", test_case.description); + let parts: LineageOutcomeParts = fast_lineage( + &PivotlessDialect, + (test_case.sql, &placeholders), + &references, + &schema, + ) + .into_parts(); + let mut lines: Vec = parts.columns.iter().map(column_line).collect(); + lines.push(format!("{} star={}", parts.status, parts.has_star)); + lines.extend(parts.detail); + assert_eq!(lines, test_case.expected_lines, "{}", test_case.description); } } diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs index d12c9f1af6..d814d9e285 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs @@ -1,5 +1,6 @@ use sqlquack_semantics::dialects::constants::{DUCKDB, POSTGRES}; use sqlquack_semantics::dialects::types::DialectProfile; +use sqlquack_semantics::lineage::models::Nullability; use sqlquack_syntax::expressions::statements::models::Command; use sqlquack_syntax::{Dialect, Expression, ParseOptions, Parsed, parse_strict}; @@ -185,17 +186,17 @@ pub(super) struct StarTestCase { pub(super) struct NativeOutcomeTestCase { pub(super) description: &'static str, pub(super) sql: &'static str, - /// `raw_orders`' shape as the session knows it; empty when it does not. - pub(super) shape: &'static [(&'static str, &'static str)], + /// `raw_orders`' columns, types and nullability as the session knows them; empty when unknown. + pub(super) shape: &'static [(&'static str, &'static str, Nullability)], /// The adapter's declared functions as `(name, return type)`. pub(super) declared_functions: &'static [(&'static str, &'static str)], /// Each output column as `name type nullability`, then success, star flags and finding codes. - pub(super) expected: &'static [&'static str], + pub(super) expected_lines: &'static [&'static str], } pub(super) struct NativeLineageTestCase { pub(super) description: &'static str, pub(super) sql: &'static str, pub(super) placeholders: &'static [(&'static str, &'static str)], - pub(super) expected: &'static [&'static str], + pub(super) expected_lines: &'static [&'static str], } diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mappings.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mappings.rs index 0018c8c5c0..d34f769e55 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mappings.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mappings.rs @@ -4,9 +4,10 @@ use std::collections::{BTreeSet, HashMap}; use std::sync::OnceLock; use sha2::{Digest, Sha256}; +use sqlquack_semantics::lineage::models::{Nullability, SchemaColumn}; -use crate::assembly::analysis_session::constants::DIALECT_ALIASES; -use crate::assembly::analysis_session::models::ModelRequest; +use crate::assembly::analysis_session::constants::{DIALECT_ALIASES, UNKNOWN_TYPE}; +use crate::assembly::analysis_session::models::{ModelRequest, SessionSchema}; use crate::assembly::analysis_session::types::{Pairs, Shapes}; use crate::semantic_validation::models::Columns; use crate::semantic_validation::types::Relations; @@ -238,3 +239,44 @@ pub(crate) fn waves(producers: &[Vec]) -> Option>> { } (scheduled == producers.len()).then_some(waves) } + +/// Each relation with a known shape, its columns typed and with the nullability the session knows. +pub(crate) fn session_schema(schema: &Shapes, nullability: &ShapeTable) -> SessionSchema { + let relations: HashMap> = schema + .iter() + .filter(|(_, columns)| !columns.is_empty()) + .map(|(relation, columns)| { + let typed: Vec = typed_columns(columns, nullability.get(relation)); + (relation.clone(), typed) + }) + .collect(); + SessionSchema { relations } +} + +/// Each column with its known type and the nullability `known` spells for it. +fn typed_columns(columns: &Pairs, known: Option<&Pairs>) -> Vec { + columns + .iter() + .map(|(name, data_type)| SchemaColumn { + name: name.clone(), + quoted: false, + data_type: (!data_type.is_empty() && data_type != UNKNOWN_TYPE) + .then(|| data_type.clone()), + nullability: column_nullability(known, name), + }) + .collect() +} + +fn column_nullability(known: Option<&Pairs>, column: &str) -> Nullability { + let spelled: Option<&str> = known.and_then(|pairs| { + pairs + .iter() + .find(|(name, _)| name == column) + .map(|(_, value)| value.as_str()) + }); + match spelled { + Some("non_null") => Nullability::NonNull, + Some("nullable") => Nullability::Nullable, + _ => Nullability::Unknown, + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mod.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mod.rs index b9e6bfc973..61615bd519 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mod.rs @@ -6,6 +6,5 @@ pub(crate) mod dict_walk; pub(crate) mod dynamic_pivot; pub(crate) mod enrichment; pub(crate) mod mappings; -pub(crate) mod native_outcome; pub(crate) mod query_columns; pub(crate) mod session; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/native_outcome.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/native_outcome.rs deleted file mode 100644 index e3c64caa45..0000000000 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/native_outcome.rs +++ /dev/null @@ -1,122 +0,0 @@ -//! One model's outcome from its dialect's native analysis module, in the session's facts. - -use std::collections::HashMap; - -use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; -use sqlquack_semantics::lineage::models::{Nullability, SchemaColumn}; - -use crate::analysis_route::models::{AnalysisInput, NativeAnalysis, ParserGap}; -use crate::analysis_route::types::NativeDialect; -use crate::assembly::analysis_session::_helpers::mappings::ShapeTable; -use crate::assembly::analysis_session::constants::UNKNOWN_TYPE; -use crate::assembly::analysis_session::models::{ - LineageFacts, ModelAnalysis, ModelOutcome, ModelRequest, SessionSchema, -}; -use crate::assembly::analysis_session::types::{Pairs, Shapes}; -use crate::semantic_validation::main::normalize_native::normalize_native_sql; - -/// The model's outcome read against `schema`, its complete input shapes, and why it failed if it did. -/// -/// `extension_functions` are the functions the adapter declares, which are never unknown. -pub(crate) fn native_outcome( - dialect: &dyn NativeDialect, - model: &ModelRequest, - (schema, nullability): (Shapes, &ShapeTable), - extension_functions: &[String], -) -> (ModelOutcome, Option) { - let (cleaned_sql, analysed): (String, Result) = - match native_cleaned_sql(dialect, model) { - Ok(sql) => { - let relations: SessionSchema = session_schema(&schema, nullability); - let analysed = dialect - .analyse(&AnalysisInput { - sql: &sql, - schema: &relations, - references: &model.lineage_references, - extension_functions, - }) - .map_err(|gap: ParserGap| gap.message); - (sql, analysed) - } - Err(reason) => (String::new(), Err(reason)), - }; - let failure: Option = analysed - .as_ref() - .err() - .map(|reason| format!("{}: {reason}", model.name)); - let analysis: ModelAnalysis = match analysed { - Ok(native) => ModelAnalysis { - analysis_succeeded: true, - columns: Some(native.columns), - lineage: LineageFacts::Native(native.lineage), - has_star: native.has_star, - star_resolved: native.star_resolved, - binding_diagnostics: native.diagnostics, - binding_validated: true, - }, - Err(_) => ModelAnalysis { - analysis_succeeded: false, - columns: None, - lineage: LineageFacts::Native(Vec::new()), - has_star: false, - star_resolved: false, - binding_diagnostics: Vec::new(), - binding_validated: true, - }, - }; - let outcome: ModelOutcome = ModelOutcome { - analysis, - cleaned_sql, - deferred_cleaning: None, - validated_schema: schema, - fused_binding_validated: true, - sql_macro_findings: Vec::new(), - }; - (outcome, failure) -} - -/// The model's SQL as its dialect's parser reads it: stubs respelt and placeholders filled. -pub(crate) fn native_cleaned_sql( - dialect: &dyn NativeDialect, - model: &ModelRequest, -) -> Result { - normalize_native_sql(&model.query_sql, dialect.name(), dialect.stub_prefix()) - .map(|normalised| placeholder_defaults(&normalised, &model.placeholders)) -} - -/// Each relation with a known shape, its columns typed and with the nullability the session knows. -fn session_schema(schema: &Shapes, nullability: &ShapeTable) -> SessionSchema { - let relations: HashMap> = schema - .iter() - .filter(|(_, columns)| !columns.is_empty()) - .map(|(relation, columns)| { - let known: Option<&Pairs> = nullability.get(relation); - let typed: Vec = columns - .iter() - .map(|(name, data_type)| SchemaColumn { - name: name.clone(), - quoted: false, - data_type: (!data_type.is_empty() && data_type != UNKNOWN_TYPE) - .then(|| data_type.clone()), - nullability: column_nullability(known, name), - }) - .collect(); - (relation.clone(), typed) - }) - .collect(); - SessionSchema { relations } -} - -fn column_nullability(known: Option<&Pairs>, column: &str) -> Nullability { - let spelled: Option<&str> = known.and_then(|pairs| { - pairs - .iter() - .find(|(name, _)| name == column) - .map(|(_, value)| value.as_str()) - }); - match spelled { - Some("non_null") => Nullability::NonNull, - Some("nullable") => Nullability::Nullable, - _ => Nullability::Unknown, - } -} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs index eaa71c0be5..395ee1dbcf 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs @@ -9,8 +9,10 @@ use sqlbuild_cache::digest::types::ContentDigest; use sqlbuild_cache::store::errors::StoreDecodeError; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use sqlbuild_core::panics::main::native_failure::native_failure; +use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::main::model_outcome::model_outcome; use crate::analysis_route::models::AnalysisRoute; use crate::analysis_route::types::NativeDialect; use crate::assembly::analysis_session::_helpers::analysis_cache::{ @@ -29,20 +31,18 @@ use crate::assembly::analysis_session::_helpers::enrichment::{ Enrichment, EnrichmentInput, enrichment, }; use crate::assembly::analysis_session::_helpers::mappings::{ - ShapeTable, dict_from_pairs, same_keys, same_relations, + ShapeTable, dict_from_pairs, same_keys, same_relations, session_schema, }; use crate::assembly::analysis_session::_helpers::mappings::{producers, waves}; -use crate::assembly::analysis_session::_helpers::native_outcome::{ - native_cleaned_sql, native_outcome, -}; use crate::assembly::analysis_session::constants::{UNKNOWN_NULLABILITY, UNKNOWN_TYPE}; use crate::assembly::analysis_session::main::resolved_cleaned_sql::resolved_cleaned_sql; use crate::assembly::analysis_session::models::{ AnalysisSession, ColumnFact, ContractProof, DeferredCleaning, FinishedSession, LineageFacts, LineageRow, ModelAnalysis, ModelOutcome, ModelRequest, Phase, PivotOutcome, PivotTables, - RelationTables, SessionModelFacts, SessionOutcome, SessionRequest, SessionStep, + RelationTables, SessionModelFacts, SessionOutcome, SessionRequest, SessionSchema, SessionStep, }; use crate::assembly::analysis_session::types::{Pairs, Shapes}; +use crate::semantic_validation::main::normalize_native::normalize_native_sql; use crate::semantic_validation::models::DiagnosticRow; use crate::semantic_validation::models::ProjectCatalog; use crate::sql_macros::main::check_sql_macro_arguments::check_sql_macro_arguments; @@ -325,7 +325,8 @@ impl AnalysisSession { .map(|position| { let model: &ModelRequest = &request.models[models[*position]]; let schema: Shapes = schemas[*position].clone(); - native_outcome(dialect, model, (schema, nullability), &declared) + let relations: SessionSchema = session_schema(&schema, nullability); + model_outcome(dialect, model, (schema, &relations), &declared) }) .collect() }); @@ -687,8 +688,11 @@ impl AnalysisSession { } if let AnalysisRoute::Native(dialect) = self.route() { for model in pending { + let request: &ModelRequest = &self.request.models[model]; let sql: String = - native_cleaned_sql(dialect, &self.request.models[model]).unwrap_or_default(); + normalize_native_sql(&request.query_sql, dialect.name(), dialect.stub_prefix()) + .map(|normalised| placeholder_defaults(&normalised, &request.placeholders)) + .unwrap_or_default(); self.outcome_mut(model)?.cleaned_sql = sql; self.pending_cleaned[model] = false; } diff --git a/crates/sqlbuild-analysis/src/lineage/_helpers/mod.rs b/crates/sqlbuild-analysis/src/lineage/_helpers/mod.rs index 31d6889d68..139d0d6c87 100644 --- a/crates/sqlbuild-analysis/src/lineage/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/lineage/_helpers/mod.rs @@ -1,7 +1,6 @@ pub(crate) mod authored_files; pub(crate) mod environment_markers; pub(crate) mod fingerprint_digest; -pub(crate) mod native_lineage; pub(crate) mod parsed_lineage; pub(crate) mod projections; pub(crate) mod python_payload; diff --git a/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs b/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs index 0d74ba11a7..96e1790c96 100644 --- a/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs +++ b/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs @@ -9,8 +9,8 @@ use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use sqlbuild_core::panics::main::native_failure::native_failure; use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::main::fast_lineage::fast_lineage; use crate::analysis_route::models::AnalysisRoute; -use crate::lineage::_helpers::native_lineage::native_model_lineage; use crate::lineage::_helpers::parsed_lineage::{ ParsedModel, parsed_model_lineage, proxy_parse_options, }; @@ -80,10 +80,10 @@ fn model_outcomes(request: &FastLineageRequest) -> Vec catch_compiler_panic(|| { - Ok(native_model_lineage( + Ok(fast_lineage( module, - query_sql, - placeholders, + (query_sql, placeholders), + &lineage_references(query_sql), &request.schema, )) }), @@ -110,3 +110,21 @@ fn model_outcomes(request: &FastLineageRequest) -> Vec Vec<(String, String, String)> { + let mut references: Vec<(String, String, String)> = Vec::new(); + for resource in physical_resources(query_sql) { + if !references + .iter() + .any(|(name, _, _)| *name == resource.resource_name) + { + references.push(( + resource.resource_name.clone(), + resource.resource_type.as_str().to_owned(), + resource.resource_name, + )); + } + } + references +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs index 18aa73d84c..0064cf828d 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs @@ -99,10 +99,7 @@ fn binding_request( (sql.to_owned(), references, overrides) } -/// Python's `_reported_diagnostic`: drop open-relation lookups, make parser rejections P001. -/// -/// On a native route the analysis gate reads model hooks with the dialect's parser and gives one -/// it cannot read P026, so Polyglot's rejection of hook SQL is never a P001 there. +/// `_reported_diagnostic`: drop open-relation lookups; parser rejections are P001, natively P026's. fn reported(resource: &ResourceSql, finding: &Finding, native_hooks: bool) -> Option { if finding.code != UNKNOWN_TABLE_CODE { return Some(finding.clone()); diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/_test_types.py index 5d57d26234..7d0eec5f34 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/_test_types.py @@ -14,6 +14,15 @@ class NativeFastLineageCliTestCase: expected_minimum_traced_edges: int +@dataclass(frozen=True) +class WarmFastLineageCliTestCase: + """Fast lineage traced after a cold compile filled the compile's stores.""" + + description: str + reuse_disabled: str + expected_minimum_cache_hits: int + + @dataclass(frozen=True) class NativeRichLineageFailureCliTestCase: """A native rich lineage failure injected at the binding, compiled through the CLI.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py index 18ab542280..546d0400a3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py @@ -29,31 +29,28 @@ def lineage_run( files: dict[str, str], targets: tuple[str, ...], mode: str = "fast", - warm: str | None = None, + cache_args: tuple[str, ...] = ("--no-cache",), + environment: tuple[tuple[str, str], ...] = ((REUSE_DISABLE_ENV_VAR, "1"),), + cold_compiles: int = 0, ) -> LineageRun: """Write the project, then compile and trace every target both ways in one lineage mode. - `warm` names the store a cold compile fills first, so the measured compile and the traces - restore from it: `analysis cache` for the analysis store alone, `compile reuse` for both. + `cold_compiles` compiles run first, so the measured compile and traces read what they stored. """ for relative_path, contents in files.items(): path: Path = project_dir / "project" / relative_path path.parent.mkdir(parents=True, exist_ok=True) _ = path.write_text(contents, encoding="utf-8") - environment: dict[str, str] = {REUSE_DISABLE_ENV_VAR: "0" if warm == "compile reuse" else "1"} - compile_args: tuple[str, ...] = ( - ("compile", "--json", "--no-cache", "--lineage-mode", mode) - if warm is None - else ("compile", "--json", "--lineage-mode", mode) - ) - if warm is not None: + env: dict[str, str] = dict(environment) + compile_args: tuple[str, ...] = ("compile", "--json", *cache_args, "--lineage-mode", mode) + for _ in range(cold_compiles): cold: subprocess.CompletedProcess[str] = run_installed_sqb( - project_dir=project_dir / "project", args=compile_args, env=environment + project_dir=project_dir / "project", args=compile_args, env=env ) assert cold.returncode == 0, cold.stdout + cold.stderr compiled: subprocess.CompletedProcess[str] = run_installed_sqb( - project_dir=project_dir / "project", args=compile_args, env=environment + project_dir=project_dir / "project", args=compile_args, env=env ) report: dict[str, object] = json.loads(compiled.stdout.replace(str(project_dir), "")) timings: dict[str, int] = cast(dict[str, int], report.pop("compile_timings", {})) @@ -62,7 +59,7 @@ def lineage_run( result: subprocess.CompletedProcess[str] = run_installed_sqb( project_dir=project_dir / "project", args=("lineage", target, "--mode", mode, "--direction", direction, "--json"), - env=environment, + env=env, ) stdout: str = result.stdout.replace(str(project_dir), "") traces.append((result.returncode, stdout, _TIMING.sub("", result.stderr))) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py index 39d6aea537..0621386f95 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py @@ -6,10 +6,12 @@ import pytest +from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR from tests.e2e.src.sqlbuild.cli.commands.main.compile.fast_lineage._test_types import ( NativeFastLineageCliTestCase, NativeRichLineageCliTestCase, NativeRichLineageFailureCliTestCase, + WarmFastLineageCliTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.fast_lineage.helpers import ( FailedLineageRun, @@ -52,6 +54,12 @@ ), } +_WARM_TARGETS: tuple[str, ...] = ( + "customer_totals.Total_Amount", + "order_copies.rounded", + "stg_orders.amount", +) + _DEEP_UNION_BRANCHES: int = 480 _DEEP_UNION_FILES: dict[str, str] = { "sqlbuild_project.toml": _PROJECT_FILES["sqlbuild_project.toml"], @@ -180,25 +188,40 @@ def test_given_project_when_compiling_then_fast_lineage_is_traced( ) -@pytest.mark.parametrize("warm", ["analysis cache", "compile reuse"]) +@pytest.mark.parametrize( + "test_case", + [ + WarmFastLineageCliTestCase( + description="the analysis cache a cold compile filled", + reuse_disabled="1", + expected_minimum_cache_hits=1, + ), + WarmFastLineageCliTestCase( + description="the compiled project a cold compile stored for reuse", + reuse_disabled="0", + expected_minimum_cache_hits=0, + ), + ], + ids=lambda case: case.description, +) def test_given_a_warm_store_when_tracing_fast_lineage_then_it_matches_the_uncached_run( - warm: str, tmp_path: Path + test_case: WarmFastLineageCliTestCase, tmp_path: Path ) -> None: - targets: tuple[str, ...] = ( - "customer_totals.Total_Amount", - "order_copies.rounded", - "stg_orders.amount", - ) uncached: LineageRun = lineage_run( - project_dir=tmp_path / "uncached", files=_PROJECT_FILES, targets=targets + project_dir=tmp_path / "uncached", files=_PROJECT_FILES, targets=_WARM_TARGETS ) restored: LineageRun = lineage_run( - project_dir=tmp_path / "warm", files=_PROJECT_FILES, targets=targets, warm=warm + project_dir=tmp_path / "warm", + files=_PROJECT_FILES, + targets=_WARM_TARGETS, + cache_args=(), + environment=((REUSE_DISABLE_ENV_VAR, test_case.reuse_disabled),), + cold_compiles=1, ) assert restored.compile_returncode == 0, restored.compile_report assert (restored.compile_report, restored.traces) == (uncached.compile_report, uncached.traces) - assert warm != "analysis cache" or restored.analysis_cache_hits > 0 + assert restored.analysis_cache_hits >= test_case.expected_minimum_cache_hits @pytest.mark.parametrize( From d9a9e16787aac77e498ad33a2b09863955f7a2db Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:36:53 +0000 Subject: [PATCH 19/29] fix(analysis): revalidate type recovery with the native analysis on native routes Type recovery revalidated each model downstream of a poisoned output through the Polyglot binding catalog. On a native route that catalog was never prepared, so preparing it while the semantic stage held a borrow on it panicked with "Already borrowed" whenever a model with its own binding error sat downstream of a blocking type error. On a native route, revalidation now reads each planned model's analysed SQL again with its dialect's analysis, over the schema the session bound and with the poisoned outputs untyped. The findings come back at the session's own positions, so type recovery matches them against the raw rows directly, and the Polyglot catalog is no longer on that path. The finished session keeps the model order and the declared functions it analysed with. The Polyglot route is unchanged. --- .../src/analysis_route/_helpers/mod.rs | 1 + .../analysis_route/_helpers/revalidation.rs | 107 ++++++++++++++++++ .../src/analysis_route/main/mod.rs | 1 + .../main/native_revalidation.rs | 31 +++++ .../analysis_session/_helpers/session.rs | 12 ++ .../src/assembly/analysis_session/models.rs | 4 + .../analysis_session/tests/helpers.rs | 2 + .../_helpers/analysis_session/session.rs | 5 + .../_helpers/semantic_checks/type_recovery.rs | 50 +++++++- .../sqlbuild/compiler/pipeline/_test_types.py | 10 ++ .../compiler/pipeline/test_semantic_triage.py | 42 +++++++ 11 files changed, 263 insertions(+), 2 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/analysis_route/_helpers/revalidation.rs create mode 100644 crates/sqlbuild-analysis/src/analysis_route/main/native_revalidation.rs diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs index 5c472fed40..393034d764 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod clickhouse_types; pub(crate) mod located_gap; pub(crate) mod native_analysis; +pub(crate) mod revalidation; pub(crate) mod whole_gap; diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/revalidation.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/revalidation.rs new file mode 100644 index 0000000000..d726434436 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/revalidation.rs @@ -0,0 +1,107 @@ +//! One model's native findings read again over its bound schema with poisoned outputs untyped. + +use std::borrow::Cow; +use std::collections::HashMap; + +use sqlquack_semantics::lineage::models::{Nullability, SchemaColumn}; + +use crate::analysis_route::models::{AnalysisInput, NativeAnalysis}; +use crate::analysis_route::types::NativeDialect; +use crate::assembly::analysis_session::constants::UNKNOWN_TYPE; +use crate::assembly::analysis_session::main::resolved_cleaned_sql::resolved_cleaned_sql; +use crate::assembly::analysis_session::models::{ + FinishedSession, ModelOutcome, PivotOutcome, SessionSchema, +}; +use crate::assembly::analysis_session::types::Pairs; +use crate::semantic_checks::types::RevisedBinding; +use crate::semantic_validation::models::DiagnosticRow; + +/// The finished outcome of the model the session analysed as `name`. +pub(crate) fn session_outcome<'a>( + finished: &FinishedSession, + outcomes: &'a [(ModelOutcome, PivotOutcome)], + name: &str, +) -> Result<&'a ModelOutcome, String> { + let position: Option = finished + .model_names + .iter() + .position(|analysed| analysed == name); + position + .and_then(|position| outcomes.get(position)) + .map(|(outcome, _)| outcome) + .ok_or_else(|| format!("the session analysed no model {name}")) +} + +/// The findings of `outcome`'s analysed SQL over its bound schema, the poisoned columns untyped. +pub(crate) fn revised( + dialect: &dyn NativeDialect, + outcome: &ModelOutcome, + poisoned: &[(String, String)], + declared: &[String], +) -> Result, String> { + let sql: Cow<'_, str> = resolved_cleaned_sql(outcome)?; + let relations: SessionSchema = SessionSchema { + relations: outcome + .validated_schema + .iter() + .filter(|(_, columns)| !columns.is_empty()) + .map(|(relation, columns)| { + ( + relation.clone(), + schema_columns(relation, columns, poisoned), + ) + }) + .collect::>>(), + }; + let analysis: NativeAnalysis = dialect + .analyse(&AnalysisInput { + sql: &sql, + schema: &relations, + references: &[], + extension_functions: declared, + }) + .map_err(|gap| gap.message)?; + analysis + .diagnostics + .into_iter() + .map(revised_binding) + .collect() +} + +/// `relation`'s columns as the session bound them, each poisoned or unknown one untyped. +fn schema_columns( + relation: &str, + columns: &Pairs, + poisoned: &[(String, String)], +) -> Vec { + let mut schema: Vec = Vec::with_capacity(columns.len()); + for (name, data_type) in columns { + let untyped: bool = data_type.is_empty() + || data_type == UNKNOWN_TYPE + || poisoned + .iter() + .any(|(model, output)| model == relation && output == name); + schema.push(SchemaColumn { + name: name.clone(), + quoted: false, + data_type: (!untyped).then(|| data_type.clone()), + nullability: Nullability::Unknown, + }); + } + schema +} + +/// A finding as type recovery matches it: its code and character span. +fn revised_binding(row: DiagnosticRow) -> Result { + let start: Option = row + .start + .map(i64::try_from) + .transpose() + .map_err(|error| error.to_string())?; + let end: Option = row + .end + .map(i64::try_from) + .transpose() + .map_err(|error| error.to_string())?; + Ok((row.code, start, end)) +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs b/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs index 66e1c9dae3..9d6f86f1f5 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs @@ -3,5 +3,6 @@ pub mod clickhouse_type_match; pub mod fast_lineage; pub(crate) mod model_outcome; pub mod native_gap; +pub mod native_revalidation; pub mod skipped_analysis; pub mod unavailable_analysis; diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/native_revalidation.rs b/crates/sqlbuild-analysis/src/analysis_route/main/native_revalidation.rs new file mode 100644 index 0000000000..84efea9ac5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/main/native_revalidation.rs @@ -0,0 +1,31 @@ +//! Type recovery's revalidation on a native route, read by the dialect's own analysis. + +use std::sync::Arc; + +use rayon::ThreadPool; +use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; + +use crate::analysis_route::_helpers::revalidation::{revised, session_outcome}; +use crate::analysis_route::types::NativeDialect; +use crate::assembly::analysis_session::models::{FinishedSession, ModelOutcome, PivotOutcome}; +use crate::semantic_checks::types::RevisedBinding; + +/// Each named model's findings, at the session's positions, with the `poisoned` outputs untyped. +pub fn native_revalidation( + dialect: &dyn NativeDialect, + (finished, outcomes): (&FinishedSession, &[(ModelOutcome, PivotOutcome)]), + models: &[String], + poisoned: &[(String, String)], +) -> Result>, String> { + let pool: Arc = finished.pool.clone()?; + pool.install(|| { + models + .par_iter() + .map(|name| { + let outcome: &ModelOutcome = session_outcome(finished, outcomes, name)?; + revised(dialect, outcome, poisoned, &finished.declared_functions) + .map_err(|reason| format!("{name}: {reason}")) + }) + .collect() + }) +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs index 395ee1dbcf..4e49c054a7 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs @@ -171,6 +171,16 @@ impl AnalysisSession { dynamic_proofs.insert(model.name.clone(), proof.clone()); } } + let model_names: Vec = request + .models + .iter() + .map(|model| model.name.clone()) + .collect(); + let declared_functions: Vec = request + .function_return_types + .iter() + .map(|(name, _)| name.clone()) + .collect(); let finished = FinishedSession { tables: PivotTables { dialect: request.dialect, @@ -183,6 +193,8 @@ impl AnalysisSession { models: model_facts, dynamic_proofs, relations, + model_names, + declared_functions, }; let outcome = SessionOutcome { models, diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs index b3f7ff410c..537ed0f04e 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs @@ -114,6 +114,10 @@ pub struct FinishedSession { pub(crate) dynamic_proofs: HashMap, /// The relation facts the session ended with, when the request asked to keep them. pub(crate) relations: Option, + /// Each requested model's name, in request order, as the session's outcomes are. + pub(crate) model_names: Vec, + /// The adapter's declared functions, which the native analysis knew. + pub(crate) declared_functions: Vec, } /// The session's typed result for one compiled model, borrowed from the finished session. diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs index 8205ca8924..3597a8c178 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs @@ -400,6 +400,8 @@ pub(crate) fn finished_session(tables: PivotTables) -> FinishedSession { models: HashMap::new(), dynamic_proofs: HashMap::new(), relations: None, + model_names: Vec::new(), + declared_functions: Vec::new(), } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs index 096934a644..83e702475c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs @@ -215,6 +215,11 @@ impl NativeModelAnalysisSession { self.finished.as_ref() } + /// Each requested model's finished outcome and dynamic pivot proof, in request order. + pub(crate) fn outcomes(&self) -> &[(ModelOutcome, PivotOutcome)] { + &self.outcomes + } + /// The finished outcome and dynamic pivot proof of the model requested at `index`. pub(crate) fn outcome(&self, index: usize) -> PyResult<&(ModelOutcome, PivotOutcome)> { self.outcomes diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs index c75a2b2af4..7317f9b6bb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs @@ -4,6 +4,10 @@ use std::collections::HashSet; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyString, PyTuple}; +use sqlbuild_analysis::analysis_route::main::analysis_route::analysis_route; +use sqlbuild_analysis::analysis_route::main::native_revalidation::native_revalidation; +use sqlbuild_analysis::analysis_route::models::AnalysisRoute; +use sqlbuild_analysis::analysis_route::types::NativeDialect; use sqlbuild_analysis::semantic_checks::main::finish_type_recovery::finish_type_recovery; use sqlbuild_analysis::semantic_checks::main::plan_type_recovery::plan_type_recovery; use sqlbuild_analysis::semantic_checks::models::{ @@ -101,8 +105,15 @@ pub(crate) fn recovered_types<'py>( .plan() .ok_or_else(|| internal_error("finishing a type recovery that has no plan"))?; let poisoned: Vec<(String, String)> = plan.poisoned_outputs(); - let revised: Vec> = - revalidated(classes, project, &plan.revalidated_models(), &poisoned)?; + let models_to_revalidate: Vec = plan.revalidated_models(); + let revised: Vec> = match analysis_route(request.dialect.as_deref()) { + AnalysisRoute::Native(dialect) => { + natively_revalidated(dialect, project, (&models_to_revalidate, &poisoned), shared)? + } + AnalysisRoute::Legacy | AnalysisRoute::Unavailable => { + revalidated(classes, project, &models_to_revalidate, &poisoned)? + } + }; let outcome = finish_type_recovery(&request, plan, &revised).map_err(|reason| internal_error(&reason))?; let mut kept: Vec> = Vec::with_capacity(outcome.kept.len()); @@ -243,6 +254,41 @@ fn recovery_model( }) } +/// Each planned model revalidated by the dialect's native analysis, never the Polyglot catalog. +fn natively_revalidated( + dialect: &'static dyn NativeDialect, + project: &Bound<'_, PyAny>, + (models, poisoned): (&[usize], &[(String, String)]), + shared: &StageShared<'_>, +) -> PyResult>> { + if models.is_empty() { + return Ok(Vec::new()); + } + let py: Python<'_> = project.py(); + let session: &NativeModelAnalysisSession = shared + .session + .ok_or_else(|| internal_error("revalidating a native route without its session"))?; + let finished = session + .finished_session() + .ok_or_else(|| internal_error("revalidating before the session finished"))?; + let project_models: Bound<'_, PyAny> = native_field(project, "models")?; + let mut names: Vec = Vec::with_capacity(models.len()); + for index in models { + names.push(native_field(&project_models.get_item(*index)?, "name")?.extract()?); + } + count_stage(py, SCHEMA_REVALIDATIONS, names.len() as u64)?; + py.compiler_detach(|| { + Ok(native_revalidation( + dialect, + (finished, session.outcomes()), + &names, + poisoned, + )) + }) + .map_err(compiler_error)? + .map_err(|reason| internal_error(&reason)) +} + /// Python's revalidation of each planned model with unknown types for poisoned outputs. fn revalidated<'py>( classes: &StageClasses<'py>, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py b/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py index ded903710f..f55a9bd700 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py @@ -390,3 +390,13 @@ class StarShapeOrderTestCase: type PreparedCompile = tuple[ CompileOutcome, tuple[str, ...], dict[str, object | None], tuple[bytes, ...] ] + + +@dataclass(frozen=True) +class AdapterRecoveryCase: + description: str + adapter: str + expected_findings: tuple[tuple[str, str], ...] = ( + ("B002", "downstream"), + ("B212", "upstream"), + ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py index 6bc4077592..ad9eeb940c 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py @@ -18,6 +18,7 @@ from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( + AdapterRecoveryCase, FunctionArgumentTypeCase, SemanticTriageCase, ) @@ -239,6 +240,47 @@ def test_given_cached_validation_when_warm_or_schema_changed_then_reuses_only_ma assert tuple(item["code"] for item in result["diagnostics"]) == test_case.expected_codes +@pytest.mark.parametrize( + "test_case", + [ + AdapterRecoveryCase("bigquery", adapter="bigquery"), + AdapterRecoveryCase("duckdb", adapter="duckdb"), + AdapterRecoveryCase("postgres", adapter="postgres"), + AdapterRecoveryCase("snowflake", adapter="snowflake"), + ], + ids=lambda case: case.description, +) +def test_given_a_type_error_upstream_of_a_model_with_its_own_error_when_compiling_then_both_are_reported( + test_case: AdapterRecoveryCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + f'name = "orders"\nadapter = "{test_case.adapter}"\n[rules]\nselect = []\n' + ) + (tmp_path / "models").mkdir() + header: str = ( + "MODEL (description 'Test model.', materialized view, database warehouse, schema analytics" + ) + files: dict[str, str] = { + "orders": f"{header});\n" + "SELECT CAST(1 AS INTEGER) AS quantity, CAST('2026-04-01' AS TIMESTAMP) AS ordered_at", + "upstream": f'{header});\nSELECT quantity, ordered_at, quantity + ordered_at AS bad FROM __ref("orders")', + "downstream": f'{header});\nSELECT bad > ordered_at AS compared, missing FROM __ref("upstream")', + } + for name, sql in files.items(): + (tmp_path / "models" / f"{name}.sql").write_text(sql) + assert main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) == 1 + result: dict[str, Any] = json.loads(capsys.readouterr().out) + found: dict[tuple[str, str], list[str]] = { + (item["code"], item["resource_name"]): item.get("notes", []) + for item in result["diagnostics"] + } + assert tuple(sorted(found)) == test_case.expected_findings, result["diagnostics"] + assert ( + "1 downstream output uses were not type-checked because of this error" + in found[("B212", "upstream")] + ) + + @pytest.mark.parametrize( "test_case", [SemanticTriageCase("distinct spans with identical messages", expected_codes=("B212", "B212"))], From db4420d26d66d8ef38f525f1fd8673363be4146a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:36:53 +0000 Subject: [PATCH 20/29] fix(analysis): keep marker lineage and SQLBuild's text checks on native routes Fast lineage on a native route passed relation markers to the parser as written. Markers now read as indexed names through the shared relation marker reader, each tied to the resource it calls, so a model and a source of the same name stay apart. A source column's relation is compared as the dialect's profile resolves names, so upper-case folding dialects such as Snowflake keep their sources too, in fast lineage and in session lineage alike, and repeated sources of one column are listed once. Native analysis now adds SQLBuild's own text checks that the Polyglot route runs: truth-value tests the warehouse rejects (B103) and SQLite's temporal casts (B104). Output column facts keep the analysis's own type spelling, such as INTEGER or VARCHAR, and use the dialect's normalised name only to decide whether the type is known. The relation marker reader is the same as on the parser train branch. --- .../_helpers/native_analysis.rs | 41 +++++++---- .../src/analysis_route/main/fast_lineage.rs | 71 +++++++++++++----- .../tests/test_native_dialect.rs | 11 ++- .../tests/test_native_outcome.rs | 70 +++++++++++++----- .../src/analysis_route/tests/test_types.rs | 43 ++++++++++- crates/sqlbuild-analysis/src/lib.rs | 1 + .../src/lineage/main/build_fast_lineage.rs | 19 ----- .../src/lineage/tests/helpers.rs | 2 +- .../lineage/tests/test_build_fast_lineage.rs | 30 ++++---- .../src/relation_markers/_helpers/mod.rs | 1 + .../src/relation_markers/_helpers/pattern.rs | 15 ++++ .../src/relation_markers/constants.rs | 9 +++ .../relation_markers/main/markers_as_names.rs | 60 ++++++++++++++++ .../src/relation_markers/main/mod.rs | 1 + .../src/relation_markers/mod.rs | 9 +++ .../src/relation_markers/tests.rs | 4 ++ .../tests/test_markers_as_names.rs | 72 +++++++++++++++++++ .../src/relation_markers/tests/test_types.rs | 10 +++ .../src/relation_markers/types.rs | 4 ++ .../semantic_validation/_helpers/catalog.rs | 2 +- .../src/semantic_validation/main.rs | 1 + .../semantic_validation/main/text_findings.rs | 19 +++++ 22 files changed, 412 insertions(+), 83 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/relation_markers/_helpers/mod.rs create mode 100644 crates/sqlbuild-analysis/src/relation_markers/_helpers/pattern.rs create mode 100644 crates/sqlbuild-analysis/src/relation_markers/constants.rs create mode 100644 crates/sqlbuild-analysis/src/relation_markers/main/markers_as_names.rs create mode 100644 crates/sqlbuild-analysis/src/relation_markers/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/relation_markers/mod.rs create mode 100644 crates/sqlbuild-analysis/src/relation_markers/tests.rs create mode 100644 crates/sqlbuild-analysis/src/relation_markers/tests/test_markers_as_names.rs create mode 100644 crates/sqlbuild-analysis/src/relation_markers/tests/test_types.rs create mode 100644 crates/sqlbuild-analysis/src/relation_markers/types.rs create mode 100644 crates/sqlbuild-analysis/src/semantic_validation/main/text_findings.rs diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/native_analysis.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/native_analysis.rs index ffe1bca27b..3f6577207f 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/_helpers/native_analysis.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/native_analysis.rs @@ -1,5 +1,6 @@ //! The provided native analysis: sqlquack-semantics lineage and findings as SQLBuild's facts. +use sqlquack_semantics::dialects::types::DialectProfile; use sqlquack_semantics::lineage::models::{ LineageContext, NameFinding, Nullability, OutputLineage, QueryLineage, SourceColumn, TransformKind, @@ -17,6 +18,7 @@ use crate::assembly::analysis_session::constants::{ }; use crate::assembly::analysis_session::models::{ColumnFact, LineageRow}; use crate::semantic_validation::main::sqlbuild_code::sqlbuild_code; +use crate::semantic_validation::main::text_findings::text_findings; use crate::semantic_validation::models::DiagnosticRow; /// The analysis of `statement`, the one statement of `input.sql`, under `dialect`'s profile. @@ -40,7 +42,7 @@ pub(crate) fn native_analysis( lineage: lineage .outputs .iter() - .map(|output| lineage_row(output, input.references)) + .map(|output| lineage_row(dialect.profile(), output, input.references)) .collect(), row_sources: lineage .row_sources @@ -52,19 +54,20 @@ pub(crate) fn native_analysis( diagnostics: statement_name_findings(statement, context) .into_iter() .filter_map(|finding| diagnostic_row(input, finding)) + .chain(text_findings(input.sql, dialect.name())) .collect(), } } -/// An output column typed through the dialect, so both routes spell its type alike. +/// An output column in the analysis's own type spelling, kept when the dialect can compare it. fn column_fact(dialect: &D, output: &OutputLineage) -> ColumnFact { ColumnFact { name: output.name.clone(), data_type: output .data_type .as_deref() - .and_then(|spelling| dialect.normalize_type(spelling)) - .map(|normalized| normalized.normalized_name), + .filter(|spelling| dialect.normalize_type(spelling).is_some()) + .map(str::to_owned), nullability: match output.nullability { Nullability::Unknown => UNKNOWN_NULLABILITY, Nullability::Nullable => NULLABLE_NULLABILITY, @@ -75,7 +78,11 @@ fn column_fact(dialect: &D, output: &OutputLineage) - } /// An output column's lineage over the model's referenced resources, as the legacy rows give it. -fn lineage_row(output: &OutputLineage, references: &[(String, String, String)]) -> LineageRow { +fn lineage_row( + profile: &dyn DialectProfile, + output: &OutputLineage, + references: &[(String, String, String)], +) -> LineageRow { let transform_code: u8 = match output.kind { TransformKind::Constant => TRANSFORM_CONSTANT, TransformKind::Direct => TRANSFORM_DIRECT, @@ -83,11 +90,16 @@ fn lineage_row(output: &OutputLineage, references: &[(String, String, String)]) TransformKind::Expression => TRANSFORM_EXPRESSION, TransformKind::Aggregation => TRANSFORM_AGGREGATION, }; - let sources: Vec<(String, String, String)> = output + let mut sources: Vec<(String, String, String)> = Vec::new(); + for source in output .sources .iter() - .filter_map(|source| referenced_source(source, references)) - .collect(); + .filter_map(|source| referenced_source(profile, source, references)) + { + if !sources.contains(&source) { + sources.push(source); + } + } let confidence_code: u8 = if sources.is_empty() && transform_code != TRANSFORM_CONSTANT { CONFIDENCE_UNKNOWN } else if output.resolved { @@ -103,14 +115,19 @@ fn lineage_row(output: &OutputLineage, references: &[(String, String, String)]) } } -/// `(resource type, resource name, column)` of a source column the model references. +/// `(resource type, resource name, column)` of a source column, each name resolved as the profile does. fn referenced_source( + profile: &dyn DialectProfile, source: &SourceColumn, references: &[(String, String, String)], ) -> Option<(String, String, String)> { - let (_, resource_type, resource_name) = references - .iter() - .find(|(analysis_name, _, _)| *analysis_name == source.relation)?; + let (_, resource_type, resource_name) = references.iter().find(|(analysis_name, _, _)| { + *analysis_name == source.relation + || profile.same_identifier( + &profile.resolved_identifier(analysis_name, false), + &source.relation, + ) + })?; Some(( resource_type.clone(), resource_name.clone(), diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/fast_lineage.rs b/crates/sqlbuild-analysis/src/analysis_route/main/fast_lineage.rs index ee96e88baa..64eb6e6a61 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/fast_lineage.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/fast_lineage.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; +use sqlbuild_sqltext::sql_scan::main::dialect_lexical_syntax::dialect_lexical_syntax; use sqlquack_semantics::lineage::models::{Nullability, SchemaColumn}; use crate::analysis_route::_helpers::located_gap::{located, unreadable_reason}; @@ -17,26 +18,56 @@ use crate::lineage::models::{ FastLineageOutcome, LineageColumn, LineageConfidence, LineageResourceType, LineageSchemaResource, LineageSource, LineageTransformKind, }; +use crate::relation_markers::constants::MARKER_NAME_PREFIX; +use crate::relation_markers::main::markers_as_names::markers_as_names; +use crate::relation_markers::types::MarkerCall; use crate::semantic_validation::main::normalize_native::normalize_native_sql; -/// Lineage over `schema` and the `(name, type, resource)` references, or why it is unreadable. +/// Lineage over `schema` of `query_sql`, its relation markers read as names, or why it is unreadable. pub fn fast_lineage( dialect: &dyn NativeDialect, (query_sql, placeholders): (&str, &[(String, String)]), - references: &[(String, String, String)], schema: &[LineageSchemaResource], ) -> FastLineageOutcome { + let (named, calls): (String, Vec) = + match markers_as_names(query_sql, &dialect_lexical_syntax(dialect.name())) { + Ok(named) => named, + Err(reason) => return FastLineageOutcome::Unparsed(reason), + }; let normalised: String = - match normalize_native_sql(query_sql, dialect.name(), dialect.stub_prefix()) { + match normalize_native_sql(&named, dialect.name(), dialect.stub_prefix()) { Ok(normalised) => normalised, Err(reason) => return FastLineageOutcome::Unparsed(reason), }; let sql: String = placeholder_defaults(&normalised, placeholders); - let relations: SessionSchema = names_schema(schema); + let referenced: Vec<(String, LineageResourceType, String)> = calls + .into_iter() + .enumerate() + .filter_map(|(index, (function, name))| { + let resource_type: LineageResourceType = match function.as_str() { + "__ref" => LineageResourceType::Model, + "__source" => LineageResourceType::Source, + "__seed" => LineageResourceType::Seed, + _ => return None, + }; + Some((format!("{MARKER_NAME_PREFIX}{index}"), resource_type, name)) + }) + .collect(); + let references: Vec<(String, String, String)> = referenced + .iter() + .map(|(marker, resource_type, name)| { + ( + marker.clone(), + resource_type.as_str().to_owned(), + name.clone(), + ) + }) + .collect(); + let relations: SessionSchema = markers_schema(&referenced, schema); let analysed: Result = dialect.analyse(&AnalysisInput { sql: &sql, schema: &relations, - references, + references: &references, extension_functions: &[], }); match analysed { @@ -50,25 +81,33 @@ pub fn fast_lineage( } } -/// Each resource with known columns, untyped and of unknown nullability; the first of a name wins. -fn names_schema(schema: &[LineageSchemaResource]) -> SessionSchema { +/// Each marker name's resource columns, untyped and of unknown nullability, when the schema knows them. +fn markers_schema( + referenced: &[(String, LineageResourceType, String)], + schema: &[LineageSchemaResource], +) -> SessionSchema { let mut relations: HashMap> = HashMap::new(); - for resource in schema - .iter() - .filter(|resource| !resource.columns.is_empty()) - { - let _ = relations.entry(resource.name.clone()).or_insert_with(|| { + for (marker, resource_type, name) in referenced { + let Some(resource) = schema.iter().find(|resource| { + resource.resource_type == *resource_type + && resource.name == *name + && !resource.columns.is_empty() + }) else { + continue; + }; + let _ = relations.insert( + marker.clone(), resource .columns .iter() - .map(|name| SchemaColumn { - name: name.clone(), + .map(|column| SchemaColumn { + name: column.clone(), quoted: false, data_type: None, nullability: Nullability::Unknown, }) - .collect() - }); + .collect(), + ); } SessionSchema { relations } } diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs index 9f8c470052..20eabe24a1 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs @@ -2,8 +2,8 @@ use crate::analysis_route::main::unavailable_analysis::unavailable_analysis; use crate::analysis_route::models::{NativeAnalysis, TypeMatch}; use crate::analysis_route::tests::helpers::{analysed, resource_columns}; use crate::analysis_route::tests::test_types::{ - CataloguedDialect, ColumnTestCase, FindingTestCase, LineageTestCase, MacroBodyTestCase, - NativeTypeMatchTestCase, PivotlessDialect, ReadWholeTestCase, StarTestCase, + CataloguedDialect, ColumnTestCase, FindingTestCase, FoldingDialect, LineageTestCase, + MacroBodyTestCase, NativeTypeMatchTestCase, PivotlessDialect, ReadWholeTestCase, StarTestCase, UnavailableAnalysisTestCase, }; use crate::analysis_route::types::NativeDialect; @@ -161,6 +161,13 @@ fn given_findings_when_analysing_then_they_take_sqlbuild_codes_and_positions() { extension_functions: &[], expected_findings: &[("B002", Some((2, 3)))], }, + FindingTestCase { + description: "a truth-value test the warehouse rejects is SQLBuild's own B103, after the parse's findings", + dialect: &FoldingDialect, + sql: "SELECT id, nope FROM orders WHERE (id > 1) IS NOT TRUE", + extension_functions: &[], + expected_findings: &[("B002", Some((1, 12))), ("B103", Some((1, 44)))], + }, FindingTestCase { description: "a function the catalogue does not know is B101", dialect: &CataloguedDialect, diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs index a1d77c535b..20eaba37c2 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs @@ -2,7 +2,8 @@ use crate::analysis_route::main::fast_lineage::fast_lineage; use crate::analysis_route::main::model_outcome::model_outcome; use crate::analysis_route::tests::helpers::{column_line, session_relations}; use crate::analysis_route::tests::test_types::{ - CataloguedDialect, NativeLineageTestCase, NativeOutcomeTestCase, PivotlessDialect, + CataloguedDialect, FoldingDialect, MarkerLineageTestCase, NativeLineageTestCase, + NativeOutcomeTestCase, PivotlessDialect, }; use crate::assembly::analysis_session::models::{ModelOutcome, ModelRequest, SessionSchema}; use crate::assembly::analysis_session::types::Shapes; @@ -127,14 +128,6 @@ fn given_a_model_without_facts_when_fast_lineage_reads_it_natively_then_it_has_l columns: vec!["payment_id".to_owned()], }, ]; - let references: Vec<(String, String, String)> = [ - ("orders", "model"), - ("unknown_model", "model"), - ("raw_payments", "source"), - ] - .iter() - .map(|(name, kind)| ((*name).to_owned(), (*kind).to_owned(), (*name).to_owned())) - .collect(); let test_cases = [ NativeLineageTestCase { description: "a referenced model's columns are the upstreams of direct and computed outputs", @@ -178,13 +171,58 @@ fn given_a_model_without_facts_when_fast_lineage_reads_it_natively_then_it_has_l .iter() .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) .collect(); - let parts: LineageOutcomeParts = fast_lineage( - &PivotlessDialect, - (test_case.sql, &placeholders), - &references, - &schema, - ) - .into_parts(); + let parts: LineageOutcomeParts = + fast_lineage(&PivotlessDialect, (test_case.sql, &placeholders), &schema).into_parts(); + let mut lines: Vec = parts.columns.iter().map(column_line).collect(); + lines.push(format!("{} star={}", parts.status, parts.has_star)); + lines.extend(parts.detail); + assert_eq!(lines, test_case.expected_lines, "{}", test_case.description); + } +} + +#[test] +fn given_relation_markers_when_each_dialect_reads_fast_lineage_then_sources_keep_their_resource() { + let schema: Vec = vec![ + LineageSchemaResource { + resource_type: LineageResourceType::Source, + name: "orders".to_owned(), + columns: vec!["order_id".to_owned(), "loaded_at".to_owned()], + }, + LineageSchemaResource { + resource_type: LineageResourceType::Model, + name: "orders".to_owned(), + columns: vec!["order_id".to_owned(), "amount".to_owned()], + }, + ]; + let sql: &str = r#"SELECT o.order_id, o.amount, s.loaded_at +FROM __ref("orders") AS o JOIN __source("orders") AS s ON o.order_id = s.order_id"#; + let expected_lines: &[&str] = &[ + "order_id direct high [model:orders.order_id]", + "amount direct high [model:orders.amount]", + "loaded_at direct high [source:orders.loaded_at]", + "built star=false", + ]; + let test_cases = [ + MarkerLineageTestCase { + description: "a lower-case folding dialect matches each marker's name", + dialect: &PivotlessDialect, + expected_lines, + }, + MarkerLineageTestCase { + description: "a case-preserving dialect matches each marker's name", + dialect: &CataloguedDialect, + expected_lines, + }, + MarkerLineageTestCase { + description: "an upper-case folding dialect matches each marker's name", + dialect: &FoldingDialect, + expected_lines, + }, + ]; + + for test_case in test_cases { + let parts: LineageOutcomeParts = + fast_lineage(test_case.dialect, (sql, &[]), &schema).into_parts(); let mut lines: Vec = parts.columns.iter().map(column_line).collect(); lines.push(format!("{} star={}", parts.status, parts.has_star)); lines.extend(parts.detail); diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs index d814d9e285..a9b789e105 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs @@ -1,4 +1,4 @@ -use sqlquack_semantics::dialects::constants::{DUCKDB, POSTGRES}; +use sqlquack_semantics::dialects::constants::{DUCKDB, POSTGRES, SNOWFLAKE}; use sqlquack_semantics::dialects::types::DialectProfile; use sqlquack_semantics::lineage::models::Nullability; use sqlquack_syntax::expressions::statements::models::Command; @@ -116,6 +116,47 @@ impl NativeDialect for CataloguedDialect { } } +/// Snowflake's shared parser under the Snowflake profile, which folds unquoted names to upper case. +pub(super) struct FoldingDialect; + +impl NativeDialect for FoldingDialect { + fn name(&self) -> &'static str { + "snowflake" + } + + fn stub_prefix(&self) -> &'static str { + "__sqlbuild_" + } + + fn parse(&self, sql: &str) -> Result { + parse_strict(sql, Dialect::Snowflake, &ParseOptions::default()) + .map(|statements| Parsed { + statements, + gaps: Vec::new(), + }) + .map_err(|failure| ParserGap { + construct: failure.to_string(), + message: failure.to_string(), + offset: 0, + syntax: false, + }) + } + + fn profile(&self) -> &'static dyn DialectProfile { + &SNOWFLAKE + } + + fn normalize_type(&self, spelling: &str) -> Option { + PivotlessDialect.normalize_type(spelling) + } +} + +pub(super) struct MarkerLineageTestCase { + pub(super) description: &'static str, + pub(super) dialect: &'static dyn NativeDialect, + pub(super) expected_lines: &'static [&'static str], +} + pub(super) struct ReadWholeTestCase { pub(super) description: &'static str, pub(super) sql: &'static str, diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 5ad16bf8b6..e8d6a2920b 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -37,6 +37,7 @@ pub mod python_plan; pub mod python_selection; pub mod python_values; pub mod query_analysis; +pub mod relation_markers; pub mod relation_names; pub mod required_columns; pub mod resource_diagnostics; diff --git a/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs b/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs index 96e1790c96..64418fc609 100644 --- a/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs +++ b/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs @@ -83,7 +83,6 @@ fn model_outcomes(request: &FastLineageRequest) -> Vec Vec Vec<(String, String, String)> { - let mut references: Vec<(String, String, String)> = Vec::new(); - for resource in physical_resources(query_sql) { - if !references - .iter() - .any(|(name, _, _)| *name == resource.resource_name) - { - references.push(( - resource.resource_name.clone(), - resource.resource_type.as_str().to_owned(), - resource.resource_name, - )); - } - } - references -} diff --git a/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs b/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs index 0553e0e346..5a0bbaff02 100644 --- a/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs @@ -36,7 +36,7 @@ pub(crate) fn deep_union_outcome_lines( stack_bytes: usize, ) -> Vec<(&'static str, Vec, bool, Option)> { let sql: String = - vec!["SELECT order_id, amount FROM __ref('orders')"; branches].join(" UNION ALL "); + vec!["SELECT order_id, amount FROM __ref(\"orders\")"; branches].join(" UNION ALL "); let worker = std::thread::Builder::new() .stack_size(stack_bytes) .spawn(move || { diff --git a/crates/sqlbuild-analysis/src/lineage/tests/test_build_fast_lineage.rs b/crates/sqlbuild-analysis/src/lineage/tests/test_build_fast_lineage.rs index 9e9f1a77de..6d225a9b5a 100644 --- a/crates/sqlbuild-analysis/src/lineage/tests/test_build_fast_lineage.rs +++ b/crates/sqlbuild-analysis/src/lineage/tests/test_build_fast_lineage.rs @@ -11,7 +11,7 @@ fn given_models_without_compact_facts_when_building_then_matches_python_lineage( description: "joins, aggregation, cast, constant and expression columns", sql: "SELECT o.order_id, c.name AS customer_name, SUM(o.amount) AS total, \ CAST(o.amount AS BIGINT) AS amount_big, 1 AS one, o.amount + c.customer_id AS mix \ - FROM __ref('orders') o JOIN __ref('customers') c ON o.customer_id = c.customer_id \ + FROM __ref(\"orders\") o JOIN __ref(\"customers\") c ON o.customer_id = c.customer_id \ GROUP BY 1, 2", dialect: None, inferred_columns: &[], @@ -29,7 +29,7 @@ fn given_models_without_compact_facts_when_building_then_matches_python_lineage( }, ParsedLineageTestCase { description: "unqualified columns of one resource and an unknown qualifier", - sql: "SELECT order_id, amount * 2 AS doubled, missing.thing FROM __ref('orders')", + sql: "SELECT order_id, amount * 2 AS doubled, missing.thing FROM __ref(\"orders\")", dialect: None, inferred_columns: &[], expected_status: "built", @@ -43,7 +43,7 @@ fn given_models_without_compact_facts_when_building_then_matches_python_lineage( }, ParsedLineageTestCase { description: "a root star expands known columns after explicit outputs", - sql: "SELECT *, amount AS amt FROM __ref('orders')", + sql: "SELECT *, amount AS amt FROM __ref(\"orders\")", dialect: None, inferred_columns: &[], expected_status: "built", @@ -58,7 +58,7 @@ fn given_models_without_compact_facts_when_building_then_matches_python_lineage( }, ParsedLineageTestCase { description: "a qualified star expands every referenced resource", - sql: "SELECT o.* FROM __ref('orders') o JOIN __seed('regions') r ON TRUE", + sql: "SELECT o.* FROM __ref(\"orders\") o JOIN __seed(\"regions\") r ON TRUE", dialect: None, inferred_columns: &[], expected_status: "built", @@ -74,8 +74,8 @@ fn given_models_without_compact_facts_when_building_then_matches_python_lineage( }, ParsedLineageTestCase { description: "a three-branch union merges branch sources", - sql: "SELECT order_id FROM __ref('orders') UNION ALL SELECT customer_id \ - FROM __ref('customers') UNION ALL SELECT region_id + 1 FROM __seed('regions')", + sql: "SELECT order_id FROM __ref(\"orders\") UNION ALL SELECT customer_id \ + FROM __ref(\"customers\") UNION ALL SELECT region_id + 1 FROM __seed(\"regions\")", dialect: None, inferred_columns: &[], expected_status: "built", @@ -88,7 +88,7 @@ fn given_models_without_compact_facts_when_building_then_matches_python_lineage( }, ParsedLineageTestCase { description: "a union with a star branch takes inferred names", - sql: "SELECT order_id, amount FROM __ref('orders') UNION SELECT * FROM __ref('customers')", + sql: "SELECT order_id, amount FROM __ref(\"orders\") UNION SELECT * FROM __ref(\"customers\")", dialect: None, inferred_columns: &["first", "second"], expected_status: "built", @@ -105,7 +105,7 @@ fn given_models_without_compact_facts_when_building_then_matches_python_lineage( }, ParsedLineageTestCase { description: "a CTE collapses onto its referenced model", - sql: "WITH base AS (SELECT order_id, amount FROM __ref('orders')) \ + sql: "WITH base AS (SELECT order_id, amount FROM __ref(\"orders\")) \ SELECT order_id, amount FROM base", dialect: None, inferred_columns: &[], @@ -119,7 +119,7 @@ fn given_models_without_compact_facts_when_building_then_matches_python_lineage( }, ParsedLineageTestCase { description: "unnamed projections take inferred names", - sql: "SELECT 1 + 1, COUNT(*) FROM __ref('orders')", + sql: "SELECT 1 + 1, COUNT(*) FROM __ref(\"orders\")", dialect: None, inferred_columns: &["a", "b"], expected_status: "built", @@ -129,7 +129,7 @@ fn given_models_without_compact_facts_when_building_then_matches_python_lineage( }, ParsedLineageTestCase { description: "quoted and case-varied identifiers keep their spelling", - sql: "SELECT \"Order_ID\", O.Amount FROM __ref('orders') AS O", + sql: "SELECT \"Order_ID\", O.Amount FROM __ref(\"orders\") AS O", dialect: None, inferred_columns: &[], expected_status: "built", @@ -148,7 +148,7 @@ fn given_models_without_compact_facts_when_building_then_matches_python_lineage( order_id IN (1, customer_id) AS listed, \ ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY amount) AS rn, \ EXTRACT(YEAR FROM amount) AS yr, order_id || '-' || amount AS joined \ - FROM __ref('orders')", + FROM __ref(\"orders\")", dialect: None, inferred_columns: &[], expected_status: "built", @@ -168,7 +168,7 @@ fn given_models_without_compact_facts_when_building_then_matches_python_lineage( ParsedLineageTestCase { description: "a top-level EXCEPT has no fast lineage", dialect: None, - sql: "SELECT order_id FROM __ref('orders') EXCEPT SELECT customer_id FROM __ref('customers')", + sql: "SELECT order_id FROM __ref(\"orders\") EXCEPT SELECT customer_id FROM __ref(\"customers\")", inferred_columns: &[], expected_status: "omitted", expected_lines: &[], @@ -256,8 +256,8 @@ fn given_compact_facts_with_unresolved_star_when_building_then_appends_unseen_co let test_cases = [ StarExpansionTestCase { description: "existing outputs and repeated references are skipped", - sql: "SELECT *, 1 AS amount FROM __ref('orders') JOIN __ref('orders') USING (order_id) \ - JOIN __seed('regions') USING (region_id)", + sql: "SELECT *, 1 AS amount FROM __ref(\"orders\") JOIN __ref(\"orders\") USING (order_id) \ + JOIN __seed(\"regions\") USING (region_id)", existing_columns: &["amount"], expected_lines: &[ "order_id star medium [model:orders.order_id]", @@ -268,7 +268,7 @@ fn given_compact_facts_with_unresolved_star_when_building_then_appends_unseen_co }, StarExpansionTestCase { description: "resources without known columns add nothing", - sql: "SELECT * FROM __ref('empty') JOIN __source('raw') ON TRUE", + sql: "SELECT * FROM __ref(\"empty\") JOIN __source(\"raw\") ON TRUE", existing_columns: &[], expected_lines: &[], }, diff --git a/crates/sqlbuild-analysis/src/relation_markers/_helpers/mod.rs b/crates/sqlbuild-analysis/src/relation_markers/_helpers/mod.rs new file mode 100644 index 0000000000..b13857eb1c --- /dev/null +++ b/crates/sqlbuild-analysis/src/relation_markers/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod pattern; diff --git a/crates/sqlbuild-analysis/src/relation_markers/_helpers/pattern.rs b/crates/sqlbuild-analysis/src/relation_markers/_helpers/pattern.rs new file mode 100644 index 0000000000..4fa72a78b6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/relation_markers/_helpers/pattern.rs @@ -0,0 +1,15 @@ +//! The compiled relation marker call pattern. + +use std::sync::LazyLock; + +use regex::Regex; + +use crate::relation_markers::constants::RELATION_MARKER_PATTERN; + +static RELATION_MARKER: LazyLock> = + LazyLock::new(|| Regex::new(RELATION_MARKER_PATTERN).map_err(|error| error.to_string())); + +/// The compiled relation marker call pattern, or its compile error. +pub(crate) fn relation_marker_pattern() -> Result<&'static Regex, String> { + RELATION_MARKER.as_ref().map_err(Clone::clone) +} diff --git a/crates/sqlbuild-analysis/src/relation_markers/constants.rs b/crates/sqlbuild-analysis/src/relation_markers/constants.rs new file mode 100644 index 0000000000..4fef92b98e --- /dev/null +++ b/crates/sqlbuild-analysis/src/relation_markers/constants.rs @@ -0,0 +1,9 @@ +//! The relation marker call pattern and the names marker calls stand as. + +/// The prefix of the name a marker call stands as while parsed; the call's index follows it. +pub const MARKER_NAME_PREFIX: &str = "__sqb_"; +/// Any relation marker call: `__name("x")`, or `__name("a", "b")` as `__dbt_ref` writes it. +pub(crate) const RELATION_MARKER_PATTERN: &str = + r#"(?i)\b(__[a-z_]+)\(\s*"([^"]+)"\s*(?:,\s*"([^"]+)"\s*)?\)"#; +/// The marker whose two names read as one, joined by `__`. +pub(crate) const DBT_REF_FUNCTION: &str = "__dbt_ref"; diff --git a/crates/sqlbuild-analysis/src/relation_markers/main/markers_as_names.rs b/crates/sqlbuild-analysis/src/relation_markers/main/markers_as_names.rs new file mode 100644 index 0000000000..9330c94939 --- /dev/null +++ b/crates/sqlbuild-analysis/src/relation_markers/main/markers_as_names.rs @@ -0,0 +1,60 @@ +//! `markers_as_names`: relation marker calls in code as plain names that keep every position. + +use regex::Regex; +use sqlbuild_sqltext::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::relation_markers::_helpers::pattern::relation_marker_pattern; +use crate::relation_markers::constants::{DBT_REF_FUNCTION, MARKER_NAME_PREFIX}; +use crate::relation_markers::types::MarkerCall; + +/// `sql` with each marker call in code as `__sqb_`, padded so later text keeps its line and column, and the calls by index. +pub fn markers_as_names( + sql: &str, + syntax: &LexicalSyntax, +) -> Result<(String, Vec), String> { + let pattern: &Regex = relation_marker_pattern()?; + let mut protected: Option> = None; + let mut named: String = String::with_capacity(sql.len()); + let mut calls: Vec = Vec::new(); + let mut cursor: usize = 0; + for captures in pattern.captures_iter(sql) { + let (Some(full), Some(function), Some(first)) = + (captures.get(0), captures.get(1), captures.get(2)) + else { + continue; + }; + let ranges = protected.get_or_insert_with(|| dialect_non_code_ranges(sql, syntax)); + let following: usize = ranges.partition_point(|(start, _)| *start <= full.start()); + if following > 0 && full.start() < ranges[following - 1].1 { + continue; + } + let function_name: String = function.as_str().to_ascii_lowercase(); + let referenced_name: String = match (captures.get(3), function_name == DBT_REF_FUNCTION) { + (None, _) => first.as_str().to_owned(), + (Some(second), true) => format!("{}__{}", first.as_str(), second.as_str()), + (Some(_), false) => continue, + }; + named.push_str(&sql[cursor..full.start()]); + named.push_str(&padded_name( + &format!("{MARKER_NAME_PREFIX}{}", calls.len()), + full.as_str(), + )); + cursor = full.end(); + calls.push((function_name, referenced_name)); + } + named.push_str(&sql[cursor..]); + Ok((named, calls)) +} + +/// `name`, then the call's line breaks, then spaces as wide as the call's last line. +fn padded_name(name: &str, call: &str) -> String { + let breaks: usize = call.matches('\n').count(); + let last_line: &str = call.rsplit('\n').next().unwrap_or(call); + let width: usize = if breaks == 0 { + call.chars().count().saturating_sub(name.chars().count()) + } else { + last_line.chars().count() + }; + format!("{name}{}{}", "\n".repeat(breaks), " ".repeat(width)) +} diff --git a/crates/sqlbuild-analysis/src/relation_markers/main/mod.rs b/crates/sqlbuild-analysis/src/relation_markers/main/mod.rs new file mode 100644 index 0000000000..32d7644fcb --- /dev/null +++ b/crates/sqlbuild-analysis/src/relation_markers/main/mod.rs @@ -0,0 +1 @@ +pub mod markers_as_names; diff --git a/crates/sqlbuild-analysis/src/relation_markers/mod.rs b/crates/sqlbuild-analysis/src/relation_markers/mod.rs new file mode 100644 index 0000000000..af27b6661d --- /dev/null +++ b/crates/sqlbuild-analysis/src/relation_markers/mod.rs @@ -0,0 +1,9 @@ +//! Relation marker calls, such as `__ref("orders")`, read as plain relation names by strict parsers. + +mod _helpers; +pub mod constants; +pub mod main; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/relation_markers/tests.rs b/crates/sqlbuild-analysis/src/relation_markers/tests.rs new file mode 100644 index 0000000000..c7c4334980 --- /dev/null +++ b/crates/sqlbuild-analysis/src/relation_markers/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_markers_as_names.rs"] +mod test_markers_as_names; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/relation_markers/tests/test_markers_as_names.rs b/crates/sqlbuild-analysis/src/relation_markers/tests/test_markers_as_names.rs new file mode 100644 index 0000000000..6df1acbc7a --- /dev/null +++ b/crates/sqlbuild-analysis/src/relation_markers/tests/test_markers_as_names.rs @@ -0,0 +1,72 @@ +use sqlbuild_sqltext::sql_scan::main::dialect_lexical_syntax::dialect_lexical_syntax; + +use crate::relation_markers::main::markers_as_names::markers_as_names; +use crate::relation_markers::tests::test_types::MarkersAsNamesTestCase; + +#[test] +fn given_marker_calls_when_reading_them_as_names_then_code_calls_become_names_and_positions_hold() { + let test_cases = [ + MarkersAsNamesTestCase { + description: "each call becomes its indexed name, padded to the call's width", + dialect: "snowflake", + sql: r#"SELECT * FROM __ref("orders") o JOIN __source("raw") r ON TRUE"#, + expected_sql: format!( + "SELECT * FROM __sqb_0{} o JOIN __sqb_1{} r ON TRUE", + " ".repeat(8), + " ".repeat(8) + ), + expected_calls: &[("__ref", "orders"), ("__source", "raw")], + }, + MarkersAsNamesTestCase { + description: "calls in quoted text and comments stay; an upper-case call folds", + dialect: "snowflake", + sql: "SELECT '__ref(\"a\")' AS q -- __ref(\"b\")\nFROM __REF(\"Orders\") // __ref(\"c\")", + expected_sql: format!( + "SELECT '__ref(\"a\")' AS q -- __ref(\"b\")\nFROM __sqb_0{} // __ref(\"c\")", + " ".repeat(8) + ), + expected_calls: &[("__ref", "Orders")], + }, + MarkersAsNamesTestCase { + description: "a call over three lines keeps its line breaks and its last line's width", + dialect: "duckdb", + sql: "FROM __ref(\n \"orders\"\n) AS o", + expected_sql: "FROM __sqb_0\n\n AS o".to_owned(), + expected_calls: &[("__ref", "orders")], + }, + MarkersAsNamesTestCase { + description: "two dbt names join; a non-ASCII name pads by characters, not bytes", + dialect: "postgres", + sql: r#"FROM __dbt_ref("shop", "customers") d JOIN __ref("café") c ON TRUE"#, + expected_sql: format!( + "FROM __sqb_0{} d JOIN __sqb_1{} c ON TRUE", + " ".repeat(23), + " ".repeat(6) + ), + expected_calls: &[("__dbt_ref", "shop__customers"), ("__ref", "café")], + }, + MarkersAsNamesTestCase { + description: "calls with a second name, or a non-literal argument, stay calls", + dialect: "duckdb", + sql: r#"FROM __ref("a", "b") JOIN __table_fn("orders", 2) ON TRUE"#, + expected_sql: r#"FROM __ref("a", "b") JOIN __table_fn("orders", 2) ON TRUE"#.to_owned(), + expected_calls: &[], + }, + ]; + for test_case in test_cases { + let (named, calls) = + markers_as_names(test_case.sql, &dialect_lexical_syntax(test_case.dialect)) + .expect(test_case.description); + let expected_calls: Vec<(String, String)> = test_case + .expected_calls + .iter() + .map(|(function, name)| ((*function).to_owned(), (*name).to_owned())) + .collect(); + assert_eq!( + (named, calls), + (test_case.expected_sql, expected_calls), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/relation_markers/tests/test_types.rs b/crates/sqlbuild-analysis/src/relation_markers/tests/test_types.rs new file mode 100644 index 0000000000..b92a6d74fa --- /dev/null +++ b/crates/sqlbuild-analysis/src/relation_markers/tests/test_types.rs @@ -0,0 +1,10 @@ +//! Test case types for relation markers read as names. + +/// SQL in one dialect, the SQL its markers read as, and the calls they stand for. +pub(crate) struct MarkersAsNamesTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_sql: String, + pub(crate) expected_calls: &'static [(&'static str, &'static str)], +} diff --git a/crates/sqlbuild-analysis/src/relation_markers/types.rs b/crates/sqlbuild-analysis/src/relation_markers/types.rs new file mode 100644 index 0000000000..9b90a22d2a --- /dev/null +++ b/crates/sqlbuild-analysis/src/relation_markers/types.rs @@ -0,0 +1,4 @@ +//! Shapes of relation marker calls. + +/// A marker call's lower-case function, such as `__ref`, and the name it references. +pub type MarkerCall = (String, String); diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs index 32fd8e8198..ff89c238fe 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs @@ -656,7 +656,7 @@ fn messaged_row(error: ValidationError) -> DiagnosticRow { row } -pub(super) fn diagnostic_row(error: ValidationError) -> DiagnosticRow { +pub(crate) fn diagnostic_row(error: ValidationError) -> DiagnosticRow { let severity = match error.severity { polyglot_sql::ValidationSeverity::Error => "error", _ => "warning", diff --git a/crates/sqlbuild-analysis/src/semantic_validation/main.rs b/crates/sqlbuild-analysis/src/semantic_validation/main.rs index 2db687f9e5..3805b70c71 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/main.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/main.rs @@ -9,6 +9,7 @@ pub mod normalize_dialect; pub mod normalize_mapped; pub mod normalize_native; pub mod sqlbuild_code; +pub mod text_findings; pub mod truth_test_spans; use crate::semantic_validation::models::ProjectCatalog; diff --git a/crates/sqlbuild-analysis/src/semantic_validation/main/text_findings.rs b/crates/sqlbuild-analysis/src/semantic_validation/main/text_findings.rs new file mode 100644 index 0000000000..7ff13fcb00 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_validation/main/text_findings.rs @@ -0,0 +1,19 @@ +//! SQLBuild's own checks of SQL text that every analysis route adds to the parse's findings. + +use crate::lineage::main::parser_dialect::parser_dialect; +use crate::semantic_validation::_helpers::catalog::diagnostic_row; +use crate::semantic_validation::_helpers::sqlite_temporal_casts::sqlite_temporal_cast_warnings; +use crate::semantic_validation::_helpers::truth_tests::unsupported_truth_test_errors; +use crate::semantic_validation::models::DiagnosticRow; + +/// Truth-value tests the warehouse rejects (B103) and SQLite's temporal casts (B104) in `sql`. +pub fn text_findings(sql: &str, dialect: &str) -> Vec { + let Ok(dialect) = parser_dialect(Some(dialect)) else { + return Vec::new(); + }; + unsupported_truth_test_errors(sql, dialect) + .into_iter() + .chain(sqlite_temporal_cast_warnings(sql, dialect)) + .map(diagnostic_row) + .collect() +} From 5813dd12b5a6abaf23f69dc7b7553d946695b445 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:01:37 +0000 Subject: [PATCH 21/29] fix(sqltext): read T-SQL bracket names and ClickHouse quoting in dialect scans A T-SQL `[...]` name was code to the dialect scan, so an apostrophe, `--` or `/*` inside one opened a string or comment that hid everything after it. On a native route a cursor stub after such a name was not respelt, and a valid model was skipped with P026. Lexical rules gain bracket identifiers, closing at `]` with `]]` for one bracket, and the T-SQL rules set them. Adapter records never set the flag, so their keys and scans are unchanged. ClickHouse's rules now follow its lexer: a backslash escapes inside single quotes, double quotes and backticks, and `#` starts a comment only before a space or `!`. --- .../tests/test_normalize_batch.rs | 12 ++++++ .../_helpers/sqltext/lexical_syntax.rs | 2 + .../src/sql_scan/_helpers/dialect.rs | 15 +++++++ .../src/sql_scan/constants.rs | 2 + .../sql_scan/main/dialect_lexical_syntax.rs | 5 ++- .../src/sql_scan/main/lexical_syntax_key.rs | 12 ++++-- .../main/reads_differently_from_generic.rs | 3 ++ .../sqlbuild-sqltext/src/sql_scan/models.rs | 3 ++ .../src/sql_scan/tests/helpers.rs | 1 + .../src/sql_scan/tests/test_dialect_scan.rs | 43 +++++++++++++++++++ 10 files changed, 92 insertions(+), 6 deletions(-) diff --git a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs index 05db7af22e..95dcadd3f3 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs @@ -162,6 +162,18 @@ fn given_cursor_stubs_when_normalizing_for_a_native_parser_then_only_whole_names sql: r"SELECT 'it\'s __cursor_end()', __cursor_end() FROM t", expected_sql: r"SELECT 'it\'s __cursor_end()', __sqlbuild.cursor_end() FROM t", }, + NativeStubTestCase { + dialect: "tsql", + description: "a T-SQL bracket name's apostrophe opens no string, so the stub after it is code", + sql: "SELECT [customer's name] AS n FROM t WHERE ts >= __cursor_start()", + expected_sql: "SELECT [customer's name] AS n FROM t WHERE ts >= __sqlbuild.cursor_start()", + }, + NativeStubTestCase { + dialect: "clickhouse", + description: "a ClickHouse backslash-quoted quote in an identifier keeps the stub after it in code", + sql: r#"SELECT 1 AS "a\"b", __cursor_end() FROM t"#, + expected_sql: r#"SELECT 1 AS "a\"b", __sqlbuild.cursor_end() FROM t"#, + }, NativeStubTestCase { dialect: "snowflake", description: "a cursor stub's text inside a Snowflake // comment is left as written", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/lexical_syntax.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/lexical_syntax.rs index 600dd863ed..39148bb37b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/lexical_syntax.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/lexical_syntax.rs @@ -30,6 +30,7 @@ impl From for LexicalSyntax { triple_quoted_strings: input.triple_quoted_strings, nested_block_comments: input.nested_block_comments, line_comment_prefixes: input.line_comment_prefixes, + bracket_identifiers: false, } } } @@ -54,6 +55,7 @@ pub(crate) fn record_syntax(instance: &Bound<'_, PyAny>) -> PyResult dollar_quote_end(sql, index), Some(b'\'' | b'"' | b'`') => quoted_text_end(sql, index, syntax).map(Some), + Some(b'[') if syntax.bracket_identifiers => bracket_identifier_end(sql, index).map(Some), _ => Ok(None), } } @@ -44,6 +45,7 @@ pub(crate) fn non_code_end( /// Whether a comment or quoted text may open with `byte`; most code bytes are rejected here cheaply. fn may_open_non_code(byte: u8, syntax: &LexicalSyntax) -> bool { matches!(byte, b'/' | b'$' | b'\'' | b'"' | b'`') + || (byte == b'[' && syntax.bracket_identifiers) || syntax .line_comment_prefixes .iter() @@ -133,6 +135,19 @@ fn quoted_text_end(sql: &[u8], start: usize, syntax: &LexicalSyntax) -> Result Result { + let mut index = start + 1; + while let Some(offset) = sql[index..].iter().position(|byte| *byte == b']') { + let close = index + offset; + if sql.get(close + 1) != Some(&b']') { + return Ok(close + 1); + } + index = close + 2; + } + Err(Unclosed::Quote) +} + fn string_literal_prefix(sql: &[u8], start: usize) -> Vec { let mut prefix_start = start; while prefix_start > 0 diff --git a/crates/sqlbuild-sqltext/src/sql_scan/constants.rs b/crates/sqlbuild-sqltext/src/sql_scan/constants.rs index f7d11818a4..b604b49f12 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/constants.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/constants.rs @@ -14,3 +14,5 @@ pub const GENERIC_LINE_COMMENT_PREFIX: &str = "--"; pub const TRIPLE_QUOTES: [&str; 2] = ["'''", "\"\"\""]; pub const ESCAPE_CHARACTER: char = '\\'; pub const BLOCK_COMMENT_OPEN: &str = "/*"; +/// What a lexical key adds for bracket identifiers. +pub(crate) const BRACKET_IDENTIFIERS_KEY: &str = "brackets"; diff --git a/crates/sqlbuild-sqltext/src/sql_scan/main/dialect_lexical_syntax.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/dialect_lexical_syntax.rs index ff94e94b32..3028fc6dbb 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/main/dialect_lexical_syntax.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/dialect_lexical_syntax.rs @@ -20,8 +20,8 @@ pub fn dialect_lexical_syntax(dialect: &str) -> LexicalSyntax { ..LexicalSyntax::default() }, "clickhouse" => LexicalSyntax { - backslash_escape_quotes: strings(&["'"]), - line_comment_prefixes: lines(&["--", "#", "//"]), + backslash_escape_quotes: strings(&["'", "\"", "`"]), + line_comment_prefixes: lines(&["--", "# ", "#!", "//"]), ..LexicalSyntax::default() }, "databricks" | "duckdb" => LexicalSyntax { @@ -31,6 +31,7 @@ pub fn dialect_lexical_syntax(dialect: &str) -> LexicalSyntax { "mssql" | "sqlserver" | "tsql" => LexicalSyntax { nested_block_comments: true, line_comment_prefixes: lines(&["--"]), + bracket_identifiers: true, ..LexicalSyntax::default() }, _ => LexicalSyntax { diff --git a/crates/sqlbuild-sqltext/src/sql_scan/main/lexical_syntax_key.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/lexical_syntax_key.rs index ff2cd882d3..5bcdb7be2a 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/main/lexical_syntax_key.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/lexical_syntax_key.rs @@ -1,8 +1,9 @@ //! The process-stable text key of one dialect's lexical rules. +use crate::sql_scan::constants::BRACKET_IDENTIFIERS_KEY; use crate::sql_scan::models::LexicalSyntax; -/// `SqlLexicalSyntax.cache_key`: sorted quotes, the four flags as Python booleans, sorted prefixes. +/// `SqlLexicalSyntax.cache_key`: quotes, the four flags, prefixes, then `brackets` only when set. #[must_use] pub fn lexical_syntax_key(syntax: &LexicalSyntax) -> String { let flag = |value: bool| if value { "True" } else { "False" }; @@ -18,13 +19,16 @@ pub fn lexical_syntax_key(syntax: &LexicalSyntax) -> String { .map(String::as_str) .collect(); prefixes.sort_unstable(); - [ + let mut parts: Vec = vec![ quotes.join(","), flag(syntax.escape_string_prefix).to_owned(), flag(syntax.raw_string_prefix).to_owned(), flag(syntax.triple_quoted_strings).to_owned(), flag(syntax.nested_block_comments).to_owned(), prefixes.join(","), - ] - .join("|") + ]; + if syntax.bracket_identifiers { + parts.push(BRACKET_IDENTIFIERS_KEY.to_owned()); + } + parts.join("|") } diff --git a/crates/sqlbuild-sqltext/src/sql_scan/main/reads_differently_from_generic.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/reads_differently_from_generic.rs index 05479da834..3ea203edde 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/main/reads_differently_from_generic.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/reads_differently_from_generic.rs @@ -19,6 +19,9 @@ pub fn reads_differently_from_generic(syntax: &LexicalSyntax, sql: &str) -> bool if syntax.nested_block_comments && sql.matches(BLOCK_COMMENT_OPEN).count() > 1 { return true; } + if syntax.bracket_identifiers && sql.contains('[') { + return true; + } syntax .line_comment_prefixes .iter() diff --git a/crates/sqlbuild-sqltext/src/sql_scan/models.rs b/crates/sqlbuild-sqltext/src/sql_scan/models.rs index 632b1b5b07..b89ffe1f8f 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/models.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/models.rs @@ -81,6 +81,9 @@ pub struct LexicalSyntax { pub nested_block_comments: bool, /// Prefixes that start a line comment, such as `--`, `#` or `//`. pub line_comment_prefixes: Vec, + /// `[...]` quotes an identifier, `]]` standing for one `]`, as in T-SQL. + #[serde(default)] + pub bracket_identifiers: bool, } impl LexicalSyntax { diff --git a/crates/sqlbuild-sqltext/src/sql_scan/tests/helpers.rs b/crates/sqlbuild-sqltext/src/sql_scan/tests/helpers.rs index 911a581657..ad9f12b511 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/tests/helpers.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/tests/helpers.rs @@ -127,5 +127,6 @@ pub(crate) fn syntax(quotes: &[&str], flags: [bool; 4], prefixes: &[&str]) -> Le triple_quoted_strings: flags[2], nested_block_comments: flags[3], line_comment_prefixes: prefixes.iter().map(|prefix| (*prefix).to_owned()).collect(), + bracket_identifiers: false, } } diff --git a/crates/sqlbuild-sqltext/src/sql_scan/tests/test_dialect_scan.rs b/crates/sqlbuild-sqltext/src/sql_scan/tests/test_dialect_scan.rs index b23fd27f99..24354b62d3 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/tests/test_dialect_scan.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/tests/test_dialect_scan.rs @@ -1,3 +1,4 @@ +use crate::sql_scan::main::dialect_lexical_syntax::dialect_lexical_syntax; use crate::sql_scan::main::dialect_non_code_end::dialect_non_code_end; use crate::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges; use crate::sql_scan::models::{LexicalSyntax, Unclosed}; @@ -151,6 +152,48 @@ fn given_dialect_lexical_syntax_when_scanning_non_code_then_dialect_decides_boun syntax: generic(), expected_first_range: Some((0, 12)), }, + DialectFirstNonCodeRangeTestCase { + description: "a T-SQL bracket identifier hides the apostrophe inside it", + sql: "[customer's name] x", + syntax: dialect_lexical_syntax("tsql"), + expected_first_range: Some((0, 17)), + }, + DialectFirstNonCodeRangeTestCase { + description: "a doubled closing bracket stays inside the identifier", + sql: "[a]]'b] x", + syntax: dialect_lexical_syntax("tsql"), + expected_first_range: Some((0, 7)), + }, + DialectFirstNonCodeRangeTestCase { + description: "a comment prefix inside a bracket identifier opens no comment", + sql: "[a--b] x", + syntax: dialect_lexical_syntax("tsql"), + expected_first_range: Some((0, 6)), + }, + DialectFirstNonCodeRangeTestCase { + description: "outside T-SQL a bracket is code, so the apostrophe opens a quote", + sql: "[customer's name] x", + syntax: generic(), + expected_first_range: Some((9, 19)), + }, + DialectFirstNonCodeRangeTestCase { + description: "ClickHouse backslash escapes hold in double-quoted identifiers", + sql: r#""a\"b" x"#, + syntax: dialect_lexical_syntax("clickhouse"), + expected_first_range: Some((0, 6)), + }, + DialectFirstNonCodeRangeTestCase { + description: "a ClickHouse hash before a word is code, not a comment", + sql: "#x 'a'", + syntax: dialect_lexical_syntax("clickhouse"), + expected_first_range: Some((3, 6)), + }, + DialectFirstNonCodeRangeTestCase { + description: "a ClickHouse hash before a space opens a comment", + sql: "# it's\nx", + syntax: dialect_lexical_syntax("clickhouse"), + expected_first_range: Some((0, 7)), + }, DialectFirstNonCodeRangeTestCase { description: "an unclosed backslash-escaped quote extends to the end", sql: r"'O\'", From 1eaa0f1de6eeea410723a9831015bedb6d70238a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:01:37 +0000 Subject: [PATCH 22/29] fix(analysis): read repeated markers as one relation and keep unreadable revalidations Native fast lineage gave every relation marker call its own name, so a union of two branches over one model read two relations and its columns became expressions. Calls of the same marker now share their first call's name, padded to keep positions. Native revalidation in type recovery failed the compile when the dialect's parser could not read a planned model again. Such a model now keeps its own findings, as an unrevised model would. --- .../analysis_route/_helpers/marker_names.rs | 50 +++++++++++ .../src/analysis_route/_helpers/mod.rs | 1 + .../analysis_route/_helpers/revalidation.rs | 34 +++++--- .../src/analysis_route/main/fast_lineage.rs | 21 +++-- .../main/native_revalidation.rs | 18 ++-- .../src/analysis_route/tests.rs | 2 + .../src/analysis_route/tests/helpers.rs | 42 +++++++++- .../tests/test_native_outcome.rs | 9 ++ .../tests/test_native_revalidation.rs | 84 +++++++++++++++++++ .../src/analysis_route/tests/test_types.rs | 7 ++ .../_helpers/semantic_checks/type_recovery.rs | 28 ++++--- 11 files changed, 258 insertions(+), 38 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/analysis_route/_helpers/marker_names.rs create mode 100644 crates/sqlbuild-analysis/src/analysis_route/tests/test_native_revalidation.rs diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/marker_names.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/marker_names.rs new file mode 100644 index 0000000000..c5e547fe75 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/marker_names.rs @@ -0,0 +1,50 @@ +//! One analysis name per distinct relation marker call, so repeated calls read as one relation. + +use std::sync::LazyLock; + +use regex::{Captures, Regex}; + +use crate::relation_markers::constants::MARKER_NAME_PREFIX; +use crate::relation_markers::types::MarkerCall; + +static MARKER_NAME: LazyLock> = LazyLock::new(|| { + Regex::new(&format!(r"\b{MARKER_NAME_PREFIX}(\d+)\b")).map_err(|error| error.to_string()) +}); + +/// `named` with each marker name renamed to its call's first index, padded to keep positions. +pub(crate) fn first_call_names(named: &str, calls: &[MarkerCall]) -> Result { + let pattern: &Regex = MARKER_NAME.as_ref().map_err(Clone::clone)?; + Ok(pattern + .replace_all(named, |captures: &Captures<'_>| { + let written: &str = captures.get(0).map_or("", |found| found.as_str()); + let first: Option = match captures.get(1).map(|index| index.as_str().parse()) { + Some(Ok(index)) => first_index(calls, index), + Some(Err(_)) | None => None, + }; + match first { + Some(first) => { + let name: String = format!("{MARKER_NAME_PREFIX}{first}"); + format!("{name: written.to_owned(), + } + }) + .into_owned()) +} + +/// The calls whose index is their call's first, in order, with that index. +pub(crate) fn distinct_calls(calls: &[MarkerCall]) -> Vec<(usize, &MarkerCall)> { + let mut distinct: Vec<(usize, &MarkerCall)> = Vec::with_capacity(calls.len()); + for (index, call) in calls.iter().enumerate() { + if first_index(calls, index) == Some(index) { + distinct.push((index, call)); + } + } + distinct +} + +/// The first index of the call at `index`, which is `index` unless an earlier call is the same. +fn first_index(calls: &[MarkerCall], index: usize) -> Option { + let call: &MarkerCall = calls.get(index)?; + calls.iter().position(|earlier| earlier == call) +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs index 393034d764..279c10ae95 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod clickhouse_types; pub(crate) mod located_gap; +pub(crate) mod marker_names; pub(crate) mod native_analysis; pub(crate) mod revalidation; pub(crate) mod whole_gap; diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/revalidation.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/revalidation.rs index d726434436..44d3976120 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/_helpers/revalidation.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/revalidation.rs @@ -5,7 +5,7 @@ use std::collections::HashMap; use sqlquack_semantics::lineage::models::{Nullability, SchemaColumn}; -use crate::analysis_route::models::{AnalysisInput, NativeAnalysis}; +use crate::analysis_route::models::{AnalysisInput, NativeAnalysis, ParserGap}; use crate::analysis_route::types::NativeDialect; use crate::assembly::analysis_session::constants::UNKNOWN_TYPE; use crate::assembly::analysis_session::main::resolved_cleaned_sql::resolved_cleaned_sql; @@ -13,6 +13,7 @@ use crate::assembly::analysis_session::models::{ FinishedSession, ModelOutcome, PivotOutcome, SessionSchema, }; use crate::assembly::analysis_session::types::Pairs; +use crate::semantic_checks::models::RecoveryModel; use crate::semantic_checks::types::RevisedBinding; use crate::semantic_validation::models::DiagnosticRow; @@ -32,13 +33,13 @@ pub(crate) fn session_outcome<'a>( .ok_or_else(|| format!("the session analysed no model {name}")) } -/// The findings of `outcome`'s analysed SQL over its bound schema, the poisoned columns untyped. +/// The findings of `outcome`'s SQL with the poisoned columns untyped; `None` when it is unreadable. pub(crate) fn revised( dialect: &dyn NativeDialect, outcome: &ModelOutcome, poisoned: &[(String, String)], declared: &[String], -) -> Result, String> { +) -> Result>, String> { let sql: Cow<'_, str> = resolved_cleaned_sql(outcome)?; let relations: SessionSchema = SessionSchema { relations: outcome @@ -53,18 +54,29 @@ pub(crate) fn revised( }) .collect::>>(), }; - let analysis: NativeAnalysis = dialect - .analyse(&AnalysisInput { - sql: &sql, - schema: &relations, - references: &[], - extension_functions: declared, - }) - .map_err(|gap| gap.message)?; + let analysed: Result = dialect.analyse(&AnalysisInput { + sql: &sql, + schema: &relations, + references: &[], + extension_functions: declared, + }); + let Ok(analysis) = analysed else { + return Ok(None); + }; analysis .diagnostics .into_iter() .map(revised_binding) + .collect::, String>>() + .map(Some) +} + +/// The model's own findings, as type recovery matches them, kept as they are. +pub(crate) fn unrevised(model: &RecoveryModel) -> Vec { + model + .raw_bindings + .iter() + .map(|raw| (raw.code.clone(), raw.start, raw.end)) .collect() } diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/fast_lineage.rs b/crates/sqlbuild-analysis/src/analysis_route/main/fast_lineage.rs index 64eb6e6a61..e95f61f52a 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/fast_lineage.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/fast_lineage.rs @@ -7,6 +7,7 @@ use sqlbuild_sqltext::sql_scan::main::dialect_lexical_syntax::dialect_lexical_sy use sqlquack_semantics::lineage::models::{Nullability, SchemaColumn}; use crate::analysis_route::_helpers::located_gap::{located, unreadable_reason}; +use crate::analysis_route::_helpers::marker_names::{distinct_calls, first_call_names}; use crate::analysis_route::models::{AnalysisInput, NativeAnalysis, ParserGap}; use crate::analysis_route::types::NativeDialect; use crate::assembly::analysis_session::constants::{ @@ -34,15 +35,15 @@ pub fn fast_lineage( Ok(named) => named, Err(reason) => return FastLineageOutcome::Unparsed(reason), }; - let normalised: String = - match normalize_native_sql(&named, dialect.name(), dialect.stub_prefix()) { - Ok(normalised) => normalised, - Err(reason) => return FastLineageOutcome::Unparsed(reason), - }; + let normalised: Result = first_call_names(&named, &calls) + .and_then(|named| normalize_native_sql(&named, dialect.name(), dialect.stub_prefix())); + let normalised: String = match normalised { + Ok(normalised) => normalised, + Err(reason) => return FastLineageOutcome::Unparsed(reason), + }; let sql: String = placeholder_defaults(&normalised, placeholders); - let referenced: Vec<(String, LineageResourceType, String)> = calls + let referenced: Vec<(String, LineageResourceType, String)> = distinct_calls(&calls) .into_iter() - .enumerate() .filter_map(|(index, (function, name))| { let resource_type: LineageResourceType = match function.as_str() { "__ref" => LineageResourceType::Model, @@ -50,7 +51,11 @@ pub fn fast_lineage( "__seed" => LineageResourceType::Seed, _ => return None, }; - Some((format!("{MARKER_NAME_PREFIX}{index}"), resource_type, name)) + Some(( + format!("{MARKER_NAME_PREFIX}{index}"), + resource_type, + name.clone(), + )) }) .collect(); let references: Vec<(String, String, String)> = referenced diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/native_revalidation.rs b/crates/sqlbuild-analysis/src/analysis_route/main/native_revalidation.rs index 84efea9ac5..67ae52f5cd 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/native_revalidation.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/native_revalidation.rs @@ -5,26 +5,30 @@ use std::sync::Arc; use rayon::ThreadPool; use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; -use crate::analysis_route::_helpers::revalidation::{revised, session_outcome}; +use crate::analysis_route::_helpers::revalidation::{revised, session_outcome, unrevised}; use crate::analysis_route::types::NativeDialect; use crate::assembly::analysis_session::models::{FinishedSession, ModelOutcome, PivotOutcome}; +use crate::semantic_checks::models::RecoveryModel; use crate::semantic_checks::types::RevisedBinding; -/// Each named model's findings, at the session's positions, with the `poisoned` outputs untyped. +/// Each model's findings with the `poisoned` outputs untyped; an unreadable model keeps its own. pub fn native_revalidation( dialect: &dyn NativeDialect, (finished, outcomes): (&FinishedSession, &[(ModelOutcome, PivotOutcome)]), - models: &[String], + models: &[&RecoveryModel], poisoned: &[(String, String)], ) -> Result>, String> { let pool: Arc = finished.pool.clone()?; pool.install(|| { models .par_iter() - .map(|name| { - let outcome: &ModelOutcome = session_outcome(finished, outcomes, name)?; - revised(dialect, outcome, poisoned, &finished.declared_functions) - .map_err(|reason| format!("{name}: {reason}")) + .map(|model| { + let outcome: &ModelOutcome = session_outcome(finished, outcomes, &model.name)?; + match revised(dialect, outcome, poisoned, &finished.declared_functions) { + Ok(Some(rows)) => Ok(rows), + Ok(None) => Ok(unrevised(model)), + Err(reason) => Err(format!("{}: {reason}", model.name)), + } }) .collect() }) diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests.rs b/crates/sqlbuild-analysis/src/analysis_route/tests.rs index 0b05ee5ffb..242ac2a388 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests.rs @@ -8,5 +8,7 @@ mod test_native_dialect; mod test_native_gap; #[path = "tests/test_native_outcome.rs"] mod test_native_outcome; +#[path = "tests/test_native_revalidation.rs"] +mod test_native_revalidation; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs index f3a99b4bbd..9dd60dfa2d 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs @@ -4,9 +4,12 @@ use sqlquack_semantics::lineage::models::{Nullability, SchemaColumn}; use crate::analysis_route::models::{AnalysisInput, NativeAnalysis, StandInSchema}; use crate::analysis_route::types::NativeDialect; -use crate::assembly::analysis_session::models::{LineageRow, SessionSchema}; +use crate::assembly::analysis_session::models::{ + LineageFacts, LineageRow, ModelAnalysis, ModelOutcome, SessionSchema, +}; use crate::assembly::analysis_session::types::Shapes; use crate::lineage::models::{LineageColumn, LineageSource}; +use crate::semantic_checks::types::RevisedBinding; /// `orders` with a non-null INTEGER id and a nullable VARCHAR status. pub(super) fn orders() -> StandInSchema { @@ -104,3 +107,40 @@ fn schema_column(&(name, data_type, nullability): &(&str, &str, Nullability)) -> nullability, } } + +/// A finished outcome of `sql` over `orders`, as the session bound it. +pub(super) fn outcome(sql: &str) -> ModelOutcome { + ModelOutcome { + analysis: ModelAnalysis { + analysis_succeeded: true, + columns: None, + lineage: LineageFacts::Native(Vec::new()), + has_star: false, + star_resolved: false, + binding_diagnostics: Vec::new(), + binding_validated: true, + }, + cleaned_sql: sql.to_owned(), + deferred_cleaning: None, + validated_schema: vec![( + "orders".to_owned(), + vec![ + ("id".to_owned(), "INTEGER".to_owned()), + ("status".to_owned(), "VARCHAR".to_owned()), + ], + )], + fused_binding_validated: true, + sql_macro_findings: Vec::new(), + } +} + +/// Each revised finding of every model as `(code, start, end)`, in order. +pub(super) fn finding_rows( + models: &[Vec], +) -> Vec<(&str, Option, Option)> { + let mut rows: Vec<(&str, Option, Option)> = Vec::new(); + for (code, start, end) in models.iter().flatten() { + rows.push((code.as_str(), *start, *end)); + } + rows +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs index 20eaba37c2..095d840e14 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs @@ -148,6 +148,15 @@ fn given_a_model_without_facts_when_fast_lineage_reads_it_natively_then_it_has_l "built star=false", ], }, + NativeLineageTestCase { + description: "repeated calls of one marker read as one relation, so a union stays direct", + sql: r#"SELECT order_id FROM __ref("orders") UNION ALL SELECT order_id FROM __ref("orders")"#, + placeholders: &[], + expected_lines: &[ + "order_id direct high [model:orders.order_id]", + "built star=false", + ], + }, NativeLineageTestCase { description: "a star over a relation of unknown columns is a star without outputs", sql: r#"SELECT * FROM __ref("unknown_model")"#, diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_revalidation.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_revalidation.rs new file mode 100644 index 0000000000..0e68c40a87 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_revalidation.rs @@ -0,0 +1,84 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use crate::analysis_route::main::native_revalidation::native_revalidation; +use crate::analysis_route::tests::helpers::{finding_rows, outcome}; +use crate::analysis_route::tests::test_types::{PivotlessDialect, RevalidationTestCase}; +use crate::assembly::analysis_session::models::{ + FinishedSession, ModelOutcome, PivotOutcome, PivotTables, +}; +use crate::semantic_checks::models::{RawBinding, RecoveryModel}; +use crate::semantic_checks::types::RevisedBinding; +use crate::semantic_validation::main::new_analysis_pool::new_analysis_pool; + +#[test] +fn given_poisoned_outputs_when_revalidating_natively_then_findings_keep_session_positions() { + let test_cases = [ + RevalidationTestCase { + description: "a finding on a typed column stays at the session's span", + sql: "SELECT id FROM orders WHERE status", + poisoned: &[], + expected_rows: &[("B211", Some(28), Some(34))], + }, + RevalidationTestCase { + description: "a finding only a poisoned column causes drops out", + sql: "SELECT id FROM orders WHERE status", + poisoned: &[("orders", "status")], + expected_rows: &[], + }, + RevalidationTestCase { + description: "a model the parser cannot read again keeps its own findings", + sql: "SELECT id FROM orders PIVOT (sum(id) FOR status IN (1))", + poisoned: &[("orders", "status")], + expected_rows: &[("B002", Some(7), Some(9))], + }, + ]; + + for test_case in test_cases { + let finished: FinishedSession = FinishedSession { + tables: PivotTables::default(), + pool: new_analysis_pool().map(Arc::new), + models: HashMap::new(), + dynamic_proofs: HashMap::new(), + relations: None, + model_names: vec!["other".to_owned(), "downstream".to_owned()], + declared_functions: Vec::new(), + }; + let outcomes: Vec<(ModelOutcome, PivotOutcome)> = vec![ + (outcome("SELECT 1"), PivotOutcome::Absent), + (outcome(test_case.sql), PivotOutcome::Absent), + ]; + let model: RecoveryModel = RecoveryModel { + name: "downstream".to_owned(), + query_sql: test_case.sql.to_owned(), + inferred_columns: None, + references: vec!["orders".to_owned()], + lineage: Vec::new(), + bindings: Vec::new(), + raw_bindings: vec![RawBinding { + id: 0, + code: "B002".to_owned(), + message: "Unknown column".to_owned(), + start: Some(7), + end: Some(9), + }], + }; + let poisoned: Vec<(String, String)> = test_case + .poisoned + .iter() + .map(|(relation, column)| ((*relation).to_owned(), (*column).to_owned())) + .collect(); + let revised: Result>, String> = native_revalidation( + &PivotlessDialect, + (&finished, &outcomes), + &[&model], + &poisoned, + ); + assert_eq!( + revised.as_deref().map(finding_rows), + Ok(test_case.expected_rows.to_vec()), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs index a9b789e105..3bd8d16851 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs @@ -157,6 +157,13 @@ pub(super) struct MarkerLineageTestCase { pub(super) expected_lines: &'static [&'static str], } +pub(super) struct RevalidationTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) poisoned: &'static [(&'static str, &'static str)], + pub(super) expected_rows: &'static [(&'static str, Option, Option)], +} + pub(super) struct ReadWholeTestCase { pub(super) description: &'static str, pub(super) sql: &'static str, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs index 7317f9b6bb..e658566966 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs @@ -107,9 +107,12 @@ pub(crate) fn recovered_types<'py>( let poisoned: Vec<(String, String)> = plan.poisoned_outputs(); let models_to_revalidate: Vec = plan.revalidated_models(); let revised: Vec> = match analysis_route(request.dialect.as_deref()) { - AnalysisRoute::Native(dialect) => { - natively_revalidated(dialect, project, (&models_to_revalidate, &poisoned), shared)? - } + AnalysisRoute::Native(dialect) => natively_revalidated( + py, + dialect, + (&request, &models_to_revalidate, &poisoned), + shared, + )?, AnalysisRoute::Legacy | AnalysisRoute::Unavailable => { revalidated(classes, project, &models_to_revalidate, &poisoned)? } @@ -256,32 +259,35 @@ fn recovery_model( /// Each planned model revalidated by the dialect's native analysis, never the Polyglot catalog. fn natively_revalidated( + py: Python<'_>, dialect: &'static dyn NativeDialect, - project: &Bound<'_, PyAny>, - (models, poisoned): (&[usize], &[(String, String)]), + (request, models, poisoned): (&TypeRecoveryRequest, &[usize], &[(String, String)]), shared: &StageShared<'_>, ) -> PyResult>> { if models.is_empty() { return Ok(Vec::new()); } - let py: Python<'_> = project.py(); let session: &NativeModelAnalysisSession = shared .session .ok_or_else(|| internal_error("revalidating a native route without its session"))?; let finished = session .finished_session() .ok_or_else(|| internal_error("revalidating before the session finished"))?; - let project_models: Bound<'_, PyAny> = native_field(project, "models")?; - let mut names: Vec = Vec::with_capacity(models.len()); + let mut planned: Vec<&RecoveryModel> = Vec::with_capacity(models.len()); for index in models { - names.push(native_field(&project_models.get_item(*index)?, "name")?.extract()?); + planned.push( + request + .models + .get(*index) + .ok_or_else(|| internal_error("type recovery planned a model it does not hold"))?, + ); } - count_stage(py, SCHEMA_REVALIDATIONS, names.len() as u64)?; + count_stage(py, SCHEMA_REVALIDATIONS, planned.len() as u64)?; py.compiler_detach(|| { Ok(native_revalidation( dialect, (finished, session.outcomes()), - &names, + &planned, poisoned, )) }) From 195fa99b430f29127cbfc4fdf1bfea54d86200d9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:30:44 +0000 Subject: [PATCH 23/29] fix(analysis): name the hook and the model file line in P026 A native route's P026 for a hook now names the hook and its file, as the Polyglot syntax error did, such as post_hooks[0] sql("record_refresh") (hooks/sql/record_refresh.sql), and places the gap in the hook's SQL. A gap in the model's query is placed at its line and column in the model file when the text maps back through stub spelling, placeholders and expansion; otherwise the message says the position is in the analysed SQL. --- .../analysis_route/_helpers/located_gap.rs | 51 +++++- .../src/analysis_route/main/aligned_offset.rs | 5 + .../src/analysis_route/main/fast_lineage.rs | 7 +- .../src/analysis_route/main/mod.rs | 1 + .../src/analysis_route/main/native_gap.rs | 2 +- .../analysis_route/main/skipped_analysis.rs | 15 +- .../src/analysis_route/models.rs | 23 +++ .../analysis_route/tests/test_native_gap.rs | 95 ++++++++-- .../src/analysis_route/tests/test_types.rs | 3 +- .../_helpers/input_loops/model_loop.rs | 16 +- .../input_loops/model_loop_opt_out.rs | 166 +++++++++++++++--- .../project_assembly/syntax_validation.rs | 3 +- 12 files changed, 330 insertions(+), 57 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/analysis_route/main/aligned_offset.rs diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/located_gap.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/located_gap.rs index 40d950161c..852c8a28ad 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/_helpers/located_gap.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/located_gap.rs @@ -1,31 +1,70 @@ //! Where a native parser's gap is, and why the parser cannot read the SQL there. -use crate::analysis_route::models::{LocatedGap, ParserGap}; +use crate::analysis_route::models::{GapPosition, GapSite, LocatedGap, ParserGap}; use crate::compiled_project::main::text_position::text_position; use crate::compiled_project::models::TextPosition; -/// The gap with the line and column of its offset; an offset past the end is the end. -pub(crate) fn located(sql: &str, gap: ParserGap) -> LocatedGap { +/// The gap at its line and column in `sql`, and its offset in `source` when the text holds. +pub(crate) fn located(sql: &str, gap: ParserGap, source: Option<&str>) -> LocatedGap { let before: &str = sql.get(..gap.offset).unwrap_or(sql); - let position: TextPosition = text_position(sql, before.chars().count()); + let offset: usize = before.chars().count(); + let position: TextPosition = text_position(sql, offset); LocatedGap { construct: gap.construct, message: gap.message, line: position.line, column: position.column, syntax: gap.syntax, + source_offset: source.and_then(|source| aligned(sql, offset, source)), } } +/// The character of `source` that `derived`'s `offset` came from, when the text before or after holds. +pub(crate) fn aligned(derived: &str, offset: usize, source: &str) -> Option { + let derived: Vec = derived.chars().collect(); + let source: Vec = source.chars().collect(); + if offset > derived.len() { + return None; + } + if source.starts_with(&derived[..offset]) { + return Some(offset); + } + source + .ends_with(&derived[offset..]) + .then(|| source.len() - (derived.len() - offset)) +} + /// Why the dialect's native parser cannot read SQL, as P026 words it. pub(crate) fn unreadable_reason(dialect: &str, gap: &LocatedGap) -> String { + unreadable_at(dialect, gap, (gap.line, gap.column, "")) +} + +/// P026's text for a gap at its site: the hook and file, then the parser's reason there. +pub(crate) fn sited_reason(dialect: &str, gap: &LocatedGap, site: &GapSite<'_>) -> String { + let label: String = site + .label + .map_or_else(String::new, |label| format!(" {label}")); + let at: (usize, usize, &str) = match site.position { + GapPosition::File(position) => (position.line, position.column, ""), + GapPosition::Statement(position) => (position.line, position.column, " of its SQL"), + GapPosition::Analysed => (gap.line, gap.column, " of the analysed SQL"), + }; + format!( + "{label} ({}): {}", + site.path, + unreadable_at(dialect, gap, at) + ) +} + +fn unreadable_at(dialect: &str, gap: &LocatedGap, at: (usize, usize, &str)) -> String { + let (line, column, within) = at; let detail: String = if gap.message.is_empty() || gap.message == gap.construct { String::new() } else { format!(": {}", gap.message) }; format!( - "the {dialect} parser cannot read {} at line {}, column {}{detail}", - gap.construct, gap.line, gap.column + "the {dialect} parser cannot read {} at line {line}, column {column}{within}{detail}", + gap.construct ) } diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/aligned_offset.rs b/crates/sqlbuild-analysis/src/analysis_route/main/aligned_offset.rs new file mode 100644 index 0000000000..4d603f0cea --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/main/aligned_offset.rs @@ -0,0 +1,5 @@ +/// The character of `source` that `derived`'s `offset` came from, when the text before or after holds. +#[must_use] +pub fn aligned_offset(derived: &str, offset: usize, source: &str) -> Option { + crate::analysis_route::_helpers::located_gap::aligned(derived, offset, source) +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/fast_lineage.rs b/crates/sqlbuild-analysis/src/analysis_route/main/fast_lineage.rs index e95f61f52a..3f3fbf2627 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/fast_lineage.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/fast_lineage.rs @@ -80,9 +80,10 @@ pub fn fast_lineage( columns: native.lineage.iter().map(lineage_column).collect(), has_star: native.has_star, }, - Err(gap) => { - FastLineageOutcome::Unparsed(unreadable_reason(dialect.name(), &located(&sql, gap))) - } + Err(gap) => FastLineageOutcome::Unparsed(unreadable_reason( + dialect.name(), + &located(&sql, gap, None), + )), } } diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs b/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs index 9d6f86f1f5..5783055737 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs @@ -1,3 +1,4 @@ +pub mod aligned_offset; pub mod analysis_route; pub mod clickhouse_type_match; pub mod fast_lineage; diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/native_gap.rs b/crates/sqlbuild-analysis/src/analysis_route/main/native_gap.rs index e484e7074a..aa68f67059 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/native_gap.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/native_gap.rs @@ -17,5 +17,5 @@ pub fn native_gap( Ok(dialect .read_whole(&analysed) .err() - .map(|gap| located(&analysed, gap))) + .map(|gap| located(&analysed, gap, Some(&check.sql)))) } diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/skipped_analysis.rs b/crates/sqlbuild-analysis/src/analysis_route/main/skipped_analysis.rs index d170a54955..961719fc8c 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/skipped_analysis.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/skipped_analysis.rs @@ -1,15 +1,20 @@ -use crate::analysis_route::_helpers::located_gap::unreadable_reason; +use crate::analysis_route::_helpers::located_gap::sited_reason; use crate::analysis_route::constants::SKIPPED_ANALYSIS_CODE; -use crate::analysis_route::models::LocatedGap; +use crate::analysis_route::models::{GapSite, LocatedGap}; /// P026's code and message for a model whose SQL the dialect's native parser cannot read. #[must_use] -pub fn skipped_analysis(model: &str, dialect: &str, gap: &LocatedGap) -> (&'static str, String) { +pub fn skipped_analysis( + model: &str, + dialect: &str, + gap: &LocatedGap, + site: &GapSite<'_>, +) -> (&'static str, String) { ( SKIPPED_ANALYSIS_CODE, format!( - "SQL analysis skipped for model '{model}': {}", - unreadable_reason(dialect, gap) + "SQL analysis skipped for model '{model}'{}", + sited_reason(dialect, gap, site) ), ) } diff --git a/crates/sqlbuild-analysis/src/analysis_route/models.rs b/crates/sqlbuild-analysis/src/analysis_route/models.rs index 7330e008d6..838fd60c2c 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/models.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/models.rs @@ -5,6 +5,7 @@ use sqlquack_semantics::lineage::types::LineageSchema; use crate::analysis_route::types::NativeDialect; use crate::assembly::analysis_session::models::{ColumnFact, LineageRow}; +use crate::compiled_project::models::TextPosition; use crate::semantic_validation::models::DiagnosticRow; /// The engine that analyses a dialect's SQL. @@ -38,6 +39,28 @@ pub struct LocatedGap { pub line: usize, pub column: usize, pub syntax: bool, + /// The gap's character offset in the SQL as checked, before normalisation, when its text holds. + pub source_offset: Option, +} + +/// Where P026 places a gap. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum GapPosition { + /// In the file the site names. + File(TextPosition), + /// In the site's own SQL statement, such as a hook's. + Statement(TextPosition), + /// Only in the analysed SQL, at the gap's own line and column. + Analysed, +} + +/// The SQL a gap stops: the hook it belongs to, if any, its file, and where the gap is. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct GapSite<'a> { + /// Such as `post_hooks[0] sql("record_refresh")`; none for the model's query. + pub label: Option<&'a str>, + pub path: &'a str, + pub position: GapPosition, } /// How an inferred column type meets its declared type: the type itself, then its nullability. diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_gap.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_gap.rs index 5c395c6d1f..fc714b2bd1 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_gap.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_gap.rs @@ -1,11 +1,12 @@ use crate::analysis_route::main::analysis_route::analysis_route; use crate::analysis_route::main::native_gap::native_gap; use crate::analysis_route::main::skipped_analysis::skipped_analysis; -use crate::analysis_route::models::{AnalysisRoute, LocatedGap}; +use crate::analysis_route::models::{AnalysisRoute, GapPosition, GapSite, LocatedGap}; use crate::analysis_route::tests::test_types::{ NativeGapTestCase, PivotlessDialect, RouteTestCase, SkippedAnalysisTestCase, }; use crate::assembly::project::models::SyntaxCheck; +use crate::compiled_project::models::TextPosition; #[test] fn given_any_dialect_when_routing_analysis_then_it_stays_on_the_legacy_path() { @@ -109,6 +110,33 @@ fn given_model_sql_when_probing_the_native_parser_then_its_gap_is_located_after_ line: 3, column: 3, syntax: false, + source_offset: Some(25), + }), + }, + NativeGapTestCase { + description: "a stub respelt before the gap keeps its offset in the checked SQL", + sql: "SELECT __cursor_start() AS a FROM t PIVOT (sum(x) FOR y IN (1))", + placeholders: &[], + expected_gap: Some(LocatedGap { + construct: "PIVOT".to_owned(), + message: "PIVOT".to_owned(), + line: 1, + column: 46, + syntax: false, + source_offset: Some(36), + }), + }, + NativeGapTestCase { + description: "text changed on both sides of the gap leaves it only in the analysed SQL", + sql: "SELECT __cursor_start() AS a FROM t PIVOT (sum(x) FOR y IN (@@@n))", + placeholders: &[("n", "1")], + expected_gap: Some(LocatedGap { + construct: "PIVOT".to_owned(), + message: "PIVOT".to_owned(), + line: 1, + column: 46, + syntax: false, + source_offset: None, }), }, ]; @@ -131,10 +159,10 @@ fn given_model_sql_when_probing_the_native_parser_then_its_gap_is_located_after_ } #[test] -fn given_a_located_gap_when_skipping_analysis_then_p026_names_the_model_and_the_gap() { +fn given_a_located_gap_when_skipping_analysis_then_p026_names_the_model_its_site_and_the_gap() { let test_cases = [ SkippedAnalysisTestCase { - description: "the parser's message follows the construct", + description: "a query gap placed in the model file, with the parser's message", model: "orders", dialect: "postgres", gap: LocatedGap { @@ -143,14 +171,49 @@ fn given_a_located_gap_when_skipping_analysis_then_p026_names_the_model_and_the_ line: 1, column: 15, syntax: true, + source_offset: Some(14), + }, + site: GapSite { + label: None, + path: "models/orders.sql", + position: GapPosition::File(TextPosition { + line: 5, + column: 18, + }), + }, + expected_code: "P026", + expected_message: "SQL analysis skipped for model 'orders' (models/orders.sql): the \ + postgres parser cannot read syntax error at line 5, column 18: \ + syntax error at or near \"FROM\"", + }, + SkippedAnalysisTestCase { + description: "a hook gap names the hook and is placed in its statement", + model: "customer_totals", + dialect: "duckdb", + gap: LocatedGap { + construct: "Syntax error at or near 'WHERE'".to_owned(), + message: "Syntax error at or near 'WHERE'".to_owned(), + line: 1, + column: 13, + syntax: true, + source_offset: Some(12), + }, + site: GapSite { + label: Some("post_hooks[0] sql(\"record_refresh\")"), + path: "hooks/sql/record_refresh.sql", + position: GapPosition::Statement(TextPosition { + line: 1, + column: 13, + }), }, expected_code: "P026", - expected_message: "SQL analysis skipped for model 'orders': the postgres parser \ - cannot read syntax error at line 1, column 15: syntax error at \ - or near \"FROM\"", + expected_message: "SQL analysis skipped for model 'customer_totals' post_hooks[0] \ + sql(\"record_refresh\") (hooks/sql/record_refresh.sql): the duckdb \ + parser cannot read Syntax error at or near 'WHERE' at line 1, \ + column 13 of its SQL", }, SkippedAnalysisTestCase { - description: "a PIVOT gap on line 3", + description: "a gap only the analysed SQL places says so", model: "orders_pivot", dialect: "postgres", gap: LocatedGap { @@ -159,15 +222,27 @@ fn given_a_located_gap_when_skipping_analysis_then_p026_names_the_model_and_the_ line: 3, column: 3, syntax: false, + source_offset: None, + }, + site: GapSite { + label: None, + path: "models/orders_pivot.sql", + position: GapPosition::Analysed, }, expected_code: "P026", - expected_message: "SQL analysis skipped for model 'orders_pivot': the postgres parser \ - cannot read PIVOT at line 3, column 3", + expected_message: "SQL analysis skipped for model 'orders_pivot' \ + (models/orders_pivot.sql): the postgres parser cannot read PIVOT at \ + line 3, column 3 of the analysed SQL", }, ]; for test_case in test_cases { assert_eq!( - skipped_analysis(test_case.model, test_case.dialect, &test_case.gap), + skipped_analysis( + test_case.model, + test_case.dialect, + &test_case.gap, + &test_case.site + ), ( test_case.expected_code, test_case.expected_message.to_owned() diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs index 3bd8d16851..382cb4133f 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs @@ -4,7 +4,7 @@ use sqlquack_semantics::lineage::models::Nullability; use sqlquack_syntax::expressions::statements::models::Command; use sqlquack_syntax::{Dialect, Expression, ParseOptions, Parsed, parse_strict}; -use crate::analysis_route::models::{LocatedGap, ParserGap, TypeMatch}; +use crate::analysis_route::models::{GapSite, LocatedGap, ParserGap, TypeMatch}; use crate::analysis_route::types::NativeDialect; use crate::type_system::main::normalize_text_type::normalize_text_type; use crate::type_system::models::NormalizedType; @@ -27,6 +27,7 @@ pub(super) struct SkippedAnalysisTestCase { pub(super) model: &'static str, pub(super) dialect: &'static str, pub(super) gap: LocatedGap, + pub(super) site: GapSite<'static>, pub(super) expected_code: &'static str, pub(super) expected_message: &'static str, } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs index 4d9a371152..5d5df952e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs @@ -73,8 +73,8 @@ use crate::bindings::_helpers::input_loops::model_loop_macros::{ }; use crate::bindings::_helpers::input_loops::model_loop_named::{SCHEMA_DECLARATION, named_usages}; use crate::bindings::_helpers::input_loops::model_loop_opt_out::{ - OptOut, OptOutModel, reject_unparseable_query, rejected_opt_out, skip_unavailable, - skip_unreadable, + AuthoredQuery, OptOut, OptOutModel, reject_unparseable_query, rejected_opt_out, + skip_unavailable, skip_unreadable, }; use crate::bindings::_helpers::input_loops::model_loop_prepass::{ NativeHeader, all_header_facts, native_header, @@ -708,6 +708,16 @@ impl NativeModelLoop { (&declarations.spliced.sql, ¯o_sql, &query_sql), ¯o_spans, ); + let authored_passes: Vec> = vec![ + declarations.spliced.spans.clone(), + spliced_spans(¯o_spans), + ]; + let authored_body: Option> = (interpolated.as_str() == &**authored_query) + .then_some(AuthoredQuery { + query: authored_query, + macro_sql: ¯o_sql, + passes: &authored_passes, + }); let opt_out_model: OptOutModel<'_, 'py> = OptOutModel { name, file_path, @@ -716,6 +726,7 @@ impl NativeModelLoop { query_sql: &query_sql, relative_path, contents, + authored: authored_body, }; let legacy_gate = match self.route { AnalysisRoute::Legacy => Some(self.analysis(context, &opt_out_model, true)?), @@ -797,6 +808,7 @@ impl NativeModelLoop { query_sql: &query_sql, relative_path, contents, + authored: authored_body, }; self.analysis(context, &expanded_model, scanned.invalid_calls.is_empty())? } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs index 4020b6e5f1..322134ab17 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs @@ -4,15 +4,18 @@ use std::path::Path; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyBoolMethods, PyDict, PyDictMethods, PyList, PyString, PyTuple}; +use sqlbuild_analysis::analysis_route::main::aligned_offset::aligned_offset; use sqlbuild_analysis::analysis_route::main::native_gap::native_gap; use sqlbuild_analysis::analysis_route::main::skipped_analysis::skipped_analysis; use sqlbuild_analysis::analysis_route::main::unavailable_analysis::unavailable_analysis; -use sqlbuild_analysis::analysis_route::models::{AnalysisRoute, LocatedGap}; +use sqlbuild_analysis::analysis_route::models::{AnalysisRoute, GapPosition, GapSite, LocatedGap}; use sqlbuild_analysis::analysis_route::types::NativeDialect; use sqlbuild_analysis::assembly::project::models::{SyntaxCheck, SyntaxMode}; +use sqlbuild_analysis::compiled_project::main::authored_model_offset::authored_model_offset; use sqlbuild_analysis::compiled_project::main::opt_out_position::opt_out_position; +use sqlbuild_analysis::compiled_project::main::text_position::text_position; use sqlbuild_analysis::compiled_project::models::ModelLoopSettings; -use sqlbuild_analysis::compiled_project::models::TextPosition; +use sqlbuild_analysis::compiled_project::models::{SplicedSpan, TextPosition}; use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics::replace_intrinsics; use crate::bindings::_helpers::input_loops::model_loop::{ @@ -20,6 +23,7 @@ use crate::bindings::_helpers::input_loops::model_loop::{ }; use crate::bindings::_helpers::input_loops::model_loop_inputs::{HOOK_KEYS, present}; use crate::bindings::_helpers::project_assembly::resources::{syntax_error_message, syntax_valid}; +use crate::bindings::_helpers::project_assembly::syntax_validation::sql_entry_label; use crate::bindings::_helpers::records::construct::{new_helped_error, new_record_with}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; @@ -42,6 +46,7 @@ const LINE_FIELD: &str = "line"; const COLUMN_FIELD: &str = "column"; const HELP_ARGUMENT: &str = "help"; const MODEL_RESOURCE: &str = "model"; +const RELATIVE_PATH_ATTRIBUTE: &str = "relative_path"; /// Where the opt-out was authored, or a check the attachment still answers. pub(crate) enum OptOut<'py> { @@ -58,6 +63,32 @@ pub(crate) struct OptOutModel<'a, 'py> { pub(crate) query_sql: &'a str, pub(crate) relative_path: &'a str, pub(crate) contents: &'a str, + /// How `query_sql` maps back to the model file; none when interpolation rewrote the query. + pub(crate) authored: Option>, +} + +/// The query as the model file holds it, as macros expanded it, and the passes between them. +#[derive(Clone, Copy)] +pub(crate) struct AuthoredQuery<'a> { + pub(crate) query: &'a str, + pub(crate) macro_sql: &'a str, + /// The declaration and macro expansion passes, in that order. + pub(crate) passes: &'a [Vec], +} + +/// One statement a native parser must read, with the site P026 names it by. +struct SiteCheck { + check: SyntaxCheck, + /// The hook, such as `post_hooks[0] sql("record_refresh")`; none for the model's query. + label: Option, + path: String, +} + +/// A hook value being walked: its key, its file and the label its statements are named by. +struct HookSite<'a> { + hook_name: &'a str, + path: String, + label: Option, } /// Whether the opt-out is rejected, and where it was authored. @@ -74,9 +105,13 @@ pub(crate) fn rejected_opt_out<'py>( { return Ok(OptOut::Accepted); } - let checks: Vec = model_checks(context, model)?; + let checks: Vec = model_checks(context, model)?; let readable: bool = match model_loop.route { - AnalysisRoute::Legacy => syntax_valid(model.config.py(), GENERIC_DIALECT, &checks)?, + AnalysisRoute::Legacy => { + let statements: Vec = + checks.iter().map(|site| site.check.clone()).collect(); + syntax_valid(model.config.py(), GENERIC_DIALECT, &statements)? + } AnalysisRoute::Native(dialect) => first_gap(dialect, context, &checks)?.is_none(), AnalysisRoute::Unavailable => false, }; @@ -94,19 +129,49 @@ pub(crate) fn skip_unreadable( context: &LoopContext<'_, '_>, model: &OptOutModel<'_, '_>, ) -> PyResult { - let checks: Vec = model_checks(context, model)?; - let Some(gap) = first_gap(dialect, context, &checks)? else { + let checks: Vec = model_checks(context, model)?; + let Some((check, gap)) = first_gap(dialect, context, &checks)? else { return Ok(true); }; + let site: GapSite<'_> = GapSite { + label: check.label.as_deref(), + path: &check.path, + position: gap_position(model, check, &gap), + }; report_skip( settings, context, model, - skipped_analysis(model.name, dialect.name(), &gap), + skipped_analysis(model.name, dialect.name(), &gap, &site), )?; Ok(false) } +/// A hook's gap in its statement; the query's in the model file, when its text maps back there. +fn gap_position(model: &OptOutModel<'_, '_>, check: &SiteCheck, gap: &LocatedGap) -> GapPosition { + let Some(offset) = gap.source_offset else { + return GapPosition::Analysed; + }; + if check.label.is_some() { + return GapPosition::Statement(text_position(&check.check.sql, offset)); + } + model + .authored + .and_then(|authored| { + let query: usize = aligned_offset(&check.check.sql, offset, model.query_sql)?; + let expanded: usize = aligned_offset(model.query_sql, query, authored.macro_sql)?; + authored_model_offset( + model.contents, + authored.query, + (authored.macro_sql, expanded), + authored.passes, + ) + }) + .map_or(GapPosition::Analysed, |file| { + GapPosition::File(text_position(model.contents, file)) + }) +} + /// P026 for a model whose dialect no native parser reads; it warns, or raises, and is not analysed. pub(crate) fn skip_unavailable( settings: ModelLoopSettings, @@ -173,34 +238,47 @@ fn report_skip( fn model_checks( context: &LoopContext<'_, '_>, model: &OptOutModel<'_, '_>, -) -> PyResult> { - let mut statements: Vec = vec![analysis_sql(context, model)?]; +) -> PyResult> { + let mut statements: Vec<(Option, String, String)> = vec![( + None, + model.file_path.to_owned(), + analysis_sql(context, model)?, + )]; for key in HOOK_KEYS { if let Some(hooks) = model.values.get_item(key)? { - statements.extend(hook_statements(context, &hooks)?); + let site: HookSite<'_> = HookSite { + hook_name: key, + path: model.file_path.to_owned(), + label: None, + }; + statements.extend(hook_statements(context, &hooks, site)?); } } let placeholders: Vec<(String, String)> = placeholders(model.values)?; Ok(statements .into_iter() - .map(|sql| SyntaxCheck { - sql, - placeholders: placeholders.clone(), + .map(|(label, path, sql)| SiteCheck { + check: SyntaxCheck { + sql, + placeholders: placeholders.clone(), + }, + label, + path, }) .collect()) } -/// The native parser's first gap across `checks`; none when it reads each one whole. -fn first_gap( +/// The native parser's first gap across `checks`, with its check; none when it reads each whole. +fn first_gap<'a>( dialect: &dyn NativeDialect, context: &LoopContext<'_, '_>, - checks: &[SyntaxCheck], -) -> PyResult> { - for check in checks { - if let Some(gap) = native_gap(dialect, check) + checks: &'a [SiteCheck], +) -> PyResult> { + for site in checks { + if let Some(gap) = native_gap(dialect, &site.check) .map_err(|message| compile_error(context.classes, &message, None))? { - return Ok(Some(gap)); + return Ok(Some((site, gap))); } } Ok(None) @@ -262,21 +340,53 @@ fn opted_out(values: &Bound<'_, PyDict>) -> PyResult { Ok(false) } -/// `hook_sql_statements`: strings, SQL hook statements, and nested lists or tuples, in order. +/// `hook_sql_statements` as `(label, file, SQL)`, labelled as `validate_hook_sql_syntax` names them. fn hook_statements( context: &LoopContext<'_, '_>, value: &Bound<'_, PyAny>, -) -> PyResult> { + site: HookSite<'_>, +) -> PyResult, String, String)>> { + let sql_hook_entry: &Bound<'_, PyAny> = &context.classes.sql_hook_entry; if let Ok(text) = value.downcast::() { - return Ok(vec![text.extract()?]); + let label: String = site.label.unwrap_or_else(|| site.hook_name.to_owned()); + return Ok(vec![(Some(label), site.path, text.extract()?)]); } - if value.is_instance(&context.classes.sql_hook_entry)? { - return hook_statements(context, &native_field(value, STATEMENT_ATTRIBUTE)?); + if value.is_instance(sql_hook_entry)? { + let label: String = match site.label { + Some(label) => label, + None => sql_entry_label(site.hook_name, value)?, + }; + let relative_path: Bound<'_, PyAny> = native_field(value, RELATIVE_PATH_ATTRIBUTE)?; + let path: String = if relative_path.is_truthy()? { + relative_path.str()?.to_string() + } else { + site.path + }; + let inner: HookSite<'_> = HookSite { + hook_name: site.hook_name, + path, + label: Some(label), + }; + return hook_statements(context, &native_field(value, STATEMENT_ATTRIBUTE)?, inner); } - let mut statements: Vec = Vec::new(); + let mut statements: Vec<(Option, String, String)> = Vec::new(); if value.is_instance_of::() || value.is_instance_of::() { - for item in value.try_iter()? { - statements.extend(hook_statements(context, &item?)?); + for (index, item) in value.try_iter()?.enumerate() { + let item: Bound<'_, PyAny> = item?; + let prefix: String = format!("{}[{index}]", site.hook_name); + let label: Option = if item.is_instance(sql_hook_entry)? { + Some(sql_entry_label(&prefix, &item)?) + } else if item.is_instance_of::() { + Some(format!("{prefix} inline_sql(\"...\")")) + } else { + None + }; + let inner: HookSite<'_> = HookSite { + hook_name: site.hook_name, + path: site.path.clone(), + label, + }; + statements.extend(hook_statements(context, &item, inner)?); } } Ok(statements) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs index b36fb23fb4..1a14e9d133 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs @@ -64,7 +64,8 @@ fn parse_error(context: &SyntaxContext<'_, '_>, sql: &str) -> PyResult) -> PyResult { +/// How errors name a SQL hook entry: `{prefix} sql("name")`, or `{prefix} inline_sql("...")`. +pub(crate) fn sql_entry_label(prefix: &str, entry: &Bound<'_, PyAny>) -> PyResult { let name: Bound<'_, PyAny> = native_field(entry, "name")?; Ok(if name.is_none() { format!("{prefix} inline_sql(\"...\")") From 4409f42fae12a5ba02fbe820214a794d5a66a894 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:34:52 +0000 Subject: [PATCH 24/29] feat(analysis): let a native dialect respell session column types NativeDialect gains session_type, the identity by default: the session schema and revalidation give each known column type to the parser in the spelling the dialect reads, for dialects whose parser cannot read every type SQLBuild keeps. --- .../analysis_route/_helpers/revalidation.rs | 7 ++++--- .../src/analysis_route/types.rs | 5 +++++ .../analysis_session/_helpers/mappings.rs | 20 ++++++++++++++----- .../analysis_session/_helpers/session.rs | 2 +- 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/revalidation.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/revalidation.rs index 44d3976120..96530c5d28 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/_helpers/revalidation.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/revalidation.rs @@ -49,7 +49,7 @@ pub(crate) fn revised( .map(|(relation, columns)| { ( relation.clone(), - schema_columns(relation, columns, poisoned), + schema_columns(dialect, relation, columns, poisoned), ) }) .collect::>>(), @@ -80,8 +80,9 @@ pub(crate) fn unrevised(model: &RecoveryModel) -> Vec { .collect() } -/// `relation`'s columns as the session bound them, each poisoned or unknown one untyped. +/// `relation`'s columns as the session bound them and `dialect` reads them, poisoned ones untyped. fn schema_columns( + dialect: &dyn NativeDialect, relation: &str, columns: &Pairs, poisoned: &[(String, String)], @@ -96,7 +97,7 @@ fn schema_columns( schema.push(SchemaColumn { name: name.clone(), quoted: false, - data_type: (!untyped).then(|| data_type.clone()), + data_type: (!untyped).then(|| dialect.session_type(data_type)), nullability: Nullability::Unknown, }); } diff --git a/crates/sqlbuild-analysis/src/analysis_route/types.rs b/crates/sqlbuild-analysis/src/analysis_route/types.rs index d28a0b24fc..d22d4c9a31 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/types.rs @@ -29,6 +29,11 @@ pub trait NativeDialect: Sync { /// A type as written in this dialect, as SQLBuild compares it; `None` when unknown or refused. fn normalize_type(&self, spelling: &str) -> Option; + /// A session column type, spelled as SQLBuild keeps it, as this dialect's parser reads it. + fn session_type(&self, spelling: &str) -> String { + spelling.to_owned() + } + /// The one statement of `sql` when the parser reads it whole: no gap, and not raw text. fn read_whole(&self, sql: &str) -> Result { let parsed: Parsed = self.parse(sql)?; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mappings.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mappings.rs index d34f769e55..fc2bdaeb87 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mappings.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mappings.rs @@ -6,6 +6,7 @@ use std::sync::OnceLock; use sha2::{Digest, Sha256}; use sqlquack_semantics::lineage::models::{Nullability, SchemaColumn}; +use crate::analysis_route::types::NativeDialect; use crate::assembly::analysis_session::constants::{DIALECT_ALIASES, UNKNOWN_TYPE}; use crate::assembly::analysis_session::models::{ModelRequest, SessionSchema}; use crate::assembly::analysis_session::types::{Pairs, Shapes}; @@ -241,27 +242,36 @@ pub(crate) fn waves(producers: &[Vec]) -> Option>> { } /// Each relation with a known shape, its columns typed and with the nullability the session knows. -pub(crate) fn session_schema(schema: &Shapes, nullability: &ShapeTable) -> SessionSchema { +pub(crate) fn session_schema( + dialect: &dyn NativeDialect, + schema: &Shapes, + nullability: &ShapeTable, +) -> SessionSchema { let relations: HashMap> = schema .iter() .filter(|(_, columns)| !columns.is_empty()) .map(|(relation, columns)| { - let typed: Vec = typed_columns(columns, nullability.get(relation)); + let typed: Vec = + typed_columns(dialect, columns, nullability.get(relation)); (relation.clone(), typed) }) .collect(); SessionSchema { relations } } -/// Each column with its known type and the nullability `known` spells for it. -fn typed_columns(columns: &Pairs, known: Option<&Pairs>) -> Vec { +/// Each column with its known type as `dialect` reads it, and the nullability `known` spells. +fn typed_columns( + dialect: &dyn NativeDialect, + columns: &Pairs, + known: Option<&Pairs>, +) -> Vec { columns .iter() .map(|(name, data_type)| SchemaColumn { name: name.clone(), quoted: false, data_type: (!data_type.is_empty() && data_type != UNKNOWN_TYPE) - .then(|| data_type.clone()), + .then(|| dialect.session_type(data_type)), nullability: column_nullability(known, name), }) .collect() diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs index 4e49c054a7..97dcb6abc7 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs @@ -337,7 +337,7 @@ impl AnalysisSession { .map(|position| { let model: &ModelRequest = &request.models[models[*position]]; let schema: Shapes = schemas[*position].clone(); - let relations: SessionSchema = session_schema(&schema, nullability); + let relations: SessionSchema = session_schema(dialect, &schema, nullability); model_outcome(dialect, model, (schema, &relations), &declared) }) .collect() From b7b6cd65c6cc6f5a7bca5ef26fa3235f5221641f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:07:46 +0000 Subject: [PATCH 25/29] fix(analysis): give native routes the session's functions, quoting and case setting A native route analysed every model against a schema with no function facts, quoted output names lost their quoting, and Snowflake's QUOTED_IDENTIFIERS_IGNORE_CASE was ignored. - The session schema now answers the adapter's declared function return types, respelt for the dialect, and its first_arg, conditional_result and safe_cast nullability rules. - A stored name in double quotes becomes a quoted schema column, with "" read as one quote, wherever the session, revalidation or fast lineage builds one. - The session request carries the profile's quoted-identifier case setting into the cache key and each dialect's analysis, so a dialect can pick a profile for it. - Semantic completion no longer panics when the binding catalog is already borrowed: unknown relations stay per-request. - An adapter's own Python nullability rules are not called on a native route; compile now warns, naming the adapter and the functions. The callback tests run on the Polyglot route, and the payload-only completion check expects a native route to require its session. --- .../_helpers/native_analysis.rs | 5 +- .../analysis_route/_helpers/revalidation.rs | 18 ++-- .../src/analysis_route/main/fast_lineage.rs | 16 ++-- .../src/analysis_route/main/mod.rs | 2 + .../src/analysis_route/main/model_outcome.rs | 3 +- .../main/native_revalidation.rs | 2 +- .../src/analysis_route/main/schema_column.rs | 25 ++++++ .../main/unapplied_nullability_rules.rs | 18 ++++ .../src/analysis_route/models.rs | 2 + .../src/analysis_route/tests/helpers.rs | 16 ++-- .../tests/test_native_dialect.rs | 52 ++++++++++- .../tests/test_native_outcome.rs | 86 +++++++++++++++++-- .../tests/test_native_revalidation.rs | 4 +- .../src/analysis_route/tests/test_types.rs | 13 ++- .../src/analysis_route/types.rs | 6 ++ .../_helpers/analysis_cache.rs | 1 + .../analysis_session/_helpers/mappings.rs | 60 ++++++++++--- .../analysis_session/_helpers/session.rs | 16 +++- .../src/assembly/analysis_session/models.rs | 34 +++++++- .../analysis_session/tests/helpers.rs | 7 +- .../_helpers/analysis/binding_catalog.rs | 15 ++-- .../analysis_session/model_analyses.rs | 1 + .../analysis_session/profile_inputs.rs | 2 +- .../_helpers/analysis_session/session.rs | 3 + .../compile_driver/assembly_analysis.rs | 42 ++++++++- .../src/bindings/_helpers/native_host/host.rs | 9 ++ .../compiler/analysis_session/_test_types.py | 2 +- .../compiler/analysis_session/helpers.py | 7 +- .../compiler/semantic_checks/helpers.py | 12 +++ .../test_native_semantic_checks.py | 11 ++- 30 files changed, 420 insertions(+), 70 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/analysis_route/main/schema_column.rs create mode 100644 crates/sqlbuild-analysis/src/analysis_route/main/unapplied_nullability_rules.rs diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/native_analysis.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/native_analysis.rs index 3f6577207f..417cb2deff 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/_helpers/native_analysis.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/native_analysis.rs @@ -27,8 +27,9 @@ pub(crate) fn native_analysis( statement: &Expression, input: &AnalysisInput<'_>, ) -> NativeAnalysis { + let profile: &'static dyn DialectProfile = dialect.profile_for(input); let context: LineageContext<'_> = LineageContext { - profile: dialect.profile(), + profile, schema: input.schema, sql: Some(input.sql), }; @@ -42,7 +43,7 @@ pub(crate) fn native_analysis( lineage: lineage .outputs .iter() - .map(|output| lineage_row(dialect.profile(), output, input.references)) + .map(|output| lineage_row(profile, output, input.references)) .collect(), row_sources: lineage .row_sources diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/revalidation.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/revalidation.rs index 96530c5d28..96c507d37d 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/_helpers/revalidation.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/revalidation.rs @@ -5,6 +5,7 @@ use std::collections::HashMap; use sqlquack_semantics::lineage::models::{Nullability, SchemaColumn}; +use crate::analysis_route::main::schema_column::schema_column; use crate::analysis_route::models::{AnalysisInput, NativeAnalysis, ParserGap}; use crate::analysis_route::types::NativeDialect; use crate::assembly::analysis_session::constants::UNKNOWN_TYPE; @@ -38,7 +39,7 @@ pub(crate) fn revised( dialect: &dyn NativeDialect, outcome: &ModelOutcome, poisoned: &[(String, String)], - declared: &[String], + finished: &FinishedSession, ) -> Result>, String> { let sql: Cow<'_, str> = resolved_cleaned_sql(outcome)?; let relations: SessionSchema = SessionSchema { @@ -53,12 +54,14 @@ pub(crate) fn revised( ) }) .collect::>>(), + functions: finished.functions.clone(), }; let analysed: Result = dialect.analyse(&AnalysisInput { sql: &sql, schema: &relations, references: &[], - extension_functions: declared, + extension_functions: &finished.declared_functions, + quoted_ignore_case: finished.quoted_ignore_case, }); let Ok(analysis) = analysed else { return Ok(None); @@ -94,12 +97,11 @@ fn schema_columns( || poisoned .iter() .any(|(model, output)| model == relation && output == name); - schema.push(SchemaColumn { - name: name.clone(), - quoted: false, - data_type: (!untyped).then(|| dialect.session_type(data_type)), - nullability: Nullability::Unknown, - }); + schema.push(schema_column( + name, + (!untyped).then(|| dialect.session_type(data_type)), + Nullability::Unknown, + )); } schema } diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/fast_lineage.rs b/crates/sqlbuild-analysis/src/analysis_route/main/fast_lineage.rs index 3f3fbf2627..bcde53cba0 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/fast_lineage.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/fast_lineage.rs @@ -8,13 +8,14 @@ use sqlquack_semantics::lineage::models::{Nullability, SchemaColumn}; use crate::analysis_route::_helpers::located_gap::{located, unreadable_reason}; use crate::analysis_route::_helpers::marker_names::{distinct_calls, first_call_names}; +use crate::analysis_route::main::schema_column::schema_column; use crate::analysis_route::models::{AnalysisInput, NativeAnalysis, ParserGap}; use crate::analysis_route::types::NativeDialect; use crate::assembly::analysis_session::constants::{ CONFIDENCE_HIGH, CONFIDENCE_MEDIUM, TRANSFORM_AGGREGATION, TRANSFORM_CAST, TRANSFORM_CONSTANT, TRANSFORM_DIRECT, TRANSFORM_STAR, }; -use crate::assembly::analysis_session::models::{LineageRow, SessionSchema}; +use crate::assembly::analysis_session::models::{LineageRow, SessionFunctions, SessionSchema}; use crate::lineage::models::{ FastLineageOutcome, LineageColumn, LineageConfidence, LineageResourceType, LineageSchemaResource, LineageSource, LineageTransformKind, @@ -74,6 +75,7 @@ pub fn fast_lineage( schema: &relations, references: &references, extension_functions: &[], + quoted_ignore_case: false, }); match analysed { Ok(native) => FastLineageOutcome::Built { @@ -106,16 +108,14 @@ fn markers_schema( resource .columns .iter() - .map(|column| SchemaColumn { - name: column.clone(), - quoted: false, - data_type: None, - nullability: Nullability::Unknown, - }) + .map(|column| schema_column(column, None, Nullability::Unknown)) .collect(), ); } - SessionSchema { relations } + SessionSchema { + relations, + functions: SessionFunctions::default(), + } } /// A session lineage row as a fast lineage column; sources of other resource kinds are dropped. diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs b/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs index 5783055737..d85ed3ce17 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs @@ -5,5 +5,7 @@ pub mod fast_lineage; pub(crate) mod model_outcome; pub mod native_gap; pub mod native_revalidation; +pub mod schema_column; pub mod skipped_analysis; +pub mod unapplied_nullability_rules; pub mod unavailable_analysis; diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/model_outcome.rs b/crates/sqlbuild-analysis/src/analysis_route/main/model_outcome.rs index 1395539f72..504ab04f81 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/model_outcome.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/model_outcome.rs @@ -15,7 +15,7 @@ pub(crate) fn model_outcome( dialect: &dyn NativeDialect, model: &ModelRequest, (schema, relations): (Shapes, &SessionSchema), - extension_functions: &[String], + (extension_functions, quoted_ignore_case): (&[String], bool), ) -> (ModelOutcome, Option) { let (cleaned_sql, analysed): (String, Result) = match normalize_native_sql(&model.query_sql, dialect.name(), dialect.stub_prefix()) { @@ -27,6 +27,7 @@ pub(crate) fn model_outcome( schema: relations, references: &model.lineage_references, extension_functions, + quoted_ignore_case, }) .map_err(|gap: ParserGap| gap.message); (sql, analysed) diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/native_revalidation.rs b/crates/sqlbuild-analysis/src/analysis_route/main/native_revalidation.rs index 67ae52f5cd..35cd3db370 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/native_revalidation.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/native_revalidation.rs @@ -24,7 +24,7 @@ pub fn native_revalidation( .par_iter() .map(|model| { let outcome: &ModelOutcome = session_outcome(finished, outcomes, &model.name)?; - match revised(dialect, outcome, poisoned, &finished.declared_functions) { + match revised(dialect, outcome, poisoned, finished) { Ok(Some(rows)) => Ok(rows), Ok(None) => Ok(unrevised(model)), Err(reason) => Err(format!("{}: {reason}", model.name)), diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/schema_column.rs b/crates/sqlbuild-analysis/src/analysis_route/main/schema_column.rs new file mode 100644 index 0000000000..d2a89894a0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/main/schema_column.rs @@ -0,0 +1,25 @@ +use sqlquack_semantics::lineage::models::{Nullability, SchemaColumn}; + +use crate::assembly::analysis_session::constants::QUOTED_IDENTIFIER_DELIMITER; + +/// A stored column name as SQLQuack reads it: `"…"` is quoted, its inner `""` one quote. +#[must_use] +pub fn schema_column( + name: &str, + data_type: Option, + nullability: Nullability, +) -> SchemaColumn { + let delimiter: String = QUOTED_IDENTIFIER_DELIMITER.to_string(); + let inner: Option<&str> = name + .strip_prefix(QUOTED_IDENTIFIER_DELIMITER) + .and_then(|rest| rest.strip_suffix(QUOTED_IDENTIFIER_DELIMITER)); + SchemaColumn { + name: inner.map_or_else( + || name.to_owned(), + |inner| inner.replace(&delimiter.repeat(2), &delimiter), + ), + quoted: inner.is_some(), + data_type, + nullability, + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/unapplied_nullability_rules.rs b/crates/sqlbuild-analysis/src/analysis_route/main/unapplied_nullability_rules.rs new file mode 100644 index 0000000000..1a5b2b593d --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/main/unapplied_nullability_rules.rs @@ -0,0 +1,18 @@ +use crate::analysis_route::models::AnalysisRoute; + +/// The warning for an adapter's own nullability rules on a native route, which does not call them. +#[must_use] +pub fn unapplied_nullability_rules( + route: &AnalysisRoute, + adapter: &str, + functions: &[String], +) -> Option { + match route { + AnalysisRoute::Native(_) if !functions.is_empty() => Some(format!( + "Adapter '{adapter}' defines its own nullability rule for {}; native SQL analysis \ + does not apply it, so these calls take the dialect's own nullability", + functions.join(", ") + )), + _ => None, + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/models.rs b/crates/sqlbuild-analysis/src/analysis_route/models.rs index 838fd60c2c..bc886000ce 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/models.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/models.rs @@ -89,6 +89,8 @@ pub struct AnalysisInput<'a> { pub references: &'a [(String, String, String)], /// Functions the catalogue cannot know, such as MotherDuck's `md_*`; never reported unknown. pub extension_functions: &'a [String], + /// Whether quoted identifiers compare without case, as Snowflake's session parameter sets. + pub quoted_ignore_case: bool, } /// One query's analysis, in the facts the analysis session keeps for a model. diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs index 9dd60dfa2d..ed28a13c28 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs @@ -2,10 +2,11 @@ use std::collections::HashMap; use sqlquack_semantics::lineage::models::{Nullability, SchemaColumn}; +use crate::analysis_route::main::schema_column::schema_column as stored_column; use crate::analysis_route::models::{AnalysisInput, NativeAnalysis, StandInSchema}; use crate::analysis_route::types::NativeDialect; use crate::assembly::analysis_session::models::{ - LineageFacts, LineageRow, ModelAnalysis, ModelOutcome, SessionSchema, + LineageFacts, LineageRow, ModelAnalysis, ModelOutcome, SessionFunctions, SessionSchema, }; use crate::assembly::analysis_session::types::Shapes; use crate::lineage::models::{LineageColumn, LineageSource}; @@ -41,6 +42,7 @@ pub(super) fn analysed( schema: &schema, references: &references, extension_functions: &extension_functions, + quoted_ignore_case: false, }) .expect("the test query is read whole") } @@ -95,17 +97,15 @@ pub(super) fn session_relations(shape: &[(&str, &str, Nullability)]) -> (Shapes, .collect(); ( vec![("raw_orders".to_owned(), pairs)], - SessionSchema { relations }, + SessionSchema { + relations, + functions: SessionFunctions::default(), + }, ) } fn schema_column(&(name, data_type, nullability): &(&str, &str, Nullability)) -> SchemaColumn { - SchemaColumn { - name: name.to_owned(), - quoted: false, - data_type: Some(data_type.to_owned()), - nullability, - } + stored_column(name, Some(data_type.to_owned()), nullability) } /// A finished outcome of `sql` over `orders`, as the session bound it. diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs index 20eabe24a1..bb18b63c67 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs @@ -1,10 +1,11 @@ +use crate::analysis_route::main::unapplied_nullability_rules::unapplied_nullability_rules; use crate::analysis_route::main::unavailable_analysis::unavailable_analysis; -use crate::analysis_route::models::{NativeAnalysis, TypeMatch}; +use crate::analysis_route::models::{AnalysisRoute, NativeAnalysis, TypeMatch}; use crate::analysis_route::tests::helpers::{analysed, resource_columns}; use crate::analysis_route::tests::test_types::{ CataloguedDialect, ColumnTestCase, FindingTestCase, FoldingDialect, LineageTestCase, MacroBodyTestCase, NativeTypeMatchTestCase, PivotlessDialect, ReadWholeTestCase, StarTestCase, - UnavailableAnalysisTestCase, + UnappliedRulesTestCase, UnavailableAnalysisTestCase, }; use crate::analysis_route::types::NativeDialect; @@ -137,6 +138,53 @@ fn given_no_native_parser_when_skipping_analysis_then_p026_names_the_dialect() { } } +#[test] +fn given_adapter_nullability_rules_when_routing_then_only_a_native_route_warns_they_are_unapplied() +{ + let test_cases = [ + UnappliedRulesTestCase { + description: "a native route names the adapter and each function it leaves alone", + route: AnalysisRoute::Native(&PivotlessDialect), + functions: &["UPPER", "IFF"], + expected_warning: Some( + "Adapter 'acme' defines its own nullability rule for UPPER, IFF; native SQL \ + analysis does not apply it, so these calls take the dialect's own nullability", + ), + }, + UnappliedRulesTestCase { + description: "Polyglot calls the adapter's rules, so nothing is unapplied", + route: AnalysisRoute::Legacy, + functions: &["UPPER"], + expected_warning: None, + }, + UnappliedRulesTestCase { + description: "no model is analysed where no parser reads the dialect", + route: AnalysisRoute::Unavailable, + functions: &["UPPER"], + expected_warning: None, + }, + UnappliedRulesTestCase { + description: "an adapter with no rules of its own has nothing to warn about", + route: AnalysisRoute::Native(&PivotlessDialect), + functions: &[], + expected_warning: None, + }, + ]; + for test_case in test_cases { + let functions: Vec = test_case + .functions + .iter() + .map(|f| (*f).to_owned()) + .collect(); + assert_eq!( + unapplied_nullability_rules(&test_case.route, "acme", &functions), + test_case.expected_warning.map(str::to_owned), + "{}", + test_case.description + ); + } +} + #[test] fn given_findings_when_analysing_then_they_take_sqlbuild_codes_and_positions() { let test_cases = [ diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs index 095d840e14..e592759a37 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs @@ -5,12 +5,14 @@ use crate::analysis_route::tests::test_types::{ CataloguedDialect, FoldingDialect, MarkerLineageTestCase, NativeLineageTestCase, NativeOutcomeTestCase, PivotlessDialect, }; -use crate::assembly::analysis_session::models::{ModelOutcome, ModelRequest, SessionSchema}; +use crate::assembly::analysis_session::models::{ + ModelOutcome, ModelRequest, SessionFunctions, SessionSchema, +}; use crate::assembly::analysis_session::types::Shapes; use crate::lineage::models::{ LineageColumn, LineageOutcomeParts, LineageResourceType, LineageSchemaResource, }; -use sqlquack_semantics::lineage::models::Nullability; +use sqlquack_semantics::lineage::models::{Nullability, NullabilityRule}; #[test] fn given_session_shapes_when_a_native_route_analyses_a_model_then_its_facts_follow_them() { @@ -23,6 +25,7 @@ fn given_session_shapes_when_a_native_route_analyses_a_model_then_its_facts_foll ("status", "VARCHAR", Nullability::Nullable), ], declared_functions: &[], + nullability_rules: &[], expected_lines: &[ "order_id INTEGER non_null", "status VARCHAR nullable", @@ -34,6 +37,7 @@ fn given_session_shapes_when_a_native_route_analyses_a_model_then_its_facts_foll sql: "SELECT * FROM raw_orders", shape: &[], declared_functions: &[], + nullability_rules: &[], expected_lines: &["succeeded=true star=true/false findings=[]"], }, NativeOutcomeTestCase { @@ -41,6 +45,7 @@ fn given_session_shapes_when_a_native_route_analyses_a_model_then_its_facts_foll sql: "SELECT FROM WHERE", shape: &[("order_id", "INTEGER", Nullability::NonNull)], declared_functions: &[], + nullability_rules: &[], expected_lines: &["succeeded=false star=false/false findings=[]"], }, NativeOutcomeTestCase { @@ -48,6 +53,7 @@ fn given_session_shapes_when_a_native_route_analyses_a_model_then_its_facts_foll sql: "SELECT CURRENT_TIMESTAMP() AS at FROM raw_orders", shape: &[("order_id", "INTEGER", Nullability::NonNull)], declared_functions: &[("CURRENT_TIMESTAMP", "TIMESTAMPTZ")], + nullability_rules: &[], expected_lines: &[ "at TIMESTAMPTZ unknown", "succeeded=true star=false/true findings=[]", @@ -58,15 +64,81 @@ fn given_session_shapes_when_a_native_route_analyses_a_model_then_its_facts_foll sql: "SELECT CURRENT_TIMESTAMP() AS at FROM raw_orders", shape: &[("order_id", "INTEGER", Nullability::NonNull)], declared_functions: &[], + nullability_rules: &[], expected_lines: &[ "at TIMESTAMPTZ unknown", "succeeded=true star=false/true findings=[\"B101\"]", ], }, + NativeOutcomeTestCase { + description: "a declared function's return type types its call", + sql: "SELECT shop_day(order_id) AS d FROM raw_orders", + shape: &[("order_id", "INTEGER", Nullability::NonNull)], + declared_functions: &[("SHOP_DAY", "DATE")], + nullability_rules: &[], + expected_lines: &[ + "d DATE unknown", + "succeeded=true star=false/true findings=[]", + ], + }, + NativeOutcomeTestCase { + description: "a declared function without a return type is untyped", + sql: "SELECT shop_day(order_id) AS d FROM raw_orders", + shape: &[("order_id", "INTEGER", Nullability::NonNull)], + declared_functions: &[("SHOP_DAY", "")], + nullability_rules: &[], + expected_lines: &["d - unknown", "succeeded=true star=false/true findings=[]"], + }, + NativeOutcomeTestCase { + description: "an adapter's first-argument rule keeps a non-null argument non-null", + sql: "SELECT shop_label(status) AS s FROM raw_orders", + shape: &[("status", "VARCHAR", Nullability::NonNull)], + declared_functions: &[("SHOP_LABEL", "VARCHAR")], + nullability_rules: &[("SHOP_LABEL", NullabilityRule::FirstArgument)], + expected_lines: &[ + "s VARCHAR non_null", + "succeeded=true star=false/true findings=[]", + ], + }, + NativeOutcomeTestCase { + description: "without the rule the same call's nullability is unknown", + sql: "SELECT shop_label(status) AS s FROM raw_orders", + shape: &[("status", "VARCHAR", Nullability::NonNull)], + declared_functions: &[("SHOP_LABEL", "VARCHAR")], + nullability_rules: &[], + expected_lines: &[ + "s VARCHAR unknown", + "succeeded=true star=false/true findings=[]", + ], + }, + NativeOutcomeTestCase { + description: "a quoted stored name is the quoted column, its doubled quote one quote", + sql: "SELECT \"Say \"\"hi\"\"\" AS c FROM raw_orders", + shape: &[("\"Say \"\"hi\"\"\"", "VARCHAR", Nullability::NonNull)], + declared_functions: &[], + nullability_rules: &[], + expected_lines: &[ + "c VARCHAR non_null", + "succeeded=true star=false/true findings=[]", + ], + }, ]; for test_case in test_cases { - let (schema, relations): (Shapes, SessionSchema) = session_relations(test_case.shape); + let (schema, mut relations): (Shapes, SessionSchema) = session_relations(test_case.shape); + relations.functions = SessionFunctions { + return_types: test_case + .declared_functions + .iter() + .filter(|(_, data_type)| !data_type.is_empty()) + .map(|(name, data_type)| ((*name).to_owned(), (*data_type).to_owned())) + .collect(), + nullability: test_case + .nullability_rules + .iter() + .map(|(name, rule)| ((*name).to_owned(), *rule)) + .collect(), + }; let model: ModelRequest = ModelRequest { name: "orders".to_owned(), query_sql: test_case.sql.to_owned(), @@ -78,8 +150,12 @@ fn given_session_shapes_when_a_native_route_analyses_a_model_then_its_facts_foll .iter() .map(|(name, _)| (*name).to_owned()) .collect(); - let (outcome, failure): (ModelOutcome, Option) = - model_outcome(&CataloguedDialect, &model, (schema, &relations), &declared); + let (outcome, failure): (ModelOutcome, Option) = model_outcome( + &CataloguedDialect, + &model, + (schema, &relations), + (&declared, false), + ); let analysis = &outcome.analysis; let mut lines: Vec = analysis .columns diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_revalidation.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_revalidation.rs index 0e68c40a87..6ad508bc45 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_revalidation.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_revalidation.rs @@ -5,7 +5,7 @@ use crate::analysis_route::main::native_revalidation::native_revalidation; use crate::analysis_route::tests::helpers::{finding_rows, outcome}; use crate::analysis_route::tests::test_types::{PivotlessDialect, RevalidationTestCase}; use crate::assembly::analysis_session::models::{ - FinishedSession, ModelOutcome, PivotOutcome, PivotTables, + FinishedSession, ModelOutcome, PivotOutcome, PivotTables, SessionFunctions, }; use crate::semantic_checks::models::{RawBinding, RecoveryModel}; use crate::semantic_checks::types::RevisedBinding; @@ -43,6 +43,8 @@ fn given_poisoned_outputs_when_revalidating_natively_then_findings_keep_session_ relations: None, model_names: vec!["other".to_owned(), "downstream".to_owned()], declared_functions: Vec::new(), + functions: SessionFunctions::default(), + quoted_ignore_case: false, }; let outcomes: Vec<(ModelOutcome, PivotOutcome)> = vec![ (outcome("SELECT 1"), PivotOutcome::Absent), diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs index 382cb4133f..1a6567c526 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs @@ -1,10 +1,10 @@ use sqlquack_semantics::dialects::constants::{DUCKDB, POSTGRES, SNOWFLAKE}; use sqlquack_semantics::dialects::types::DialectProfile; -use sqlquack_semantics::lineage::models::Nullability; +use sqlquack_semantics::lineage::models::{Nullability, NullabilityRule}; use sqlquack_syntax::expressions::statements::models::Command; use sqlquack_syntax::{Dialect, Expression, ParseOptions, Parsed, parse_strict}; -use crate::analysis_route::models::{GapSite, LocatedGap, ParserGap, TypeMatch}; +use crate::analysis_route::models::{AnalysisRoute, GapSite, LocatedGap, ParserGap, TypeMatch}; use crate::analysis_route::types::NativeDialect; use crate::type_system::main::normalize_text_type::normalize_text_type; use crate::type_system::models::NormalizedType; @@ -208,6 +208,13 @@ pub(super) struct UnavailableAnalysisTestCase { pub(super) expected_message: &'static str, } +pub(super) struct UnappliedRulesTestCase { + pub(super) description: &'static str, + pub(super) route: AnalysisRoute, + pub(super) functions: &'static [&'static str], + pub(super) expected_warning: Option<&'static str>, +} + /// Each output's name, transform code, confidence code and the columns it reads. type ExpectedLineage = &'static [(&'static str, u8, u8, &'static [&'static str])]; @@ -239,6 +246,8 @@ pub(super) struct NativeOutcomeTestCase { pub(super) shape: &'static [(&'static str, &'static str, Nullability)], /// The adapter's declared functions as `(name, return type)`. pub(super) declared_functions: &'static [(&'static str, &'static str)], + /// The adapter's built-in nullability rules as `(function name, rule)`. + pub(super) nullability_rules: &'static [(&'static str, NullabilityRule)], /// Each output column as `name type nullability`, then success, star flags and finding codes. pub(super) expected_lines: &'static [&'static str], } diff --git a/crates/sqlbuild-analysis/src/analysis_route/types.rs b/crates/sqlbuild-analysis/src/analysis_route/types.rs index d22d4c9a31..f64582c207 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/types.rs @@ -26,6 +26,12 @@ pub trait NativeDialect: Sync { /// The dialect's sqlquack-semantics profile: how its names resolve, compare and type. fn profile(&self) -> &'static dyn DialectProfile; + /// The profile `input` is analysed under; a dialect whose name rules follow a setting overrides it. + fn profile_for(&self, input: &AnalysisInput<'_>) -> &'static dyn DialectProfile { + let _ = input; + self.profile() + } + /// A type as written in this dialect, as SQLBuild compares it; `None` when unknown or refused. fn normalize_type(&self, spelling: &str) -> Option; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs index 757a894a90..a814d3bfaf 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs @@ -222,6 +222,7 @@ fn session_digest(request: &SessionRequest) -> ContentDigest { key.text(ANALYSIS_CACHE_FORMAT); key.text(&request.dialect); key.flag(request.case_sensitive_shapes); + key.flag(request.quoted_ignore_case); key.pairs(&request.function_return_types); key.optional_pairs(request.nullability_rules.as_ref()); key.flag(request.rich_type_inference); diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mappings.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mappings.rs index fc2bdaeb87..a4f2495e7f 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mappings.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mappings.rs @@ -4,11 +4,16 @@ use std::collections::{BTreeSet, HashMap}; use std::sync::OnceLock; use sha2::{Digest, Sha256}; -use sqlquack_semantics::lineage::models::{Nullability, SchemaColumn}; +use sqlquack_semantics::lineage::models::{Nullability, NullabilityRule, SchemaColumn}; +use crate::analysis_route::main::schema_column::schema_column; use crate::analysis_route::types::NativeDialect; -use crate::assembly::analysis_session::constants::{DIALECT_ALIASES, UNKNOWN_TYPE}; -use crate::assembly::analysis_session::models::{ModelRequest, SessionSchema}; +use crate::assembly::analysis_session::constants::{ + CONDITIONAL_RESULT_RULE, DIALECT_ALIASES, FIRST_ARG_RULE, SAFE_CAST_RULE, UNKNOWN_TYPE, +}; +use crate::assembly::analysis_session::models::{ + ModelRequest, SessionFunctions, SessionRequest, SessionSchema, +}; use crate::assembly::analysis_session::types::{Pairs, Shapes}; use crate::semantic_validation::models::Columns; use crate::semantic_validation::types::Relations; @@ -241,11 +246,33 @@ pub(crate) fn waves(producers: &[Vec]) -> Option>> { (scheduled == producers.len()).then_some(waves) } -/// Each relation with a known shape, its columns typed and with the nullability the session knows. +/// The request's function return types, and its built-in nullability rules; adapter callbacks are not. +pub(crate) fn session_functions(request: &SessionRequest) -> SessionFunctions { + SessionFunctions { + return_types: request.function_return_types.clone(), + nullability: request + .nullability_rules + .iter() + .flatten() + .filter_map(|(function, id)| { + let rule: NullabilityRule = match id.as_str() { + FIRST_ARG_RULE => NullabilityRule::FirstArgument, + CONDITIONAL_RESULT_RULE => NullabilityRule::ConditionalResult, + SAFE_CAST_RULE => NullabilityRule::SafeCast, + _ => return None, + }; + Some((function.clone(), rule)) + }) + .collect(), + } +} + +/// Each relation with a known shape, typed with the nullability the session knows, and the functions. pub(crate) fn session_schema( dialect: &dyn NativeDialect, schema: &Shapes, nullability: &ShapeTable, + functions: &SessionFunctions, ) -> SessionSchema { let relations: HashMap> = schema .iter() @@ -256,7 +283,17 @@ pub(crate) fn session_schema( (relation.clone(), typed) }) .collect(); - SessionSchema { relations } + SessionSchema { + relations, + functions: SessionFunctions { + return_types: functions + .return_types + .iter() + .map(|(function, data_type)| (function.clone(), dialect.session_type(data_type))) + .collect(), + nullability: functions.nullability.clone(), + }, + } } /// Each column with its known type as `dialect` reads it, and the nullability `known` spells. @@ -267,12 +304,13 @@ fn typed_columns( ) -> Vec { columns .iter() - .map(|(name, data_type)| SchemaColumn { - name: name.clone(), - quoted: false, - data_type: (!data_type.is_empty() && data_type != UNKNOWN_TYPE) - .then(|| dialect.session_type(data_type)), - nullability: column_nullability(known, name), + .map(|(name, data_type)| { + schema_column( + name, + (!data_type.is_empty() && data_type != UNKNOWN_TYPE) + .then(|| dialect.session_type(data_type)), + column_nullability(known, name), + ) }) .collect() } diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs index 97dcb6abc7..48999b826a 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs @@ -31,7 +31,7 @@ use crate::assembly::analysis_session::_helpers::enrichment::{ Enrichment, EnrichmentInput, enrichment, }; use crate::assembly::analysis_session::_helpers::mappings::{ - ShapeTable, dict_from_pairs, same_keys, same_relations, session_schema, + ShapeTable, dict_from_pairs, same_keys, same_relations, session_functions, session_schema, }; use crate::assembly::analysis_session::_helpers::mappings::{producers, waves}; use crate::assembly::analysis_session::constants::{UNKNOWN_NULLABILITY, UNKNOWN_TYPE}; @@ -39,7 +39,8 @@ use crate::assembly::analysis_session::main::resolved_cleaned_sql::resolved_clea use crate::assembly::analysis_session::models::{ AnalysisSession, ColumnFact, ContractProof, DeferredCleaning, FinishedSession, LineageFacts, LineageRow, ModelAnalysis, ModelOutcome, ModelRequest, Phase, PivotOutcome, PivotTables, - RelationTables, SessionModelFacts, SessionOutcome, SessionRequest, SessionSchema, SessionStep, + RelationTables, SessionFunctions, SessionModelFacts, SessionOutcome, SessionRequest, + SessionSchema, SessionStep, }; use crate::assembly::analysis_session::types::{Pairs, Shapes}; use crate::semantic_validation::main::normalize_native::normalize_native_sql; @@ -181,6 +182,8 @@ impl AnalysisSession { .iter() .map(|(name, _)| name.clone()) .collect(); + let functions: SessionFunctions = session_functions(&request); + let quoted_ignore_case: bool = request.quoted_ignore_case; let finished = FinishedSession { tables: PivotTables { dialect: request.dialect, @@ -195,6 +198,8 @@ impl AnalysisSession { relations, model_names, declared_functions, + functions, + quoted_ignore_case, }; let outcome = SessionOutcome { models, @@ -331,14 +336,17 @@ impl AnalysisSession { .iter() .map(|(name, _)| name.clone()) .collect(); + let functions: SessionFunctions = session_functions(request); let outcomes: Vec<(ModelOutcome, Option)> = pool.install(|| { misses .par_iter() .map(|position| { let model: &ModelRequest = &request.models[models[*position]]; let schema: Shapes = schemas[*position].clone(); - let relations: SessionSchema = session_schema(dialect, &schema, nullability); - model_outcome(dialect, model, (schema, &relations), &declared) + let relations: SessionSchema = + session_schema(dialect, &schema, nullability, &functions); + let analysis = (declared.as_slice(), request.quoted_ignore_case); + model_outcome(dialect, model, (schema, &relations), analysis) }) .collect() }); diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs index 537ed0f04e..6f79793ac2 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use rayon::ThreadPool; use sqlbuild_cache::digest::types::ContentDigest; use sqlbuild_cache::store::models::NativeStore; -use sqlquack_semantics::lineage::models::SchemaColumn; +use sqlquack_semantics::lineage::models::{NullabilityRule, SchemaColumn}; use sqlquack_semantics::lineage::types::LineageSchema; use crate::assembly::analysis_session::_helpers::catalog_state::SessionCatalog; @@ -118,6 +118,10 @@ pub struct FinishedSession { pub(crate) model_names: Vec, /// The adapter's declared functions, which the native analysis knew. pub(crate) declared_functions: Vec, + /// The adapter's function return types and nullability rules the native analysis read. + pub(crate) functions: SessionFunctions, + /// Whether quoted identifiers compared without case in the session's analysis. + pub(crate) quoted_ignore_case: bool, } /// The session's typed result for one compiled model, borrowed from the finished session. @@ -176,6 +180,8 @@ pub struct SessionRequest { pub dialect: String, /// Whether published shapes keep authored quoting, Python's `inferred_binding_shape` test. pub case_sensitive_shapes: bool, + /// Whether quoted identifiers compare without case, as Snowflake's session parameter sets. + pub quoted_ignore_case: bool, pub function_return_types: Pairs, /// Adapter nullability rules as `(function name, rule id)`. pub nullability_rules: Option, @@ -342,11 +348,21 @@ pub struct AnalysisCache { pub(crate) stats: AnalysisCacheStats, } +/// The adapter's function facts a native route gives SQLQuack: return types and nullability rules. +#[derive(Clone, Debug, Default)] +pub(crate) struct SessionFunctions { + /// `(function name, type)` of each function whose return type the adapter declares. + pub(crate) return_types: Vec<(String, String)>, + /// `(function name, rule)` of each built-in nullability rule the adapter names. + pub(crate) nullability: Vec<(String, NullabilityRule)>, +} + /// The relations one model reads, as the session knows them, for a native route's analysis. #[derive(Debug, Default)] pub(crate) struct SessionSchema { /// Columns of each relation with a known shape, by the analysis name the query reads. pub(crate) relations: HashMap>, + pub(crate) functions: SessionFunctions, } impl LineageSchema for SessionSchema { @@ -361,6 +377,22 @@ impl LineageSchema for SessionSchema { }) .map(Vec::as_slice) } + + fn function_type(&self, name: &str) -> Option<&str> { + self.functions + .return_types + .iter() + .find(|(function, _)| function.eq_ignore_ascii_case(name)) + .map(|(_, data_type)| data_type.as_str()) + } + + fn function_nullability(&self, name: &str) -> Option { + self.functions + .nullability + .iter() + .find(|(function, _)| function.eq_ignore_ascii_case(name)) + .map(|(_, rule)| *rule) + } } /// One compile's model analysis, advanced phase by phase. diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs index 3597a8c178..7687674a6d 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs @@ -19,8 +19,8 @@ use crate::assembly::analysis_session::main::take_analysis_cache::take_analysis_ use crate::assembly::analysis_session::models::{ AnalysisCacheStats, AnalysisSession, ColumnFact, ContractProof, DynamicFamily, FinishedSession, LineageRow, ModelColumns, ModelOutcome, ModelReference, ModelRequest, PivotBatchRequest, - PivotModel, PivotOutcome, PivotTables, SessionModelAnalysis, SessionModelFacts, SessionOutcome, - SessionRequest, SessionStep, + PivotModel, PivotOutcome, PivotTables, SessionFunctions, SessionModelAnalysis, + SessionModelFacts, SessionOutcome, SessionRequest, SessionStep, }; use crate::assembly::analysis_session::tests::test_types::{ CachedRun, ColumnView, ModelSpec, RecoveredFacts, @@ -121,6 +121,7 @@ pub(crate) fn orders_request(models: Vec) -> SessionRequest { SessionRequest { dialect: "duckdb".to_owned(), case_sensitive_shapes: false, + quoted_ignore_case: false, function_return_types: Vec::new(), nullability_rules: Some(Vec::new()), nullability_callback: None, @@ -402,6 +403,8 @@ pub(crate) fn finished_session(tables: PivotTables) -> FinishedSession { relations: None, model_names: Vec::new(), declared_functions: Vec::new(), + functions: SessionFunctions::default(), + quoted_ignore_case: false, } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs index 5a0dab6dd3..3aa2be4bab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs @@ -109,7 +109,7 @@ impl BindingCatalog { }) } - /// `BindingCatalog.prepare`: add unknown closed relations, then override changed ones. + /// `BindingCatalog.prepare`: add unknown closed relations (overrides while in use), then changed ones. fn prepared( &self, py: Python<'_>, @@ -121,16 +121,21 @@ impl BindingCatalog { let schema: Bound<'_, PyAny> = native_field(request, "schema")?; for item in mapping_items(&schema)?.try_iter()? { let (name, columns): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; - if columns.is_truthy()? && !schemas.contains(&name)? { + if columns.is_truthy()? + && !schemas.contains(&name)? + && !additions.contains(&name)? + { additions.set_item(&name, &columns)?; - schemas.set_item(&name, &columns)?; } } } - if !additions.is_empty() { + if !additions.is_empty() + && let Ok(mut native) = self.native.try_borrow_mut(py) + { self.extended.store(true, Ordering::Relaxed); + schemas.update(additions.as_mapping())?; let relations: Relations = conversions::relations(additions.as_any())?; - self.native.borrow_mut(py).inner.update_relations(relations); + native.inner.update_relations(relations); } let mut prepared: Vec = Vec::with_capacity(requests.len()); for request in requests { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index e964962a2e..cd4096e763 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -129,6 +129,7 @@ fn session_request<'py>( profile, &native_field(profile, "sql_analysis_dialect")?, )?, + quoted_ignore_case: native_field(profile, "quoted_identifiers_ignore_case")?.is_truthy()?, function_return_types: return_types(profile)?, nullability_rules: Some(rules), rich_type_inference: request.rich_type_inference, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs index 94a47c4c04..045cc2861b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs @@ -15,7 +15,7 @@ use crate::bindings::_helpers::records::fields::mapping_items; use crate::bindings::_helpers::records::ids::RecordId; const GENERIC_DIALECT: &str = "generic"; -const ADAPTER_RULE_ID: &str = "python"; +pub(crate) const ADAPTER_RULE_ID: &str = "python"; /// The nullability rules the native analysis knows, with their native rule ids. const KNOWN_RULES: [(AdapterHost, &str); 3] = [ (AdapterHost::FirstArgNullability, "first_arg"), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs index 83e702475c..8716b69b5e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs @@ -99,6 +99,7 @@ pub(crate) struct ModelRow { pub(crate) struct RequestRow { pub(crate) dialect: String, pub(crate) case_sensitive_shapes: bool, + pub(crate) quoted_ignore_case: bool, pub(crate) function_return_types: Pairs, pub(crate) nullability_rules: Option, pub(crate) rich_type_inference: bool, @@ -456,6 +457,7 @@ fn session_request(request: RequestRow) -> SessionRequest { let RequestRow { dialect, case_sensitive_shapes, + quoted_ignore_case, function_return_types, nullability_rules, rich_type_inference, @@ -469,6 +471,7 @@ fn session_request(request: RequestRow) -> SessionRequest { SessionRequest { dialect, case_sensitive_shapes, + quoted_ignore_case, function_return_types, nullability_rules, nullability_callback: None, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index 642ba9517f..562800f14a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -4,6 +4,9 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple, PyTupleMethods}; +use sqlbuild_analysis::analysis_route::main::analysis_route::analysis_route; +use sqlbuild_analysis::analysis_route::main::unapplied_nullability_rules::unapplied_nullability_rules; +use sqlbuild_analysis::analysis_route::models::AnalysisRoute; use sqlbuild_analysis::assembly::analysis_session::main::native_dialect::native_dialect; use sqlbuild_analysis::assembly::analysis_session::types::Pairs; use sqlbuild_host::compile_context::main::record_compile_cpu_timing::record_compile_cpu_timing; @@ -16,7 +19,9 @@ use crate::bindings::_helpers::analysis::binding_catalog::{ use crate::bindings::_helpers::analysis_session::model_analyses::{ ModelAnalyses, ModelAnalysisRequest, analyze_models, }; -use crate::bindings::_helpers::analysis_session::profile_inputs::text_pairs; +use crate::bindings::_helpers::analysis_session::profile_inputs::{ + ADAPTER_RULE_ID, nullability_rules, text_pairs, +}; use crate::bindings::_helpers::analysis_session::stage_functions::expression_source_shapes; use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::compile_context::compile_timings::PythonProcessClock; @@ -27,6 +32,7 @@ use crate::bindings::_helpers::compile_driver::stage_work::{ }; use crate::bindings::_helpers::compiled_views::model_input_view::attached_schema; use crate::bindings::_helpers::compiled_views::source_views::read_source_view; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::project_assembly::declared_relations::{ DeclaredRelations, DeclaredShapes, declared_relations, @@ -45,6 +51,7 @@ use std::collections::HashMap; const GENERIC_DIALECT: &str = "generic"; const UDF_PREFIX: &str = "__sqlbuild_udf_"; +const COMPILE_LOGGER: &str = "sqlbuild.compile"; /// The analysis state every assembled model and the project share. pub(crate) struct AnalysisContext<'py> { @@ -278,6 +285,38 @@ fn function_return_types<'py>( Ok(types) } +/// Warn that a native route leaves the adapter's own nullability rules uncalled. +fn warn_unapplied_nullability_rules( + inputs: &Bound<'_, PyAny>, + profile: &Bound<'_, PyAny>, +) -> PyResult<()> { + let (rows, adapter_rules) = nullability_rules(profile)?; + if adapter_rules.is_none() { + return Ok(()); + } + let mut functions: Vec = rows + .into_iter() + .filter(|(_, id)| id == ADAPTER_RULE_ID) + .map(|(function, _)| function) + .collect(); + functions.sort(); + let dialect: Option = native_field(profile, "sql_analysis_dialect")?.extract()?; + let adapter: Option = resolve_effective_adapter_name( + &native_field(inputs, "project_config")?, + &native_field(inputs, "local_config")?, + )? + .extract()?; + let route: AnalysisRoute = analysis_route(dialect.as_deref()); + let named: &str = adapter + .as_deref() + .or(dialect.as_deref()) + .unwrap_or(GENERIC_DIALECT); + match unapplied_nullability_rules(&route, named, &functions) { + Some(warning) => host::log_warning_message(inputs.py(), COMPILE_LOGGER, &warning), + None => Ok(()), + } +} + /// Record each expression source's analysed shape as its declared relation. fn record_expression_shapes<'py>( request: &AssemblyRequest, @@ -391,6 +430,7 @@ fn model_analyses<'py>( return Ok(None); } let inputs: &Bound<'py, PyAny> = request.inputs.bind(py); + warn_unapplied_nullability_rules(inputs, &context.profile)?; let names: &Bound<'py, PyAny> = request.analysis_model_names.bind(py); let mode: &Bound<'py, PyAny> = request.column_lineage_mode.bind(py); let lineage_mode: Bound<'py, PyAny> = record_class(py, RecordId::ColumnLineageMode)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs index babcc38c91..84d87dcdcb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs @@ -97,3 +97,12 @@ pub(crate) fn log_debug_message(py: Python<'_>, logger: &str, message: &str) -> .call((), Some(&kwargs)) .map(|_| ()) } + +/// `logging.getLogger(logger).warning(message)`. +pub(crate) fn log_warning_message(py: Python<'_>, logger: &str, message: &str) -> PyResult<()> { + py.import("logging")? + .getattr("getLogger")? + .call1((logger,))? + .call_method1("warning", (message,)) + .map(|_| ()) +} diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py b/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py index 350086450d..3d3515d7c3 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py @@ -88,7 +88,7 @@ class CteFactRecoveryParityTestCase: @dataclass(frozen=True) class AdapterRuleCallbackTestCase: - """A project-local adapter rule the native session calls back into, and what it raises.""" + """A project-local adapter rule the Polyglot route calls back into, and what it raises.""" description: str seed: int diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index a4f2c1993f..7ffe4d1f00 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -757,7 +757,10 @@ def __call__(self, arguments: tuple[InferredNullability, ...]) -> InferredNullab def adapter_rule_project( *, test_case: AdapterRuleCallbackTestCase, rule: RecordingNullabilityRule, tmp_path: Path ) -> tuple[CompileInputsView, ExpressionInferenceProfile]: - """A generated project with the adapter-rule models, and a profile calling `rule`.""" + """A generated project with the adapter-rule models, and a Polyglot profile calling `rule`. + + A native route does not call an adapter's own rules; compile warns instead. + """ return ( compile_inputs( @@ -770,7 +773,7 @@ def adapter_rule_project( }, ), ExpressionInferenceProfile( - sql_analysis_dialect="duckdb", + sql_analysis_dialect=None, function_nullability_rules={"UPPER": rule, "IFF": rule}, ), ) diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py index a0488f1fa2..9aba77650f 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py @@ -26,6 +26,7 @@ if TYPE_CHECKING: from sqlbuild._native import DiagnosticRow +_NATIVE_SESSION_REQUIRED: str = "revalidating a native route without its session" _RELATIONS: dict[str, tuple[str, ...]] = { '__source("raw_orders")': ( "order_id", @@ -262,6 +263,17 @@ def without_session(inputs: SemanticInputs) -> SemanticInputs: return replace(inputs, session=None) +def payload_completion_view(inputs: SemanticInputs) -> tuple[object, ...] | None: + """The completion from payload facts alone; `None` where a native route requires the session.""" + + try: + completed: CompiledProjectView = completed_natively(without_session(inputs)) + except ValueError as error: + assert _NATIVE_SESSION_REQUIRED in str(error) + return None + return completion_view(completed) + + def session_fact_models(inputs: SemanticInputs) -> frozenset[str]: """Models whose output names and lineage the finished session supplies to the stage.""" diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py index 1f2c4f7dcb..da2259b155 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py @@ -38,12 +38,12 @@ native_completion, note_kinds, outcome_label, + payload_completion_view, proven_output_count, semantic_outcome, session_corpus_files, session_fact_models, with_dialect, - without_session, ) _NOTE_KINDS: tuple[str, ...] = ("downstream output uses", "downstream uses of", " has: ") @@ -261,14 +261,14 @@ def test_given_compiles_when_completing_from_the_session_then_matches_payload_an selections: list[frozenset[str]] = [session_fact_models(inputs) for inputs in captured] codes: Counter[str] = Counter() golden: list[GoldenEntry] = [] - payload_views: list[tuple[object, ...]] = [] + payload_views: list[tuple[object, ...] | None] = [] session_views: list[tuple[object, ...]] = [] for index, inputs in enumerate(captured): completed: CompiledProjectView = completed_natively(inputs) codes.update(item.code for item in completed.diagnostics) codes.update(note_kinds(completed, _NOTE_KINDS)) session_views.append(completion_view(completed)) - payload_views.append(completion_view(completed_natively(without_session(inputs)))) + payload_views.append(payload_completion_view(inputs)) golden.append( golden_entry( "completion", session_views[-1], masked=(str(tmp_path / f"project_{index}"),) @@ -284,7 +284,10 @@ def test_given_compiles_when_completing_from_the_session_then_matches_payload_an ) == [] ) - assert payload_views == session_views + assert all( + payload in (None, session) + for payload, session in zip(payload_views, session_views, strict=True) + ) assert ( session_models, model_count - session_models, From 99785e65eb0ea607f711934457378ac4c6d3902f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:12:32 +0000 Subject: [PATCH 26/29] fix(analysis): quote an unknown quoted column as written SQLQuack quotes an unknown column's name in lower case, so a quoted "MiSsPeLlEd" was reported as 'misspelled'. A native route now puts the authored name back into the message. --- .../_helpers/native_analysis.rs | 18 ++++++++-- .../src/analysis_route/constants.rs | 2 ++ .../tests/test_native_dialect.rs | 35 +++++++++++++++++-- .../src/analysis_route/tests/test_types.rs | 6 ++++ 4 files changed, 56 insertions(+), 5 deletions(-) diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/native_analysis.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/native_analysis.rs index 417cb2deff..9e2730994f 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/_helpers/native_analysis.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/native_analysis.rs @@ -8,7 +8,9 @@ use sqlquack_semantics::lineage::models::{ use sqlquack_semantics::{statement_lineage, statement_name_findings}; use sqlquack_syntax::Expression; -use crate::analysis_route::constants::{ERROR_SEVERITY, UNKNOWN_FUNCTION_CODE}; +use crate::analysis_route::constants::{ + ERROR_SEVERITY, UNKNOWN_COLUMN_CODE, UNKNOWN_FUNCTION_CODE, +}; use crate::analysis_route::models::{AnalysisInput, NativeAnalysis}; use crate::analysis_route::types::NativeDialect; use crate::assembly::analysis_session::constants::{ @@ -156,7 +158,7 @@ fn diagnostic_row(input: &AnalysisInput<'_>, finding: NameFinding) -> Option, finding: NameFinding) -> Option String { + if finding.code != UNKNOWN_COLUMN_CODE || finding.written.is_empty() { + return finding.message.clone(); + } + finding.message.replacen( + &format!("'{}'", finding.written.to_lowercase()), + &format!("'{}'", finding.written), + 1, + ) +} + /// The function an unknown-function finding names, as written or as its message quotes it. fn function_name(finding: &NameFinding) -> &str { if finding.written.is_empty() { diff --git a/crates/sqlbuild-analysis/src/analysis_route/constants.rs b/crates/sqlbuild-analysis/src/analysis_route/constants.rs index 9e622059ae..70cba12bec 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/constants.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/constants.rs @@ -20,6 +20,8 @@ pub(crate) const NULLABLE_WRAPPER: &str = "Nullable"; pub(crate) const LOW_CARDINALITY_WRAPPER: &str = "LowCardinality"; /// The finding of a function the dialect's catalogue does not know. pub(crate) const UNKNOWN_FUNCTION_CODE: &str = "E202"; +/// The finding of a column no referenced relation has. +pub(crate) const UNKNOWN_COLUMN_CODE: &str = "E201"; /// The severity of every finding the native analysis reports, as the Polyglot path blocks them. pub(crate) const ERROR_SEVERITY: &str = "error"; /// What `macro_body` puts before a body to read it as the one projection of a query. diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs index bb18b63c67..8dbdd79904 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs @@ -3,9 +3,9 @@ use crate::analysis_route::main::unavailable_analysis::unavailable_analysis; use crate::analysis_route::models::{AnalysisRoute, NativeAnalysis, TypeMatch}; use crate::analysis_route::tests::helpers::{analysed, resource_columns}; use crate::analysis_route::tests::test_types::{ - CataloguedDialect, ColumnTestCase, FindingTestCase, FoldingDialect, LineageTestCase, - MacroBodyTestCase, NativeTypeMatchTestCase, PivotlessDialect, ReadWholeTestCase, StarTestCase, - UnappliedRulesTestCase, UnavailableAnalysisTestCase, + CataloguedDialect, ColumnTestCase, FindingMessageTestCase, FindingTestCase, FoldingDialect, + LineageTestCase, MacroBodyTestCase, NativeTypeMatchTestCase, PivotlessDialect, + ReadWholeTestCase, StarTestCase, UnappliedRulesTestCase, UnavailableAnalysisTestCase, }; use crate::analysis_route::types::NativeDialect; @@ -138,6 +138,35 @@ fn given_no_native_parser_when_skipping_analysis_then_p026_names_the_dialect() { } } +#[test] +fn given_an_unknown_quoted_column_when_analysing_then_its_message_quotes_the_name_as_written() { + let test_cases = [ + FindingMessageTestCase { + description: "a quoted name keeps its authored case", + sql: "SELECT \"MiSsPeLlEd\" FROM orders", + expected_messages: &[("B002", "Unknown column 'MiSsPeLlEd' in table 'orders'")], + }, + FindingMessageTestCase { + description: "a qualified quoted name keeps its authored case", + sql: "SELECT o.\"Missing\" FROM orders AS o", + expected_messages: &[("B002", "Unknown column 'Missing' in table 'orders'")], + }, + ]; + for test_case in test_cases { + let analysis: NativeAnalysis = analysed(&PivotlessDialect, test_case.sql, &[]); + let messages: Vec<(&str, &str)> = analysis + .diagnostics + .iter() + .map(|row| (row.code.as_str(), row.message.as_str())) + .collect(); + assert_eq!( + messages, test_case.expected_messages, + "{}", + test_case.description + ); + } +} + #[test] fn given_adapter_nullability_rules_when_routing_then_only_a_native_route_warns_they_are_unapplied() { diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs index 1a6567c526..9caa9ddf2c 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs @@ -208,6 +208,12 @@ pub(super) struct UnavailableAnalysisTestCase { pub(super) expected_message: &'static str, } +pub(super) struct FindingMessageTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) expected_messages: &'static [(&'static str, &'static str)], +} + pub(super) struct UnappliedRulesTestCase { pub(super) description: &'static str, pub(super) route: AnalysisRoute, From 9e9d911b794605f222ce84883ffed6e6942ebbff Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:18:21 +0000 Subject: [PATCH 27/29] test(analysis): expect the native refusal to complete without a session as a compiler error --- .../src/sqlbuild/compiler/semantic_checks/helpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py index 9aba77650f..80cd8c7dda 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py @@ -13,6 +13,7 @@ import sqlbuild._native as native_module from sqlbuild._native import ( CompilerDiagnostic, + NativeCompilerError, project_with, ) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile @@ -268,7 +269,7 @@ def payload_completion_view(inputs: SemanticInputs) -> tuple[object, ...] | None try: completed: CompiledProjectView = completed_natively(without_session(inputs)) - except ValueError as error: + except NativeCompilerError as error: assert _NATIVE_SESSION_REQUIRED in str(error) return None return completion_view(completed) From fc816af1a8dde5bf958c2c2bfc6c9c02e3335522 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:34:44 +0000 Subject: [PATCH 28/29] fix(analysis): type a native route's reads of declared but open relations A native route gave SQLQuack only relations whose full shape was known, so a source or seed with declared column types but no authoritative shape was read untyped: `status + 1` over a declared VARCHAR went unreported, where Polyglot's rich analysis used the declared types. The session schema now passes each such relation's declared columns, with the nullability the session knows, as open columns. They type the columns a query reads without making the relation's column list complete, so star expansion and unknown-column checks are unchanged. Revalidation passes the same open columns. The analysis cache format moves to v4, as outcomes for the same inputs change. --- .../analysis_route/_helpers/revalidation.rs | 39 ++++++--- .../src/analysis_route/main/fast_lineage.rs | 1 + .../src/analysis_route/tests/helpers.rs | 1 + .../tests/test_native_outcome.rs | 87 ++++++++++++++++++- .../src/analysis_route/tests/test_types.rs | 9 ++ .../analysis_session/_helpers/mappings.rs | 28 +++--- .../analysis_session/_helpers/session.rs | 5 +- .../assembly/analysis_session/constants.rs | 2 +- .../src/assembly/analysis_session/models.rs | 14 +++ 9 files changed, 160 insertions(+), 26 deletions(-) diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/revalidation.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/revalidation.rs index 96c507d37d..6f5bfde6e3 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/_helpers/revalidation.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/revalidation.rs @@ -42,18 +42,25 @@ pub(crate) fn revised( finished: &FinishedSession, ) -> Result>, String> { let sql: Cow<'_, str> = resolved_cleaned_sql(outcome)?; + let mut relations: HashMap> = HashMap::new(); + let mut open: HashMap> = HashMap::new(); + for (relation, columns) in &outcome.validated_schema { + let (table, columns) = if columns.is_empty() { + match declared_columns(finished, relation) { + Some(declared) => (&mut open, declared), + None => continue, + } + } else { + (&mut relations, columns) + }; + let _ = table.insert( + relation.clone(), + schema_columns(dialect, relation, columns, poisoned), + ); + } let relations: SessionSchema = SessionSchema { - relations: outcome - .validated_schema - .iter() - .filter(|(_, columns)| !columns.is_empty()) - .map(|(relation, columns)| { - ( - relation.clone(), - schema_columns(dialect, relation, columns, poisoned), - ) - }) - .collect::>>(), + relations, + open, functions: finished.functions.clone(), }; let analysed: Result = dialect.analyse(&AnalysisInput { @@ -74,6 +81,16 @@ pub(crate) fn revised( .map(Some) } +/// The declared columns of `relation`, whose full shape the session did not know, when it has any. +fn declared_columns<'a>(finished: &'a FinishedSession, relation: &str) -> Option<&'a Pairs> { + finished + .tables + .column_types + .iter() + .find(|(name, columns)| name == relation && !columns.is_empty()) + .map(|(_, columns)| columns) +} + /// The model's own findings, as type recovery matches them, kept as they are. pub(crate) fn unrevised(model: &RecoveryModel) -> Vec { model diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/fast_lineage.rs b/crates/sqlbuild-analysis/src/analysis_route/main/fast_lineage.rs index bcde53cba0..3b179b4acd 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/fast_lineage.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/fast_lineage.rs @@ -114,6 +114,7 @@ fn markers_schema( } SessionSchema { relations, + open: HashMap::new(), functions: SessionFunctions::default(), } } diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs index ed28a13c28..29831d22d3 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs @@ -99,6 +99,7 @@ pub(super) fn session_relations(shape: &[(&str, &str, Nullability)]) -> (Shapes, vec![("raw_orders".to_owned(), pairs)], SessionSchema { relations, + open: HashMap::new(), functions: SessionFunctions::default(), }, ) diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs index e592759a37..bbeaa37d63 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs @@ -3,8 +3,9 @@ use crate::analysis_route::main::model_outcome::model_outcome; use crate::analysis_route::tests::helpers::{column_line, session_relations}; use crate::analysis_route::tests::test_types::{ CataloguedDialect, FoldingDialect, MarkerLineageTestCase, NativeLineageTestCase, - NativeOutcomeTestCase, PivotlessDialect, + NativeOutcomeTestCase, OpenRelationTestCase, PivotlessDialect, }; +use crate::assembly::analysis_session::_helpers::mappings::{ShapeTable, session_schema}; use crate::assembly::analysis_session::models::{ ModelOutcome, ModelRequest, SessionFunctions, SessionSchema, }; @@ -189,6 +190,90 @@ fn given_session_shapes_when_a_native_route_analyses_a_model_then_its_facts_foll } } +#[test] +fn given_a_relation_with_declared_columns_only_when_analysing_natively_then_they_type_its_reads() { + let test_cases = [ + OpenRelationTestCase { + description: "a declared VARCHAR read in arithmetic is a type error", + sql: "SELECT status + 1 AS bumped FROM raw_orders", + declared: &[("status", "VARCHAR"), ("amount", "DOUBLE")], + expected_lines: &["bumped - unknown", "findings=[\"B212\"]"], + }, + OpenRelationTestCase { + description: "a declared column read as it is keeps its declared type", + sql: "SELECT amount FROM raw_orders", + declared: &[("status", "VARCHAR"), ("amount", "DOUBLE")], + expected_lines: &["amount DOUBLE unknown", "findings=[]"], + }, + OpenRelationTestCase { + description: "an undeclared column of an open relation is still not refused", + sql: "SELECT region FROM raw_orders", + declared: &[("status", "VARCHAR")], + expected_lines: &["region - unknown", "findings=[]"], + }, + OpenRelationTestCase { + description: "with nothing declared, nothing is typed", + sql: "SELECT status + 1 AS bumped FROM raw_orders", + declared: &[], + expected_lines: &["bumped - unknown", "findings=[]"], + }, + ]; + for test_case in test_cases { + let schema: Shapes = vec![("raw_orders".to_owned(), Vec::new())]; + let declared: Shapes = vec![( + "raw_orders".to_owned(), + test_case + .declared + .iter() + .map(|(name, data_type)| ((*name).to_owned(), (*data_type).to_owned())) + .collect(), + )]; + let relations: SessionSchema = session_schema( + &CataloguedDialect, + &schema, + ( + &ShapeTable::from_shapes(&declared), + &ShapeTable::from_shapes(&Vec::new()), + ), + &SessionFunctions::default(), + ); + let model: ModelRequest = ModelRequest { + name: "orders".to_owned(), + query_sql: test_case.sql.to_owned(), + required_names: vec!["raw_orders".to_owned()], + ..ModelRequest::default() + }; + let (outcome, _): (ModelOutcome, Option) = model_outcome( + &CataloguedDialect, + &model, + (schema, &relations), + (&[], false), + ); + let mut lines: Vec = outcome + .analysis + .columns + .iter() + .flatten() + .map(|column| { + format!( + "{} {} {}", + column.name, + column.data_type.as_deref().unwrap_or("-"), + column.nullability + ) + }) + .collect(); + let findings: Vec<&str> = outcome + .analysis + .binding_diagnostics + .iter() + .map(|row| row.code.as_str()) + .collect(); + lines.push(format!("findings={findings:?}")); + assert_eq!(lines, test_case.expected_lines, "{}", test_case.description); + } +} + #[test] fn given_a_model_without_facts_when_fast_lineage_reads_it_natively_then_it_has_lineage_or_a_reason() { diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs index 9caa9ddf2c..0ec0a299b4 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs @@ -258,6 +258,15 @@ pub(super) struct NativeOutcomeTestCase { pub(super) expected_lines: &'static [&'static str], } +pub(super) struct OpenRelationTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + /// `raw_orders`' declared columns and types; its full shape is never known. + pub(super) declared: &'static [(&'static str, &'static str)], + /// Each output column as `name type`, then the finding codes. + pub(super) expected_lines: &'static [&'static str], +} + pub(super) struct NativeLineageTestCase { pub(super) description: &'static str, pub(super) sql: &'static str, diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mappings.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mappings.rs index a4f2495e7f..3a76ba7a57 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mappings.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mappings.rs @@ -267,24 +267,30 @@ pub(crate) fn session_functions(request: &SessionRequest) -> SessionFunctions { } } -/// Each relation with a known shape, typed with the nullability the session knows, and the functions. +/// Each relation's known shape, or else its declared columns, typed, with the session's functions. pub(crate) fn session_schema( dialect: &dyn NativeDialect, schema: &Shapes, - nullability: &ShapeTable, + (types, nullability): (&ShapeTable, &ShapeTable), functions: &SessionFunctions, ) -> SessionSchema { - let relations: HashMap> = schema - .iter() - .filter(|(_, columns)| !columns.is_empty()) - .map(|(relation, columns)| { - let typed: Vec = - typed_columns(dialect, columns, nullability.get(relation)); - (relation.clone(), typed) - }) - .collect(); + let mut relations: HashMap> = HashMap::new(); + let mut open: HashMap> = HashMap::new(); + for (relation, columns) in schema { + let (table, columns) = if columns.is_empty() { + match types.get(relation) { + Some(declared) if !declared.is_empty() => (&mut open, declared), + _ => continue, + } + } else { + (&mut relations, columns) + }; + let typed: Vec = typed_columns(dialect, columns, nullability.get(relation)); + let _ = table.insert(relation.clone(), typed); + } SessionSchema { relations, + open, functions: SessionFunctions { return_types: functions .return_types diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs index 48999b826a..8328531601 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs @@ -330,7 +330,8 @@ impl AnalysisSession { ) -> Result<(), String> { let pool = self.catalog.native.analysis_pool()?; let request: &SessionRequest = &self.request; - let nullability: &ShapeTable = &self.available_nullability; + let tables: (&ShapeTable, &ShapeTable) = + (&self.available_types, &self.available_nullability); let declared: Vec = request .function_return_types .iter() @@ -344,7 +345,7 @@ impl AnalysisSession { let model: &ModelRequest = &request.models[models[*position]]; let schema: Shapes = schemas[*position].clone(); let relations: SessionSchema = - session_schema(dialect, &schema, nullability, &functions); + session_schema(dialect, &schema, tables, &functions); let analysis = (declared.as_slice(), request.quoted_ignore_case); model_outcome(dialect, model, (schema, &relations), analysis) }) diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs index 5da96a56e2..0473e9c3f1 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs @@ -196,7 +196,7 @@ pub(crate) const PYTHON_RULE: &str = "python"; pub(crate) const NON_NULL_NULLABILITY: &str = "non_null"; pub(crate) const NULLABLE_NULLABILITY: &str = "nullable"; /// Bumped whenever the stored outcome or a key's fields change; older entries never match. -pub(crate) const ANALYSIS_CACHE_FORMAT: &str = "native-model-analysis-v3"; +pub(crate) const ANALYSIS_CACHE_FORMAT: &str = "native-model-analysis-v4"; /// A stored varint byte at or above this value continues into the next byte. pub(crate) const VARINT_CONTINUATION: u64 = 0x80; /// The value bits each stored varint byte carries. diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs index 6f79793ac2..1752e43587 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs @@ -362,6 +362,8 @@ pub(crate) struct SessionFunctions { pub(crate) struct SessionSchema { /// Columns of each relation with a known shape, by the analysis name the query reads. pub(crate) relations: HashMap>, + /// Declared columns of each relation whose full shape is not known, which type the ones read. + pub(crate) open: HashMap>, pub(crate) functions: SessionFunctions, } @@ -378,6 +380,18 @@ impl LineageSchema for SessionSchema { .map(Vec::as_slice) } + fn open_columns(&self, relation: &str) -> Option<&[SchemaColumn]> { + self.open + .get(relation) + .or_else(|| { + self.open + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(relation)) + .map(|(_, columns)| columns) + }) + .map(Vec::as_slice) + } + fn function_type(&self, name: &str) -> Option<&str> { self.functions .return_types From 098499d2245739fa515c0eb6131f21cf577d780b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 18:02:57 +0000 Subject: [PATCH 29/29] fix(deps): resolve tempfile against the workspace's windows-sys again The native route's dependency update moved tempfile onto windows-sys 0.52, which sqb and the Python extension then built with different features. --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 7d467327d9..3471841ece 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4773,7 +4773,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]]