diff --git a/Cargo.lock b/Cargo.lock index 4edc9a920b..3471841ece 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4091,7 +4091,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -4142,6 +4142,7 @@ dependencies = [ "sqlbuild-scopes", "sqlbuild-sqltext", "sqlclean", + "sqlquack-semantics", "sqlquack-syntax", "sqlquack-syntax-postgres", ] @@ -4496,6 +4497,37 @@ dependencies = [ "sha2 0.10.9", ] +[[package]] +name = "sqlquack-core" +version = "0.2.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5#19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" +dependencies = [ + "serde", + "serde_json", + "sqlquack-syntax", +] + +[[package]] +name = "sqlquack-semantics" +version = "0.2.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5#19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" +dependencies = [ + "serde", + "serde_json", + "sqlquack-core", + "sqlquack-snowflake", + "sqlquack-syntax", +] + +[[package]] +name = "sqlquack-snowflake" +version = "0.2.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5#19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" +dependencies = [ + "serde", + "sqlquack-core", +] + [[package]] name = "sqlquack-syntax" version = "0.2.0" @@ -4619,7 +4651,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -4741,7 +4773,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5055,7 +5087,7 @@ checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" dependencies = [ "memoffset", "tempfile", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 9e8e7cfd75..c4475e8387 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -176,6 +176,8 @@ sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" } sqlbuild-state = { path = "crates/sqlbuild-state" } # SQL lint Rules, quality Rules and the formatter. sqlclean = { git = "https://github.com/sqlstack-io/sqlclean.git", rev = "cf79656d247277a30bc81ef39b6dee420e721221" } +# Scopes, name resolution, types and lineage for the native analysis route. +sqlquack-semantics = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" } # The lossless SQL tokenizer behind query fingerprints. sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" } sqlquack-syntax-postgres = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" } diff --git a/crates/sqlbuild-analysis/Cargo.toml b/crates/sqlbuild-analysis/Cargo.toml index 80a5ae2313..95c643ebd3 100644 --- a/crates/sqlbuild-analysis/Cargo.toml +++ b/crates/sqlbuild-analysis/Cargo.toml @@ -26,6 +26,7 @@ sqlbuild-render.workspace = true sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true sqlclean.workspace = true +sqlquack-semantics.workspace = true sqlquack-syntax.workspace = true sqlquack-syntax-postgres.workspace = true diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/clickhouse_types.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/clickhouse_types.rs new file mode 100644 index 0000000000..9018bfc67b --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/clickhouse_types.rs @@ -0,0 +1,53 @@ +use crate::analysis_route::constants::{LOW_CARDINALITY_WRAPPER, NULLABLE_WRAPPER}; + +/// `type_sql` without whitespace outside its quoted strings. +pub(crate) fn unspaced(type_sql: &str) -> String { + let mut quoted: bool = false; + type_sql + .chars() + .filter(|character| { + if *character == '\'' { + quoted = !quoted; + } + quoted || !character.is_whitespace() + }) + .collect() +} + +/// The type inside its top-level `Nullable` and `LowCardinality` wrappers, and whether it is nullable. +pub(crate) fn unwrapped(type_sql: &str) -> (&str, bool) { + let mut current: &str = type_sql.trim(); + let mut nullable: bool = false; + loop { + if let Some(inner) = wrapped(current, NULLABLE_WRAPPER) { + nullable = true; + current = inner; + } else if let Some(inner) = wrapped(current, LOW_CARDINALITY_WRAPPER) { + current = inner; + } else { + return (current, nullable); + } + } +} + +/// The argument of `wrapper(...)` when the whole of `type_sql` is that one call. +fn wrapped<'a>(type_sql: &'a str, wrapper: &str) -> Option<&'a str> { + let head: &str = type_sql.get(..wrapper.len())?; + if !head.eq_ignore_ascii_case(wrapper) { + return None; + } + let inner: &str = type_sql[wrapper.len()..] + .trim_start() + .strip_prefix('(')? + .strip_suffix(')')?; + let mut depth: usize = 0; + for character in inner.chars() { + match character { + '(' => depth += 1, + ')' if depth == 0 => return None, + ')' => depth -= 1, + _ => {} + } + } + Some(inner.trim()) +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/located_gap.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/located_gap.rs new file mode 100644 index 0000000000..852c8a28ad --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/located_gap.rs @@ -0,0 +1,70 @@ +//! Where a native parser's gap is, and why the parser cannot read the SQL there. + +use crate::analysis_route::models::{GapPosition, GapSite, LocatedGap, ParserGap}; +use crate::compiled_project::main::text_position::text_position; +use crate::compiled_project::models::TextPosition; + +/// The gap at its line and column in `sql`, and its offset in `source` when the text holds. +pub(crate) fn located(sql: &str, gap: ParserGap, source: Option<&str>) -> LocatedGap { + let before: &str = sql.get(..gap.offset).unwrap_or(sql); + let offset: usize = before.chars().count(); + let position: TextPosition = text_position(sql, offset); + LocatedGap { + construct: gap.construct, + message: gap.message, + line: position.line, + column: position.column, + syntax: gap.syntax, + source_offset: source.and_then(|source| aligned(sql, offset, source)), + } +} + +/// The character of `source` that `derived`'s `offset` came from, when the text before or after holds. +pub(crate) fn aligned(derived: &str, offset: usize, source: &str) -> Option { + let derived: Vec = derived.chars().collect(); + let source: Vec = source.chars().collect(); + if offset > derived.len() { + return None; + } + if source.starts_with(&derived[..offset]) { + return Some(offset); + } + source + .ends_with(&derived[offset..]) + .then(|| source.len() - (derived.len() - offset)) +} + +/// Why the dialect's native parser cannot read SQL, as P026 words it. +pub(crate) fn unreadable_reason(dialect: &str, gap: &LocatedGap) -> String { + unreadable_at(dialect, gap, (gap.line, gap.column, "")) +} + +/// P026's text for a gap at its site: the hook and file, then the parser's reason there. +pub(crate) fn sited_reason(dialect: &str, gap: &LocatedGap, site: &GapSite<'_>) -> String { + let label: String = site + .label + .map_or_else(String::new, |label| format!(" {label}")); + let at: (usize, usize, &str) = match site.position { + GapPosition::File(position) => (position.line, position.column, ""), + GapPosition::Statement(position) => (position.line, position.column, " of its SQL"), + GapPosition::Analysed => (gap.line, gap.column, " of the analysed SQL"), + }; + format!( + "{label} ({}): {}", + site.path, + unreadable_at(dialect, gap, at) + ) +} + +fn unreadable_at(dialect: &str, gap: &LocatedGap, at: (usize, usize, &str)) -> String { + let (line, column, within) = at; + let detail: String = if gap.message.is_empty() || gap.message == gap.construct { + String::new() + } else { + format!(": {}", gap.message) + }; + format!( + "the {dialect} parser cannot read {} at line {line}, column {column}{within}{detail}", + gap.construct + ) +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/marker_names.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/marker_names.rs new file mode 100644 index 0000000000..c5e547fe75 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/marker_names.rs @@ -0,0 +1,50 @@ +//! One analysis name per distinct relation marker call, so repeated calls read as one relation. + +use std::sync::LazyLock; + +use regex::{Captures, Regex}; + +use crate::relation_markers::constants::MARKER_NAME_PREFIX; +use crate::relation_markers::types::MarkerCall; + +static MARKER_NAME: LazyLock> = LazyLock::new(|| { + Regex::new(&format!(r"\b{MARKER_NAME_PREFIX}(\d+)\b")).map_err(|error| error.to_string()) +}); + +/// `named` with each marker name renamed to its call's first index, padded to keep positions. +pub(crate) fn first_call_names(named: &str, calls: &[MarkerCall]) -> Result { + let pattern: &Regex = MARKER_NAME.as_ref().map_err(Clone::clone)?; + Ok(pattern + .replace_all(named, |captures: &Captures<'_>| { + let written: &str = captures.get(0).map_or("", |found| found.as_str()); + let first: Option = match captures.get(1).map(|index| index.as_str().parse()) { + Some(Ok(index)) => first_index(calls, index), + Some(Err(_)) | None => None, + }; + match first { + Some(first) => { + let name: String = format!("{MARKER_NAME_PREFIX}{first}"); + format!("{name: written.to_owned(), + } + }) + .into_owned()) +} + +/// The calls whose index is their call's first, in order, with that index. +pub(crate) fn distinct_calls(calls: &[MarkerCall]) -> Vec<(usize, &MarkerCall)> { + let mut distinct: Vec<(usize, &MarkerCall)> = Vec::with_capacity(calls.len()); + for (index, call) in calls.iter().enumerate() { + if first_index(calls, index) == Some(index) { + distinct.push((index, call)); + } + } + distinct +} + +/// The first index of the call at `index`, which is `index` unless an earlier call is the same. +fn first_index(calls: &[MarkerCall], index: usize) -> Option { + let call: &MarkerCall = calls.get(index)?; + calls.iter().position(|earlier| earlier == call) +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs new file mode 100644 index 0000000000..279c10ae95 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod clickhouse_types; +pub(crate) mod located_gap; +pub(crate) mod marker_names; +pub(crate) mod native_analysis; +pub(crate) mod revalidation; +pub(crate) mod whole_gap; diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/native_analysis.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/native_analysis.rs new file mode 100644 index 0000000000..9e2730994f --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/native_analysis.rs @@ -0,0 +1,202 @@ +//! The provided native analysis: sqlquack-semantics lineage and findings as SQLBuild's facts. + +use sqlquack_semantics::dialects::types::DialectProfile; +use sqlquack_semantics::lineage::models::{ + LineageContext, NameFinding, Nullability, OutputLineage, QueryLineage, SourceColumn, + TransformKind, +}; +use sqlquack_semantics::{statement_lineage, statement_name_findings}; +use sqlquack_syntax::Expression; + +use crate::analysis_route::constants::{ + ERROR_SEVERITY, UNKNOWN_COLUMN_CODE, UNKNOWN_FUNCTION_CODE, +}; +use crate::analysis_route::models::{AnalysisInput, NativeAnalysis}; +use crate::analysis_route::types::NativeDialect; +use crate::assembly::analysis_session::constants::{ + CONFIDENCE_HIGH, CONFIDENCE_MEDIUM, CONFIDENCE_UNKNOWN, NON_NULL_NULLABILITY, + NULLABLE_NULLABILITY, TRANSFORM_AGGREGATION, TRANSFORM_CAST, TRANSFORM_CONSTANT, + TRANSFORM_DIRECT, TRANSFORM_EXPRESSION, UNKNOWN_NULLABILITY, +}; +use crate::assembly::analysis_session::models::{ColumnFact, LineageRow}; +use crate::semantic_validation::main::sqlbuild_code::sqlbuild_code; +use crate::semantic_validation::main::text_findings::text_findings; +use crate::semantic_validation::models::DiagnosticRow; + +/// The analysis of `statement`, the one statement of `input.sql`, under `dialect`'s profile. +pub(crate) fn native_analysis( + dialect: &D, + statement: &Expression, + input: &AnalysisInput<'_>, +) -> NativeAnalysis { + let profile: &'static dyn DialectProfile = dialect.profile_for(input); + let context: LineageContext<'_> = LineageContext { + profile, + schema: input.schema, + sql: Some(input.sql), + }; + let lineage: QueryLineage = statement_lineage(statement, context); + NativeAnalysis { + columns: lineage + .outputs + .iter() + .map(|output| column_fact(dialect, output)) + .collect(), + lineage: lineage + .outputs + .iter() + .map(|output| lineage_row(profile, output, input.references)) + .collect(), + row_sources: lineage + .row_sources + .into_iter() + .map(|source| (source.relation, source.column)) + .collect(), + has_star: lineage.projects_star, + star_resolved: !lineage.open_star, + diagnostics: statement_name_findings(statement, context) + .into_iter() + .filter_map(|finding| diagnostic_row(input, finding)) + .chain(text_findings(input.sql, dialect.name())) + .collect(), + } +} + +/// An output column in the analysis's own type spelling, kept when the dialect can compare it. +fn column_fact(dialect: &D, output: &OutputLineage) -> ColumnFact { + ColumnFact { + name: output.name.clone(), + data_type: output + .data_type + .as_deref() + .filter(|spelling| dialect.normalize_type(spelling).is_some()) + .map(str::to_owned), + nullability: match output.nullability { + Nullability::Unknown => UNKNOWN_NULLABILITY, + Nullability::Nullable => NULLABLE_NULLABILITY, + Nullability::NonNull => NON_NULL_NULLABILITY, + } + .to_owned(), + } +} + +/// An output column's lineage over the model's referenced resources, as the legacy rows give it. +fn lineage_row( + profile: &dyn DialectProfile, + output: &OutputLineage, + references: &[(String, String, String)], +) -> LineageRow { + let transform_code: u8 = match output.kind { + TransformKind::Constant => TRANSFORM_CONSTANT, + TransformKind::Direct => TRANSFORM_DIRECT, + TransformKind::Cast => TRANSFORM_CAST, + TransformKind::Expression => TRANSFORM_EXPRESSION, + TransformKind::Aggregation => TRANSFORM_AGGREGATION, + }; + let mut sources: Vec<(String, String, String)> = Vec::new(); + for source in output + .sources + .iter() + .filter_map(|source| referenced_source(profile, source, references)) + { + if !sources.contains(&source) { + sources.push(source); + } + } + let confidence_code: u8 = if sources.is_empty() && transform_code != TRANSFORM_CONSTANT { + CONFIDENCE_UNKNOWN + } else if output.resolved { + CONFIDENCE_HIGH + } else { + CONFIDENCE_MEDIUM + }; + LineageRow { + output_column: output.name.clone(), + transform_code, + confidence_code, + sources, + } +} + +/// `(resource type, resource name, column)` of a source column, each name resolved as the profile does. +fn referenced_source( + profile: &dyn DialectProfile, + source: &SourceColumn, + references: &[(String, String, String)], +) -> Option<(String, String, String)> { + let (_, resource_type, resource_name) = references.iter().find(|(analysis_name, _, _)| { + *analysis_name == source.relation + || profile.same_identifier( + &profile.resolved_identifier(analysis_name, false), + &source.relation, + ) + })?; + Some(( + resource_type.clone(), + resource_name.clone(), + source.column.clone(), + )) +} + +/// A finding in SQLBuild's code at its position in `input.sql`, unless SQLBuild does not report it. +fn diagnostic_row(input: &AnalysisInput<'_>, finding: NameFinding) -> Option { + if finding.code == UNKNOWN_FUNCTION_CODE + && input + .extension_functions + .iter() + .any(|name| name.eq_ignore_ascii_case(function_name(&finding))) + { + return None; + } + let code: String = sqlbuild_code(finding.code)?; + let (line, column, start, end) = match finding.span { + Some(span) => { + let (line, column) = line_column(input.sql, span.start); + (Some(line), Some(column), Some(span.start), Some(span.end)) + } + None => (None, None, None, None), + }; + Some(DiagnosticRow { + code, + message: authored_message(&finding), + line, + column, + start, + end, + severity: ERROR_SEVERITY.to_owned(), + }) +} + +/// An unknown column's message with a quoted name as written, which SQLQuack quotes in lower case. +fn authored_message(finding: &NameFinding) -> String { + if finding.code != UNKNOWN_COLUMN_CODE || finding.written.is_empty() { + return finding.message.clone(); + } + finding.message.replacen( + &format!("'{}'", finding.written.to_lowercase()), + &format!("'{}'", finding.written), + 1, + ) +} + +/// The function an unknown-function finding names, as written or as its message quotes it. +fn function_name(finding: &NameFinding) -> &str { + if finding.written.is_empty() { + finding.message.split('\'').nth(1).unwrap_or_default() + } else { + &finding.written + } +} + +/// The line and column, from 1, of character `offset` in `sql`. +fn line_column(sql: &str, offset: usize) -> (usize, usize) { + sql.chars() + .take(offset) + .fold((1, 1), |(line, column), character| { + if character == '\n' { + (line + 1, 1) + } else { + (line, column + 1) + } + }) +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/revalidation.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/revalidation.rs new file mode 100644 index 0000000000..6f5bfde6e3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/revalidation.rs @@ -0,0 +1,139 @@ +//! One model's native findings read again over its bound schema with poisoned outputs untyped. + +use std::borrow::Cow; +use std::collections::HashMap; + +use sqlquack_semantics::lineage::models::{Nullability, SchemaColumn}; + +use crate::analysis_route::main::schema_column::schema_column; +use crate::analysis_route::models::{AnalysisInput, NativeAnalysis, ParserGap}; +use crate::analysis_route::types::NativeDialect; +use crate::assembly::analysis_session::constants::UNKNOWN_TYPE; +use crate::assembly::analysis_session::main::resolved_cleaned_sql::resolved_cleaned_sql; +use crate::assembly::analysis_session::models::{ + FinishedSession, ModelOutcome, PivotOutcome, SessionSchema, +}; +use crate::assembly::analysis_session::types::Pairs; +use crate::semantic_checks::models::RecoveryModel; +use crate::semantic_checks::types::RevisedBinding; +use crate::semantic_validation::models::DiagnosticRow; + +/// The finished outcome of the model the session analysed as `name`. +pub(crate) fn session_outcome<'a>( + finished: &FinishedSession, + outcomes: &'a [(ModelOutcome, PivotOutcome)], + name: &str, +) -> Result<&'a ModelOutcome, String> { + let position: Option = finished + .model_names + .iter() + .position(|analysed| analysed == name); + position + .and_then(|position| outcomes.get(position)) + .map(|(outcome, _)| outcome) + .ok_or_else(|| format!("the session analysed no model {name}")) +} + +/// The findings of `outcome`'s SQL with the poisoned columns untyped; `None` when it is unreadable. +pub(crate) fn revised( + dialect: &dyn NativeDialect, + outcome: &ModelOutcome, + poisoned: &[(String, String)], + finished: &FinishedSession, +) -> Result>, String> { + let sql: Cow<'_, str> = resolved_cleaned_sql(outcome)?; + let mut relations: HashMap> = HashMap::new(); + let mut open: HashMap> = HashMap::new(); + for (relation, columns) in &outcome.validated_schema { + let (table, columns) = if columns.is_empty() { + match declared_columns(finished, relation) { + Some(declared) => (&mut open, declared), + None => continue, + } + } else { + (&mut relations, columns) + }; + let _ = table.insert( + relation.clone(), + schema_columns(dialect, relation, columns, poisoned), + ); + } + let relations: SessionSchema = SessionSchema { + relations, + open, + functions: finished.functions.clone(), + }; + let analysed: Result = dialect.analyse(&AnalysisInput { + sql: &sql, + schema: &relations, + references: &[], + extension_functions: &finished.declared_functions, + quoted_ignore_case: finished.quoted_ignore_case, + }); + let Ok(analysis) = analysed else { + return Ok(None); + }; + analysis + .diagnostics + .into_iter() + .map(revised_binding) + .collect::, String>>() + .map(Some) +} + +/// The declared columns of `relation`, whose full shape the session did not know, when it has any. +fn declared_columns<'a>(finished: &'a FinishedSession, relation: &str) -> Option<&'a Pairs> { + finished + .tables + .column_types + .iter() + .find(|(name, columns)| name == relation && !columns.is_empty()) + .map(|(_, columns)| columns) +} + +/// The model's own findings, as type recovery matches them, kept as they are. +pub(crate) fn unrevised(model: &RecoveryModel) -> Vec { + model + .raw_bindings + .iter() + .map(|raw| (raw.code.clone(), raw.start, raw.end)) + .collect() +} + +/// `relation`'s columns as the session bound them and `dialect` reads them, poisoned ones untyped. +fn schema_columns( + dialect: &dyn NativeDialect, + relation: &str, + columns: &Pairs, + poisoned: &[(String, String)], +) -> Vec { + let mut schema: Vec = Vec::with_capacity(columns.len()); + for (name, data_type) in columns { + let untyped: bool = data_type.is_empty() + || data_type == UNKNOWN_TYPE + || poisoned + .iter() + .any(|(model, output)| model == relation && output == name); + schema.push(schema_column( + name, + (!untyped).then(|| dialect.session_type(data_type)), + Nullability::Unknown, + )); + } + schema +} + +/// A finding as type recovery matches it: its code and character span. +fn revised_binding(row: DiagnosticRow) -> Result { + let start: Option = row + .start + .map(i64::try_from) + .transpose() + .map_err(|error| error.to_string())?; + let end: Option = row + .end + .map(i64::try_from) + .transpose() + .map_err(|error| error.to_string())?; + Ok((row.code, start, end)) +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/whole_gap.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/whole_gap.rs new file mode 100644 index 0000000000..7b4fda1d47 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/whole_gap.rs @@ -0,0 +1,11 @@ +use crate::analysis_route::models::ParserGap; + +/// A gap `read_whole` names itself, which no parser reports as invalid syntax. +pub(crate) fn whole_gap(construct: &str, offset: usize) -> ParserGap { + ParserGap { + construct: construct.to_owned(), + message: construct.to_owned(), + offset, + syntax: false, + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/constants.rs b/crates/sqlbuild-analysis/src/analysis_route/constants.rs index 1e31d63436..70cba12bec 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/constants.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/constants.rs @@ -4,9 +4,27 @@ use crate::analysis_route::types::NativeDialect; /// Each dialect, by its adapter's analysis dialect name, whose SQL a native module analyses. pub(crate) const NATIVE_DIALECTS: &[(&str, &dyn NativeDialect)] = &[]; +/// The dialect a project with no analysis dialect is named by. +pub(crate) const GENERIC_DIALECT: &str = "generic"; +/// The gap of SQL that holds more than one statement. +pub(crate) const MORE_THAN_ONE_STATEMENT: &str = "more than one statement"; +/// The gap of SQL that holds no statement. +pub(crate) const NO_STATEMENT: &str = "no statement"; +/// The gap of a statement the parser kept as raw text. +pub(crate) const UNPARSED_STATEMENT: &str = "unparsed statement"; /// The warning, or under `require_sql_analysis` the error, for a model analysis skips. pub(crate) const SKIPPED_ANALYSIS_CODE: &str = "P026"; /// ClickHouse's top-level nullability wrapper, which the native profile spells as a flag. pub(crate) const NULLABLE_WRAPPER: &str = "Nullable"; /// ClickHouse's top-level dictionary-encoding wrapper, which leaves the column's type as it is. pub(crate) const LOW_CARDINALITY_WRAPPER: &str = "LowCardinality"; +/// The finding of a function the dialect's catalogue does not know. +pub(crate) const UNKNOWN_FUNCTION_CODE: &str = "E202"; +/// The finding of a column no referenced relation has. +pub(crate) const UNKNOWN_COLUMN_CODE: &str = "E201"; +/// The severity of every finding the native analysis reports, as the Polyglot path blocks them. +pub(crate) const ERROR_SEVERITY: &str = "error"; +/// What `macro_body` puts before a body to read it as the one projection of a query. +pub(crate) const MACRO_BODY_PREFIX: &str = "SELECT "; +/// The gap of a macro body that is not one expression. +pub(crate) const NOT_ONE_EXPRESSION: &str = "not one expression"; diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/aligned_offset.rs b/crates/sqlbuild-analysis/src/analysis_route/main/aligned_offset.rs new file mode 100644 index 0000000000..4d603f0cea --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/main/aligned_offset.rs @@ -0,0 +1,5 @@ +/// The character of `source` that `derived`'s `offset` came from, when the text before or after holds. +#[must_use] +pub fn aligned_offset(derived: &str, offset: usize, source: &str) -> Option { + crate::analysis_route::_helpers::located_gap::aligned(derived, offset, source) +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/clickhouse_type_match.rs b/crates/sqlbuild-analysis/src/analysis_route/main/clickhouse_type_match.rs index 91223ddd45..2a497aa248 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/clickhouse_type_match.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/clickhouse_type_match.rs @@ -1,51 +1,22 @@ -use crate::analysis_route::constants::{LOW_CARDINALITY_WRAPPER, NULLABLE_WRAPPER}; +use crate::analysis_route::_helpers::clickhouse_types::{unspaced, unwrapped}; use crate::analysis_route::models::TypeMatch; -/// A declared and an inferred ClickHouse type compared without top-level wrappers, nullability apart. +/// A declared and an inferred ClickHouse type compared without top-level wrappers or spacing. #[must_use] -pub fn clickhouse_type_match(declared: &str, inferred: &str, inferred_nullable: bool) -> TypeMatch { +pub fn clickhouse_type_match( + declared: &str, + inferred: Option<&str>, + inferred_nullable: bool, +) -> TypeMatch { let (declared_base, declared_nullable) = unwrapped(declared); + let Some(inferred) = inferred else { + return TypeMatch::NotCompared { + nullable_into_non_null: !declared_nullable && inferred_nullable, + }; + }; let (inferred_base, wrapped_nullable) = unwrapped(inferred); - TypeMatch { - same_type: declared_base == inferred_base, + TypeMatch::Compared { + same_type: unspaced(declared_base) == unspaced(inferred_base), nullable_into_non_null: !declared_nullable && (inferred_nullable || wrapped_nullable), } } - -/// The type inside its top-level `Nullable` and `LowCardinality` wrappers, and whether it is nullable. -fn unwrapped(type_sql: &str) -> (&str, bool) { - let mut current: &str = type_sql.trim(); - let mut nullable: bool = false; - loop { - if let Some(inner) = wrapped(current, NULLABLE_WRAPPER) { - nullable = true; - current = inner; - } else if let Some(inner) = wrapped(current, LOW_CARDINALITY_WRAPPER) { - current = inner; - } else { - return (current, nullable); - } - } -} - -/// The argument of `wrapper(...)` when the whole of `type_sql` is that one call. -fn wrapped<'a>(type_sql: &'a str, wrapper: &str) -> Option<&'a str> { - let head: &str = type_sql.get(..wrapper.len())?; - if !head.eq_ignore_ascii_case(wrapper) { - return None; - } - let inner: &str = type_sql[wrapper.len()..] - .trim_start() - .strip_prefix('(')? - .strip_suffix(')')?; - let mut depth: usize = 0; - for character in inner.chars() { - match character { - '(' => depth += 1, - ')' if depth == 0 => return None, - ')' => depth -= 1, - _ => {} - } - } - Some(inner.trim()) -} diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/fast_lineage.rs b/crates/sqlbuild-analysis/src/analysis_route/main/fast_lineage.rs new file mode 100644 index 0000000000..3b179b4acd --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/main/fast_lineage.rs @@ -0,0 +1,151 @@ +//! Fast lineage of a model without facts, read by its dialect's native analysis module. + +use std::collections::HashMap; + +use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; +use sqlbuild_sqltext::sql_scan::main::dialect_lexical_syntax::dialect_lexical_syntax; +use sqlquack_semantics::lineage::models::{Nullability, SchemaColumn}; + +use crate::analysis_route::_helpers::located_gap::{located, unreadable_reason}; +use crate::analysis_route::_helpers::marker_names::{distinct_calls, first_call_names}; +use crate::analysis_route::main::schema_column::schema_column; +use crate::analysis_route::models::{AnalysisInput, NativeAnalysis, ParserGap}; +use crate::analysis_route::types::NativeDialect; +use crate::assembly::analysis_session::constants::{ + CONFIDENCE_HIGH, CONFIDENCE_MEDIUM, TRANSFORM_AGGREGATION, TRANSFORM_CAST, TRANSFORM_CONSTANT, + TRANSFORM_DIRECT, TRANSFORM_STAR, +}; +use crate::assembly::analysis_session::models::{LineageRow, SessionFunctions, SessionSchema}; +use crate::lineage::models::{ + FastLineageOutcome, LineageColumn, LineageConfidence, LineageResourceType, + LineageSchemaResource, LineageSource, LineageTransformKind, +}; +use crate::relation_markers::constants::MARKER_NAME_PREFIX; +use crate::relation_markers::main::markers_as_names::markers_as_names; +use crate::relation_markers::types::MarkerCall; +use crate::semantic_validation::main::normalize_native::normalize_native_sql; + +/// Lineage over `schema` of `query_sql`, its relation markers read as names, or why it is unreadable. +pub fn fast_lineage( + dialect: &dyn NativeDialect, + (query_sql, placeholders): (&str, &[(String, String)]), + schema: &[LineageSchemaResource], +) -> FastLineageOutcome { + let (named, calls): (String, Vec) = + match markers_as_names(query_sql, &dialect_lexical_syntax(dialect.name())) { + Ok(named) => named, + Err(reason) => return FastLineageOutcome::Unparsed(reason), + }; + let normalised: Result = first_call_names(&named, &calls) + .and_then(|named| normalize_native_sql(&named, dialect.name(), dialect.stub_prefix())); + let normalised: String = match normalised { + Ok(normalised) => normalised, + Err(reason) => return FastLineageOutcome::Unparsed(reason), + }; + let sql: String = placeholder_defaults(&normalised, placeholders); + let referenced: Vec<(String, LineageResourceType, String)> = distinct_calls(&calls) + .into_iter() + .filter_map(|(index, (function, name))| { + let resource_type: LineageResourceType = match function.as_str() { + "__ref" => LineageResourceType::Model, + "__source" => LineageResourceType::Source, + "__seed" => LineageResourceType::Seed, + _ => return None, + }; + Some(( + format!("{MARKER_NAME_PREFIX}{index}"), + resource_type, + name.clone(), + )) + }) + .collect(); + let references: Vec<(String, String, String)> = referenced + .iter() + .map(|(marker, resource_type, name)| { + ( + marker.clone(), + resource_type.as_str().to_owned(), + name.clone(), + ) + }) + .collect(); + let relations: SessionSchema = markers_schema(&referenced, schema); + let analysed: Result = dialect.analyse(&AnalysisInput { + sql: &sql, + schema: &relations, + references: &references, + extension_functions: &[], + quoted_ignore_case: false, + }); + match analysed { + Ok(native) => FastLineageOutcome::Built { + columns: native.lineage.iter().map(lineage_column).collect(), + has_star: native.has_star, + }, + Err(gap) => FastLineageOutcome::Unparsed(unreadable_reason( + dialect.name(), + &located(&sql, gap, None), + )), + } +} + +/// Each marker name's resource columns, untyped and of unknown nullability, when the schema knows them. +fn markers_schema( + referenced: &[(String, LineageResourceType, String)], + schema: &[LineageSchemaResource], +) -> SessionSchema { + let mut relations: HashMap> = HashMap::new(); + for (marker, resource_type, name) in referenced { + let Some(resource) = schema.iter().find(|resource| { + resource.resource_type == *resource_type + && resource.name == *name + && !resource.columns.is_empty() + }) else { + continue; + }; + let _ = relations.insert( + marker.clone(), + resource + .columns + .iter() + .map(|column| schema_column(column, None, Nullability::Unknown)) + .collect(), + ); + } + SessionSchema { + relations, + open: HashMap::new(), + functions: SessionFunctions::default(), + } +} + +/// A session lineage row as a fast lineage column; sources of other resource kinds are dropped. +fn lineage_column(row: &LineageRow) -> LineageColumn { + LineageColumn { + output_column: row.output_column.clone(), + transform_kind: match row.transform_code { + TRANSFORM_DIRECT => LineageTransformKind::Direct, + TRANSFORM_CAST => LineageTransformKind::Cast, + TRANSFORM_AGGREGATION => LineageTransformKind::Aggregation, + TRANSFORM_STAR => LineageTransformKind::Star, + TRANSFORM_CONSTANT => LineageTransformKind::Constant, + _ => LineageTransformKind::Expression, + }, + confidence: match row.confidence_code { + CONFIDENCE_HIGH => LineageConfidence::High, + CONFIDENCE_MEDIUM => LineageConfidence::Medium, + _ => LineageConfidence::Unknown, + }, + upstream_columns: row + .sources + .iter() + .filter_map(|(resource_type, resource_name, column_name)| { + Some(LineageSource { + resource_type: LineageResourceType::from_value(resource_type)?, + resource_name: resource_name.clone(), + column_name: column_name.clone(), + }) + }) + .collect(), + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs b/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs index 87c5c0cfd3..d85ed3ce17 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs @@ -1,4 +1,11 @@ +pub mod aligned_offset; pub mod analysis_route; pub mod clickhouse_type_match; +pub mod fast_lineage; +pub(crate) mod model_outcome; pub mod native_gap; +pub mod native_revalidation; +pub mod schema_column; pub mod skipped_analysis; +pub mod unapplied_nullability_rules; +pub mod unavailable_analysis; diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/model_outcome.rs b/crates/sqlbuild-analysis/src/analysis_route/main/model_outcome.rs new file mode 100644 index 0000000000..504ab04f81 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/main/model_outcome.rs @@ -0,0 +1,70 @@ +//! One model's outcome from its dialect's native analysis module, in the session's facts. + +use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; + +use crate::analysis_route::models::{AnalysisInput, NativeAnalysis, ParserGap}; +use crate::analysis_route::types::NativeDialect; +use crate::assembly::analysis_session::models::{ + LineageFacts, ModelAnalysis, ModelOutcome, ModelRequest, SessionSchema, +}; +use crate::assembly::analysis_session::types::Shapes; +use crate::semantic_validation::main::normalize_native::normalize_native_sql; + +/// The model's outcome over `schema` with the adapter's declared functions, and why it failed. +pub(crate) fn model_outcome( + dialect: &dyn NativeDialect, + model: &ModelRequest, + (schema, relations): (Shapes, &SessionSchema), + (extension_functions, quoted_ignore_case): (&[String], bool), +) -> (ModelOutcome, Option) { + let (cleaned_sql, analysed): (String, Result) = + match normalize_native_sql(&model.query_sql, dialect.name(), dialect.stub_prefix()) { + Ok(normalised) => { + let sql: String = placeholder_defaults(&normalised, &model.placeholders); + let analysed = dialect + .analyse(&AnalysisInput { + sql: &sql, + schema: relations, + references: &model.lineage_references, + extension_functions, + quoted_ignore_case, + }) + .map_err(|gap: ParserGap| gap.message); + (sql, analysed) + } + Err(reason) => (String::new(), Err(reason)), + }; + let failure: Option = analysed + .as_ref() + .err() + .map(|reason| format!("{}: {reason}", model.name)); + let analysis: ModelAnalysis = match analysed { + Ok(native) => ModelAnalysis { + analysis_succeeded: true, + columns: Some(native.columns), + lineage: LineageFacts::Native(native.lineage), + has_star: native.has_star, + star_resolved: native.star_resolved, + binding_diagnostics: native.diagnostics, + binding_validated: true, + }, + Err(_) => ModelAnalysis { + analysis_succeeded: false, + columns: None, + lineage: LineageFacts::Native(Vec::new()), + has_star: false, + star_resolved: false, + binding_diagnostics: Vec::new(), + binding_validated: true, + }, + }; + let outcome: ModelOutcome = ModelOutcome { + analysis, + cleaned_sql, + deferred_cleaning: None, + validated_schema: schema, + fused_binding_validated: true, + sql_macro_findings: Vec::new(), + }; + (outcome, failure) +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/native_gap.rs b/crates/sqlbuild-analysis/src/analysis_route/main/native_gap.rs index 2d2c02726f..aa68f67059 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/native_gap.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/native_gap.rs @@ -1,10 +1,9 @@ use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; -use crate::analysis_route::models::{LocatedGap, ParserGap}; +use crate::analysis_route::_helpers::located_gap::located; +use crate::analysis_route::models::LocatedGap; use crate::analysis_route::types::NativeDialect; use crate::assembly::project::models::SyntaxCheck; -use crate::compiled_project::main::text_position::text_position; -use crate::compiled_project::models::TextPosition; use crate::semantic_validation::main::normalize_native::normalize_native_sql; /// The native parser's first gap in `check`'s analysis SQL, with the dialect's stub spelling. @@ -18,16 +17,5 @@ pub fn native_gap( Ok(dialect .read_whole(&analysed) .err() - .map(|gap| located(&analysed, gap))) -} - -/// The gap with the line and column of its offset; an offset past the end is the end. -fn located(sql: &str, gap: ParserGap) -> LocatedGap { - let before: &str = sql.get(..gap.offset).unwrap_or(sql); - let position: TextPosition = text_position(sql, before.chars().count()); - LocatedGap { - construct: gap.construct, - line: position.line, - column: position.column, - } + .map(|gap| located(&analysed, gap, Some(&check.sql)))) } diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/native_revalidation.rs b/crates/sqlbuild-analysis/src/analysis_route/main/native_revalidation.rs new file mode 100644 index 0000000000..35cd3db370 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/main/native_revalidation.rs @@ -0,0 +1,35 @@ +//! Type recovery's revalidation on a native route, read by the dialect's own analysis. + +use std::sync::Arc; + +use rayon::ThreadPool; +use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; + +use crate::analysis_route::_helpers::revalidation::{revised, session_outcome, unrevised}; +use crate::analysis_route::types::NativeDialect; +use crate::assembly::analysis_session::models::{FinishedSession, ModelOutcome, PivotOutcome}; +use crate::semantic_checks::models::RecoveryModel; +use crate::semantic_checks::types::RevisedBinding; + +/// Each model's findings with the `poisoned` outputs untyped; an unreadable model keeps its own. +pub fn native_revalidation( + dialect: &dyn NativeDialect, + (finished, outcomes): (&FinishedSession, &[(ModelOutcome, PivotOutcome)]), + models: &[&RecoveryModel], + poisoned: &[(String, String)], +) -> Result>, String> { + let pool: Arc = finished.pool.clone()?; + pool.install(|| { + models + .par_iter() + .map(|model| { + let outcome: &ModelOutcome = session_outcome(finished, outcomes, &model.name)?; + match revised(dialect, outcome, poisoned, finished) { + Ok(Some(rows)) => Ok(rows), + Ok(None) => Ok(unrevised(model)), + Err(reason) => Err(format!("{}: {reason}", model.name)), + } + }) + .collect() + }) +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/schema_column.rs b/crates/sqlbuild-analysis/src/analysis_route/main/schema_column.rs new file mode 100644 index 0000000000..d2a89894a0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/main/schema_column.rs @@ -0,0 +1,25 @@ +use sqlquack_semantics::lineage::models::{Nullability, SchemaColumn}; + +use crate::assembly::analysis_session::constants::QUOTED_IDENTIFIER_DELIMITER; + +/// A stored column name as SQLQuack reads it: `"…"` is quoted, its inner `""` one quote. +#[must_use] +pub fn schema_column( + name: &str, + data_type: Option, + nullability: Nullability, +) -> SchemaColumn { + let delimiter: String = QUOTED_IDENTIFIER_DELIMITER.to_string(); + let inner: Option<&str> = name + .strip_prefix(QUOTED_IDENTIFIER_DELIMITER) + .and_then(|rest| rest.strip_suffix(QUOTED_IDENTIFIER_DELIMITER)); + SchemaColumn { + name: inner.map_or_else( + || name.to_owned(), + |inner| inner.replace(&delimiter.repeat(2), &delimiter), + ), + quoted: inner.is_some(), + data_type, + nullability, + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/skipped_analysis.rs b/crates/sqlbuild-analysis/src/analysis_route/main/skipped_analysis.rs index 3fba7a616f..961719fc8c 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/skipped_analysis.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/skipped_analysis.rs @@ -1,15 +1,20 @@ +use crate::analysis_route::_helpers::located_gap::sited_reason; use crate::analysis_route::constants::SKIPPED_ANALYSIS_CODE; -use crate::analysis_route::models::LocatedGap; +use crate::analysis_route::models::{GapSite, LocatedGap}; /// P026's code and message for a model whose SQL the dialect's native parser cannot read. #[must_use] -pub fn skipped_analysis(model: &str, dialect: &str, gap: &LocatedGap) -> (&'static str, String) { +pub fn skipped_analysis( + model: &str, + dialect: &str, + gap: &LocatedGap, + site: &GapSite<'_>, +) -> (&'static str, String) { ( SKIPPED_ANALYSIS_CODE, format!( - "SQL analysis skipped for model '{model}': the {dialect} parser cannot read {} at \ - line {}, column {}", - gap.construct, gap.line, gap.column + "SQL analysis skipped for model '{model}'{}", + sited_reason(dialect, gap, site) ), ) } diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/unapplied_nullability_rules.rs b/crates/sqlbuild-analysis/src/analysis_route/main/unapplied_nullability_rules.rs new file mode 100644 index 0000000000..1a5b2b593d --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/main/unapplied_nullability_rules.rs @@ -0,0 +1,18 @@ +use crate::analysis_route::models::AnalysisRoute; + +/// The warning for an adapter's own nullability rules on a native route, which does not call them. +#[must_use] +pub fn unapplied_nullability_rules( + route: &AnalysisRoute, + adapter: &str, + functions: &[String], +) -> Option { + match route { + AnalysisRoute::Native(_) if !functions.is_empty() => Some(format!( + "Adapter '{adapter}' defines its own nullability rule for {}; native SQL analysis \ + does not apply it, so these calls take the dialect's own nullability", + functions.join(", ") + )), + _ => None, + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/unavailable_analysis.rs b/crates/sqlbuild-analysis/src/analysis_route/main/unavailable_analysis.rs new file mode 100644 index 0000000000..97257e54f6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/main/unavailable_analysis.rs @@ -0,0 +1,13 @@ +use crate::analysis_route::constants::{GENERIC_DIALECT, SKIPPED_ANALYSIS_CODE}; + +/// P026's code and message for a model whose dialect no native parser reads. +#[must_use] +pub fn unavailable_analysis(model: &str, dialect: Option<&str>) -> (&'static str, String) { + ( + SKIPPED_ANALYSIS_CODE, + format!( + "SQL analysis skipped for model '{model}': no native parser reads the {} dialect", + dialect.unwrap_or(GENERIC_DIALECT) + ), + ) +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/mod.rs b/crates/sqlbuild-analysis/src/analysis_route/mod.rs index 76ff5d49a0..4d7376c049 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/mod.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/mod.rs @@ -1,5 +1,6 @@ //! Which engine analyses a dialect's SQL: the Polyglot path, or a native module over SQLQuack. +pub(crate) mod _helpers; pub(crate) mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-analysis/src/analysis_route/models.rs b/crates/sqlbuild-analysis/src/analysis_route/models.rs index a93a53d297..bc886000ce 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/models.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/models.rs @@ -1,6 +1,12 @@ //! The route a dialect's analysis takes, and what stops a native parser reading SQL whole. +use sqlquack_semantics::lineage::models::{Nullability, SchemaColumn}; +use sqlquack_semantics::lineage::types::LineageSchema; + use crate::analysis_route::types::NativeDialect; +use crate::assembly::analysis_session::models::{ColumnFact, LineageRow}; +use crate::compiled_project::models::TextPosition; +use crate::semantic_validation::models::DiagnosticRow; /// The engine that analyses a dialect's SQL. #[derive(Clone, Copy)] @@ -9,27 +15,135 @@ pub enum AnalysisRoute { Legacy, /// The dialect's native module; a model it cannot read whole is not analysed. Native(&'static dyn NativeDialect), + /// No native module reads the dialect: nothing is analysed, and no Polyglot path stands in. + Unavailable, } -/// The first construct a native parser cannot read, and its byte offset in the SQL it read. +/// The first construct or error that stops a native parser, at its byte offset in the SQL it read. #[derive(Clone, Debug, PartialEq, Eq)] pub struct ParserGap { + /// The construct, such as `PIVOT`, or the parser's error message. pub construct: String, + /// The parser's bare message, without the line and column an error's `Display` adds. + pub message: String, pub offset: usize, + /// Whether the parser reports invalid syntax; false for an unsupported construct or a limit. + pub syntax: bool, } /// A gap with the line and column, from 1, of its offset in the analysed SQL. #[derive(Clone, Debug, PartialEq, Eq)] pub struct LocatedGap { pub construct: String, + pub message: String, pub line: usize, pub column: usize, + pub syntax: bool, + /// The gap's character offset in the SQL as checked, before normalisation, when its text holds. + pub source_offset: Option, +} + +/// Where P026 places a gap. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum GapPosition { + /// In the file the site names. + File(TextPosition), + /// In the site's own SQL statement, such as a hook's. + Statement(TextPosition), + /// Only in the analysed SQL, at the gap's own line and column. + Analysed, +} + +/// The SQL a gap stops: the hook it belongs to, if any, its file, and where the gap is. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct GapSite<'a> { + /// Such as `post_hooks[0] sql("record_refresh")`; none for the model's query. + pub label: Option<&'a str>, + pub path: &'a str, + pub position: GapPosition, } /// How an inferred column type meets its declared type: the type itself, then its nullability. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct TypeMatch { - pub same_type: bool, - /// The inferred column may be null, but its declared type does not allow it. - pub nullable_into_non_null: bool, +pub enum TypeMatch { + /// One side's type is unknown or refused: no type finding, though null may still be forbidden. + NotCompared { + /// The inferred column may be null, but its declared type does not allow it. + nullable_into_non_null: bool, + }, + Compared { + same_type: bool, + /// The inferred column may be null, but its declared type does not allow it. + nullable_into_non_null: bool, + }, +} + +/// What a native module analyses: one query, the relations it may read, and how to name them. +#[derive(Clone, Copy, Debug)] +pub struct AnalysisInput<'a> { + /// The analysis SQL, normalised with the dialect's stubs and its placeholders filled. + pub sql: &'a str, + /// The columns, types and nullability of the relations the query may read. + pub schema: &'a dyn LineageSchema, + /// `(analysis name, resource type, resource name)` of each relation the model references. + pub references: &'a [(String, String, String)], + /// Functions the catalogue cannot know, such as MotherDuck's `md_*`; never reported unknown. + pub extension_functions: &'a [String], + /// Whether quoted identifiers compare without case, as Snowflake's session parameter sets. + pub quoted_ignore_case: bool, +} + +/// One query's analysis, in the facts the analysis session keeps for a model. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct NativeAnalysis { + /// The output columns in order, each typed through the dialect's `normalize_type`. + pub columns: Vec, + /// One row per output column: how it is derived and the referenced columns it reads. + pub lineage: Vec, + /// `(relation, column)` of the columns that decide which rows the query returns. + pub row_sources: Vec<(String, String)>, + /// The outermost query level projects a star. + pub has_star: bool, + /// Every column a star projects is known. + pub star_resolved: bool, + /// The findings SQLBuild reports, in its codes and at character positions in `sql`. + pub diagnostics: Vec, +} + +/// One relation of known columns, standing in for what a probe query such as a macro's reads. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StandInSchema { + relation: String, + columns: Vec, +} + +impl StandInSchema { + /// `relation` with each `(column, type as the dialect spells it, nullable)` in order. + #[must_use] + pub fn new(relation: &str, columns: &[(&str, &str, bool)]) -> Self { + Self { + relation: relation.to_owned(), + columns: columns + .iter() + .map(|&(name, data_type, nullable)| SchemaColumn { + name: name.to_owned(), + quoted: false, + data_type: Some(data_type.to_owned()), + nullability: if nullable { + Nullability::Nullable + } else { + Nullability::NonNull + }, + }) + .collect(), + } + } +} + +impl LineageSchema for StandInSchema { + fn columns(&self, relation: &str) -> Option<&[SchemaColumn]> { + relation + .eq_ignore_ascii_case(&self.relation) + .then_some(self.columns.as_slice()) + } } diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests.rs b/crates/sqlbuild-analysis/src/analysis_route/tests.rs index b8464bdb55..242ac2a388 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests.rs @@ -1,6 +1,14 @@ +#[path = "tests/helpers.rs"] +mod helpers; #[path = "tests/test_clickhouse_type_match.rs"] mod test_clickhouse_type_match; +#[path = "tests/test_native_dialect.rs"] +mod test_native_dialect; #[path = "tests/test_native_gap.rs"] mod test_native_gap; +#[path = "tests/test_native_outcome.rs"] +mod test_native_outcome; +#[path = "tests/test_native_revalidation.rs"] +mod test_native_revalidation; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs new file mode 100644 index 0000000000..29831d22d3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs @@ -0,0 +1,147 @@ +use std::collections::HashMap; + +use sqlquack_semantics::lineage::models::{Nullability, SchemaColumn}; + +use crate::analysis_route::main::schema_column::schema_column as stored_column; +use crate::analysis_route::models::{AnalysisInput, NativeAnalysis, StandInSchema}; +use crate::analysis_route::types::NativeDialect; +use crate::assembly::analysis_session::models::{ + LineageFacts, LineageRow, ModelAnalysis, ModelOutcome, SessionFunctions, SessionSchema, +}; +use crate::assembly::analysis_session::types::Shapes; +use crate::lineage::models::{LineageColumn, LineageSource}; +use crate::semantic_checks::types::RevisedBinding; + +/// `orders` with a non-null INTEGER id and a nullable VARCHAR status. +pub(super) fn orders() -> StandInSchema { + StandInSchema::new( + "orders", + &[("id", "INTEGER", false), ("status", "VARCHAR", true)], + ) +} + +/// The analysis of `sql` over `orders`, which the model references as the model `shop.orders`. +pub(super) fn analysed( + dialect: &dyn NativeDialect, + sql: &str, + extension_functions: &[&str], +) -> NativeAnalysis { + let schema: StandInSchema = orders(); + let references: [(String, String, String); 1] = [( + "orders".to_owned(), + "model".to_owned(), + "shop.orders".to_owned(), + )]; + let extension_functions: Vec = extension_functions + .iter() + .map(|name| (*name).to_owned()) + .collect(); + dialect + .analyse(&AnalysisInput { + sql, + schema: &schema, + references: &references, + extension_functions: &extension_functions, + quoted_ignore_case: false, + }) + .expect("the test query is read whole") +} + +/// The columns a lineage row reads, each of which must come from the model `shop.orders`. +pub(super) fn resource_columns(row: &LineageRow) -> Vec<&str> { + row.sources + .iter() + .map(|(resource_type, resource_name, column)| { + assert_eq!( + (resource_type.as_str(), resource_name.as_str()), + ("model", "shop.orders"), + "a source of {}", + row.output_column + ); + column.as_str() + }) + .collect() +} + +/// `output transform confidence [type:resource.column, ...]`. +pub(super) fn column_line(column: &LineageColumn) -> String { + let sources: Vec = column.upstream_columns.iter().map(source_text).collect(); + format!( + "{} {} {} [{}]", + column.output_column, + column.transform_kind.as_str(), + column.confidence.as_str(), + sources.join(", ") + ) +} + +fn source_text(source: &LineageSource) -> String { + format!( + "{}:{}.{}", + source.resource_type.as_str(), + source.resource_name, + source.column_name + ) +} + +/// `raw_orders` with `shape` as the session's input shapes and its typed schema; none when empty. +pub(super) fn session_relations(shape: &[(&str, &str, Nullability)]) -> (Shapes, SessionSchema) { + let pairs: Vec<(String, String)> = shape + .iter() + .map(|(name, data_type, _)| ((*name).to_owned(), (*data_type).to_owned())) + .collect(); + let columns: Vec = shape.iter().map(schema_column).collect(); + let relations: HashMap> = std::iter::once(("raw_orders", columns)) + .filter(|(_, columns)| !columns.is_empty()) + .map(|(name, columns)| (name.to_owned(), columns)) + .collect(); + ( + vec![("raw_orders".to_owned(), pairs)], + SessionSchema { + relations, + open: HashMap::new(), + functions: SessionFunctions::default(), + }, + ) +} + +fn schema_column(&(name, data_type, nullability): &(&str, &str, Nullability)) -> SchemaColumn { + stored_column(name, Some(data_type.to_owned()), nullability) +} + +/// A finished outcome of `sql` over `orders`, as the session bound it. +pub(super) fn outcome(sql: &str) -> ModelOutcome { + ModelOutcome { + analysis: ModelAnalysis { + analysis_succeeded: true, + columns: None, + lineage: LineageFacts::Native(Vec::new()), + has_star: false, + star_resolved: false, + binding_diagnostics: Vec::new(), + binding_validated: true, + }, + cleaned_sql: sql.to_owned(), + deferred_cleaning: None, + validated_schema: vec![( + "orders".to_owned(), + vec![ + ("id".to_owned(), "INTEGER".to_owned()), + ("status".to_owned(), "VARCHAR".to_owned()), + ], + )], + fused_binding_validated: true, + sql_macro_findings: Vec::new(), + } +} + +/// Each revised finding of every model as `(code, start, end)`, in order. +pub(super) fn finding_rows( + models: &[Vec], +) -> Vec<(&str, Option, Option)> { + let mut rows: Vec<(&str, Option, Option)> = Vec::new(); + for (code, start, end) in models.iter().flatten() { + rows.push((code.as_str(), *start, *end)); + } + rows +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_clickhouse_type_match.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_clickhouse_type_match.rs index e23bab65e4..c6cab99ac9 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_clickhouse_type_match.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_clickhouse_type_match.rs @@ -9,9 +9,9 @@ fn given_clickhouse_column_types_when_matching_then_wrappers_are_stripped_and_nu TypeMatchTestCase { description: "a nullable String flag matches DESCRIBE's Nullable(String)", declared: "Nullable(String)", - inferred: "String", + inferred: Some("String"), inferred_nullable: true, - expected_match: TypeMatch { + expected_match: TypeMatch::Compared { same_type: true, nullable_into_non_null: false, }, @@ -19,9 +19,9 @@ fn given_clickhouse_column_types_when_matching_then_wrappers_are_stripped_and_nu TypeMatchTestCase { description: "a declared LowCardinality(String) is a String", declared: "LowCardinality(String)", - inferred: "String", + inferred: Some("String"), inferred_nullable: false, - expected_match: TypeMatch { + expected_match: TypeMatch::Compared { same_type: true, nullable_into_non_null: false, }, @@ -29,9 +29,9 @@ fn given_clickhouse_column_types_when_matching_then_wrappers_are_stripped_and_nu TypeMatchTestCase { description: "LowCardinality around Nullable allows null", declared: "LowCardinality(Nullable(String))", - inferred: "Nullable(String)", + inferred: Some("Nullable(String)"), inferred_nullable: false, - expected_match: TypeMatch { + expected_match: TypeMatch::Compared { same_type: true, nullable_into_non_null: false, }, @@ -39,9 +39,9 @@ fn given_clickhouse_column_types_when_matching_then_wrappers_are_stripped_and_nu TypeMatchTestCase { description: "a nullable inference into a plain String declaration is a nullability conflict", declared: "String", - inferred: "String", + inferred: Some("String"), inferred_nullable: true, - expected_match: TypeMatch { + expected_match: TypeMatch::Compared { same_type: true, nullable_into_non_null: true, }, @@ -49,9 +49,9 @@ fn given_clickhouse_column_types_when_matching_then_wrappers_are_stripped_and_nu TypeMatchTestCase { description: "a non-null inference into a Nullable declaration is no conflict", declared: "Nullable(Int64)", - inferred: "Int64", + inferred: Some("Int64"), inferred_nullable: false, - expected_match: TypeMatch { + expected_match: TypeMatch::Compared { same_type: true, nullable_into_non_null: false, }, @@ -59,9 +59,9 @@ fn given_clickhouse_column_types_when_matching_then_wrappers_are_stripped_and_nu TypeMatchTestCase { description: "different base types differ whatever their wrappers", declared: "Nullable(Int32)", - inferred: "Int64", + inferred: Some("Int64"), inferred_nullable: true, - expected_match: TypeMatch { + expected_match: TypeMatch::Compared { same_type: false, nullable_into_non_null: false, }, @@ -69,13 +69,60 @@ fn given_clickhouse_column_types_when_matching_then_wrappers_are_stripped_and_nu TypeMatchTestCase { description: "a Nullable nested inside Array is part of the type, not stripped", declared: "Array(Nullable(String))", - inferred: "Array(String)", + inferred: Some("Array(String)"), inferred_nullable: false, - expected_match: TypeMatch { + expected_match: TypeMatch::Compared { same_type: false, nullable_into_non_null: false, }, }, + TypeMatchTestCase { + description: "spacing inside a type's arguments is not a difference", + declared: "Decimal(10, 2)", + inferred: Some("Decimal(10,2)"), + inferred_nullable: false, + expected_match: TypeMatch::Compared { + same_type: true, + nullable_into_non_null: false, + }, + }, + TypeMatchTestCase { + description: "spacing inside a quoted enum value is part of the type", + declared: "Enum8('a b' = 1)", + inferred: Some("Enum8('ab' = 1)"), + inferred_nullable: false, + expected_match: TypeMatch::Compared { + same_type: false, + nullable_into_non_null: false, + }, + }, + TypeMatchTestCase { + description: "a nullable value of unknown type into a plain String is a nullability conflict", + declared: "String", + inferred: None, + inferred_nullable: true, + expected_match: TypeMatch::NotCompared { + nullable_into_non_null: true, + }, + }, + TypeMatchTestCase { + description: "a nullable value of unknown type into a Nullable column is no conflict", + declared: "Nullable(String)", + inferred: None, + inferred_nullable: true, + expected_match: TypeMatch::NotCompared { + nullable_into_non_null: false, + }, + }, + TypeMatchTestCase { + description: "a non-null value of unknown type is no conflict", + declared: "String", + inferred: None, + inferred_nullable: false, + expected_match: TypeMatch::NotCompared { + nullable_into_non_null: false, + }, + }, ]; for test_case in test_cases { assert_eq!( diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs new file mode 100644 index 0000000000..8dbdd79904 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_dialect.rs @@ -0,0 +1,450 @@ +use crate::analysis_route::main::unapplied_nullability_rules::unapplied_nullability_rules; +use crate::analysis_route::main::unavailable_analysis::unavailable_analysis; +use crate::analysis_route::models::{AnalysisRoute, NativeAnalysis, TypeMatch}; +use crate::analysis_route::tests::helpers::{analysed, resource_columns}; +use crate::analysis_route::tests::test_types::{ + CataloguedDialect, ColumnTestCase, FindingMessageTestCase, FindingTestCase, FoldingDialect, + LineageTestCase, MacroBodyTestCase, NativeTypeMatchTestCase, PivotlessDialect, + ReadWholeTestCase, StarTestCase, UnappliedRulesTestCase, UnavailableAnalysisTestCase, +}; +use crate::analysis_route::types::NativeDialect; + +#[test] +fn given_sql_when_reading_it_whole_then_only_one_parsed_statement_without_a_gap_is_read() { + let test_cases = [ + ReadWholeTestCase { + description: "one query is read whole", + sql: "SELECT a FROM t", + expected_gap: None, + }, + ReadWholeTestCase { + description: "two statements are not one model", + sql: "SELECT 1; SELECT 2", + expected_gap: Some("more than one statement"), + }, + ReadWholeTestCase { + description: "no statement at all", + sql: " ", + expected_gap: Some("no statement"), + }, + ReadWholeTestCase { + description: "a comment alone holds no statement, not several", + sql: "-- nothing to read", + expected_gap: Some("no statement"), + }, + ReadWholeTestCase { + description: "a statement kept as raw text is not read", + sql: "RAW VACUUM t", + expected_gap: Some("unparsed statement"), + }, + ReadWholeTestCase { + description: "a construct the parser refuses stops it first", + sql: "SELECT * FROM t PIVOT (sum(x) FOR y IN (1))", + expected_gap: Some("PIVOT"), + }, + ]; + for test_case in test_cases { + assert_eq!( + PivotlessDialect + .read_whole(test_case.sql) + .err() + .map(|gap| gap.construct), + test_case.expected_gap.map(str::to_owned), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_type_the_dialect_refuses_when_matching_types_then_nothing_is_compared() { + let test_cases = [ + NativeTypeMatchTestCase { + description: "a refused declared type is not compared", + declared: "NOT_A_TYPE", + inferred: Some("INTEGER"), + expected_match: TypeMatch::NotCompared { + nullable_into_non_null: false, + }, + }, + NativeTypeMatchTestCase { + description: "a refused inferred type is not compared", + declared: "INTEGER", + inferred: Some("NOT_A_TYPE"), + expected_match: TypeMatch::NotCompared { + nullable_into_non_null: false, + }, + }, + NativeTypeMatchTestCase { + description: "an unknown inferred type is not compared", + declared: "INTEGER", + inferred: None, + expected_match: TypeMatch::NotCompared { + nullable_into_non_null: false, + }, + }, + NativeTypeMatchTestCase { + description: "two known, different types still mismatch", + declared: "INTEGER", + inferred: Some("TEXT"), + expected_match: TypeMatch::Compared { + same_type: false, + nullable_into_non_null: false, + }, + }, + NativeTypeMatchTestCase { + description: "the same type matches", + declared: "INTEGER", + inferred: Some("integer"), + expected_match: TypeMatch::Compared { + same_type: true, + nullable_into_non_null: false, + }, + }, + ]; + for test_case in test_cases { + assert_eq!( + PivotlessDialect.type_match(test_case.declared, test_case.inferred, true), + test_case.expected_match, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_no_native_parser_when_skipping_analysis_then_p026_names_the_dialect() { + let test_cases = [ + UnavailableAnalysisTestCase { + description: "an unknown dialect is named", + dialect: Some("trino"), + expected_message: "SQL analysis skipped for model 'orders': no native parser reads \ + the trino dialect", + }, + UnavailableAnalysisTestCase { + description: "no dialect is the generic dialect", + dialect: None, + expected_message: "SQL analysis skipped for model 'orders': no native parser reads \ + the generic dialect", + }, + ]; + for test_case in test_cases { + assert_eq!( + unavailable_analysis("orders", test_case.dialect), + ("P026", test_case.expected_message.to_owned()), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_an_unknown_quoted_column_when_analysing_then_its_message_quotes_the_name_as_written() { + let test_cases = [ + FindingMessageTestCase { + description: "a quoted name keeps its authored case", + sql: "SELECT \"MiSsPeLlEd\" FROM orders", + expected_messages: &[("B002", "Unknown column 'MiSsPeLlEd' in table 'orders'")], + }, + FindingMessageTestCase { + description: "a qualified quoted name keeps its authored case", + sql: "SELECT o.\"Missing\" FROM orders AS o", + expected_messages: &[("B002", "Unknown column 'Missing' in table 'orders'")], + }, + ]; + for test_case in test_cases { + let analysis: NativeAnalysis = analysed(&PivotlessDialect, test_case.sql, &[]); + let messages: Vec<(&str, &str)> = analysis + .diagnostics + .iter() + .map(|row| (row.code.as_str(), row.message.as_str())) + .collect(); + assert_eq!( + messages, test_case.expected_messages, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_adapter_nullability_rules_when_routing_then_only_a_native_route_warns_they_are_unapplied() +{ + let test_cases = [ + UnappliedRulesTestCase { + description: "a native route names the adapter and each function it leaves alone", + route: AnalysisRoute::Native(&PivotlessDialect), + functions: &["UPPER", "IFF"], + expected_warning: Some( + "Adapter 'acme' defines its own nullability rule for UPPER, IFF; native SQL \ + analysis does not apply it, so these calls take the dialect's own nullability", + ), + }, + UnappliedRulesTestCase { + description: "Polyglot calls the adapter's rules, so nothing is unapplied", + route: AnalysisRoute::Legacy, + functions: &["UPPER"], + expected_warning: None, + }, + UnappliedRulesTestCase { + description: "no model is analysed where no parser reads the dialect", + route: AnalysisRoute::Unavailable, + functions: &["UPPER"], + expected_warning: None, + }, + UnappliedRulesTestCase { + description: "an adapter with no rules of its own has nothing to warn about", + route: AnalysisRoute::Native(&PivotlessDialect), + functions: &[], + expected_warning: None, + }, + ]; + for test_case in test_cases { + let functions: Vec = test_case + .functions + .iter() + .map(|f| (*f).to_owned()) + .collect(); + assert_eq!( + unapplied_nullability_rules(&test_case.route, "acme", &functions), + test_case.expected_warning.map(str::to_owned), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_findings_when_analysing_then_they_take_sqlbuild_codes_and_positions() { + let test_cases = [ + FindingTestCase { + description: "a VARCHAR column as a condition is an implicit conversion, blocked as B211", + dialect: &PivotlessDialect, + sql: "SELECT id FROM orders WHERE status", + extension_functions: &[], + expected_findings: &[("B211", Some((1, 29)))], + }, + FindingTestCase { + description: "a cast to a type the engine lacks is blocked as B218, at the value cast", + dialect: &PivotlessDialect, + sql: "SELECT CAST(id AS NOT_A_TYPE) AS x FROM orders", + extension_functions: &[], + expected_findings: &[("B218", Some((1, 13)))], + }, + FindingTestCase { + description: "an unknown column on its own line keeps its line and column", + dialect: &PivotlessDialect, + sql: "SELECT id,\n nope\nFROM orders", + extension_functions: &[], + expected_findings: &[("B002", Some((2, 3)))], + }, + FindingTestCase { + description: "a truth-value test the warehouse rejects is SQLBuild's own B103, after the parse's findings", + dialect: &FoldingDialect, + sql: "SELECT id, nope FROM orders WHERE (id > 1) IS NOT TRUE", + extension_functions: &[], + expected_findings: &[("B002", Some((1, 12))), ("B103", Some((1, 44)))], + }, + FindingTestCase { + description: "a function the catalogue does not know is B101", + dialect: &CataloguedDialect, + sql: "SELECT CURRENT_TIMESTAMP() AS created_at FROM orders", + extension_functions: &[], + expected_findings: &[("B101", None)], + }, + FindingTestCase { + description: "a declared extension function is known, matched without case", + dialect: &CataloguedDialect, + sql: "SELECT CURRENT_TIMESTAMP() AS created_at FROM orders", + extension_functions: &["current_timestamp"], + expected_findings: &[], + }, + FindingTestCase { + description: "an undeclared function is still unknown beside declared extensions", + dialect: &CataloguedDialect, + sql: "SELECT CURRENT_TIMESTAMP() AS created_at FROM orders", + extension_functions: &["md_user_info"], + expected_findings: &[("B101", None)], + }, + ]; + for test_case in test_cases { + let analysis: NativeAnalysis = analysed( + test_case.dialect, + test_case.sql, + test_case.extension_functions, + ); + let findings: Vec<(&str, Option<(usize, usize)>)> = analysis + .diagnostics + .iter() + .map(|row| (row.code.as_str(), row.line.zip(row.column))) + .collect(); + assert_eq!( + findings, test_case.expected_findings, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_query_over_referenced_columns_when_analysing_then_lineage_reads_the_resources() { + let test_cases = [ + LineageTestCase { + description: "direct, cast and constant columns read the model; an unresolved one is \ + read from the one relation at medium confidence", + sql: "SELECT id, CAST(id AS TEXT) AS id_text, 1 AS one, nope FROM orders \ + WHERE status = 'paid'", + expected_lineage: &[ + ("id", 0, 1, &["id"]), + ("id_text", 1, 1, &["id"]), + ("one", 5, 1, &[]), + ("nope", 0, 2, &["nope"]), + ], + expected_row_sources: &[("orders", "status")], + }, + LineageTestCase { + description: "a query without a filter has no columns deciding rows", + sql: "SELECT status FROM orders", + expected_lineage: &[("status", 0, 1, &["status"])], + expected_row_sources: &[], + }, + ]; + for test_case in test_cases { + let analysis: NativeAnalysis = analysed(&PivotlessDialect, test_case.sql, &[]); + let lineage: Vec<(&str, u8, u8, Vec<&str>)> = analysis + .lineage + .iter() + .map(|row| { + ( + row.output_column.as_str(), + row.transform_code, + row.confidence_code, + resource_columns(row), + ) + }) + .collect(); + let row_sources: Vec<(&str, &str)> = analysis + .row_sources + .iter() + .map(|(relation, column)| (relation.as_str(), column.as_str())) + .collect(); + assert_eq!( + (lineage, row_sources), + ( + test_case + .expected_lineage + .iter() + .map(|&(name, transform, confidence, columns)| { + (name, transform, confidence, columns.to_vec()) + }) + .collect::>(), + test_case.expected_row_sources.to_vec() + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_schema_columns_when_analysing_then_they_keep_their_types_and_nullability() { + let test_cases = [ + ColumnTestCase { + description: "a non-null INTEGER column", + sql: "SELECT id FROM orders", + expected_columns: &[("id", Some("INTEGER"), "non_null")], + }, + ColumnTestCase { + description: "a nullable VARCHAR column", + sql: "SELECT status FROM orders", + expected_columns: &[("status", Some("VARCHAR"), "nullable")], + }, + ]; + for test_case in test_cases { + let analysis: NativeAnalysis = analysed(&PivotlessDialect, test_case.sql, &[]); + let columns: Vec<(&str, Option<&str>, &str)> = analysis + .columns + .iter() + .map(|column| { + ( + column.name.as_str(), + column.data_type.as_deref(), + column.nullability.as_str(), + ) + }) + .collect(); + assert_eq!( + columns, test_case.expected_columns, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_star_when_analysing_then_it_is_resolved_only_over_known_columns() { + let test_cases = [ + StarTestCase { + description: "a star over a known relation projects its columns", + sql: "SELECT * FROM orders", + expected_star: (true, true, 2), + }, + StarTestCase { + description: "a star over an unknown relation leaves columns out", + sql: "SELECT * FROM elsewhere", + expected_star: (true, false, 0), + }, + ]; + for test_case in test_cases { + let analysis: NativeAnalysis = analysed(&PivotlessDialect, test_case.sql, &[]); + assert_eq!( + ( + analysis.has_star, + analysis.star_resolved, + analysis.columns.len() + ), + test_case.expected_star, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_macro_body_when_reading_it_then_only_one_expression_is_read() { + let test_cases = [ + MacroBodyTestCase { + description: "one expression is the body", + body: "a + 1", + expected_gap: None, + }, + MacroBodyTestCase { + description: "two expressions are not one body", + body: "a, b", + expected_gap: Some(("not one expression", 0)), + }, + MacroBodyTestCase { + description: "a query is not a body", + body: "1 FROM t", + expected_gap: Some(("not one expression", 0)), + }, + MacroBodyTestCase { + description: "a refused construct is placed in the body, not the query around it", + body: "x PIVOT", + expected_gap: Some(("PIVOT", 2)), + }, + ]; + for test_case in test_cases { + assert_eq!( + PivotlessDialect + .macro_body(test_case.body) + .err() + .map(|gap| (gap.construct, gap.offset)), + test_case + .expected_gap + .map(|(construct, offset)| (construct.to_owned(), offset)), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_gap.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_gap.rs index 75a1600999..fc714b2bd1 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_gap.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_gap.rs @@ -1,11 +1,12 @@ use crate::analysis_route::main::analysis_route::analysis_route; use crate::analysis_route::main::native_gap::native_gap; use crate::analysis_route::main::skipped_analysis::skipped_analysis; -use crate::analysis_route::models::{AnalysisRoute, LocatedGap}; +use crate::analysis_route::models::{AnalysisRoute, GapPosition, GapSite, LocatedGap}; use crate::analysis_route::tests::test_types::{ NativeGapTestCase, PivotlessDialect, RouteTestCase, SkippedAnalysisTestCase, }; use crate::assembly::project::models::SyntaxCheck; +use crate::compiled_project::models::TextPosition; #[test] fn given_any_dialect_when_routing_analysis_then_it_stays_on_the_legacy_path() { @@ -105,8 +106,37 @@ fn given_model_sql_when_probing_the_native_parser_then_its_gap_is_located_after_ placeholders: &[], expected_gap: Some(LocatedGap { construct: "PIVOT".to_owned(), + message: "PIVOT".to_owned(), line: 3, column: 3, + syntax: false, + source_offset: Some(25), + }), + }, + NativeGapTestCase { + description: "a stub respelt before the gap keeps its offset in the checked SQL", + sql: "SELECT __cursor_start() AS a FROM t PIVOT (sum(x) FOR y IN (1))", + placeholders: &[], + expected_gap: Some(LocatedGap { + construct: "PIVOT".to_owned(), + message: "PIVOT".to_owned(), + line: 1, + column: 46, + syntax: false, + source_offset: Some(36), + }), + }, + NativeGapTestCase { + description: "text changed on both sides of the gap leaves it only in the analysed SQL", + sql: "SELECT __cursor_start() AS a FROM t PIVOT (sum(x) FOR y IN (@@@n))", + placeholders: &[("n", "1")], + expected_gap: Some(LocatedGap { + construct: "PIVOT".to_owned(), + message: "PIVOT".to_owned(), + line: 1, + column: 46, + syntax: false, + source_offset: None, }), }, ]; @@ -129,23 +159,90 @@ fn given_model_sql_when_probing_the_native_parser_then_its_gap_is_located_after_ } #[test] -fn given_a_located_gap_when_skipping_analysis_then_p026_names_the_model_and_the_gap() { - let test_cases = [SkippedAnalysisTestCase { - description: "a PIVOT gap on line 3", - model: "orders_pivot", - dialect: "postgres", - gap: LocatedGap { - construct: "PIVOT".to_owned(), - line: 3, - column: 3, - }, - expected_code: "P026", - expected_message: "SQL analysis skipped for model 'orders_pivot': the postgres parser \ - cannot read PIVOT at line 3, column 3", - }]; +fn given_a_located_gap_when_skipping_analysis_then_p026_names_the_model_its_site_and_the_gap() { + let test_cases = [ + SkippedAnalysisTestCase { + description: "a query gap placed in the model file, with the parser's message", + model: "orders", + dialect: "postgres", + gap: LocatedGap { + construct: "syntax error".to_owned(), + message: "syntax error at or near \"FROM\"".to_owned(), + line: 1, + column: 15, + syntax: true, + source_offset: Some(14), + }, + site: GapSite { + label: None, + path: "models/orders.sql", + position: GapPosition::File(TextPosition { + line: 5, + column: 18, + }), + }, + expected_code: "P026", + expected_message: "SQL analysis skipped for model 'orders' (models/orders.sql): the \ + postgres parser cannot read syntax error at line 5, column 18: \ + syntax error at or near \"FROM\"", + }, + SkippedAnalysisTestCase { + description: "a hook gap names the hook and is placed in its statement", + model: "customer_totals", + dialect: "duckdb", + gap: LocatedGap { + construct: "Syntax error at or near 'WHERE'".to_owned(), + message: "Syntax error at or near 'WHERE'".to_owned(), + line: 1, + column: 13, + syntax: true, + source_offset: Some(12), + }, + site: GapSite { + label: Some("post_hooks[0] sql(\"record_refresh\")"), + path: "hooks/sql/record_refresh.sql", + position: GapPosition::Statement(TextPosition { + line: 1, + column: 13, + }), + }, + expected_code: "P026", + expected_message: "SQL analysis skipped for model 'customer_totals' post_hooks[0] \ + sql(\"record_refresh\") (hooks/sql/record_refresh.sql): the duckdb \ + parser cannot read Syntax error at or near 'WHERE' at line 1, \ + column 13 of its SQL", + }, + SkippedAnalysisTestCase { + description: "a gap only the analysed SQL places says so", + model: "orders_pivot", + dialect: "postgres", + gap: LocatedGap { + construct: "PIVOT".to_owned(), + message: "PIVOT".to_owned(), + line: 3, + column: 3, + syntax: false, + source_offset: None, + }, + site: GapSite { + label: None, + path: "models/orders_pivot.sql", + position: GapPosition::Analysed, + }, + expected_code: "P026", + expected_message: "SQL analysis skipped for model 'orders_pivot' \ + (models/orders_pivot.sql): the postgres parser cannot read PIVOT at \ + line 3, column 3 of the analysed SQL", + }, + ]; for test_case in test_cases { assert_eq!( - skipped_analysis(test_case.model, test_case.dialect, &test_case.gap), + skipped_analysis( + test_case.model, + test_case.dialect, + &test_case.gap, + &test_case.site + ), ( test_case.expected_code, test_case.expected_message.to_owned() diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs new file mode 100644 index 0000000000..bbeaa37d63 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_outcome.rs @@ -0,0 +1,401 @@ +use crate::analysis_route::main::fast_lineage::fast_lineage; +use crate::analysis_route::main::model_outcome::model_outcome; +use crate::analysis_route::tests::helpers::{column_line, session_relations}; +use crate::analysis_route::tests::test_types::{ + CataloguedDialect, FoldingDialect, MarkerLineageTestCase, NativeLineageTestCase, + NativeOutcomeTestCase, OpenRelationTestCase, PivotlessDialect, +}; +use crate::assembly::analysis_session::_helpers::mappings::{ShapeTable, session_schema}; +use crate::assembly::analysis_session::models::{ + ModelOutcome, ModelRequest, SessionFunctions, SessionSchema, +}; +use crate::assembly::analysis_session::types::Shapes; +use crate::lineage::models::{ + LineageColumn, LineageOutcomeParts, LineageResourceType, LineageSchemaResource, +}; +use sqlquack_semantics::lineage::models::{Nullability, NullabilityRule}; + +#[test] +fn given_session_shapes_when_a_native_route_analyses_a_model_then_its_facts_follow_them() { + let test_cases = [ + NativeOutcomeTestCase { + description: "a closed source expands a star with its types and known nullability", + sql: "SELECT * FROM raw_orders", + shape: &[ + ("order_id", "INTEGER", Nullability::NonNull), + ("status", "VARCHAR", Nullability::Nullable), + ], + declared_functions: &[], + nullability_rules: &[], + expected_lines: &[ + "order_id INTEGER non_null", + "status VARCHAR nullable", + "succeeded=true star=true/true findings=[]", + ], + }, + NativeOutcomeTestCase { + description: "a relation the session has no shape for is unknown, not empty", + sql: "SELECT * FROM raw_orders", + shape: &[], + declared_functions: &[], + nullability_rules: &[], + expected_lines: &["succeeded=true star=true/false findings=[]"], + }, + NativeOutcomeTestCase { + description: "SQL the dialect cannot read fails the analysis", + sql: "SELECT FROM WHERE", + shape: &[("order_id", "INTEGER", Nullability::NonNull)], + declared_functions: &[], + nullability_rules: &[], + expected_lines: &["succeeded=false star=false/false findings=[]"], + }, + NativeOutcomeTestCase { + description: "a function the adapter declares is not unknown", + sql: "SELECT CURRENT_TIMESTAMP() AS at FROM raw_orders", + shape: &[("order_id", "INTEGER", Nullability::NonNull)], + declared_functions: &[("CURRENT_TIMESTAMP", "TIMESTAMPTZ")], + nullability_rules: &[], + expected_lines: &[ + "at TIMESTAMPTZ unknown", + "succeeded=true star=false/true findings=[]", + ], + }, + NativeOutcomeTestCase { + description: "the same function without the declaration is unknown", + sql: "SELECT CURRENT_TIMESTAMP() AS at FROM raw_orders", + shape: &[("order_id", "INTEGER", Nullability::NonNull)], + declared_functions: &[], + nullability_rules: &[], + expected_lines: &[ + "at TIMESTAMPTZ unknown", + "succeeded=true star=false/true findings=[\"B101\"]", + ], + }, + NativeOutcomeTestCase { + description: "a declared function's return type types its call", + sql: "SELECT shop_day(order_id) AS d FROM raw_orders", + shape: &[("order_id", "INTEGER", Nullability::NonNull)], + declared_functions: &[("SHOP_DAY", "DATE")], + nullability_rules: &[], + expected_lines: &[ + "d DATE unknown", + "succeeded=true star=false/true findings=[]", + ], + }, + NativeOutcomeTestCase { + description: "a declared function without a return type is untyped", + sql: "SELECT shop_day(order_id) AS d FROM raw_orders", + shape: &[("order_id", "INTEGER", Nullability::NonNull)], + declared_functions: &[("SHOP_DAY", "")], + nullability_rules: &[], + expected_lines: &["d - unknown", "succeeded=true star=false/true findings=[]"], + }, + NativeOutcomeTestCase { + description: "an adapter's first-argument rule keeps a non-null argument non-null", + sql: "SELECT shop_label(status) AS s FROM raw_orders", + shape: &[("status", "VARCHAR", Nullability::NonNull)], + declared_functions: &[("SHOP_LABEL", "VARCHAR")], + nullability_rules: &[("SHOP_LABEL", NullabilityRule::FirstArgument)], + expected_lines: &[ + "s VARCHAR non_null", + "succeeded=true star=false/true findings=[]", + ], + }, + NativeOutcomeTestCase { + description: "without the rule the same call's nullability is unknown", + sql: "SELECT shop_label(status) AS s FROM raw_orders", + shape: &[("status", "VARCHAR", Nullability::NonNull)], + declared_functions: &[("SHOP_LABEL", "VARCHAR")], + nullability_rules: &[], + expected_lines: &[ + "s VARCHAR unknown", + "succeeded=true star=false/true findings=[]", + ], + }, + NativeOutcomeTestCase { + description: "a quoted stored name is the quoted column, its doubled quote one quote", + sql: "SELECT \"Say \"\"hi\"\"\" AS c FROM raw_orders", + shape: &[("\"Say \"\"hi\"\"\"", "VARCHAR", Nullability::NonNull)], + declared_functions: &[], + nullability_rules: &[], + expected_lines: &[ + "c VARCHAR non_null", + "succeeded=true star=false/true findings=[]", + ], + }, + ]; + + for test_case in test_cases { + let (schema, mut relations): (Shapes, SessionSchema) = session_relations(test_case.shape); + relations.functions = SessionFunctions { + return_types: test_case + .declared_functions + .iter() + .filter(|(_, data_type)| !data_type.is_empty()) + .map(|(name, data_type)| ((*name).to_owned(), (*data_type).to_owned())) + .collect(), + nullability: test_case + .nullability_rules + .iter() + .map(|(name, rule)| ((*name).to_owned(), *rule)) + .collect(), + }; + let model: ModelRequest = ModelRequest { + name: "orders".to_owned(), + query_sql: test_case.sql.to_owned(), + required_names: vec!["raw_orders".to_owned()], + ..ModelRequest::default() + }; + let declared: Vec = test_case + .declared_functions + .iter() + .map(|(name, _)| (*name).to_owned()) + .collect(); + let (outcome, failure): (ModelOutcome, Option) = model_outcome( + &CataloguedDialect, + &model, + (schema, &relations), + (&declared, false), + ); + let analysis = &outcome.analysis; + let mut lines: Vec = analysis + .columns + .iter() + .flatten() + .map(|column| { + format!( + "{} {} {}", + column.name, + column.data_type.as_deref().unwrap_or("-"), + column.nullability + ) + }) + .collect(); + let findings: Vec<&str> = analysis + .binding_diagnostics + .iter() + .map(|row| row.code.as_str()) + .collect(); + lines.push(format!( + "succeeded={} star={}/{} findings={findings:?}", + analysis.analysis_succeeded, analysis.has_star, analysis.star_resolved + )); + assert_eq!(lines, test_case.expected_lines, "{}", test_case.description); + assert_eq!( + failure.is_some(), + !analysis.analysis_succeeded, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_relation_with_declared_columns_only_when_analysing_natively_then_they_type_its_reads() { + let test_cases = [ + OpenRelationTestCase { + description: "a declared VARCHAR read in arithmetic is a type error", + sql: "SELECT status + 1 AS bumped FROM raw_orders", + declared: &[("status", "VARCHAR"), ("amount", "DOUBLE")], + expected_lines: &["bumped - unknown", "findings=[\"B212\"]"], + }, + OpenRelationTestCase { + description: "a declared column read as it is keeps its declared type", + sql: "SELECT amount FROM raw_orders", + declared: &[("status", "VARCHAR"), ("amount", "DOUBLE")], + expected_lines: &["amount DOUBLE unknown", "findings=[]"], + }, + OpenRelationTestCase { + description: "an undeclared column of an open relation is still not refused", + sql: "SELECT region FROM raw_orders", + declared: &[("status", "VARCHAR")], + expected_lines: &["region - unknown", "findings=[]"], + }, + OpenRelationTestCase { + description: "with nothing declared, nothing is typed", + sql: "SELECT status + 1 AS bumped FROM raw_orders", + declared: &[], + expected_lines: &["bumped - unknown", "findings=[]"], + }, + ]; + for test_case in test_cases { + let schema: Shapes = vec![("raw_orders".to_owned(), Vec::new())]; + let declared: Shapes = vec![( + "raw_orders".to_owned(), + test_case + .declared + .iter() + .map(|(name, data_type)| ((*name).to_owned(), (*data_type).to_owned())) + .collect(), + )]; + let relations: SessionSchema = session_schema( + &CataloguedDialect, + &schema, + ( + &ShapeTable::from_shapes(&declared), + &ShapeTable::from_shapes(&Vec::new()), + ), + &SessionFunctions::default(), + ); + let model: ModelRequest = ModelRequest { + name: "orders".to_owned(), + query_sql: test_case.sql.to_owned(), + required_names: vec!["raw_orders".to_owned()], + ..ModelRequest::default() + }; + let (outcome, _): (ModelOutcome, Option) = model_outcome( + &CataloguedDialect, + &model, + (schema, &relations), + (&[], false), + ); + let mut lines: Vec = outcome + .analysis + .columns + .iter() + .flatten() + .map(|column| { + format!( + "{} {} {}", + column.name, + column.data_type.as_deref().unwrap_or("-"), + column.nullability + ) + }) + .collect(); + let findings: Vec<&str> = outcome + .analysis + .binding_diagnostics + .iter() + .map(|row| row.code.as_str()) + .collect(); + lines.push(format!("findings={findings:?}")); + assert_eq!(lines, test_case.expected_lines, "{}", test_case.description); + } +} + +#[test] +fn given_a_model_without_facts_when_fast_lineage_reads_it_natively_then_it_has_lineage_or_a_reason() +{ + let schema: Vec = vec![ + LineageSchemaResource { + resource_type: LineageResourceType::Model, + name: "orders".to_owned(), + columns: vec!["order_id".to_owned(), "amount".to_owned()], + }, + LineageSchemaResource { + resource_type: LineageResourceType::Source, + name: "raw_payments".to_owned(), + columns: vec!["payment_id".to_owned()], + }, + ]; + let test_cases = [ + NativeLineageTestCase { + description: "a referenced model's columns are the upstreams of direct and computed outputs", + sql: r#"SELECT o.order_id, o.amount * 2 AS doubled FROM __ref("orders") AS o"#, + placeholders: &[], + expected_lines: &[ + "order_id direct high [model:orders.order_id]", + "doubled expression high [model:orders.amount]", + "built star=false", + ], + }, + NativeLineageTestCase { + description: "a placeholder takes its default before the parse, and a source is a source", + sql: r#"SELECT payment_id FROM __source("raw_payments") WHERE payment_id > @@@min_id"#, + placeholders: &[("min_id", "0")], + expected_lines: &[ + "payment_id direct high [source:raw_payments.payment_id]", + "built star=false", + ], + }, + NativeLineageTestCase { + description: "repeated calls of one marker read as one relation, so a union stays direct", + sql: r#"SELECT order_id FROM __ref("orders") UNION ALL SELECT order_id FROM __ref("orders")"#, + placeholders: &[], + expected_lines: &[ + "order_id direct high [model:orders.order_id]", + "built star=false", + ], + }, + NativeLineageTestCase { + description: "a star over a relation of unknown columns is a star without outputs", + sql: r#"SELECT * FROM __ref("unknown_model")"#, + placeholders: &[], + expected_lines: &["built star=true"], + }, + NativeLineageTestCase { + description: "SQL the parser cannot read gives P026's reason, located", + sql: "SELECT 1\nFROM t PIVOT (sum(x) FOR y IN (1))", + placeholders: &[], + expected_lines: &[ + "unparsed star=false", + "the postgres parser cannot read PIVOT at line 2, column 8", + ], + }, + ]; + + for test_case in test_cases { + let placeholders: Vec<(String, String)> = test_case + .placeholders + .iter() + .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) + .collect(); + let parts: LineageOutcomeParts = + fast_lineage(&PivotlessDialect, (test_case.sql, &placeholders), &schema).into_parts(); + let mut lines: Vec = parts.columns.iter().map(column_line).collect(); + lines.push(format!("{} star={}", parts.status, parts.has_star)); + lines.extend(parts.detail); + assert_eq!(lines, test_case.expected_lines, "{}", test_case.description); + } +} + +#[test] +fn given_relation_markers_when_each_dialect_reads_fast_lineage_then_sources_keep_their_resource() { + let schema: Vec = vec![ + LineageSchemaResource { + resource_type: LineageResourceType::Source, + name: "orders".to_owned(), + columns: vec!["order_id".to_owned(), "loaded_at".to_owned()], + }, + LineageSchemaResource { + resource_type: LineageResourceType::Model, + name: "orders".to_owned(), + columns: vec!["order_id".to_owned(), "amount".to_owned()], + }, + ]; + let sql: &str = r#"SELECT o.order_id, o.amount, s.loaded_at +FROM __ref("orders") AS o JOIN __source("orders") AS s ON o.order_id = s.order_id"#; + let expected_lines: &[&str] = &[ + "order_id direct high [model:orders.order_id]", + "amount direct high [model:orders.amount]", + "loaded_at direct high [source:orders.loaded_at]", + "built star=false", + ]; + let test_cases = [ + MarkerLineageTestCase { + description: "a lower-case folding dialect matches each marker's name", + dialect: &PivotlessDialect, + expected_lines, + }, + MarkerLineageTestCase { + description: "a case-preserving dialect matches each marker's name", + dialect: &CataloguedDialect, + expected_lines, + }, + MarkerLineageTestCase { + description: "an upper-case folding dialect matches each marker's name", + dialect: &FoldingDialect, + expected_lines, + }, + ]; + + for test_case in test_cases { + let parts: LineageOutcomeParts = + fast_lineage(test_case.dialect, (sql, &[]), &schema).into_parts(); + let mut lines: Vec = parts.columns.iter().map(column_line).collect(); + lines.push(format!("{} star={}", parts.status, parts.has_star)); + lines.extend(parts.detail); + assert_eq!(lines, test_case.expected_lines, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_revalidation.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_revalidation.rs new file mode 100644 index 0000000000..6ad508bc45 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_revalidation.rs @@ -0,0 +1,86 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use crate::analysis_route::main::native_revalidation::native_revalidation; +use crate::analysis_route::tests::helpers::{finding_rows, outcome}; +use crate::analysis_route::tests::test_types::{PivotlessDialect, RevalidationTestCase}; +use crate::assembly::analysis_session::models::{ + FinishedSession, ModelOutcome, PivotOutcome, PivotTables, SessionFunctions, +}; +use crate::semantic_checks::models::{RawBinding, RecoveryModel}; +use crate::semantic_checks::types::RevisedBinding; +use crate::semantic_validation::main::new_analysis_pool::new_analysis_pool; + +#[test] +fn given_poisoned_outputs_when_revalidating_natively_then_findings_keep_session_positions() { + let test_cases = [ + RevalidationTestCase { + description: "a finding on a typed column stays at the session's span", + sql: "SELECT id FROM orders WHERE status", + poisoned: &[], + expected_rows: &[("B211", Some(28), Some(34))], + }, + RevalidationTestCase { + description: "a finding only a poisoned column causes drops out", + sql: "SELECT id FROM orders WHERE status", + poisoned: &[("orders", "status")], + expected_rows: &[], + }, + RevalidationTestCase { + description: "a model the parser cannot read again keeps its own findings", + sql: "SELECT id FROM orders PIVOT (sum(id) FOR status IN (1))", + poisoned: &[("orders", "status")], + expected_rows: &[("B002", Some(7), Some(9))], + }, + ]; + + for test_case in test_cases { + let finished: FinishedSession = FinishedSession { + tables: PivotTables::default(), + pool: new_analysis_pool().map(Arc::new), + models: HashMap::new(), + dynamic_proofs: HashMap::new(), + relations: None, + model_names: vec!["other".to_owned(), "downstream".to_owned()], + declared_functions: Vec::new(), + functions: SessionFunctions::default(), + quoted_ignore_case: false, + }; + let outcomes: Vec<(ModelOutcome, PivotOutcome)> = vec![ + (outcome("SELECT 1"), PivotOutcome::Absent), + (outcome(test_case.sql), PivotOutcome::Absent), + ]; + let model: RecoveryModel = RecoveryModel { + name: "downstream".to_owned(), + query_sql: test_case.sql.to_owned(), + inferred_columns: None, + references: vec!["orders".to_owned()], + lineage: Vec::new(), + bindings: Vec::new(), + raw_bindings: vec![RawBinding { + id: 0, + code: "B002".to_owned(), + message: "Unknown column".to_owned(), + start: Some(7), + end: Some(9), + }], + }; + let poisoned: Vec<(String, String)> = test_case + .poisoned + .iter() + .map(|(relation, column)| ((*relation).to_owned(), (*column).to_owned())) + .collect(); + let revised: Result>, String> = native_revalidation( + &PivotlessDialect, + (&finished, &outcomes), + &[&model], + &poisoned, + ); + assert_eq!( + revised.as_deref().map(finding_rows), + Ok(test_case.expected_rows.to_vec()), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs index 9587ebda22..0ec0a299b4 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs @@ -1,5 +1,13 @@ -use crate::analysis_route::models::{LocatedGap, ParserGap, TypeMatch}; +use sqlquack_semantics::dialects::constants::{DUCKDB, POSTGRES, SNOWFLAKE}; +use sqlquack_semantics::dialects::types::DialectProfile; +use sqlquack_semantics::lineage::models::{Nullability, NullabilityRule}; +use sqlquack_syntax::expressions::statements::models::Command; +use sqlquack_syntax::{Dialect, Expression, ParseOptions, Parsed, parse_strict}; + +use crate::analysis_route::models::{AnalysisRoute, GapSite, LocatedGap, ParserGap, TypeMatch}; use crate::analysis_route::types::NativeDialect; +use crate::type_system::main::normalize_text_type::normalize_text_type; +use crate::type_system::models::NormalizedType; pub(super) struct NativeGapTestCase { pub(super) description: &'static str, @@ -19,11 +27,12 @@ pub(super) struct SkippedAnalysisTestCase { pub(super) model: &'static str, pub(super) dialect: &'static str, pub(super) gap: LocatedGap, + pub(super) site: GapSite<'static>, pub(super) expected_code: &'static str, pub(super) expected_message: &'static str, } -/// A test dialect whose parser refuses PIVOT, unfilled placeholders and one-part stubs. +/// The PostgreSQL parser, refusing PIVOT, placeholders, one-part stubs and the type `NOT_A_TYPE`. pub(super) struct PivotlessDialect; impl NativeDialect for PivotlessDialect { @@ -31,29 +40,236 @@ impl NativeDialect for PivotlessDialect { "postgres" } - fn read_whole(&self, sql: &str) -> Result<(), ParserGap> { + fn stub_prefix(&self) -> &'static str { + "__sqlbuild." + } + + fn parse(&self, sql: &str) -> Result { let refused = ["PIVOT", "@@@", "__sqlbuild_udf_", "__cursor_", "@'"] .iter() .filter_map(|word| sql.find(word).map(|offset| (offset, *word))) .min(); - match refused { - Some((offset, word)) => Err(ParserGap { + if let Some(text) = sql.strip_prefix("RAW ") { + return Ok(Parsed { + statements: vec![Expression::Command(Box::new(Command { + this: text.to_owned(), + }))], + gaps: Vec::new(), + }); + } + if let Some((offset, word)) = refused { + return Err(ParserGap { construct: word.to_owned(), + message: word.to_owned(), offset, - }), - None => Ok(()), + syntax: false, + }); } + sqlquack_syntax_postgres::parse(sql).map_err(|error| ParserGap { + construct: error.message.clone(), + message: error.message, + offset: error.position, + syntax: error.syntax, + }) + } + + fn profile(&self) -> &'static dyn DialectProfile { + &POSTGRES + } + + fn normalize_type(&self, spelling: &str) -> Option { + (!spelling.eq_ignore_ascii_case("NOT_A_TYPE")).then(|| normalize_text_type(spelling, None)) + } +} + +/// DuckDB's shared parser under the DuckDB profile, as MotherDuck projects are analysed. +pub(super) struct CataloguedDialect; + +impl NativeDialect for CataloguedDialect { + fn name(&self) -> &'static str { + "duckdb" } fn stub_prefix(&self) -> &'static str { - "__sqlbuild." + "__sqlbuild_" + } + + fn parse(&self, sql: &str) -> Result { + parse_strict(sql, Dialect::DuckDB, &ParseOptions::default()) + .map(|statements| Parsed { + statements, + gaps: Vec::new(), + }) + .map_err(|failure| ParserGap { + construct: failure.to_string(), + message: failure.to_string(), + offset: 0, + syntax: false, + }) + } + + fn profile(&self) -> &'static dyn DialectProfile { + &DUCKDB + } + + fn normalize_type(&self, spelling: &str) -> Option { + PivotlessDialect.normalize_type(spelling) } } +/// Snowflake's shared parser under the Snowflake profile, which folds unquoted names to upper case. +pub(super) struct FoldingDialect; + +impl NativeDialect for FoldingDialect { + fn name(&self) -> &'static str { + "snowflake" + } + + fn stub_prefix(&self) -> &'static str { + "__sqlbuild_" + } + + fn parse(&self, sql: &str) -> Result { + parse_strict(sql, Dialect::Snowflake, &ParseOptions::default()) + .map(|statements| Parsed { + statements, + gaps: Vec::new(), + }) + .map_err(|failure| ParserGap { + construct: failure.to_string(), + message: failure.to_string(), + offset: 0, + syntax: false, + }) + } + + fn profile(&self) -> &'static dyn DialectProfile { + &SNOWFLAKE + } + + fn normalize_type(&self, spelling: &str) -> Option { + PivotlessDialect.normalize_type(spelling) + } +} + +pub(super) struct MarkerLineageTestCase { + pub(super) description: &'static str, + pub(super) dialect: &'static dyn NativeDialect, + pub(super) expected_lines: &'static [&'static str], +} + +pub(super) struct RevalidationTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) poisoned: &'static [(&'static str, &'static str)], + pub(super) expected_rows: &'static [(&'static str, Option, Option)], +} + +pub(super) struct ReadWholeTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) expected_gap: Option<&'static str>, +} + +pub(super) struct NativeTypeMatchTestCase { + pub(super) description: &'static str, + pub(super) declared: &'static str, + pub(super) inferred: Option<&'static str>, + pub(super) expected_match: TypeMatch, +} + pub(super) struct TypeMatchTestCase { pub(super) description: &'static str, pub(super) declared: &'static str, - pub(super) inferred: &'static str, + pub(super) inferred: Option<&'static str>, pub(super) inferred_nullable: bool, pub(super) expected_match: TypeMatch, } + +pub(super) struct FindingTestCase { + pub(super) description: &'static str, + pub(super) dialect: &'static dyn NativeDialect, + pub(super) sql: &'static str, + pub(super) extension_functions: &'static [&'static str], + /// The code of each finding in order, with its line and column when the parse placed it. + pub(super) expected_findings: &'static [(&'static str, Option<(usize, usize)>)], +} + +pub(super) struct MacroBodyTestCase { + pub(super) description: &'static str, + pub(super) body: &'static str, + /// `(construct, offset in the body)` of the gap, when the body is not read. + pub(super) expected_gap: Option<(&'static str, usize)>, +} + +pub(super) struct UnavailableAnalysisTestCase { + pub(super) description: &'static str, + pub(super) dialect: Option<&'static str>, + pub(super) expected_message: &'static str, +} + +pub(super) struct FindingMessageTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) expected_messages: &'static [(&'static str, &'static str)], +} + +pub(super) struct UnappliedRulesTestCase { + pub(super) description: &'static str, + pub(super) route: AnalysisRoute, + pub(super) functions: &'static [&'static str], + pub(super) expected_warning: Option<&'static str>, +} + +/// Each output's name, transform code, confidence code and the columns it reads. +type ExpectedLineage = &'static [(&'static str, u8, u8, &'static [&'static str])]; + +pub(super) struct LineageTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) expected_lineage: ExpectedLineage, + pub(super) expected_row_sources: &'static [(&'static str, &'static str)], +} + +pub(super) struct ColumnTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + /// Each output's name, type and nullability. + pub(super) expected_columns: &'static [(&'static str, Option<&'static str>, &'static str)], +} + +pub(super) struct StarTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + /// Whether it projects a star, whether every star column is known, and the column count. + pub(super) expected_star: (bool, bool, usize), +} + +pub(super) struct NativeOutcomeTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + /// `raw_orders`' columns, types and nullability as the session knows them; empty when unknown. + pub(super) shape: &'static [(&'static str, &'static str, Nullability)], + /// The adapter's declared functions as `(name, return type)`. + pub(super) declared_functions: &'static [(&'static str, &'static str)], + /// The adapter's built-in nullability rules as `(function name, rule)`. + pub(super) nullability_rules: &'static [(&'static str, NullabilityRule)], + /// Each output column as `name type nullability`, then success, star flags and finding codes. + pub(super) expected_lines: &'static [&'static str], +} + +pub(super) struct OpenRelationTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + /// `raw_orders`' declared columns and types; its full shape is never known. + pub(super) declared: &'static [(&'static str, &'static str)], + /// Each output column as `name type`, then the finding codes. + pub(super) expected_lines: &'static [&'static str], +} + +pub(super) struct NativeLineageTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) placeholders: &'static [(&'static str, &'static str)], + pub(super) expected_lines: &'static [&'static str], +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/types.rs b/crates/sqlbuild-analysis/src/analysis_route/types.rs index fedc80ad24..f64582c207 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/types.rs @@ -1,15 +1,110 @@ //! What a dialect's native analysis module provides. -use crate::analysis_route::models::ParserGap; +use sqlquack_semantics::dialects::types::DialectProfile; +use sqlquack_syntax::{Expression, Parsed}; + +use crate::analysis_route::_helpers::native_analysis::native_analysis; +use crate::analysis_route::_helpers::whole_gap::whole_gap; +use crate::analysis_route::constants::{ + MACRO_BODY_PREFIX, MORE_THAN_ONE_STATEMENT, NO_STATEMENT, NOT_ONE_EXPRESSION, + UNPARSED_STATEMENT, +}; +use crate::analysis_route::models::{AnalysisInput, NativeAnalysis, ParserGap, TypeMatch}; +use crate::type_system::models::NormalizedType; /// A dialect whose SQL SQLQuack analyses in place of Polyglot. pub trait NativeDialect: Sync { /// The adapter's analysis dialect name, such as `postgres`. fn name(&self) -> &'static str; - /// Whether the native parser reads `sql` whole: one statement, no gap and no unparsed bytes. - fn read_whole(&self, sql: &str) -> Result<(), ParserGap>; - /// The engine-accepted prefix of UDF and cursor stubs: `__sqlbuild_`, or `__sqlbuild.` for T-SQL. fn stub_prefix(&self) -> &'static str; + + /// Every statement of `sql` with the parser's named gaps, or the first error that stops it. + fn parse(&self, sql: &str) -> Result; + + /// The dialect's sqlquack-semantics profile: how its names resolve, compare and type. + fn profile(&self) -> &'static dyn DialectProfile; + + /// The profile `input` is analysed under; a dialect whose name rules follow a setting overrides it. + fn profile_for(&self, input: &AnalysisInput<'_>) -> &'static dyn DialectProfile { + let _ = input; + self.profile() + } + + /// A type as written in this dialect, as SQLBuild compares it; `None` when unknown or refused. + fn normalize_type(&self, spelling: &str) -> Option; + + /// A session column type, spelled as SQLBuild keeps it, as this dialect's parser reads it. + fn session_type(&self, spelling: &str) -> String { + spelling.to_owned() + } + + /// The one statement of `sql` when the parser reads it whole: no gap, and not raw text. + fn read_whole(&self, sql: &str) -> Result { + let parsed: Parsed = self.parse(sql)?; + if let Some(gap) = parsed.gaps.first() { + return Err(ParserGap { + construct: gap.construct.to_owned(), + message: gap.construct.to_owned(), + offset: gap.start, + syntax: false, + }); + } + let mut statements = parsed.statements.into_iter(); + match (statements.next(), statements.next()) { + (None, _) => Err(whole_gap(NO_STATEMENT, 0)), + (Some(_), Some(_)) => Err(whole_gap(MORE_THAN_ONE_STATEMENT, 0)), + (Some(Expression::Command(_)), None) => Err(whole_gap(UNPARSED_STATEMENT, 0)), + (Some(statement), None) => Ok(statement), + } + } + + /// The outputs, lineage and findings of the query `input.sql` in SQLBuild's facts and codes. + fn analyse(&self, input: &AnalysisInput<'_>) -> Result { + let statement: Expression = self.read_whole(input.sql)?; + Ok(native_analysis(self, &statement, input)) + } + + /// A macro body's one expression, read as the projection of `SELECT `. + fn macro_body(&self, body: &str) -> Result { + let statement: Expression = self + .read_whole(&format!("{MACRO_BODY_PREFIX}{body}")) + .map_err(|gap| ParserGap { + offset: gap.offset.saturating_sub(MACRO_BODY_PREFIX.len()), + ..gap + })?; + let Expression::Select(select) = statement else { + return Err(whole_gap(NOT_ONE_EXPRESSION, 0)); + }; + if select.from.is_some() { + return Err(whole_gap(NOT_ONE_EXPRESSION, 0)); + } + match <[Expression; 1]>::try_from(select.expressions) { + Ok([expression]) => Ok(expression), + Err(_) => Err(whole_gap(NOT_ONE_EXPRESSION, 0)), + } + } + + /// Whether a declared and an inferred type match; an override keeps `NotCompared` for unknowns. + fn type_match( + &self, + declared: &str, + inferred: Option<&str>, + inferred_nullable: bool, + ) -> TypeMatch { + let _ = inferred_nullable; + match ( + self.normalize_type(declared), + inferred.and_then(|inferred| self.normalize_type(inferred)), + ) { + (Some(declared), Some(inferred)) => TypeMatch::Compared { + same_type: declared == inferred, + nullable_into_non_null: false, + }, + _ => TypeMatch::NotCompared { + nullable_into_non_null: false, + }, + } + } } diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs index 757a894a90..a814d3bfaf 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs @@ -222,6 +222,7 @@ fn session_digest(request: &SessionRequest) -> ContentDigest { key.text(ANALYSIS_CACHE_FORMAT); key.text(&request.dialect); key.flag(request.case_sensitive_shapes); + key.flag(request.quoted_ignore_case); key.pairs(&request.function_return_types); key.optional_pairs(request.nullability_rules.as_ref()); key.flag(request.rich_type_inference); diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mappings.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mappings.rs index 0018c8c5c0..3a76ba7a57 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mappings.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mappings.rs @@ -4,9 +4,16 @@ use std::collections::{BTreeSet, HashMap}; use std::sync::OnceLock; use sha2::{Digest, Sha256}; +use sqlquack_semantics::lineage::models::{Nullability, NullabilityRule, SchemaColumn}; -use crate::assembly::analysis_session::constants::DIALECT_ALIASES; -use crate::assembly::analysis_session::models::ModelRequest; +use crate::analysis_route::main::schema_column::schema_column; +use crate::analysis_route::types::NativeDialect; +use crate::assembly::analysis_session::constants::{ + CONDITIONAL_RESULT_RULE, DIALECT_ALIASES, FIRST_ARG_RULE, SAFE_CAST_RULE, UNKNOWN_TYPE, +}; +use crate::assembly::analysis_session::models::{ + ModelRequest, SessionFunctions, SessionRequest, SessionSchema, +}; use crate::assembly::analysis_session::types::{Pairs, Shapes}; use crate::semantic_validation::models::Columns; use crate::semantic_validation::types::Relations; @@ -238,3 +245,92 @@ pub(crate) fn waves(producers: &[Vec]) -> Option>> { } (scheduled == producers.len()).then_some(waves) } + +/// The request's function return types, and its built-in nullability rules; adapter callbacks are not. +pub(crate) fn session_functions(request: &SessionRequest) -> SessionFunctions { + SessionFunctions { + return_types: request.function_return_types.clone(), + nullability: request + .nullability_rules + .iter() + .flatten() + .filter_map(|(function, id)| { + let rule: NullabilityRule = match id.as_str() { + FIRST_ARG_RULE => NullabilityRule::FirstArgument, + CONDITIONAL_RESULT_RULE => NullabilityRule::ConditionalResult, + SAFE_CAST_RULE => NullabilityRule::SafeCast, + _ => return None, + }; + Some((function.clone(), rule)) + }) + .collect(), + } +} + +/// Each relation's known shape, or else its declared columns, typed, with the session's functions. +pub(crate) fn session_schema( + dialect: &dyn NativeDialect, + schema: &Shapes, + (types, nullability): (&ShapeTable, &ShapeTable), + functions: &SessionFunctions, +) -> SessionSchema { + let mut relations: HashMap> = HashMap::new(); + let mut open: HashMap> = HashMap::new(); + for (relation, columns) in schema { + let (table, columns) = if columns.is_empty() { + match types.get(relation) { + Some(declared) if !declared.is_empty() => (&mut open, declared), + _ => continue, + } + } else { + (&mut relations, columns) + }; + let typed: Vec = typed_columns(dialect, columns, nullability.get(relation)); + let _ = table.insert(relation.clone(), typed); + } + SessionSchema { + relations, + open, + functions: SessionFunctions { + return_types: functions + .return_types + .iter() + .map(|(function, data_type)| (function.clone(), dialect.session_type(data_type))) + .collect(), + nullability: functions.nullability.clone(), + }, + } +} + +/// Each column with its known type as `dialect` reads it, and the nullability `known` spells. +fn typed_columns( + dialect: &dyn NativeDialect, + columns: &Pairs, + known: Option<&Pairs>, +) -> Vec { + columns + .iter() + .map(|(name, data_type)| { + schema_column( + name, + (!data_type.is_empty() && data_type != UNKNOWN_TYPE) + .then(|| dialect.session_type(data_type)), + column_nullability(known, name), + ) + }) + .collect() +} + +fn column_nullability(known: Option<&Pairs>, column: &str) -> Nullability { + let spelled: Option<&str> = known.and_then(|pairs| { + pairs + .iter() + .find(|(name, _)| name == column) + .map(|(_, value)| value.as_str()) + }); + match spelled { + Some("non_null") => Nullability::NonNull, + Some("nullable") => Nullability::Nullable, + _ => Nullability::Unknown, + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs index 19327a8c92..8328531601 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs @@ -9,7 +9,12 @@ use sqlbuild_cache::digest::types::ContentDigest; use sqlbuild_cache::store::errors::StoreDecodeError; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use sqlbuild_core::panics::main::native_failure::native_failure; +use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::main::model_outcome::model_outcome; +use crate::analysis_route::models::AnalysisRoute; +use crate::analysis_route::types::NativeDialect; use crate::assembly::analysis_session::_helpers::analysis_cache::{ CachedOutcome, decode_outcome, encode_outcome, }; @@ -26,7 +31,7 @@ use crate::assembly::analysis_session::_helpers::enrichment::{ Enrichment, EnrichmentInput, enrichment, }; use crate::assembly::analysis_session::_helpers::mappings::{ - ShapeTable, dict_from_pairs, same_keys, same_relations, + ShapeTable, dict_from_pairs, same_keys, same_relations, session_functions, session_schema, }; use crate::assembly::analysis_session::_helpers::mappings::{producers, waves}; use crate::assembly::analysis_session::constants::{UNKNOWN_NULLABILITY, UNKNOWN_TYPE}; @@ -34,9 +39,11 @@ use crate::assembly::analysis_session::main::resolved_cleaned_sql::resolved_clea use crate::assembly::analysis_session::models::{ AnalysisSession, ColumnFact, ContractProof, DeferredCleaning, FinishedSession, LineageFacts, LineageRow, ModelAnalysis, ModelOutcome, ModelRequest, Phase, PivotOutcome, PivotTables, - RelationTables, SessionModelFacts, SessionOutcome, SessionRequest, SessionStep, + RelationTables, SessionFunctions, SessionModelFacts, SessionOutcome, SessionRequest, + SessionSchema, SessionStep, }; use crate::assembly::analysis_session::types::{Pairs, Shapes}; +use crate::semantic_validation::main::normalize_native::normalize_native_sql; use crate::semantic_validation::models::DiagnosticRow; use crate::semantic_validation::models::ProjectCatalog; use crate::sql_macros::main::check_sql_macro_arguments::check_sql_macro_arguments; @@ -165,6 +172,18 @@ impl AnalysisSession { dynamic_proofs.insert(model.name.clone(), proof.clone()); } } + let model_names: Vec = request + .models + .iter() + .map(|model| model.name.clone()) + .collect(); + let declared_functions: Vec = request + .function_return_types + .iter() + .map(|(name, _)| name.clone()) + .collect(); + let functions: SessionFunctions = session_functions(&request); + let quoted_ignore_case: bool = request.quoted_ignore_case; let finished = FinishedSession { tables: PivotTables { dialect: request.dialect, @@ -177,6 +196,10 @@ impl AnalysisSession { models: model_facts, dynamic_proofs, relations, + model_names, + declared_functions, + functions, + quoted_ignore_case, }; let outcome = SessionOutcome { models, @@ -231,6 +254,9 @@ impl AnalysisSession { .map(|model| self.binding_schema(&self.request.models[*model])) .collect(); let misses: Vec = self.read_cache(models, &schemas)?; + if let AnalysisRoute::Native(dialect) = self.route() { + return self.analyze_native_wave(dialect, models, &misses, &schemas); + } let batch = Batch { dialect: &self.request.dialect, function_return_types: &self.request.function_return_types, @@ -294,6 +320,49 @@ impl AnalysisSession { self.check_sql_macros(models) } + /// Each model of the wave the cache did not answer, analysed by its dialect's native module. + fn analyze_native_wave( + &mut self, + dialect: &'static dyn NativeDialect, + models: &[usize], + misses: &[usize], + schemas: &[Shapes], + ) -> Result<(), String> { + let pool = self.catalog.native.analysis_pool()?; + let request: &SessionRequest = &self.request; + let tables: (&ShapeTable, &ShapeTable) = + (&self.available_types, &self.available_nullability); + let declared: Vec = request + .function_return_types + .iter() + .map(|(name, _)| name.clone()) + .collect(); + let functions: SessionFunctions = session_functions(request); + let outcomes: Vec<(ModelOutcome, Option)> = pool.install(|| { + misses + .par_iter() + .map(|position| { + let model: &ModelRequest = &request.models[models[*position]]; + let schema: Shapes = schemas[*position].clone(); + let relations: SessionSchema = + session_schema(dialect, &schema, tables, &functions); + let analysis = (declared.as_slice(), request.quoted_ignore_case); + model_outcome(dialect, model, (schema, &relations), analysis) + }) + .collect() + }); + for (position, (outcome, failure)) in misses.iter().zip(outcomes) { + self.failures.extend(failure); + self.outcomes[models[*position]] = Some(outcome); + } + self.check_sql_macros(models) + } + + /// The route of the session's dialect: a native module, or Polyglot. + pub(crate) fn route(&self) -> AnalysisRoute { + analysis_route(Some(&self.request.dialect)) + } + /// Each model's SQL macro call arguments checked against its analysed types, cache hits too. fn check_sql_macros(&mut self, models: &[usize]) -> Result<(), String> { let checked: Vec = models @@ -376,7 +445,7 @@ impl AnalysisSession { /// Python's `_complete_inferred_bindings` star and type pass for one wave. fn complete_wave(&mut self, wave: usize) -> Result<(), String> { - if self.cyclic { + if self.cyclic || matches!(self.route(), AnalysisRoute::Native(_)) { return Ok(()); } let mut candidates: Vec<(usize, Shapes, bool)> = Vec::new(); @@ -542,22 +611,8 @@ impl AnalysisSession { let Some(mut cache) = self.cache.take() else { return Ok((0..models.len()).collect()); }; - let mut required: Vec = Vec::new(); - let mut seen: HashSet<&str> = HashSet::new(); - for model in models { - for reference in &self.request.models[*model].references { - if seen.insert(reference.analysis_name.as_str()) { - required.push(reference.analysis_name.clone()); - } - } - } - self.catalog.prepare_analysis( - &required, - &self.available_types, - &self.available_nullability, - ); - for schema in schemas { - let _ = self.catalog.prepare(&[("", schema)]); + if matches!(self.route(), AnalysisRoute::Legacy) { + self.prepare_catalog(models, schemas); } let pool: Arc = self.catalog.native.analysis_pool()?; let session_digest: ContentDigest = cache.session_digest; @@ -621,6 +676,27 @@ impl AnalysisSession { Ok(misses) } + /// Load the relations and schemas a Polyglot analysis of `models` reads into its catalog. + fn prepare_catalog(&mut self, models: &[usize], schemas: &[Shapes]) { + let mut required: Vec = Vec::new(); + let mut seen: HashSet<&str> = HashSet::new(); + for model in models { + for reference in &self.request.models[*model].references { + if seen.insert(reference.analysis_name.as_str()) { + required.push(reference.analysis_name.clone()); + } + } + } + self.catalog.prepare_analysis( + &required, + &self.available_types, + &self.available_nullability, + ); + for schema in schemas { + let _ = self.catalog.prepare(&[("", schema)]); + } + } + /// Normalize the cleaned SQL of the cache hits among `models` that still lack it. fn ensure_cleaned(&mut self, models: &[usize]) -> Result<(), String> { let pending: Vec = models @@ -631,6 +707,18 @@ impl AnalysisSession { if pending.is_empty() { return Ok(()); } + if let AnalysisRoute::Native(dialect) = self.route() { + for model in pending { + let request: &ModelRequest = &self.request.models[model]; + let sql: String = + normalize_native_sql(&request.query_sql, dialect.name(), dialect.stub_prefix()) + .map(|normalised| placeholder_defaults(&normalised, &request.placeholders)) + .unwrap_or_default(); + self.outcome_mut(model)?.cleaned_sql = sql; + self.pending_cleaned[model] = false; + } + return Ok(()); + } let schemas: Vec = pending .iter() .map(|model| self.binding_schema(&self.request.models[*model])) diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs index 5da96a56e2..0473e9c3f1 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs @@ -196,7 +196,7 @@ pub(crate) const PYTHON_RULE: &str = "python"; pub(crate) const NON_NULL_NULLABILITY: &str = "non_null"; pub(crate) const NULLABLE_NULLABILITY: &str = "nullable"; /// Bumped whenever the stored outcome or a key's fields change; older entries never match. -pub(crate) const ANALYSIS_CACHE_FORMAT: &str = "native-model-analysis-v3"; +pub(crate) const ANALYSIS_CACHE_FORMAT: &str = "native-model-analysis-v4"; /// A stored varint byte at or above this value continues into the next byte. pub(crate) const VARINT_CONTINUATION: u64 = 0x80; /// The value bits each stored varint byte carries. diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs index 594038d064..1752e43587 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs @@ -7,6 +7,8 @@ use std::sync::Arc; use rayon::ThreadPool; use sqlbuild_cache::digest::types::ContentDigest; use sqlbuild_cache::store::models::NativeStore; +use sqlquack_semantics::lineage::models::{NullabilityRule, SchemaColumn}; +use sqlquack_semantics::lineage::types::LineageSchema; use crate::assembly::analysis_session::_helpers::catalog_state::SessionCatalog; use crate::assembly::analysis_session::_helpers::mappings::ShapeTable; @@ -112,6 +114,14 @@ pub struct FinishedSession { pub(crate) dynamic_proofs: HashMap, /// The relation facts the session ended with, when the request asked to keep them. pub(crate) relations: Option, + /// Each requested model's name, in request order, as the session's outcomes are. + pub(crate) model_names: Vec, + /// The adapter's declared functions, which the native analysis knew. + pub(crate) declared_functions: Vec, + /// The adapter's function return types and nullability rules the native analysis read. + pub(crate) functions: SessionFunctions, + /// Whether quoted identifiers compared without case in the session's analysis. + pub(crate) quoted_ignore_case: bool, } /// The session's typed result for one compiled model, borrowed from the finished session. @@ -170,6 +180,8 @@ pub struct SessionRequest { pub dialect: String, /// Whether published shapes keep authored quoting, Python's `inferred_binding_shape` test. pub case_sensitive_shapes: bool, + /// Whether quoted identifiers compare without case, as Snowflake's session parameter sets. + pub quoted_ignore_case: bool, pub function_return_types: Pairs, /// Adapter nullability rules as `(function name, rule id)`. pub nullability_rules: Option, @@ -336,6 +348,67 @@ pub struct AnalysisCache { pub(crate) stats: AnalysisCacheStats, } +/// The adapter's function facts a native route gives SQLQuack: return types and nullability rules. +#[derive(Clone, Debug, Default)] +pub(crate) struct SessionFunctions { + /// `(function name, type)` of each function whose return type the adapter declares. + pub(crate) return_types: Vec<(String, String)>, + /// `(function name, rule)` of each built-in nullability rule the adapter names. + pub(crate) nullability: Vec<(String, NullabilityRule)>, +} + +/// The relations one model reads, as the session knows them, for a native route's analysis. +#[derive(Debug, Default)] +pub(crate) struct SessionSchema { + /// Columns of each relation with a known shape, by the analysis name the query reads. + pub(crate) relations: HashMap>, + /// Declared columns of each relation whose full shape is not known, which type the ones read. + pub(crate) open: HashMap>, + pub(crate) functions: SessionFunctions, +} + +impl LineageSchema for SessionSchema { + fn columns(&self, relation: &str) -> Option<&[SchemaColumn]> { + self.relations + .get(relation) + .or_else(|| { + self.relations + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(relation)) + .map(|(_, columns)| columns) + }) + .map(Vec::as_slice) + } + + fn open_columns(&self, relation: &str) -> Option<&[SchemaColumn]> { + self.open + .get(relation) + .or_else(|| { + self.open + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(relation)) + .map(|(_, columns)| columns) + }) + .map(Vec::as_slice) + } + + fn function_type(&self, name: &str) -> Option<&str> { + self.functions + .return_types + .iter() + .find(|(function, _)| function.eq_ignore_ascii_case(name)) + .map(|(_, data_type)| data_type.as_str()) + } + + fn function_nullability(&self, name: &str) -> Option { + self.functions + .nullability + .iter() + .find(|(function, _)| function.eq_ignore_ascii_case(name)) + .map(|(_, rule)| *rule) + } +} + /// One compile's model analysis, advanced phase by phase. #[derive(Debug)] pub struct AnalysisSession { diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs index 8205ca8924..7687674a6d 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs @@ -19,8 +19,8 @@ use crate::assembly::analysis_session::main::take_analysis_cache::take_analysis_ use crate::assembly::analysis_session::models::{ AnalysisCacheStats, AnalysisSession, ColumnFact, ContractProof, DynamicFamily, FinishedSession, LineageRow, ModelColumns, ModelOutcome, ModelReference, ModelRequest, PivotBatchRequest, - PivotModel, PivotOutcome, PivotTables, SessionModelAnalysis, SessionModelFacts, SessionOutcome, - SessionRequest, SessionStep, + PivotModel, PivotOutcome, PivotTables, SessionFunctions, SessionModelAnalysis, + SessionModelFacts, SessionOutcome, SessionRequest, SessionStep, }; use crate::assembly::analysis_session::tests::test_types::{ CachedRun, ColumnView, ModelSpec, RecoveredFacts, @@ -121,6 +121,7 @@ pub(crate) fn orders_request(models: Vec) -> SessionRequest { SessionRequest { dialect: "duckdb".to_owned(), case_sensitive_shapes: false, + quoted_ignore_case: false, function_return_types: Vec::new(), nullability_rules: Some(Vec::new()), nullability_callback: None, @@ -400,6 +401,10 @@ pub(crate) fn finished_session(tables: PivotTables) -> FinishedSession { models: HashMap::new(), dynamic_proofs: HashMap::new(), relations: None, + model_names: Vec::new(), + declared_functions: Vec::new(), + functions: SessionFunctions::default(), + quoted_ignore_case: false, } } diff --git a/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs b/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs index a5273a5217..811f5ebce4 100644 --- a/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs +++ b/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs @@ -310,7 +310,7 @@ fn column_type( }, })); }; - if check.types.types_equal(declared_type, inferred_type)? == TypeComparison::Equal { + if check.types.types_equal(declared_type, inferred_type)? != TypeComparison::Different { return Ok(None); } Ok(Some(ContractDiagnostic { diff --git a/crates/sqlbuild-analysis/src/contracts/_helpers/type_comparison.rs b/crates/sqlbuild-analysis/src/contracts/_helpers/type_comparison.rs index e439b2d73e..fba3ccc4aa 100644 --- a/crates/sqlbuild-analysis/src/contracts/_helpers/type_comparison.rs +++ b/crates/sqlbuild-analysis/src/contracts/_helpers/type_comparison.rs @@ -4,9 +4,8 @@ use std::cell::RefCell; use std::collections::HashMap; use crate::analysis_route::main::analysis_route::analysis_route; -use crate::analysis_route::main::clickhouse_type_match::clickhouse_type_match; -use crate::analysis_route::models::AnalysisRoute; -use crate::contracts::constants::{CLICKHOUSE_DIALECT, UPPER_FOLDING_DIALECT}; +use crate::analysis_route::models::{AnalysisRoute, TypeMatch}; +use crate::contracts::constants::UPPER_FOLDING_DIALECT; use crate::contracts::models::{DeclaredColumn, InferredOutputColumn}; use crate::type_system::errors::TypeNormalizationError; use crate::type_system::main::normalize_type::normalize_type; @@ -17,13 +16,15 @@ use crate::type_system::models::{NormalizedType, TypeNormalization}; pub(crate) enum TypeComparison { Equal, Different, + /// A native route cannot spell one of the types, or has no parser: no finding either way. + NotCompared, } /// One request's type comparisons, normalizing each type string once as Python's cache does. pub(crate) struct TypeComparer<'a> { dialect: &'a str, - /// ClickHouse on its native route, whose types are compared without top-level wrappers. - native_clickhouse: bool, + /// The dialect's route: a native module compares types itself, and no route compares none. + route: AnalysisRoute, normalized: RefCell>>, } @@ -36,8 +37,7 @@ impl<'a> TypeComparer<'a> { pub(crate) fn new(dialect: &'a str) -> Self { Self { dialect, - native_clickhouse: dialect == CLICKHOUSE_DIALECT - && matches!(analysis_route(Some(dialect)), AnalysisRoute::Native(_)), + route: analysis_route(Some(dialect)), normalized: RefCell::new(HashMap::new()), } } @@ -48,12 +48,18 @@ impl<'a> TypeComparer<'a> { left: &str, right: &str, ) -> Result { - if self.native_clickhouse { - return Ok(if clickhouse_type_match(left, right, false).same_type { - TypeComparison::Equal - } else { - TypeComparison::Different - }); + match self.route { + AnalysisRoute::Legacy => {} + AnalysisRoute::Unavailable => return Ok(TypeComparison::NotCompared), + AnalysisRoute::Native(dialect) => { + return Ok(match dialect.type_match(left, Some(right), false) { + TypeMatch::NotCompared { .. } => TypeComparison::NotCompared, + TypeMatch::Compared { + same_type: true, .. + } => TypeComparison::Equal, + TypeMatch::Compared { .. } => TypeComparison::Different, + }); + } } let left: NormalizedType = self.normalized(left)?; let right: NormalizedType = self.normalized(right)?; @@ -73,15 +79,24 @@ impl<'a> TypeComparer<'a> { if declared.not_null && inferred.nullable { return true; } - let (true, Some(declared_type), Some(inferred_type)) = ( - self.native_clickhouse, - declared.declared_type.as_deref(), - inferred.inferred_type.as_deref(), - ) else { + let (AnalysisRoute::Native(dialect), Some(declared_type)) = + (self.route, declared.declared_type.as_deref()) + else { return false; }; - clickhouse_type_match(declared_type, inferred_type, inferred.nullable) - .nullable_into_non_null + matches!( + dialect.type_match( + declared_type, + inferred.inferred_type.as_deref(), + inferred.nullable + ), + TypeMatch::Compared { + nullable_into_non_null: true, + .. + } | TypeMatch::NotCompared { + nullable_into_non_null: true + } + ) } fn normalized(&self, type_sql: &str) -> Result { diff --git a/crates/sqlbuild-analysis/src/contracts/constants.rs b/crates/sqlbuild-analysis/src/contracts/constants.rs index 08cb6e0278..c1edad4679 100644 --- a/crates/sqlbuild-analysis/src/contracts/constants.rs +++ b/crates/sqlbuild-analysis/src/contracts/constants.rs @@ -42,7 +42,5 @@ pub(crate) const PROMOTION_REMOVE_CONTRACT_HELP: &str = "or remove the model's e pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; pub(crate) const ADDITIONAL_HELP_SEPARATOR: &str = "\n = help: "; -/// The analysis dialect name whose native route compares ClickHouse types without wrappers. -pub(crate) const CLICKHOUSE_DIALECT: &str = "clickhouse"; /// The dialect that folds unquoted identifiers to upper case. pub(crate) const UPPER_FOLDING_DIALECT: &str = "snowflake"; diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 5ad16bf8b6..e8d6a2920b 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -37,6 +37,7 @@ pub mod python_plan; pub mod python_selection; pub mod python_values; pub mod query_analysis; +pub mod relation_markers; pub mod relation_names; pub mod required_columns; pub mod resource_diagnostics; diff --git a/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs b/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs index 94dcca4ac8..64418fc609 100644 --- a/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs +++ b/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs @@ -8,6 +8,9 @@ use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use sqlbuild_core::panics::main::native_failure::native_failure; +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::main::fast_lineage::fast_lineage; +use crate::analysis_route::models::AnalysisRoute; use crate::lineage::_helpers::parsed_lineage::{ ParsedModel, parsed_model_lineage, proxy_parse_options, }; @@ -52,23 +55,48 @@ fn model_outcomes(request: &FastLineageRequest) -> Vec = proxy_parse_options(); request .models .par_iter() - .map(|model| match model { - FastLineageModel::StarExpansion { - query_sql, - existing_columns, - } => Ok(FastLineageOutcome::StarColumns(star_lineage( + .map(|model| match (model, route) { + ( + FastLineageModel::StarExpansion { + query_sql, + existing_columns, + }, + _, + ) => Ok(FastLineageOutcome::StarColumns(star_lineage( &schema, &physical_resources(query_sql), existing_columns.iter().map(String::as_str), ))), - FastLineageModel::Parse { - query_sql, - inferred_columns, - } => { + ( + FastLineageModel::Parse { + query_sql, + placeholders, + .. + }, + AnalysisRoute::Native(module), + ) => catch_compiler_panic(|| { + Ok(fast_lineage( + module, + (query_sql, placeholders), + &request.schema, + )) + }), + (FastLineageModel::Parse { .. }, AnalysisRoute::Unavailable) => { + Ok(FastLineageOutcome::Omitted) + } + ( + FastLineageModel::Parse { + query_sql, + inferred_columns, + .. + }, + AnalysisRoute::Legacy, + ) => { let model = ParsedModel { query_sql, inferred_names: inferred_columns, diff --git a/crates/sqlbuild-analysis/src/lineage/models.rs b/crates/sqlbuild-analysis/src/lineage/models.rs index d8d4a2657b..2632824ed2 100644 --- a/crates/sqlbuild-analysis/src/lineage/models.rs +++ b/crates/sqlbuild-analysis/src/lineage/models.rs @@ -106,6 +106,8 @@ pub enum FastLineageModel { Parse { query_sql: String, inferred_columns: Vec, + /// `@@@name` defaults a native parser reads in place of each placeholder. + placeholders: Vec<(String, String)>, }, } diff --git a/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs b/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs index bf332a89da..5a0bbaff02 100644 --- a/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs @@ -36,7 +36,7 @@ pub(crate) fn deep_union_outcome_lines( stack_bytes: usize, ) -> Vec<(&'static str, Vec, bool, Option)> { let sql: String = - vec!["SELECT order_id, amount FROM __ref('orders')"; branches].join(" UNION ALL "); + vec!["SELECT order_id, amount FROM __ref(\"orders\")"; branches].join(" UNION ALL "); let worker = std::thread::Builder::new() .stack_size(stack_bytes) .spawn(move || { @@ -45,6 +45,7 @@ pub(crate) fn deep_union_outcome_lines( FastLineageModel::Parse { query_sql: sql, inferred_columns: Vec::new(), + placeholders: Vec::new(), }, ) }) diff --git a/crates/sqlbuild-analysis/src/lineage/tests/test_build_fast_lineage.rs b/crates/sqlbuild-analysis/src/lineage/tests/test_build_fast_lineage.rs index 9998ac560d..6d225a9b5a 100644 --- a/crates/sqlbuild-analysis/src/lineage/tests/test_build_fast_lineage.rs +++ b/crates/sqlbuild-analysis/src/lineage/tests/test_build_fast_lineage.rs @@ -11,7 +11,7 @@ fn given_models_without_compact_facts_when_building_then_matches_python_lineage( description: "joins, aggregation, cast, constant and expression columns", sql: "SELECT o.order_id, c.name AS customer_name, SUM(o.amount) AS total, \ CAST(o.amount AS BIGINT) AS amount_big, 1 AS one, o.amount + c.customer_id AS mix \ - FROM __ref('orders') o JOIN __ref('customers') c ON o.customer_id = c.customer_id \ + FROM __ref(\"orders\") o JOIN __ref(\"customers\") c ON o.customer_id = c.customer_id \ GROUP BY 1, 2", dialect: None, inferred_columns: &[], @@ -29,7 +29,7 @@ fn given_models_without_compact_facts_when_building_then_matches_python_lineage( }, ParsedLineageTestCase { description: "unqualified columns of one resource and an unknown qualifier", - sql: "SELECT order_id, amount * 2 AS doubled, missing.thing FROM __ref('orders')", + sql: "SELECT order_id, amount * 2 AS doubled, missing.thing FROM __ref(\"orders\")", dialect: None, inferred_columns: &[], expected_status: "built", @@ -43,7 +43,7 @@ fn given_models_without_compact_facts_when_building_then_matches_python_lineage( }, ParsedLineageTestCase { description: "a root star expands known columns after explicit outputs", - sql: "SELECT *, amount AS amt FROM __ref('orders')", + sql: "SELECT *, amount AS amt FROM __ref(\"orders\")", dialect: None, inferred_columns: &[], expected_status: "built", @@ -58,7 +58,7 @@ fn given_models_without_compact_facts_when_building_then_matches_python_lineage( }, ParsedLineageTestCase { description: "a qualified star expands every referenced resource", - sql: "SELECT o.* FROM __ref('orders') o JOIN __seed('regions') r ON TRUE", + sql: "SELECT o.* FROM __ref(\"orders\") o JOIN __seed(\"regions\") r ON TRUE", dialect: None, inferred_columns: &[], expected_status: "built", @@ -74,8 +74,8 @@ fn given_models_without_compact_facts_when_building_then_matches_python_lineage( }, ParsedLineageTestCase { description: "a three-branch union merges branch sources", - sql: "SELECT order_id FROM __ref('orders') UNION ALL SELECT customer_id \ - FROM __ref('customers') UNION ALL SELECT region_id + 1 FROM __seed('regions')", + sql: "SELECT order_id FROM __ref(\"orders\") UNION ALL SELECT customer_id \ + FROM __ref(\"customers\") UNION ALL SELECT region_id + 1 FROM __seed(\"regions\")", dialect: None, inferred_columns: &[], expected_status: "built", @@ -88,7 +88,7 @@ fn given_models_without_compact_facts_when_building_then_matches_python_lineage( }, ParsedLineageTestCase { description: "a union with a star branch takes inferred names", - sql: "SELECT order_id, amount FROM __ref('orders') UNION SELECT * FROM __ref('customers')", + sql: "SELECT order_id, amount FROM __ref(\"orders\") UNION SELECT * FROM __ref(\"customers\")", dialect: None, inferred_columns: &["first", "second"], expected_status: "built", @@ -105,7 +105,7 @@ fn given_models_without_compact_facts_when_building_then_matches_python_lineage( }, ParsedLineageTestCase { description: "a CTE collapses onto its referenced model", - sql: "WITH base AS (SELECT order_id, amount FROM __ref('orders')) \ + sql: "WITH base AS (SELECT order_id, amount FROM __ref(\"orders\")) \ SELECT order_id, amount FROM base", dialect: None, inferred_columns: &[], @@ -119,7 +119,7 @@ fn given_models_without_compact_facts_when_building_then_matches_python_lineage( }, ParsedLineageTestCase { description: "unnamed projections take inferred names", - sql: "SELECT 1 + 1, COUNT(*) FROM __ref('orders')", + sql: "SELECT 1 + 1, COUNT(*) FROM __ref(\"orders\")", dialect: None, inferred_columns: &["a", "b"], expected_status: "built", @@ -129,7 +129,7 @@ fn given_models_without_compact_facts_when_building_then_matches_python_lineage( }, ParsedLineageTestCase { description: "quoted and case-varied identifiers keep their spelling", - sql: "SELECT \"Order_ID\", O.Amount FROM __ref('orders') AS O", + sql: "SELECT \"Order_ID\", O.Amount FROM __ref(\"orders\") AS O", dialect: None, inferred_columns: &[], expected_status: "built", @@ -148,7 +148,7 @@ fn given_models_without_compact_facts_when_building_then_matches_python_lineage( order_id IN (1, customer_id) AS listed, \ ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY amount) AS rn, \ EXTRACT(YEAR FROM amount) AS yr, order_id || '-' || amount AS joined \ - FROM __ref('orders')", + FROM __ref(\"orders\")", dialect: None, inferred_columns: &[], expected_status: "built", @@ -168,7 +168,7 @@ fn given_models_without_compact_facts_when_building_then_matches_python_lineage( ParsedLineageTestCase { description: "a top-level EXCEPT has no fast lineage", dialect: None, - sql: "SELECT order_id FROM __ref('orders') EXCEPT SELECT customer_id FROM __ref('customers')", + sql: "SELECT order_id FROM __ref(\"orders\") EXCEPT SELECT customer_id FROM __ref(\"customers\")", inferred_columns: &[], expected_status: "omitted", expected_lines: &[], @@ -234,6 +234,7 @@ fn given_models_without_compact_facts_when_building_then_matches_python_lineage( FastLineageModel::Parse { query_sql: test_case.sql.to_owned(), inferred_columns: strings(test_case.inferred_columns), + placeholders: Vec::new(), }, ); assert_eq!( @@ -255,8 +256,8 @@ fn given_compact_facts_with_unresolved_star_when_building_then_appends_unseen_co let test_cases = [ StarExpansionTestCase { description: "existing outputs and repeated references are skipped", - sql: "SELECT *, 1 AS amount FROM __ref('orders') JOIN __ref('orders') USING (order_id) \ - JOIN __seed('regions') USING (region_id)", + sql: "SELECT *, 1 AS amount FROM __ref(\"orders\") JOIN __ref(\"orders\") USING (order_id) \ + JOIN __seed(\"regions\") USING (region_id)", existing_columns: &["amount"], expected_lines: &[ "order_id star medium [model:orders.order_id]", @@ -267,7 +268,7 @@ fn given_compact_facts_with_unresolved_star_when_building_then_appends_unseen_co }, StarExpansionTestCase { description: "resources without known columns add nothing", - sql: "SELECT * FROM __ref('empty') JOIN __source('raw') ON TRUE", + sql: "SELECT * FROM __ref(\"empty\") JOIN __source(\"raw\") ON TRUE", existing_columns: &[], expected_lines: &[], }, diff --git a/crates/sqlbuild-analysis/src/relation_markers/_helpers/mod.rs b/crates/sqlbuild-analysis/src/relation_markers/_helpers/mod.rs new file mode 100644 index 0000000000..b13857eb1c --- /dev/null +++ b/crates/sqlbuild-analysis/src/relation_markers/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod pattern; diff --git a/crates/sqlbuild-analysis/src/relation_markers/_helpers/pattern.rs b/crates/sqlbuild-analysis/src/relation_markers/_helpers/pattern.rs new file mode 100644 index 0000000000..4fa72a78b6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/relation_markers/_helpers/pattern.rs @@ -0,0 +1,15 @@ +//! The compiled relation marker call pattern. + +use std::sync::LazyLock; + +use regex::Regex; + +use crate::relation_markers::constants::RELATION_MARKER_PATTERN; + +static RELATION_MARKER: LazyLock> = + LazyLock::new(|| Regex::new(RELATION_MARKER_PATTERN).map_err(|error| error.to_string())); + +/// The compiled relation marker call pattern, or its compile error. +pub(crate) fn relation_marker_pattern() -> Result<&'static Regex, String> { + RELATION_MARKER.as_ref().map_err(Clone::clone) +} diff --git a/crates/sqlbuild-analysis/src/relation_markers/constants.rs b/crates/sqlbuild-analysis/src/relation_markers/constants.rs new file mode 100644 index 0000000000..4fef92b98e --- /dev/null +++ b/crates/sqlbuild-analysis/src/relation_markers/constants.rs @@ -0,0 +1,9 @@ +//! The relation marker call pattern and the names marker calls stand as. + +/// The prefix of the name a marker call stands as while parsed; the call's index follows it. +pub const MARKER_NAME_PREFIX: &str = "__sqb_"; +/// Any relation marker call: `__name("x")`, or `__name("a", "b")` as `__dbt_ref` writes it. +pub(crate) const RELATION_MARKER_PATTERN: &str = + r#"(?i)\b(__[a-z_]+)\(\s*"([^"]+)"\s*(?:,\s*"([^"]+)"\s*)?\)"#; +/// The marker whose two names read as one, joined by `__`. +pub(crate) const DBT_REF_FUNCTION: &str = "__dbt_ref"; diff --git a/crates/sqlbuild-analysis/src/relation_markers/main/markers_as_names.rs b/crates/sqlbuild-analysis/src/relation_markers/main/markers_as_names.rs new file mode 100644 index 0000000000..9330c94939 --- /dev/null +++ b/crates/sqlbuild-analysis/src/relation_markers/main/markers_as_names.rs @@ -0,0 +1,60 @@ +//! `markers_as_names`: relation marker calls in code as plain names that keep every position. + +use regex::Regex; +use sqlbuild_sqltext::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::relation_markers::_helpers::pattern::relation_marker_pattern; +use crate::relation_markers::constants::{DBT_REF_FUNCTION, MARKER_NAME_PREFIX}; +use crate::relation_markers::types::MarkerCall; + +/// `sql` with each marker call in code as `__sqb_`, padded so later text keeps its line and column, and the calls by index. +pub fn markers_as_names( + sql: &str, + syntax: &LexicalSyntax, +) -> Result<(String, Vec), String> { + let pattern: &Regex = relation_marker_pattern()?; + let mut protected: Option> = None; + let mut named: String = String::with_capacity(sql.len()); + let mut calls: Vec = Vec::new(); + let mut cursor: usize = 0; + for captures in pattern.captures_iter(sql) { + let (Some(full), Some(function), Some(first)) = + (captures.get(0), captures.get(1), captures.get(2)) + else { + continue; + }; + let ranges = protected.get_or_insert_with(|| dialect_non_code_ranges(sql, syntax)); + let following: usize = ranges.partition_point(|(start, _)| *start <= full.start()); + if following > 0 && full.start() < ranges[following - 1].1 { + continue; + } + let function_name: String = function.as_str().to_ascii_lowercase(); + let referenced_name: String = match (captures.get(3), function_name == DBT_REF_FUNCTION) { + (None, _) => first.as_str().to_owned(), + (Some(second), true) => format!("{}__{}", first.as_str(), second.as_str()), + (Some(_), false) => continue, + }; + named.push_str(&sql[cursor..full.start()]); + named.push_str(&padded_name( + &format!("{MARKER_NAME_PREFIX}{}", calls.len()), + full.as_str(), + )); + cursor = full.end(); + calls.push((function_name, referenced_name)); + } + named.push_str(&sql[cursor..]); + Ok((named, calls)) +} + +/// `name`, then the call's line breaks, then spaces as wide as the call's last line. +fn padded_name(name: &str, call: &str) -> String { + let breaks: usize = call.matches('\n').count(); + let last_line: &str = call.rsplit('\n').next().unwrap_or(call); + let width: usize = if breaks == 0 { + call.chars().count().saturating_sub(name.chars().count()) + } else { + last_line.chars().count() + }; + format!("{name}{}{}", "\n".repeat(breaks), " ".repeat(width)) +} diff --git a/crates/sqlbuild-analysis/src/relation_markers/main/mod.rs b/crates/sqlbuild-analysis/src/relation_markers/main/mod.rs new file mode 100644 index 0000000000..32d7644fcb --- /dev/null +++ b/crates/sqlbuild-analysis/src/relation_markers/main/mod.rs @@ -0,0 +1 @@ +pub mod markers_as_names; diff --git a/crates/sqlbuild-analysis/src/relation_markers/mod.rs b/crates/sqlbuild-analysis/src/relation_markers/mod.rs new file mode 100644 index 0000000000..af27b6661d --- /dev/null +++ b/crates/sqlbuild-analysis/src/relation_markers/mod.rs @@ -0,0 +1,9 @@ +//! Relation marker calls, such as `__ref("orders")`, read as plain relation names by strict parsers. + +mod _helpers; +pub mod constants; +pub mod main; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/relation_markers/tests.rs b/crates/sqlbuild-analysis/src/relation_markers/tests.rs new file mode 100644 index 0000000000..c7c4334980 --- /dev/null +++ b/crates/sqlbuild-analysis/src/relation_markers/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_markers_as_names.rs"] +mod test_markers_as_names; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/relation_markers/tests/test_markers_as_names.rs b/crates/sqlbuild-analysis/src/relation_markers/tests/test_markers_as_names.rs new file mode 100644 index 0000000000..6df1acbc7a --- /dev/null +++ b/crates/sqlbuild-analysis/src/relation_markers/tests/test_markers_as_names.rs @@ -0,0 +1,72 @@ +use sqlbuild_sqltext::sql_scan::main::dialect_lexical_syntax::dialect_lexical_syntax; + +use crate::relation_markers::main::markers_as_names::markers_as_names; +use crate::relation_markers::tests::test_types::MarkersAsNamesTestCase; + +#[test] +fn given_marker_calls_when_reading_them_as_names_then_code_calls_become_names_and_positions_hold() { + let test_cases = [ + MarkersAsNamesTestCase { + description: "each call becomes its indexed name, padded to the call's width", + dialect: "snowflake", + sql: r#"SELECT * FROM __ref("orders") o JOIN __source("raw") r ON TRUE"#, + expected_sql: format!( + "SELECT * FROM __sqb_0{} o JOIN __sqb_1{} r ON TRUE", + " ".repeat(8), + " ".repeat(8) + ), + expected_calls: &[("__ref", "orders"), ("__source", "raw")], + }, + MarkersAsNamesTestCase { + description: "calls in quoted text and comments stay; an upper-case call folds", + dialect: "snowflake", + sql: "SELECT '__ref(\"a\")' AS q -- __ref(\"b\")\nFROM __REF(\"Orders\") // __ref(\"c\")", + expected_sql: format!( + "SELECT '__ref(\"a\")' AS q -- __ref(\"b\")\nFROM __sqb_0{} // __ref(\"c\")", + " ".repeat(8) + ), + expected_calls: &[("__ref", "Orders")], + }, + MarkersAsNamesTestCase { + description: "a call over three lines keeps its line breaks and its last line's width", + dialect: "duckdb", + sql: "FROM __ref(\n \"orders\"\n) AS o", + expected_sql: "FROM __sqb_0\n\n AS o".to_owned(), + expected_calls: &[("__ref", "orders")], + }, + MarkersAsNamesTestCase { + description: "two dbt names join; a non-ASCII name pads by characters, not bytes", + dialect: "postgres", + sql: r#"FROM __dbt_ref("shop", "customers") d JOIN __ref("café") c ON TRUE"#, + expected_sql: format!( + "FROM __sqb_0{} d JOIN __sqb_1{} c ON TRUE", + " ".repeat(23), + " ".repeat(6) + ), + expected_calls: &[("__dbt_ref", "shop__customers"), ("__ref", "café")], + }, + MarkersAsNamesTestCase { + description: "calls with a second name, or a non-literal argument, stay calls", + dialect: "duckdb", + sql: r#"FROM __ref("a", "b") JOIN __table_fn("orders", 2) ON TRUE"#, + expected_sql: r#"FROM __ref("a", "b") JOIN __table_fn("orders", 2) ON TRUE"#.to_owned(), + expected_calls: &[], + }, + ]; + for test_case in test_cases { + let (named, calls) = + markers_as_names(test_case.sql, &dialect_lexical_syntax(test_case.dialect)) + .expect(test_case.description); + let expected_calls: Vec<(String, String)> = test_case + .expected_calls + .iter() + .map(|(function, name)| ((*function).to_owned(), (*name).to_owned())) + .collect(); + assert_eq!( + (named, calls), + (test_case.expected_sql, expected_calls), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/relation_markers/tests/test_types.rs b/crates/sqlbuild-analysis/src/relation_markers/tests/test_types.rs new file mode 100644 index 0000000000..b92a6d74fa --- /dev/null +++ b/crates/sqlbuild-analysis/src/relation_markers/tests/test_types.rs @@ -0,0 +1,10 @@ +//! Test case types for relation markers read as names. + +/// SQL in one dialect, the SQL its markers read as, and the calls they stand for. +pub(crate) struct MarkersAsNamesTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_sql: String, + pub(crate) expected_calls: &'static [(&'static str, &'static str)], +} diff --git a/crates/sqlbuild-analysis/src/relation_markers/types.rs b/crates/sqlbuild-analysis/src/relation_markers/types.rs new file mode 100644 index 0000000000..9b90a22d2a --- /dev/null +++ b/crates/sqlbuild-analysis/src/relation_markers/types.rs @@ -0,0 +1,4 @@ +//! Shapes of relation marker calls. + +/// A marker call's lower-case function, such as `__ref`, and the name it references. +pub type MarkerCall = (String, String); diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs index 33ced0fba8..0064cf828d 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs @@ -6,6 +6,8 @@ use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics::replace_intrinsics; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; use crate::semantic_checks::_helpers::explanation::opt_outs::unneeded_opt_out; use crate::semantic_checks::_helpers::metadata_checks::positions::{text_position, word_start}; use crate::semantic_checks::_helpers::resource_sql::help::{HelpContext, resource_sql_help}; @@ -97,12 +99,14 @@ fn binding_request( (sql.to_owned(), references, overrides) } -/// Python's `_reported_diagnostic`: drop open-relation lookups, make parser rejections P001. -fn reported(resource: &ResourceSql, finding: &Finding) -> Option { +/// `_reported_diagnostic`: drop open-relation lookups; parser rejections are P001, natively P026's. +fn reported(resource: &ResourceSql, finding: &Finding, native_hooks: bool) -> Option { if finding.code != UNKNOWN_TABLE_CODE { return Some(finding.clone()); } - if finding.message.starts_with(UNKNOWN_TABLE_PREFIX) { + if finding.message.starts_with(UNKNOWN_TABLE_PREFIX) + || (native_hooks && resource.kind == ResourceSqlKind::Model) + { return None; } Some(Finding { @@ -134,8 +138,12 @@ fn resource_diagnostics( checking: &Checking<'_>, ) -> Result, ResourceSqlFailure> { let mut found: Vec<(Key, ResourceSqlDiagnostic)> = Vec::new(); + let native_hooks: bool = matches!( + analysis_route(checking.request.dialect.as_deref()), + AnalysisRoute::Native(_) + ); for (index, native) in findings.iter().enumerate() { - let Some(finding) = reported(resource, native) else { + let Some(finding) = reported(resource, native, native_hooks) else { continue; }; let span = location(resource, checking.cleaned_sql, &finding)?; diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs index 32fd8e8198..ff89c238fe 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs @@ -656,7 +656,7 @@ fn messaged_row(error: ValidationError) -> DiagnosticRow { row } -pub(super) fn diagnostic_row(error: ValidationError) -> DiagnosticRow { +pub(crate) fn diagnostic_row(error: ValidationError) -> DiagnosticRow { let severity = match error.severity { polyglot_sql::ValidationSeverity::Error => "error", _ => "warning", diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs index b198ae8a54..4440d80a4e 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs @@ -177,7 +177,7 @@ pub(crate) fn map_diagnostics( Ok(ValidationResult { valid, errors }) } -fn sqlbuild_code(code: &str) -> Option { +pub(crate) fn sqlbuild_code(code: &str) -> Option { let direct = match code { "E001" | "E002" | "E003" | "E004" | "E200" => Some("B000"), "E201" => Some("B002"), diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs index daaeed17d4..5dd9a55e9a 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs @@ -2,6 +2,8 @@ use crate::semantic_checks::constants::UDF_NAME_PREFIX; use regex::Regex; +use sqlbuild_sqltext::sql_scan::main::dialect_lexical_syntax::dialect_lexical_syntax; +use sqlbuild_sqltext::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges; use sqlbuild_sqltext::sql_scan::main::matching_paren::matching_paren; use sqlbuild_sqltext::sql_scan::main::non_code_end::non_code_end; use sqlbuild_sqltext::sql_scan::models::QuotePolicy; @@ -194,17 +196,37 @@ pub(crate) fn normalize_native_sql( let udf_prefix: String = format!("{stub_prefix}udf_"); let normalized: String = normalize(sql, dialect, (&none, &none), &udf_prefix)?.sql; let stubs: &Regex = NATIVE_STUBS.as_ref()?; - Ok(stubs - .replace_all(&normalized, |captures: ®ex::Captures<'_>| { - match (captures.get(1), captures.get(2)) { - (Some(bound), _) => format!("{stub_prefix}cursor_{}()", bound.as_str()), - (None, Some(parameter)) => { - format!("'{HOOK_PARAMETER_PREFIX}{}'", parameter.as_str()) - } - (None, None) => captures[0].to_owned(), + let non_code: Vec<(usize, usize)> = + dialect_non_code_ranges(&normalized, &dialect_lexical_syntax(dialect)); + let mut native: String = String::with_capacity(normalized.len()); + let mut copied_until: usize = 0; + for captures in stubs.captures_iter(&normalized) { + let whole = captures.get(0).ok_or("missing stub match")?; + let quoted: bool = non_code + .iter() + .any(|&(start, end)| start <= whole.start() && whole.start() < end); + let glued: bool = captures.get(1).is_some() + && normalized[..whole.start()] + .chars() + .next_back() + .is_some_and(|character| character.is_alphanumeric() || character == '_'); + if quoted || glued { + continue; + } + native.push_str(&normalized[copied_until..whole.start()]); + match (captures.get(1), captures.get(2)) { + (Some(bound), _) => { + native.push_str(&format!("{stub_prefix}cursor_{}()", bound.as_str())); } - }) - .into_owned()) + (None, Some(parameter)) => { + native.push_str(&format!("'{HOOK_PARAMETER_PREFIX}{}'", parameter.as_str())); + } + (None, None) => native.push_str(whole.as_str()), + } + copied_until = whole.end(); + } + native.push_str(&normalized[copied_until..]); + Ok(native) } pub(crate) fn normalize_dialect_sql_mapped( diff --git a/crates/sqlbuild-analysis/src/semantic_validation/main.rs b/crates/sqlbuild-analysis/src/semantic_validation/main.rs index 930655bff7..3805b70c71 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/main.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/main.rs @@ -8,6 +8,8 @@ pub mod normalize_batch; pub mod normalize_dialect; pub mod normalize_mapped; pub mod normalize_native; +pub mod sqlbuild_code; +pub mod text_findings; pub mod truth_test_spans; use crate::semantic_validation::models::ProjectCatalog; diff --git a/crates/sqlbuild-analysis/src/semantic_validation/main/sqlbuild_code.rs b/crates/sqlbuild-analysis/src/semantic_validation/main/sqlbuild_code.rs new file mode 100644 index 0000000000..0187252e45 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_validation/main/sqlbuild_code.rs @@ -0,0 +1,5 @@ +/// SQLBuild's code for an analysis finding; `None` for a code SQLBuild does not report. +#[must_use] +pub fn sqlbuild_code(code: &str) -> Option { + crate::semantic_validation::_helpers::diagnostics::sqlbuild_code(code) +} diff --git a/crates/sqlbuild-analysis/src/semantic_validation/main/text_findings.rs b/crates/sqlbuild-analysis/src/semantic_validation/main/text_findings.rs new file mode 100644 index 0000000000..7ff13fcb00 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_validation/main/text_findings.rs @@ -0,0 +1,19 @@ +//! SQLBuild's own checks of SQL text that every analysis route adds to the parse's findings. + +use crate::lineage::main::parser_dialect::parser_dialect; +use crate::semantic_validation::_helpers::catalog::diagnostic_row; +use crate::semantic_validation::_helpers::sqlite_temporal_casts::sqlite_temporal_cast_warnings; +use crate::semantic_validation::_helpers::truth_tests::unsupported_truth_test_errors; +use crate::semantic_validation::models::DiagnosticRow; + +/// Truth-value tests the warehouse rejects (B103) and SQLite's temporal casts (B104) in `sql`. +pub fn text_findings(sql: &str, dialect: &str) -> Vec { + let Ok(dialect) = parser_dialect(Some(dialect)) else { + return Vec::new(); + }; + unsupported_truth_test_errors(sql, dialect) + .into_iter() + .chain(sqlite_temporal_cast_warnings(sql, dialect)) + .map(diagnostic_row) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs index 112492fef7..95dcadd3f3 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs @@ -1,7 +1,10 @@ use std::collections::HashMap; use crate::semantic_validation::main::normalize_batch::normalize_analysis_sqls; -use crate::semantic_validation::tests::test_types::NormalizationBatchTestCase; +use crate::semantic_validation::main::normalize_native::normalize_native_sql; +use crate::semantic_validation::tests::test_types::{ + NativeStubTestCase, NormalizationBatchTestCase, +}; use crate::semantic_validation::types::NormalizationRequest; #[test] @@ -100,3 +103,90 @@ fn given_batch_when_normalizing_on_pool_or_in_turn_then_matches_single_calls_in_ ); } } + +#[test] +fn given_cursor_stubs_when_normalizing_for_a_native_parser_then_only_whole_names_in_code_are_respelt() + { + let test_cases = [ + NativeStubTestCase { + dialect: "postgres", + description: "a cursor stub in code takes the dialect's prefix", + sql: "SELECT * FROM t WHERE ts > __cursor_start()", + expected_sql: "SELECT * FROM t WHERE ts > __sqlbuild.cursor_start()", + }, + NativeStubTestCase { + dialect: "postgres", + description: "a cursor stub's text inside a string literal is left as written", + sql: "SELECT '__cursor_start()' AS s FROM t", + expected_sql: "SELECT '__cursor_start()' AS s FROM t", + }, + NativeStubTestCase { + dialect: "postgres", + description: "a cursor stub's text inside a comment is left as written", + sql: "SELECT 1 FROM t -- __cursor_end()\n", + expected_sql: "SELECT 1 FROM t -- __cursor_end()\n", + }, + NativeStubTestCase { + dialect: "postgres", + description: "a longer name ending in a cursor stub's text is another function", + sql: "SELECT x__cursor_end() FROM t", + expected_sql: "SELECT x__cursor_end() FROM t", + }, + NativeStubTestCase { + dialect: "postgres", + description: "a quoted stub beside a stub in code leaves only the code one respelt", + sql: "SELECT '__cursor_end()', __cursor_end() FROM t", + expected_sql: "SELECT '__cursor_end()', __sqlbuild.cursor_end() FROM t", + }, + NativeStubTestCase { + dialect: "duckdb", + description: "a DuckDB backslash before a closing quote is literal, so the stub after it is code", + sql: r"SELECT 'C:\' AS folder, __cursor_end() FROM t", + expected_sql: r"SELECT 'C:\' AS folder, __sqlbuild.cursor_end() FROM t", + }, + NativeStubTestCase { + dialect: "postgres", + description: "a Postgres escape string's backslash-quoted quote keeps the stub after it quoted", + sql: r"SELECT E'O\'Brien __cursor_end()', __cursor_end() FROM t", + expected_sql: r"SELECT E'O\'Brien __cursor_end()', __sqlbuild.cursor_end() FROM t", + }, + NativeStubTestCase { + dialect: "postgres", + description: "a plain Postgres string's backslash is literal, so the stub after it is code", + sql: r"SELECT 'C:\', __cursor_end() FROM t", + expected_sql: r"SELECT 'C:\', __sqlbuild.cursor_end() FROM t", + }, + NativeStubTestCase { + dialect: "snowflake", + description: "a Snowflake backslash-quoted quote keeps the stub after it quoted", + sql: r"SELECT 'it\'s __cursor_end()', __cursor_end() FROM t", + expected_sql: r"SELECT 'it\'s __cursor_end()', __sqlbuild.cursor_end() FROM t", + }, + NativeStubTestCase { + dialect: "tsql", + description: "a T-SQL bracket name's apostrophe opens no string, so the stub after it is code", + sql: "SELECT [customer's name] AS n FROM t WHERE ts >= __cursor_start()", + expected_sql: "SELECT [customer's name] AS n FROM t WHERE ts >= __sqlbuild.cursor_start()", + }, + NativeStubTestCase { + dialect: "clickhouse", + description: "a ClickHouse backslash-quoted quote in an identifier keeps the stub after it in code", + sql: r#"SELECT 1 AS "a\"b", __cursor_end() FROM t"#, + expected_sql: r#"SELECT 1 AS "a\"b", __sqlbuild.cursor_end() FROM t"#, + }, + NativeStubTestCase { + dialect: "snowflake", + description: "a cursor stub's text inside a Snowflake // comment is left as written", + sql: "SELECT 1 FROM t // __cursor_end()\n", + expected_sql: "SELECT 1 FROM t // __cursor_end()\n", + }, + ]; + for test_case in test_cases { + assert_eq!( + normalize_native_sql(test_case.sql, test_case.dialect, "__sqlbuild.").as_deref(), + Ok(test_case.expected_sql), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_types.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_types.rs index 201a5dc6fc..8a51815a13 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_types.rs @@ -53,6 +53,13 @@ pub(crate) struct NormalizationBatchTestCase { pub(crate) expected_failures: &'static [usize], } +pub(crate) struct NativeStubTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_sql: &'static str, +} + pub(crate) struct TruthTestScanTestCase { pub(crate) description: &'static str, pub(crate) sql: &'static str, diff --git a/crates/sqlbuild-analysis/src/type_system/_helpers/fallback.rs b/crates/sqlbuild-analysis/src/type_system/_helpers/fallback.rs index 6ee0ba4bf1..6cc429a7f6 100644 --- a/crates/sqlbuild-analysis/src/type_system/_helpers/fallback.rs +++ b/crates/sqlbuild-analysis/src/type_system/_helpers/fallback.rs @@ -97,7 +97,11 @@ pub(crate) fn normalize_with_fallback( let name: &str = if bigquery { "BOOL" } else { base_type }; return simple(name, TypeFamily::Boolean); } - if base_type.contains(TIMESTAMP_TYPE_TOKEN) { + let top_level: &str = base_type + .split(|character: char| !(character.is_ascii_alphanumeric() || character == '_')) + .next() + .unwrap_or(base_type); + if top_level.contains(TIMESTAMP_TYPE_TOKEN) { return simple( ×tamp_normalized_name(base_type, dialect), TypeFamily::Timestamp, diff --git a/crates/sqlbuild-analysis/src/type_system/main/mod.rs b/crates/sqlbuild-analysis/src/type_system/main/mod.rs index 7434eded53..1ac950cb8f 100644 --- a/crates/sqlbuild-analysis/src/type_system/main/mod.rs +++ b/crates/sqlbuild-analysis/src/type_system/main/mod.rs @@ -1,3 +1,4 @@ +pub mod normalize_text_type; pub mod normalize_type; pub mod polyglot_type_args; pub mod same_type; diff --git a/crates/sqlbuild-analysis/src/type_system/main/normalize_text_type.rs b/crates/sqlbuild-analysis/src/type_system/main/normalize_text_type.rs new file mode 100644 index 0000000000..51e6458938 --- /dev/null +++ b/crates/sqlbuild-analysis/src/type_system/main/normalize_text_type.rs @@ -0,0 +1,10 @@ +//! `normalize_text_type`: a type's comparison shape read from its text alone, without Polyglot. + +use crate::type_system::_helpers::fallback::normalize_with_fallback; +use crate::type_system::models::{NormalizedType, TypeDialect}; + +/// `type_sql`'s name, family and parameters as the text fallback reads them under `dialect`. +#[must_use] +pub fn normalize_text_type(type_sql: &str, dialect: Option) -> NormalizedType { + normalize_with_fallback(type_sql, dialect) +} diff --git a/crates/sqlbuild-analysis/src/type_system/tests/test_normalize_type.rs b/crates/sqlbuild-analysis/src/type_system/tests/test_normalize_type.rs index afee3fc479..73a83a8cba 100644 --- a/crates/sqlbuild-analysis/src/type_system/tests/test_normalize_type.rs +++ b/crates/sqlbuild-analysis/src/type_system/tests/test_normalize_type.rs @@ -1,7 +1,11 @@ use crate::type_system::errors::TypeNormalizationError; +use crate::type_system::main::normalize_text_type::normalize_text_type; use crate::type_system::main::normalize_type::normalize_type; +use crate::type_system::models::TypeFamily; use crate::type_system::tests::helpers::normalization_text; -use crate::type_system::tests::test_types::{BracketDepthTestCase, NormalizeTypeTestCase}; +use crate::type_system::tests::test_types::{ + BracketDepthTestCase, NormalizeTypeTestCase, TextFamilyTestCase, +}; #[test] fn given_type_strings_when_normalizing_then_python_normalization_is_returned() { @@ -179,3 +183,52 @@ fn given_array_suffixes_when_normalizing_on_a_worker_stack_then_python_supported ); } } + +#[test] +fn given_complex_types_holding_a_timestamp_when_normalizing_text_then_the_top_level_type_decides() { + let test_cases = [ + TextFamilyTestCase { + description: "an array of timestamps is an array", + type_sql: "ARRAY", + expected_family: TypeFamily::Other, + }, + TextFamilyTestCase { + description: "a map of structs holding a timestamp is a map", + type_sql: "MAP>", + expected_family: TypeFamily::Other, + }, + TextFamilyTestCase { + description: "a struct holding a timestamp is a struct", + type_sql: "STRUCT", + expected_family: TypeFamily::Other, + }, + TextFamilyTestCase { + description: "a bare timestamp is a timestamp", + type_sql: "TIMESTAMP", + expected_family: TypeFamily::Timestamp, + }, + TextFamilyTestCase { + description: "a timestamp with precision is a timestamp", + type_sql: "TIMESTAMP(3)", + expected_family: TypeFamily::Timestamp, + }, + TextFamilyTestCase { + description: "a zoned timestamp is a timestamp", + type_sql: "TIMESTAMP WITH TIME ZONE", + expected_family: TypeFamily::Timestamp, + }, + TextFamilyTestCase { + description: "Snowflake's NTZ alias is a timestamp", + type_sql: "TIMESTAMP_NTZ", + expected_family: TypeFamily::Timestamp, + }, + ]; + for test_case in test_cases { + assert_eq!( + normalize_text_type(test_case.type_sql, None).family, + test_case.expected_family, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/type_system/tests/test_types.rs b/crates/sqlbuild-analysis/src/type_system/tests/test_types.rs index 2b9aeec95d..46c3592c52 100644 --- a/crates/sqlbuild-analysis/src/type_system/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/type_system/tests/test_types.rs @@ -30,3 +30,9 @@ pub(super) struct SameTypeTestCase { pub(super) types: (&'static str, &'static str), pub(super) expected_same: bool, } + +pub(super) struct TextFamilyTestCase { + pub(super) description: &'static str, + pub(super) type_sql: &'static str, + pub(super) expected_family: crate::type_system::models::TypeFamily, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs index 5a0dab6dd3..3aa2be4bab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs @@ -109,7 +109,7 @@ impl BindingCatalog { }) } - /// `BindingCatalog.prepare`: add unknown closed relations, then override changed ones. + /// `BindingCatalog.prepare`: add unknown closed relations (overrides while in use), then changed ones. fn prepared( &self, py: Python<'_>, @@ -121,16 +121,21 @@ impl BindingCatalog { let schema: Bound<'_, PyAny> = native_field(request, "schema")?; for item in mapping_items(&schema)?.try_iter()? { let (name, columns): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; - if columns.is_truthy()? && !schemas.contains(&name)? { + if columns.is_truthy()? + && !schemas.contains(&name)? + && !additions.contains(&name)? + { additions.set_item(&name, &columns)?; - schemas.set_item(&name, &columns)?; } } } - if !additions.is_empty() { + if !additions.is_empty() + && let Ok(mut native) = self.native.try_borrow_mut(py) + { self.extended.store(true, Ordering::Relaxed); + schemas.update(additions.as_mapping())?; let relations: Relations = conversions::relations(additions.as_any())?; - self.native.borrow_mut(py).inner.update_relations(relations); + native.inner.update_relations(relations); } let mut prepared: Vec = Vec::with_capacity(requests.len()); for request in requests { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index e964962a2e..cd4096e763 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -129,6 +129,7 @@ fn session_request<'py>( profile, &native_field(profile, "sql_analysis_dialect")?, )?, + quoted_ignore_case: native_field(profile, "quoted_identifiers_ignore_case")?.is_truthy()?, function_return_types: return_types(profile)?, nullability_rules: Some(rules), rich_type_inference: request.rich_type_inference, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs index 94a47c4c04..045cc2861b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs @@ -15,7 +15,7 @@ use crate::bindings::_helpers::records::fields::mapping_items; use crate::bindings::_helpers::records::ids::RecordId; const GENERIC_DIALECT: &str = "generic"; -const ADAPTER_RULE_ID: &str = "python"; +pub(crate) const ADAPTER_RULE_ID: &str = "python"; /// The nullability rules the native analysis knows, with their native rule ids. const KNOWN_RULES: [(AdapterHost, &str); 3] = [ (AdapterHost::FirstArgNullability, "first_arg"), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs index 096934a644..8716b69b5e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs @@ -99,6 +99,7 @@ pub(crate) struct ModelRow { pub(crate) struct RequestRow { pub(crate) dialect: String, pub(crate) case_sensitive_shapes: bool, + pub(crate) quoted_ignore_case: bool, pub(crate) function_return_types: Pairs, pub(crate) nullability_rules: Option, pub(crate) rich_type_inference: bool, @@ -215,6 +216,11 @@ impl NativeModelAnalysisSession { self.finished.as_ref() } + /// Each requested model's finished outcome and dynamic pivot proof, in request order. + pub(crate) fn outcomes(&self) -> &[(ModelOutcome, PivotOutcome)] { + &self.outcomes + } + /// The finished outcome and dynamic pivot proof of the model requested at `index`. pub(crate) fn outcome(&self, index: usize) -> PyResult<&(ModelOutcome, PivotOutcome)> { self.outcomes @@ -451,6 +457,7 @@ fn session_request(request: RequestRow) -> SessionRequest { let RequestRow { dialect, case_sensitive_shapes, + quoted_ignore_case, function_return_types, nullability_rules, rich_type_inference, @@ -464,6 +471,7 @@ fn session_request(request: RequestRow) -> SessionRequest { SessionRequest { dialect, case_sensitive_shapes, + quoted_ignore_case, function_return_types, nullability_rules, nullability_callback: None, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index 642ba9517f..562800f14a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -4,6 +4,9 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple, PyTupleMethods}; +use sqlbuild_analysis::analysis_route::main::analysis_route::analysis_route; +use sqlbuild_analysis::analysis_route::main::unapplied_nullability_rules::unapplied_nullability_rules; +use sqlbuild_analysis::analysis_route::models::AnalysisRoute; use sqlbuild_analysis::assembly::analysis_session::main::native_dialect::native_dialect; use sqlbuild_analysis::assembly::analysis_session::types::Pairs; use sqlbuild_host::compile_context::main::record_compile_cpu_timing::record_compile_cpu_timing; @@ -16,7 +19,9 @@ use crate::bindings::_helpers::analysis::binding_catalog::{ use crate::bindings::_helpers::analysis_session::model_analyses::{ ModelAnalyses, ModelAnalysisRequest, analyze_models, }; -use crate::bindings::_helpers::analysis_session::profile_inputs::text_pairs; +use crate::bindings::_helpers::analysis_session::profile_inputs::{ + ADAPTER_RULE_ID, nullability_rules, text_pairs, +}; use crate::bindings::_helpers::analysis_session::stage_functions::expression_source_shapes; use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::compile_context::compile_timings::PythonProcessClock; @@ -27,6 +32,7 @@ use crate::bindings::_helpers::compile_driver::stage_work::{ }; use crate::bindings::_helpers::compiled_views::model_input_view::attached_schema; use crate::bindings::_helpers::compiled_views::source_views::read_source_view; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::project_assembly::declared_relations::{ DeclaredRelations, DeclaredShapes, declared_relations, @@ -45,6 +51,7 @@ use std::collections::HashMap; const GENERIC_DIALECT: &str = "generic"; const UDF_PREFIX: &str = "__sqlbuild_udf_"; +const COMPILE_LOGGER: &str = "sqlbuild.compile"; /// The analysis state every assembled model and the project share. pub(crate) struct AnalysisContext<'py> { @@ -278,6 +285,38 @@ fn function_return_types<'py>( Ok(types) } +/// Warn that a native route leaves the adapter's own nullability rules uncalled. +fn warn_unapplied_nullability_rules( + inputs: &Bound<'_, PyAny>, + profile: &Bound<'_, PyAny>, +) -> PyResult<()> { + let (rows, adapter_rules) = nullability_rules(profile)?; + if adapter_rules.is_none() { + return Ok(()); + } + let mut functions: Vec = rows + .into_iter() + .filter(|(_, id)| id == ADAPTER_RULE_ID) + .map(|(function, _)| function) + .collect(); + functions.sort(); + let dialect: Option = native_field(profile, "sql_analysis_dialect")?.extract()?; + let adapter: Option = resolve_effective_adapter_name( + &native_field(inputs, "project_config")?, + &native_field(inputs, "local_config")?, + )? + .extract()?; + let route: AnalysisRoute = analysis_route(dialect.as_deref()); + let named: &str = adapter + .as_deref() + .or(dialect.as_deref()) + .unwrap_or(GENERIC_DIALECT); + match unapplied_nullability_rules(&route, named, &functions) { + Some(warning) => host::log_warning_message(inputs.py(), COMPILE_LOGGER, &warning), + None => Ok(()), + } +} + /// Record each expression source's analysed shape as its declared relation. fn record_expression_shapes<'py>( request: &AssemblyRequest, @@ -391,6 +430,7 @@ fn model_analyses<'py>( return Ok(None); } let inputs: &Bound<'py, PyAny> = request.inputs.bind(py); + warn_unapplied_nullability_rules(inputs, &context.profile)?; let names: &Bound<'py, PyAny> = request.analysis_model_names.bind(py); let mode: &Bound<'py, PyAny> = request.column_lineage_mode.bind(py); let lineage_mode: Bound<'py, PyAny> = record_class(py, RecordId::ColumnLineageMode)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs index af5593c480..99db37ef08 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs @@ -4,6 +4,8 @@ use pyo3::IntoPyObject; use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSetMethods, PyList, PyString, PyTuple}; +use sqlbuild_analysis::analysis_route::main::analysis_route::analysis_route; +use sqlbuild_analysis::analysis_route::models::AnalysisRoute; use crate::bindings::_helpers::analysis_session::model_analyses::ModelAnalyses; use crate::bindings::_helpers::analysis_session::profile_inputs::profile_dialect; @@ -116,7 +118,13 @@ fn compiled_model<'py>( syntax()?, ) }; - let validates: bool = input_validated && !syntax_validated; + let analysis_dialect: Option = + native_field(&context.profile, "sql_analysis_dialect")?.extract()?; + let legacy_route: bool = matches!( + analysis_route(analysis_dialect.as_deref()), + AnalysisRoute::Legacy + ); + let validates: bool = input_validated && !syntax_validated && legacy_route; if request.sql_validation_enabled && validates { for hook_name in HOOK_NAMES { validate_hook_sql_syntax( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs index c61261dd75..5d5df952e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs @@ -73,7 +73,8 @@ use crate::bindings::_helpers::input_loops::model_loop_macros::{ }; use crate::bindings::_helpers::input_loops::model_loop_named::{SCHEMA_DECLARATION, named_usages}; use crate::bindings::_helpers::input_loops::model_loop_opt_out::{ - OptOut, OptOutModel, reject_unparseable_query, rejected_opt_out, skip_unreadable, + AuthoredQuery, OptOut, OptOutModel, reject_unparseable_query, rejected_opt_out, + skip_unavailable, skip_unreadable, }; use crate::bindings::_helpers::input_loops::model_loop_prepass::{ NativeHeader, all_header_facts, native_header, @@ -204,6 +205,8 @@ pub(crate) struct NativeModelLoop { pub(crate) project_config_path: String, /// Whether the dialect's SQL is analysed by Polyglot or by its native module. pub(crate) route: AnalysisRoute, + /// The adapter's analysis dialect name, which P026 names when no native parser reads it. + pub(crate) dialect: Option, contract_classes: Py, loop_classes: Py, } @@ -265,6 +268,7 @@ impl NativeModelLoop { }, project_config_path, route: analysis_route(dialect.as_deref()), + dialect, contract_classes, loop_classes, }) @@ -704,6 +708,16 @@ impl NativeModelLoop { (&declarations.spliced.sql, ¯o_sql, &query_sql), ¯o_spans, ); + let authored_passes: Vec> = vec![ + declarations.spliced.spans.clone(), + spliced_spans(¯o_spans), + ]; + let authored_body: Option> = (interpolated.as_str() == &**authored_query) + .then_some(AuthoredQuery { + query: authored_query, + macro_sql: ¯o_sql, + passes: &authored_passes, + }); let opt_out_model: OptOutModel<'_, 'py> = OptOutModel { name, file_path, @@ -712,9 +726,12 @@ impl NativeModelLoop { query_sql: &query_sql, relative_path, contents, + authored: authored_body, + }; + let legacy_gate = match self.route { + AnalysisRoute::Legacy => Some(self.analysis(context, &opt_out_model, true)?), + AnalysisRoute::Native(_) | AnalysisRoute::Unavailable => None, }; - let (analysed, rejected_opt_out, analysis_skipped) = - self.analysis(context, &opt_out_model)?; let scanned = match extract_sql_references(&query_sql, &self.syntax) { ReferenceExtraction::Extracted(scanned) => scanned, ReferenceExtraction::Failed(failure) => { @@ -747,10 +764,6 @@ impl NativeModelLoop { contents, }, )?; - let analysis: (bool, Option>) = ( - analysed && scanned.invalid_calls.is_empty(), - rejected_opt_out, - ); let references: Vec = merge_call_site_references( scanned.references.iter().map(reference_row).collect(), argument_references, @@ -784,6 +797,26 @@ impl NativeModelLoop { } else { (values, Vec::new()) }; + let (analysed, rejected_opt_out, analysis_skipped) = match legacy_gate { + Some(gate) => gate, + None => { + let expanded_model: OptOutModel<'_, 'py> = OptOutModel { + name, + file_path, + config: &config, + values: &values, + query_sql: &query_sql, + relative_path, + contents, + authored: authored_body, + }; + self.analysis(context, &expanded_model, scanned.invalid_calls.is_empty())? + } + }; + let analysis: (bool, Option>) = ( + analysed && scanned.invalid_calls.is_empty(), + rejected_opt_out, + ); let source: HeaderSource<'_, 'py> = HeaderSource { name, values: &values, @@ -814,11 +847,12 @@ impl NativeModelLoop { }) } - /// Whether the model's SQL is analysed, its rejected opt-out, and whether its parser skipped it. + /// Analysed, rejected opt-out, skipped; a native route reads no SQL whose calls P012 failed. fn analysis<'py>( &self, context: &LoopContext<'_, 'py>, model: &OptOutModel<'_, 'py>, + calls_valid: bool, ) -> PyResult<(bool, Option>, bool)> { let mut skipped: bool = false; let (analysed, rejected) = @@ -832,11 +866,19 @@ impl NativeModelLoop { } AnalysisGate::Enabled => match self.route { AnalysisRoute::Legacy => (true, None), + AnalysisRoute::Native(_) | AnalysisRoute::Unavailable if !calls_valid => { + (false, None) + } AnalysisRoute::Native(dialect) => { let read: bool = skip_unreadable(self.settings, dialect, context, model)?; skipped = !read; (read, None) } + AnalysisRoute::Unavailable => { + skip_unavailable(self.settings, self.dialect.as_deref(), context, model)?; + skipped = true; + (false, None) + } }, AnalysisGate::Disabled => match rejected_opt_out(self, context, model)? { OptOut::Accepted => (false, None), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs index b392bc8aaf..322134ab17 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs @@ -4,14 +4,18 @@ use std::path::Path; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyBoolMethods, PyDict, PyDictMethods, PyList, PyString, PyTuple}; +use sqlbuild_analysis::analysis_route::main::aligned_offset::aligned_offset; use sqlbuild_analysis::analysis_route::main::native_gap::native_gap; use sqlbuild_analysis::analysis_route::main::skipped_analysis::skipped_analysis; -use sqlbuild_analysis::analysis_route::models::{AnalysisRoute, LocatedGap}; +use sqlbuild_analysis::analysis_route::main::unavailable_analysis::unavailable_analysis; +use sqlbuild_analysis::analysis_route::models::{AnalysisRoute, GapPosition, GapSite, LocatedGap}; use sqlbuild_analysis::analysis_route::types::NativeDialect; use sqlbuild_analysis::assembly::project::models::{SyntaxCheck, SyntaxMode}; +use sqlbuild_analysis::compiled_project::main::authored_model_offset::authored_model_offset; use sqlbuild_analysis::compiled_project::main::opt_out_position::opt_out_position; +use sqlbuild_analysis::compiled_project::main::text_position::text_position; use sqlbuild_analysis::compiled_project::models::ModelLoopSettings; -use sqlbuild_analysis::compiled_project::models::TextPosition; +use sqlbuild_analysis::compiled_project::models::{SplicedSpan, TextPosition}; use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics::replace_intrinsics; use crate::bindings::_helpers::input_loops::model_loop::{ @@ -19,6 +23,7 @@ use crate::bindings::_helpers::input_loops::model_loop::{ }; use crate::bindings::_helpers::input_loops::model_loop_inputs::{HOOK_KEYS, present}; use crate::bindings::_helpers::project_assembly::resources::{syntax_error_message, syntax_valid}; +use crate::bindings::_helpers::project_assembly::syntax_validation::sql_entry_label; use crate::bindings::_helpers::records::construct::{new_helped_error, new_record_with}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; @@ -41,6 +46,7 @@ const LINE_FIELD: &str = "line"; const COLUMN_FIELD: &str = "column"; const HELP_ARGUMENT: &str = "help"; const MODEL_RESOURCE: &str = "model"; +const RELATIVE_PATH_ATTRIBUTE: &str = "relative_path"; /// Where the opt-out was authored, or a check the attachment still answers. pub(crate) enum OptOut<'py> { @@ -57,6 +63,32 @@ pub(crate) struct OptOutModel<'a, 'py> { pub(crate) query_sql: &'a str, pub(crate) relative_path: &'a str, pub(crate) contents: &'a str, + /// How `query_sql` maps back to the model file; none when interpolation rewrote the query. + pub(crate) authored: Option>, +} + +/// The query as the model file holds it, as macros expanded it, and the passes between them. +#[derive(Clone, Copy)] +pub(crate) struct AuthoredQuery<'a> { + pub(crate) query: &'a str, + pub(crate) macro_sql: &'a str, + /// The declaration and macro expansion passes, in that order. + pub(crate) passes: &'a [Vec], +} + +/// One statement a native parser must read, with the site P026 names it by. +struct SiteCheck { + check: SyntaxCheck, + /// The hook, such as `post_hooks[0] sql("record_refresh")`; none for the model's query. + label: Option, + path: String, +} + +/// A hook value being walked: its key, its file and the label its statements are named by. +struct HookSite<'a> { + hook_name: &'a str, + path: String, + label: Option, } /// Whether the opt-out is rejected, and where it was authored. @@ -73,10 +105,15 @@ pub(crate) fn rejected_opt_out<'py>( { return Ok(OptOut::Accepted); } - let checks: Vec = model_checks(context, model)?; + let checks: Vec = model_checks(context, model)?; let readable: bool = match model_loop.route { - AnalysisRoute::Legacy => syntax_valid(model.config.py(), GENERIC_DIALECT, &checks)?, + AnalysisRoute::Legacy => { + let statements: Vec = + checks.iter().map(|site| site.check.clone()).collect(); + syntax_valid(model.config.py(), GENERIC_DIALECT, &statements)? + } AnalysisRoute::Native(dialect) => first_gap(dialect, context, &checks)?.is_none(), + AnalysisRoute::Unavailable => false, }; if readable { Ok(OptOut::Rejected(location(model_loop, context, model)?)) @@ -92,11 +129,71 @@ pub(crate) fn skip_unreadable( context: &LoopContext<'_, '_>, model: &OptOutModel<'_, '_>, ) -> PyResult { - let checks: Vec = model_checks(context, model)?; - let Some(gap) = first_gap(dialect, context, &checks)? else { + let checks: Vec = model_checks(context, model)?; + let Some((check, gap)) = first_gap(dialect, context, &checks)? else { return Ok(true); }; - let (code, message) = skipped_analysis(model.name, dialect.name(), &gap); + let site: GapSite<'_> = GapSite { + label: check.label.as_deref(), + path: &check.path, + position: gap_position(model, check, &gap), + }; + report_skip( + settings, + context, + model, + skipped_analysis(model.name, dialect.name(), &gap, &site), + )?; + Ok(false) +} + +/// A hook's gap in its statement; the query's in the model file, when its text maps back there. +fn gap_position(model: &OptOutModel<'_, '_>, check: &SiteCheck, gap: &LocatedGap) -> GapPosition { + let Some(offset) = gap.source_offset else { + return GapPosition::Analysed; + }; + if check.label.is_some() { + return GapPosition::Statement(text_position(&check.check.sql, offset)); + } + model + .authored + .and_then(|authored| { + let query: usize = aligned_offset(&check.check.sql, offset, model.query_sql)?; + let expanded: usize = aligned_offset(model.query_sql, query, authored.macro_sql)?; + authored_model_offset( + model.contents, + authored.query, + (authored.macro_sql, expanded), + authored.passes, + ) + }) + .map_or(GapPosition::Analysed, |file| { + GapPosition::File(text_position(model.contents, file)) + }) +} + +/// P026 for a model whose dialect no native parser reads; it warns, or raises, and is not analysed. +pub(crate) fn skip_unavailable( + settings: ModelLoopSettings, + dialect: Option<&str>, + context: &LoopContext<'_, '_>, + model: &OptOutModel<'_, '_>, +) -> PyResult<()> { + report_skip( + settings, + context, + model, + unavailable_analysis(model.name, dialect), + ) +} + +/// Reports P026 as a warning, or raises it under `require_sql_analysis`. +fn report_skip( + settings: ModelLoopSettings, + context: &LoopContext<'_, '_>, + model: &OptOutModel<'_, '_>, + (code, message): (&str, String), +) -> PyResult<()> { let py: Python<'_> = model.config.py(); let help: &str = &context.classes.syntax_help; if settings.require_sql_analysis { @@ -134,41 +231,54 @@ pub(crate) fn skip_unreadable( .classes .report_diagnostic .call((), Some(&arguments))?; - Ok(false) + Ok(()) } /// The model's analysis SQL and each hook statement, as filled to run, with its placeholders. fn model_checks( context: &LoopContext<'_, '_>, model: &OptOutModel<'_, '_>, -) -> PyResult> { - let mut statements: Vec = vec![analysis_sql(context, model)?]; +) -> PyResult> { + let mut statements: Vec<(Option, String, String)> = vec![( + None, + model.file_path.to_owned(), + analysis_sql(context, model)?, + )]; for key in HOOK_KEYS { if let Some(hooks) = model.values.get_item(key)? { - statements.extend(hook_statements(context, &hooks)?); + let site: HookSite<'_> = HookSite { + hook_name: key, + path: model.file_path.to_owned(), + label: None, + }; + statements.extend(hook_statements(context, &hooks, site)?); } } let placeholders: Vec<(String, String)> = placeholders(model.values)?; Ok(statements .into_iter() - .map(|sql| SyntaxCheck { - sql, - placeholders: placeholders.clone(), + .map(|(label, path, sql)| SiteCheck { + check: SyntaxCheck { + sql, + placeholders: placeholders.clone(), + }, + label, + path, }) .collect()) } -/// The native parser's first gap across `checks`; none when it reads each one whole. -fn first_gap( +/// The native parser's first gap across `checks`, with its check; none when it reads each whole. +fn first_gap<'a>( dialect: &dyn NativeDialect, context: &LoopContext<'_, '_>, - checks: &[SyntaxCheck], -) -> PyResult> { - for check in checks { - if let Some(gap) = native_gap(dialect, check) + checks: &'a [SiteCheck], +) -> PyResult> { + for site in checks { + if let Some(gap) = native_gap(dialect, &site.check) .map_err(|message| compile_error(context.classes, &message, None))? { - return Ok(Some(gap)); + return Ok(Some((site, gap))); } } Ok(None) @@ -230,21 +340,53 @@ fn opted_out(values: &Bound<'_, PyDict>) -> PyResult { Ok(false) } -/// `hook_sql_statements`: strings, SQL hook statements, and nested lists or tuples, in order. +/// `hook_sql_statements` as `(label, file, SQL)`, labelled as `validate_hook_sql_syntax` names them. fn hook_statements( context: &LoopContext<'_, '_>, value: &Bound<'_, PyAny>, -) -> PyResult> { + site: HookSite<'_>, +) -> PyResult, String, String)>> { + let sql_hook_entry: &Bound<'_, PyAny> = &context.classes.sql_hook_entry; if let Ok(text) = value.downcast::() { - return Ok(vec![text.extract()?]); + let label: String = site.label.unwrap_or_else(|| site.hook_name.to_owned()); + return Ok(vec![(Some(label), site.path, text.extract()?)]); } - if value.is_instance(&context.classes.sql_hook_entry)? { - return hook_statements(context, &native_field(value, STATEMENT_ATTRIBUTE)?); + if value.is_instance(sql_hook_entry)? { + let label: String = match site.label { + Some(label) => label, + None => sql_entry_label(site.hook_name, value)?, + }; + let relative_path: Bound<'_, PyAny> = native_field(value, RELATIVE_PATH_ATTRIBUTE)?; + let path: String = if relative_path.is_truthy()? { + relative_path.str()?.to_string() + } else { + site.path + }; + let inner: HookSite<'_> = HookSite { + hook_name: site.hook_name, + path, + label: Some(label), + }; + return hook_statements(context, &native_field(value, STATEMENT_ATTRIBUTE)?, inner); } - let mut statements: Vec = Vec::new(); + let mut statements: Vec<(Option, String, String)> = Vec::new(); if value.is_instance_of::() || value.is_instance_of::() { - for item in value.try_iter()? { - statements.extend(hook_statements(context, &item?)?); + for (index, item) in value.try_iter()?.enumerate() { + let item: Bound<'_, PyAny> = item?; + let prefix: String = format!("{}[{index}]", site.hook_name); + let label: Option = if item.is_instance(sql_hook_entry)? { + Some(sql_entry_label(&prefix, &item)?) + } else if item.is_instance_of::() { + Some(format!("{prefix} inline_sql(\"...\")")) + } else { + None + }; + let inner: HookSite<'_> = HookSite { + hook_name: site.hook_name, + path: site.path.clone(), + label, + }; + statements.extend(hook_statements(context, &item, inner)?); } } Ok(statements) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs index 30e49894fc..ad9c50e3f5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs @@ -30,6 +30,7 @@ use crate::bindings::_helpers::lineage::lineage_objects::{LineageClasses, log_li use crate::bindings::_helpers::lineage::project_schema::{ fast_schema, inferred_names, rich_schema, }; +use crate::bindings::_helpers::project_assembly::syntax_checks::model_placeholders; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::ErrorId; @@ -154,8 +155,6 @@ struct FastModel<'py> { /// A model's lineage facts: none, Python facts, or a compiled model's native facts and edge count. enum FastFacts<'py> { Missing, - /// No facts, on a native route, where Polyglot never parses the model: no lineage. - Skipped, Python(Bound<'py, PyAny>), Native(Bound<'py, PyAny>, usize), } @@ -170,9 +169,9 @@ fn fast_lineage<'py>( let mut work: Vec> = Vec::with_capacity(models.len()); let mut requests: Vec = Vec::new(); let dialect_name: Option = dialect.map(|name| name.extract()).transpose()?; - let native: bool = matches!( + let native: bool = !matches!( analysis_route(dialect_name.as_deref()), - AnalysisRoute::Native(_) + AnalysisRoute::Legacy ); for model in models { let (has_star, star_resolved): (bool, bool) = star_flags(model)?; @@ -191,19 +190,15 @@ fn fast_lineage<'py>( continue; } let facts: Bound<'py, PyAny> = native_field(model, "fast_lineage_columns")?; - if facts.is_none() && native { - work.push(FastModel { - name: native_field(model, "name")?.extract()?, - facts: FastFacts::Skipped, - has_star, - expands_star, - }); - continue; - } if facts.is_none() { requests.push(FastLineageModel::Parse { query_sql: native_field(model, "query_sql")?.extract()?, inferred_columns: inferred_names(model)?, + placeholders: if native { + placeholder_pairs(model)? + } else { + Vec::new() + }, }); } else if expands_star { requests.push(FastLineageModel::StarExpansion { @@ -266,7 +261,6 @@ fn fast_lineage<'py>( } CompactColumns::Facts(columns.unbind()) } - FastFacts::Skipped => continue, FastFacts::Missing => { match outcomes.next() { Some(FastLineageOutcome::Built { columns, has_star }) => { @@ -282,7 +276,11 @@ fn fast_lineage<'py>( } Some(FastLineageOutcome::Unparsed(detail)) => log_lineage_event( py, - "fast column lineage parse failed; falling back", + if native { + "fast column lineage not built: the native parser cannot read the model" + } else { + "fast column lineage parse failed; falling back" + }, &model.name, Some(&detail), )?, @@ -310,6 +308,17 @@ fn fast_lineage<'py>( .map(Bound::into_any) } +/// The model's `@@@name` placeholder defaults, which a native parser reads in their place. +fn placeholder_pairs(model: &Bound<'_, PyAny>) -> PyResult> { + let Some(placeholders) = model_placeholders(model)? else { + return Ok(Vec::new()); + }; + placeholders + .iter() + .map(|(name, value)| Ok((name.extract()?, value.extract()?))) + .collect() +} + /// The output column names of compact facts or plain column facts. fn output_columns(classes: &LineageClasses<'_>, facts: &Bound<'_, PyAny>) -> PyResult> { if facts.is_instance(&classes.compact_facts)? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs index babcc38c91..84d87dcdcb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs @@ -97,3 +97,12 @@ pub(crate) fn log_debug_message(py: Python<'_>, logger: &str, message: &str) -> .call((), Some(&kwargs)) .map(|_| ()) } + +/// `logging.getLogger(logger).warning(message)`. +pub(crate) fn log_warning_message(py: Python<'_>, logger: &str, message: &str) -> PyResult<()> { + py.import("logging")? + .getattr("getLogger")? + .call1((logger,))? + .call_method1("warning", (message,)) + .map(|_| ()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs index b36fb23fb4..1a14e9d133 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs @@ -64,7 +64,8 @@ fn parse_error(context: &SyntaxContext<'_, '_>, sql: &str) -> PyResult) -> PyResult { +/// How errors name a SQL hook entry: `{prefix} sql("name")`, or `{prefix} inline_sql("...")`. +pub(crate) fn sql_entry_label(prefix: &str, entry: &Bound<'_, PyAny>) -> PyResult { let name: Bound<'_, PyAny> = native_field(entry, "name")?; Ok(if name.is_none() { format!("{prefix} inline_sql(\"...\")") diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs index c75a2b2af4..e658566966 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs @@ -4,6 +4,10 @@ use std::collections::HashSet; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyString, PyTuple}; +use sqlbuild_analysis::analysis_route::main::analysis_route::analysis_route; +use sqlbuild_analysis::analysis_route::main::native_revalidation::native_revalidation; +use sqlbuild_analysis::analysis_route::models::AnalysisRoute; +use sqlbuild_analysis::analysis_route::types::NativeDialect; use sqlbuild_analysis::semantic_checks::main::finish_type_recovery::finish_type_recovery; use sqlbuild_analysis::semantic_checks::main::plan_type_recovery::plan_type_recovery; use sqlbuild_analysis::semantic_checks::models::{ @@ -101,8 +105,18 @@ pub(crate) fn recovered_types<'py>( .plan() .ok_or_else(|| internal_error("finishing a type recovery that has no plan"))?; let poisoned: Vec<(String, String)> = plan.poisoned_outputs(); - let revised: Vec> = - revalidated(classes, project, &plan.revalidated_models(), &poisoned)?; + let models_to_revalidate: Vec = plan.revalidated_models(); + let revised: Vec> = match analysis_route(request.dialect.as_deref()) { + AnalysisRoute::Native(dialect) => natively_revalidated( + py, + dialect, + (&request, &models_to_revalidate, &poisoned), + shared, + )?, + AnalysisRoute::Legacy | AnalysisRoute::Unavailable => { + revalidated(classes, project, &models_to_revalidate, &poisoned)? + } + }; let outcome = finish_type_recovery(&request, plan, &revised).map_err(|reason| internal_error(&reason))?; let mut kept: Vec> = Vec::with_capacity(outcome.kept.len()); @@ -243,6 +257,44 @@ fn recovery_model( }) } +/// Each planned model revalidated by the dialect's native analysis, never the Polyglot catalog. +fn natively_revalidated( + py: Python<'_>, + dialect: &'static dyn NativeDialect, + (request, models, poisoned): (&TypeRecoveryRequest, &[usize], &[(String, String)]), + shared: &StageShared<'_>, +) -> PyResult>> { + if models.is_empty() { + return Ok(Vec::new()); + } + let session: &NativeModelAnalysisSession = shared + .session + .ok_or_else(|| internal_error("revalidating a native route without its session"))?; + let finished = session + .finished_session() + .ok_or_else(|| internal_error("revalidating before the session finished"))?; + let mut planned: Vec<&RecoveryModel> = Vec::with_capacity(models.len()); + for index in models { + planned.push( + request + .models + .get(*index) + .ok_or_else(|| internal_error("type recovery planned a model it does not hold"))?, + ); + } + count_stage(py, SCHEMA_REVALIDATIONS, planned.len() as u64)?; + py.compiler_detach(|| { + Ok(native_revalidation( + dialect, + (finished, session.outcomes()), + &planned, + poisoned, + )) + }) + .map_err(compiler_error)? + .map_err(|reason| internal_error(&reason)) +} + /// Python's revalidation of each planned model with unknown types for poisoned outputs. fn revalidated<'py>( classes: &StageClasses<'py>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/lexical_syntax.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/lexical_syntax.rs index 600dd863ed..39148bb37b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/lexical_syntax.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/lexical_syntax.rs @@ -30,6 +30,7 @@ impl From for LexicalSyntax { triple_quoted_strings: input.triple_quoted_strings, nested_block_comments: input.nested_block_comments, line_comment_prefixes: input.line_comment_prefixes, + bracket_identifiers: false, } } } @@ -54,6 +55,7 @@ pub(crate) fn record_syntax(instance: &Bound<'_, PyAny>) -> PyResult dollar_quote_end(sql, index), Some(b'\'' | b'"' | b'`') => quoted_text_end(sql, index, syntax).map(Some), + Some(b'[') if syntax.bracket_identifiers => bracket_identifier_end(sql, index).map(Some), _ => Ok(None), } } @@ -44,6 +45,7 @@ pub(crate) fn non_code_end( /// Whether a comment or quoted text may open with `byte`; most code bytes are rejected here cheaply. fn may_open_non_code(byte: u8, syntax: &LexicalSyntax) -> bool { matches!(byte, b'/' | b'$' | b'\'' | b'"' | b'`') + || (byte == b'[' && syntax.bracket_identifiers) || syntax .line_comment_prefixes .iter() @@ -133,6 +135,19 @@ fn quoted_text_end(sql: &[u8], start: usize, syntax: &LexicalSyntax) -> Result Result { + let mut index = start + 1; + while let Some(offset) = sql[index..].iter().position(|byte| *byte == b']') { + let close = index + offset; + if sql.get(close + 1) != Some(&b']') { + return Ok(close + 1); + } + index = close + 2; + } + Err(Unclosed::Quote) +} + fn string_literal_prefix(sql: &[u8], start: usize) -> Vec { let mut prefix_start = start; while prefix_start > 0 diff --git a/crates/sqlbuild-sqltext/src/sql_scan/constants.rs b/crates/sqlbuild-sqltext/src/sql_scan/constants.rs index f7d11818a4..b604b49f12 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/constants.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/constants.rs @@ -14,3 +14,5 @@ pub const GENERIC_LINE_COMMENT_PREFIX: &str = "--"; pub const TRIPLE_QUOTES: [&str; 2] = ["'''", "\"\"\""]; pub const ESCAPE_CHARACTER: char = '\\'; pub const BLOCK_COMMENT_OPEN: &str = "/*"; +/// What a lexical key adds for bracket identifiers. +pub(crate) const BRACKET_IDENTIFIERS_KEY: &str = "brackets"; diff --git a/crates/sqlbuild-sqltext/src/sql_scan/main/dialect_lexical_syntax.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/dialect_lexical_syntax.rs new file mode 100644 index 0000000000..3028fc6dbb --- /dev/null +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/dialect_lexical_syntax.rs @@ -0,0 +1,44 @@ +//! `dialect_lexical_syntax`: the comment and quoted-text syntax of a dialect's SQL, by dialect name. + +use crate::sql_scan::models::LexicalSyntax; + +/// The comment and quoted-text syntax that SQLQuack tokenizes for `dialect`. +pub fn dialect_lexical_syntax(dialect: &str) -> LexicalSyntax { + let strings = |quotes: &[&str]| quotes.iter().map(|quote| (*quote).to_owned()).collect(); + let lines = |prefixes: &[&str]| prefixes.iter().map(|prefix| (*prefix).to_owned()).collect(); + match dialect.to_ascii_lowercase().as_str() { + "bigquery" => LexicalSyntax { + backslash_escape_quotes: strings(&["'", "\""]), + raw_string_prefix: true, + triple_quoted_strings: true, + line_comment_prefixes: lines(&["--", "#"]), + ..LexicalSyntax::default() + }, + "snowflake" => LexicalSyntax { + backslash_escape_quotes: strings(&["'"]), + line_comment_prefixes: lines(&["--", "//"]), + ..LexicalSyntax::default() + }, + "clickhouse" => LexicalSyntax { + backslash_escape_quotes: strings(&["'", "\"", "`"]), + line_comment_prefixes: lines(&["--", "# ", "#!", "//"]), + ..LexicalSyntax::default() + }, + "databricks" | "duckdb" => LexicalSyntax { + line_comment_prefixes: lines(&["--"]), + ..LexicalSyntax::default() + }, + "mssql" | "sqlserver" | "tsql" => LexicalSyntax { + nested_block_comments: true, + line_comment_prefixes: lines(&["--"]), + bracket_identifiers: true, + ..LexicalSyntax::default() + }, + _ => LexicalSyntax { + escape_string_prefix: true, + nested_block_comments: true, + line_comment_prefixes: lines(&["--"]), + ..LexicalSyntax::default() + }, + } +} diff --git a/crates/sqlbuild-sqltext/src/sql_scan/main/lexical_syntax_key.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/lexical_syntax_key.rs index ff2cd882d3..5bcdb7be2a 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/main/lexical_syntax_key.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/lexical_syntax_key.rs @@ -1,8 +1,9 @@ //! The process-stable text key of one dialect's lexical rules. +use crate::sql_scan::constants::BRACKET_IDENTIFIERS_KEY; use crate::sql_scan::models::LexicalSyntax; -/// `SqlLexicalSyntax.cache_key`: sorted quotes, the four flags as Python booleans, sorted prefixes. +/// `SqlLexicalSyntax.cache_key`: quotes, the four flags, prefixes, then `brackets` only when set. #[must_use] pub fn lexical_syntax_key(syntax: &LexicalSyntax) -> String { let flag = |value: bool| if value { "True" } else { "False" }; @@ -18,13 +19,16 @@ pub fn lexical_syntax_key(syntax: &LexicalSyntax) -> String { .map(String::as_str) .collect(); prefixes.sort_unstable(); - [ + let mut parts: Vec = vec![ quotes.join(","), flag(syntax.escape_string_prefix).to_owned(), flag(syntax.raw_string_prefix).to_owned(), flag(syntax.triple_quoted_strings).to_owned(), flag(syntax.nested_block_comments).to_owned(), prefixes.join(","), - ] - .join("|") + ]; + if syntax.bracket_identifiers { + parts.push(BRACKET_IDENTIFIERS_KEY.to_owned()); + } + parts.join("|") } diff --git a/crates/sqlbuild-sqltext/src/sql_scan/main/mod.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/mod.rs index 4735e4b172..3cc4afefde 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/main/mod.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/mod.rs @@ -1,4 +1,5 @@ pub mod comment_end; +pub mod dialect_lexical_syntax; pub mod dialect_matching_paren; pub mod dialect_non_code_end; pub mod dialect_non_code_ranges; diff --git a/crates/sqlbuild-sqltext/src/sql_scan/main/reads_differently_from_generic.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/reads_differently_from_generic.rs index 05479da834..3ea203edde 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/main/reads_differently_from_generic.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/reads_differently_from_generic.rs @@ -19,6 +19,9 @@ pub fn reads_differently_from_generic(syntax: &LexicalSyntax, sql: &str) -> bool if syntax.nested_block_comments && sql.matches(BLOCK_COMMENT_OPEN).count() > 1 { return true; } + if syntax.bracket_identifiers && sql.contains('[') { + return true; + } syntax .line_comment_prefixes .iter() diff --git a/crates/sqlbuild-sqltext/src/sql_scan/models.rs b/crates/sqlbuild-sqltext/src/sql_scan/models.rs index 632b1b5b07..b89ffe1f8f 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/models.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/models.rs @@ -81,6 +81,9 @@ pub struct LexicalSyntax { pub nested_block_comments: bool, /// Prefixes that start a line comment, such as `--`, `#` or `//`. pub line_comment_prefixes: Vec, + /// `[...]` quotes an identifier, `]]` standing for one `]`, as in T-SQL. + #[serde(default)] + pub bracket_identifiers: bool, } impl LexicalSyntax { diff --git a/crates/sqlbuild-sqltext/src/sql_scan/tests/helpers.rs b/crates/sqlbuild-sqltext/src/sql_scan/tests/helpers.rs index 911a581657..ad9f12b511 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/tests/helpers.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/tests/helpers.rs @@ -127,5 +127,6 @@ pub(crate) fn syntax(quotes: &[&str], flags: [bool; 4], prefixes: &[&str]) -> Le triple_quoted_strings: flags[2], nested_block_comments: flags[3], line_comment_prefixes: prefixes.iter().map(|prefix| (*prefix).to_owned()).collect(), + bracket_identifiers: false, } } diff --git a/crates/sqlbuild-sqltext/src/sql_scan/tests/test_dialect_scan.rs b/crates/sqlbuild-sqltext/src/sql_scan/tests/test_dialect_scan.rs index b23fd27f99..24354b62d3 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/tests/test_dialect_scan.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/tests/test_dialect_scan.rs @@ -1,3 +1,4 @@ +use crate::sql_scan::main::dialect_lexical_syntax::dialect_lexical_syntax; use crate::sql_scan::main::dialect_non_code_end::dialect_non_code_end; use crate::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges; use crate::sql_scan::models::{LexicalSyntax, Unclosed}; @@ -151,6 +152,48 @@ fn given_dialect_lexical_syntax_when_scanning_non_code_then_dialect_decides_boun syntax: generic(), expected_first_range: Some((0, 12)), }, + DialectFirstNonCodeRangeTestCase { + description: "a T-SQL bracket identifier hides the apostrophe inside it", + sql: "[customer's name] x", + syntax: dialect_lexical_syntax("tsql"), + expected_first_range: Some((0, 17)), + }, + DialectFirstNonCodeRangeTestCase { + description: "a doubled closing bracket stays inside the identifier", + sql: "[a]]'b] x", + syntax: dialect_lexical_syntax("tsql"), + expected_first_range: Some((0, 7)), + }, + DialectFirstNonCodeRangeTestCase { + description: "a comment prefix inside a bracket identifier opens no comment", + sql: "[a--b] x", + syntax: dialect_lexical_syntax("tsql"), + expected_first_range: Some((0, 6)), + }, + DialectFirstNonCodeRangeTestCase { + description: "outside T-SQL a bracket is code, so the apostrophe opens a quote", + sql: "[customer's name] x", + syntax: generic(), + expected_first_range: Some((9, 19)), + }, + DialectFirstNonCodeRangeTestCase { + description: "ClickHouse backslash escapes hold in double-quoted identifiers", + sql: r#""a\"b" x"#, + syntax: dialect_lexical_syntax("clickhouse"), + expected_first_range: Some((0, 6)), + }, + DialectFirstNonCodeRangeTestCase { + description: "a ClickHouse hash before a word is code, not a comment", + sql: "#x 'a'", + syntax: dialect_lexical_syntax("clickhouse"), + expected_first_range: Some((3, 6)), + }, + DialectFirstNonCodeRangeTestCase { + description: "a ClickHouse hash before a space opens a comment", + sql: "# it's\nx", + syntax: dialect_lexical_syntax("clickhouse"), + expected_first_range: Some((0, 7)), + }, DialectFirstNonCodeRangeTestCase { description: "an unclosed backslash-escaped quote extends to the end", sql: r"'O\'", diff --git a/fensu.toml b/fensu.toml index bee7f18a71..3fee255b7a 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2932,3 +2932,13 @@ reason = "The override keeps queue.Queue.put's positional protocol, which every rule = "FFS201" path = "src/sqlbuild/orchestration_declarations/models/sensor_context.py" reason = "Only SensorContext lives here: invocation-local ctx.cursor assignment proposes progress, committed solely by native fenced completion. The package regression guards this declaration-only scope; class-symbol exceptions are unsupported." + +[[targets.rust.rule_exceptions]] +rule = "FPRSL105" +path = "crates/sqlbuild-analysis/src/type_system/main/normalize_text_type.rs" +reason = "Native dialect modules in analysis_route type their spellings through this entry; each dialect's module lands in its own change." + +[[targets.rust.rule_exceptions]] +rule = "FPRSL105" +path = "crates/sqlbuild-analysis/src/analysis_route/main/clickhouse_type_match.rs" +reason = "The ClickHouse native dialect module's type_match calls this entry; that module lands in its own change." diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/_test_types.py index 5d57d26234..7d0eec5f34 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/_test_types.py @@ -14,6 +14,15 @@ class NativeFastLineageCliTestCase: expected_minimum_traced_edges: int +@dataclass(frozen=True) +class WarmFastLineageCliTestCase: + """Fast lineage traced after a cold compile filled the compile's stores.""" + + description: str + reuse_disabled: str + expected_minimum_cache_hits: int + + @dataclass(frozen=True) class NativeRichLineageFailureCliTestCase: """A native rich lineage failure injected at the binding, compiled through the CLI.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py index a3222d6af4..546d0400a3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py @@ -20,6 +20,7 @@ class LineageRun(NamedTuple): compile_report: str compile_returncode: int traces: list[tuple[int, str, str]] + analysis_cache_hits: int = 0 def lineage_run( @@ -28,27 +29,37 @@ def lineage_run( files: dict[str, str], targets: tuple[str, ...], mode: str = "fast", + cache_args: tuple[str, ...] = ("--no-cache",), + environment: tuple[tuple[str, str], ...] = ((REUSE_DISABLE_ENV_VAR, "1"),), + cold_compiles: int = 0, ) -> LineageRun: - """Write the project, then compile and trace every target both ways in one lineage mode.""" + """Write the project, then compile and trace every target both ways in one lineage mode. + + `cold_compiles` compiles run first, so the measured compile and traces read what they stored. + """ for relative_path, contents in files.items(): path: Path = project_dir / "project" / relative_path path.parent.mkdir(parents=True, exist_ok=True) _ = path.write_text(contents, encoding="utf-8") - environment: dict[str, str] = {REUSE_DISABLE_ENV_VAR: "1"} + env: dict[str, str] = dict(environment) + compile_args: tuple[str, ...] = ("compile", "--json", *cache_args, "--lineage-mode", mode) + for _ in range(cold_compiles): + cold: subprocess.CompletedProcess[str] = run_installed_sqb( + project_dir=project_dir / "project", args=compile_args, env=env + ) + assert cold.returncode == 0, cold.stdout + cold.stderr compiled: subprocess.CompletedProcess[str] = run_installed_sqb( - project_dir=project_dir / "project", - args=("compile", "--json", "--no-cache", "--lineage-mode", mode), - env=environment, + project_dir=project_dir / "project", args=compile_args, env=env ) report: dict[str, object] = json.loads(compiled.stdout.replace(str(project_dir), "")) - _ = report.pop("compile_timings", None) + timings: dict[str, int] = cast(dict[str, int], report.pop("compile_timings", {})) traces: list[tuple[int, str, str]] = [] for target, direction in product(targets, _DIRECTIONS): result: subprocess.CompletedProcess[str] = run_installed_sqb( project_dir=project_dir / "project", args=("lineage", target, "--mode", mode, "--direction", direction, "--json"), - env=environment, + env=env, ) stdout: str = result.stdout.replace(str(project_dir), "") traces.append((result.returncode, stdout, _TIMING.sub("", result.stderr))) @@ -56,6 +67,8 @@ def lineage_run( compile_report=json.dumps(report, indent=2), compile_returncode=compiled.returncode, traces=traces, + analysis_cache_hits=timings.get("analysis_batch_cache_hits", 0) + + timings.get("analysis_entry_cache_hits", 0), ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py index 930da556bc..0621386f95 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py @@ -6,10 +6,12 @@ import pytest +from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR from tests.e2e.src.sqlbuild.cli.commands.main.compile.fast_lineage._test_types import ( NativeFastLineageCliTestCase, NativeRichLineageCliTestCase, NativeRichLineageFailureCliTestCase, + WarmFastLineageCliTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.fast_lineage.helpers import ( FailedLineageRun, @@ -52,6 +54,12 @@ ), } +_WARM_TARGETS: tuple[str, ...] = ( + "customer_totals.Total_Amount", + "order_copies.rounded", + "stg_orders.amount", +) + _DEEP_UNION_BRANCHES: int = 480 _DEEP_UNION_FILES: dict[str, str] = { "sqlbuild_project.toml": _PROJECT_FILES["sqlbuild_project.toml"], @@ -180,6 +188,42 @@ def test_given_project_when_compiling_then_fast_lineage_is_traced( ) +@pytest.mark.parametrize( + "test_case", + [ + WarmFastLineageCliTestCase( + description="the analysis cache a cold compile filled", + reuse_disabled="1", + expected_minimum_cache_hits=1, + ), + WarmFastLineageCliTestCase( + description="the compiled project a cold compile stored for reuse", + reuse_disabled="0", + expected_minimum_cache_hits=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_warm_store_when_tracing_fast_lineage_then_it_matches_the_uncached_run( + test_case: WarmFastLineageCliTestCase, tmp_path: Path +) -> None: + uncached: LineageRun = lineage_run( + project_dir=tmp_path / "uncached", files=_PROJECT_FILES, targets=_WARM_TARGETS + ) + restored: LineageRun = lineage_run( + project_dir=tmp_path / "warm", + files=_PROJECT_FILES, + targets=_WARM_TARGETS, + cache_args=(), + environment=((REUSE_DISABLE_ENV_VAR, test_case.reuse_disabled),), + cold_compiles=1, + ) + + assert restored.compile_returncode == 0, restored.compile_report + assert (restored.compile_report, restored.traces) == (uncached.compile_report, uncached.traces) + assert restored.analysis_cache_hits >= test_case.expected_minimum_cache_hits + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/goldens/python_oracle/types_generated_seeded_scalar_parameterized_nested_padded_and_invalid_types.json b/tests/goldens/python_oracle/types_generated_seeded_scalar_parameterized_nested_padded_and_invalid_types.json index 1729e43aef..04b9dd9f09 100644 --- a/tests/goldens/python_oracle/types_generated_seeded_scalar_parameterized_nested_padded_and_invalid_types.json +++ b/tests/goldens/python_oracle/types_generated_seeded_scalar_parameterized_nested_padded_and_invalid_types.json @@ -478,18 +478,18 @@ [ "type", { -"DuckDB": "98a80237e4e0a12f35bbf17e", -"None": "98a80237e4e0a12f35bbf17e", -"bigquery": "98a80237e4e0a12f35bbf17e", -"databricks": "98a80237e4e0a12f35bbf17e", -"duckdb": "98a80237e4e0a12f35bbf17e", -"generic": "98a80237e4e0a12f35bbf17e", +"DuckDB": "f1dcf3349867744866bb1299", +"None": "f1dcf3349867744866bb1299", +"bigquery": "f1dcf3349867744866bb1299", +"databricks": "f1dcf3349867744866bb1299", +"duckdb": "f1dcf3349867744866bb1299", +"generic": "f1dcf3349867744866bb1299", "motherduck": "901c5133ed47d4769286e860", -"mysql": "98a80237e4e0a12f35bbf17e", -"postgres": "98a80237e4e0a12f35bbf17e", -"postgresql": "98a80237e4e0a12f35bbf17e", -"snowflake": "98a80237e4e0a12f35bbf17e", -"tsql": "98a80237e4e0a12f35bbf17e" +"mysql": "f1dcf3349867744866bb1299", +"postgres": "f1dcf3349867744866bb1299", +"postgresql": "f1dcf3349867744866bb1299", +"snowflake": "f1dcf3349867744866bb1299", +"tsql": "f1dcf3349867744866bb1299" } ], [ @@ -529,18 +529,18 @@ [ "type", { -"DuckDB": "9a79cb58680bebe49009a72d", -"None": "9a79cb58680bebe49009a72d", -"bigquery": "9a79cb58680bebe49009a72d", -"databricks": "9a79cb58680bebe49009a72d", -"duckdb": "9a79cb58680bebe49009a72d", -"generic": "9a79cb58680bebe49009a72d", +"DuckDB": "7c70b66e45566b521b449adc", +"None": "7c70b66e45566b521b449adc", +"bigquery": "7c70b66e45566b521b449adc", +"databricks": "7c70b66e45566b521b449adc", +"duckdb": "7c70b66e45566b521b449adc", +"generic": "7c70b66e45566b521b449adc", "motherduck": "901c5133ed47d4769286e860", -"mysql": "9a79cb58680bebe49009a72d", -"postgres": "9a79cb58680bebe49009a72d", -"postgresql": "9a79cb58680bebe49009a72d", -"snowflake": "9a79cb58680bebe49009a72d", -"tsql": "9a79cb58680bebe49009a72d" +"mysql": "7c70b66e45566b521b449adc", +"postgres": "7c70b66e45566b521b449adc", +"postgresql": "7c70b66e45566b521b449adc", +"snowflake": "7c70b66e45566b521b449adc", +"tsql": "7c70b66e45566b521b449adc" } ], [ @@ -954,18 +954,18 @@ [ "type", { -"DuckDB": "1950e32420d1d40d41ffae64", -"None": "1950e32420d1d40d41ffae64", -"bigquery": "1950e32420d1d40d41ffae64", -"databricks": "1950e32420d1d40d41ffae64", -"duckdb": "1950e32420d1d40d41ffae64", -"generic": "1950e32420d1d40d41ffae64", +"DuckDB": "8f2569067b02a6a41c0caf62", +"None": "8f2569067b02a6a41c0caf62", +"bigquery": "8f2569067b02a6a41c0caf62", +"databricks": "8f2569067b02a6a41c0caf62", +"duckdb": "8f2569067b02a6a41c0caf62", +"generic": "8f2569067b02a6a41c0caf62", "motherduck": "901c5133ed47d4769286e860", -"mysql": "1950e32420d1d40d41ffae64", -"postgres": "1950e32420d1d40d41ffae64", -"postgresql": "1950e32420d1d40d41ffae64", -"snowflake": "1950e32420d1d40d41ffae64", -"tsql": "1950e32420d1d40d41ffae64" +"mysql": "8f2569067b02a6a41c0caf62", +"postgres": "8f2569067b02a6a41c0caf62", +"postgresql": "8f2569067b02a6a41c0caf62", +"snowflake": "8f2569067b02a6a41c0caf62", +"tsql": "8f2569067b02a6a41c0caf62" } ], [ @@ -1311,18 +1311,18 @@ [ "type", { -"DuckDB": "06aec7cca4b30489405665e2", -"None": "06aec7cca4b30489405665e2", -"bigquery": "06aec7cca4b30489405665e2", -"databricks": "06aec7cca4b30489405665e2", -"duckdb": "06aec7cca4b30489405665e2", -"generic": "06aec7cca4b30489405665e2", +"DuckDB": "da06e77421c17a0eee87e6cc", +"None": "da06e77421c17a0eee87e6cc", +"bigquery": "da06e77421c17a0eee87e6cc", +"databricks": "da06e77421c17a0eee87e6cc", +"duckdb": "da06e77421c17a0eee87e6cc", +"generic": "da06e77421c17a0eee87e6cc", "motherduck": "901c5133ed47d4769286e860", -"mysql": "06aec7cca4b30489405665e2", -"postgres": "06aec7cca4b30489405665e2", -"postgresql": "06aec7cca4b30489405665e2", -"snowflake": "06aec7cca4b30489405665e2", -"tsql": "06aec7cca4b30489405665e2" +"mysql": "da06e77421c17a0eee87e6cc", +"postgres": "da06e77421c17a0eee87e6cc", +"postgresql": "da06e77421c17a0eee87e6cc", +"snowflake": "da06e77421c17a0eee87e6cc", +"tsql": "da06e77421c17a0eee87e6cc" } ], [ @@ -1396,35 +1396,35 @@ [ "type", { -"DuckDB": "7aa59eed1bba74a9dee48e59", -"None": "7aa59eed1bba74a9dee48e59", -"bigquery": "7aa59eed1bba74a9dee48e59", -"databricks": "7aa59eed1bba74a9dee48e59", -"duckdb": "7aa59eed1bba74a9dee48e59", -"generic": "7aa59eed1bba74a9dee48e59", +"DuckDB": "85679d39a40088722a25077b", +"None": "85679d39a40088722a25077b", +"bigquery": "85679d39a40088722a25077b", +"databricks": "85679d39a40088722a25077b", +"duckdb": "85679d39a40088722a25077b", +"generic": "85679d39a40088722a25077b", "motherduck": "901c5133ed47d4769286e860", -"mysql": "7aa59eed1bba74a9dee48e59", -"postgres": "7aa59eed1bba74a9dee48e59", -"postgresql": "7aa59eed1bba74a9dee48e59", -"snowflake": "7aa59eed1bba74a9dee48e59", -"tsql": "7aa59eed1bba74a9dee48e59" +"mysql": "85679d39a40088722a25077b", +"postgres": "85679d39a40088722a25077b", +"postgresql": "85679d39a40088722a25077b", +"snowflake": "85679d39a40088722a25077b", +"tsql": "85679d39a40088722a25077b" } ], [ "type", { -"DuckDB": "66e7166890ccedeaad6c241a", -"None": "66e7166890ccedeaad6c241a", -"bigquery": "66e7166890ccedeaad6c241a", -"databricks": "66e7166890ccedeaad6c241a", -"duckdb": "66e7166890ccedeaad6c241a", -"generic": "66e7166890ccedeaad6c241a", +"DuckDB": "af83290b9888e9ea215c9c2a", +"None": "af83290b9888e9ea215c9c2a", +"bigquery": "af83290b9888e9ea215c9c2a", +"databricks": "af83290b9888e9ea215c9c2a", +"duckdb": "af83290b9888e9ea215c9c2a", +"generic": "af83290b9888e9ea215c9c2a", "motherduck": "901c5133ed47d4769286e860", -"mysql": "66e7166890ccedeaad6c241a", -"postgres": "66e7166890ccedeaad6c241a", -"postgresql": "66e7166890ccedeaad6c241a", -"snowflake": "66e7166890ccedeaad6c241a", -"tsql": "66e7166890ccedeaad6c241a" +"mysql": "af83290b9888e9ea215c9c2a", +"postgres": "af83290b9888e9ea215c9c2a", +"postgresql": "af83290b9888e9ea215c9c2a", +"snowflake": "af83290b9888e9ea215c9c2a", +"tsql": "af83290b9888e9ea215c9c2a" } ], [ @@ -1889,18 +1889,18 @@ [ "type", { -"DuckDB": "caf173733defe796376d4efe", -"None": "caf173733defe796376d4efe", -"bigquery": "caf173733defe796376d4efe", -"databricks": "caf173733defe796376d4efe", -"duckdb": "caf173733defe796376d4efe", -"generic": "caf173733defe796376d4efe", +"DuckDB": "a6458f2eeceb4012953d4be0", +"None": "a6458f2eeceb4012953d4be0", +"bigquery": "a6458f2eeceb4012953d4be0", +"databricks": "a6458f2eeceb4012953d4be0", +"duckdb": "a6458f2eeceb4012953d4be0", +"generic": "a6458f2eeceb4012953d4be0", "motherduck": "901c5133ed47d4769286e860", -"mysql": "caf173733defe796376d4efe", -"postgres": "caf173733defe796376d4efe", -"postgresql": "caf173733defe796376d4efe", -"snowflake": "caf173733defe796376d4efe", -"tsql": "caf173733defe796376d4efe" +"mysql": "a6458f2eeceb4012953d4be0", +"postgres": "a6458f2eeceb4012953d4be0", +"postgresql": "a6458f2eeceb4012953d4be0", +"snowflake": "a6458f2eeceb4012953d4be0", +"tsql": "a6458f2eeceb4012953d4be0" } ], [ @@ -1974,18 +1974,18 @@ [ "type", { -"DuckDB": "4f8af4650ce9c18ed0cb0bca", -"None": "4f8af4650ce9c18ed0cb0bca", -"bigquery": "4f8af4650ce9c18ed0cb0bca", -"databricks": "4f8af4650ce9c18ed0cb0bca", -"duckdb": "4f8af4650ce9c18ed0cb0bca", -"generic": "4f8af4650ce9c18ed0cb0bca", +"DuckDB": "a5e56c4a2e3c796d3dca7042", +"None": "a5e56c4a2e3c796d3dca7042", +"bigquery": "a5e56c4a2e3c796d3dca7042", +"databricks": "a5e56c4a2e3c796d3dca7042", +"duckdb": "a5e56c4a2e3c796d3dca7042", +"generic": "a5e56c4a2e3c796d3dca7042", "motherduck": "901c5133ed47d4769286e860", -"mysql": "4f8af4650ce9c18ed0cb0bca", -"postgres": "4f8af4650ce9c18ed0cb0bca", -"postgresql": "4f8af4650ce9c18ed0cb0bca", -"snowflake": "4f8af4650ce9c18ed0cb0bca", -"tsql": "4f8af4650ce9c18ed0cb0bca" +"mysql": "a5e56c4a2e3c796d3dca7042", +"postgres": "a5e56c4a2e3c796d3dca7042", +"postgresql": "a5e56c4a2e3c796d3dca7042", +"snowflake": "a5e56c4a2e3c796d3dca7042", +"tsql": "a5e56c4a2e3c796d3dca7042" } ], [ @@ -2603,18 +2603,18 @@ [ "type", { -"DuckDB": "ad35c4a2523ce8c2f57acf51", -"None": "ad35c4a2523ce8c2f57acf51", -"bigquery": "ad35c4a2523ce8c2f57acf51", -"databricks": "ad35c4a2523ce8c2f57acf51", -"duckdb": "ad35c4a2523ce8c2f57acf51", -"generic": "ad35c4a2523ce8c2f57acf51", +"DuckDB": "e7bc9109e1c25d865732885c", +"None": "e7bc9109e1c25d865732885c", +"bigquery": "e7bc9109e1c25d865732885c", +"databricks": "e7bc9109e1c25d865732885c", +"duckdb": "e7bc9109e1c25d865732885c", +"generic": "e7bc9109e1c25d865732885c", "motherduck": "901c5133ed47d4769286e860", -"mysql": "ad35c4a2523ce8c2f57acf51", -"postgres": "ad35c4a2523ce8c2f57acf51", -"postgresql": "ad35c4a2523ce8c2f57acf51", -"snowflake": "ad35c4a2523ce8c2f57acf51", -"tsql": "ad35c4a2523ce8c2f57acf51" +"mysql": "e7bc9109e1c25d865732885c", +"postgres": "e7bc9109e1c25d865732885c", +"postgresql": "e7bc9109e1c25d865732885c", +"snowflake": "e7bc9109e1c25d865732885c", +"tsql": "e7bc9109e1c25d865732885c" } ], [ @@ -4252,18 +4252,18 @@ [ "type", { -"DuckDB": "96ecb0bd0c27167bf4e31a08", -"None": "96ecb0bd0c27167bf4e31a08", -"bigquery": "96ecb0bd0c27167bf4e31a08", -"databricks": "96ecb0bd0c27167bf4e31a08", -"duckdb": "96ecb0bd0c27167bf4e31a08", -"generic": "96ecb0bd0c27167bf4e31a08", +"DuckDB": "a1ccf5c0db81a578bab8cda9", +"None": "a1ccf5c0db81a578bab8cda9", +"bigquery": "a1ccf5c0db81a578bab8cda9", +"databricks": "a1ccf5c0db81a578bab8cda9", +"duckdb": "a1ccf5c0db81a578bab8cda9", +"generic": "a1ccf5c0db81a578bab8cda9", "motherduck": "901c5133ed47d4769286e860", -"mysql": "96ecb0bd0c27167bf4e31a08", -"postgres": "96ecb0bd0c27167bf4e31a08", -"postgresql": "96ecb0bd0c27167bf4e31a08", -"snowflake": "96ecb0bd0c27167bf4e31a08", -"tsql": "96ecb0bd0c27167bf4e31a08" +"mysql": "a1ccf5c0db81a578bab8cda9", +"postgres": "a1ccf5c0db81a578bab8cda9", +"postgresql": "a1ccf5c0db81a578bab8cda9", +"snowflake": "a1ccf5c0db81a578bab8cda9", +"tsql": "a1ccf5c0db81a578bab8cda9" } ], [ @@ -4423,17 +4423,17 @@ "type", { "DuckDB": "c344d36a6751b773df74ee3c", -"None": "87fed6659d6324c5ef0c6ee2", -"bigquery": "87fed6659d6324c5ef0c6ee2", -"databricks": "87fed6659d6324c5ef0c6ee2", +"None": "2d031f8bbd88fca9a9492b54", +"bigquery": "2d031f8bbd88fca9a9492b54", +"databricks": "2d031f8bbd88fca9a9492b54", "duckdb": "c344d36a6751b773df74ee3c", -"generic": "87fed6659d6324c5ef0c6ee2", +"generic": "2d031f8bbd88fca9a9492b54", "motherduck": "901c5133ed47d4769286e860", -"mysql": "87fed6659d6324c5ef0c6ee2", -"postgres": "87fed6659d6324c5ef0c6ee2", -"postgresql": "87fed6659d6324c5ef0c6ee2", -"snowflake": "87fed6659d6324c5ef0c6ee2", -"tsql": "87fed6659d6324c5ef0c6ee2" +"mysql": "2d031f8bbd88fca9a9492b54", +"postgres": "2d031f8bbd88fca9a9492b54", +"postgresql": "2d031f8bbd88fca9a9492b54", +"snowflake": "2d031f8bbd88fca9a9492b54", +"tsql": "2d031f8bbd88fca9a9492b54" } ], [ @@ -4779,18 +4779,18 @@ [ "type", { -"DuckDB": "2795ccedbfd3fa0bed5fdfcd", -"None": "2795ccedbfd3fa0bed5fdfcd", -"bigquery": "2795ccedbfd3fa0bed5fdfcd", -"databricks": "2795ccedbfd3fa0bed5fdfcd", -"duckdb": "2795ccedbfd3fa0bed5fdfcd", -"generic": "2795ccedbfd3fa0bed5fdfcd", +"DuckDB": "ebd0935007c5cca9d0c14021", +"None": "ebd0935007c5cca9d0c14021", +"bigquery": "ebd0935007c5cca9d0c14021", +"databricks": "ebd0935007c5cca9d0c14021", +"duckdb": "ebd0935007c5cca9d0c14021", +"generic": "ebd0935007c5cca9d0c14021", "motherduck": "901c5133ed47d4769286e860", -"mysql": "2795ccedbfd3fa0bed5fdfcd", -"postgres": "2795ccedbfd3fa0bed5fdfcd", -"postgresql": "2795ccedbfd3fa0bed5fdfcd", -"snowflake": "2795ccedbfd3fa0bed5fdfcd", -"tsql": "2795ccedbfd3fa0bed5fdfcd" +"mysql": "ebd0935007c5cca9d0c14021", +"postgres": "ebd0935007c5cca9d0c14021", +"postgresql": "ebd0935007c5cca9d0c14021", +"snowflake": "ebd0935007c5cca9d0c14021", +"tsql": "ebd0935007c5cca9d0c14021" } ], [ @@ -5357,18 +5357,18 @@ [ "type", { -"DuckDB": "c4860fb394ecca5e4515f1c9", -"None": "c4860fb394ecca5e4515f1c9", -"bigquery": "c4860fb394ecca5e4515f1c9", -"databricks": "c4860fb394ecca5e4515f1c9", -"duckdb": "c4860fb394ecca5e4515f1c9", -"generic": "c4860fb394ecca5e4515f1c9", +"DuckDB": "8e8e75c867befbf0dd5b522f", +"None": "8e8e75c867befbf0dd5b522f", +"bigquery": "8e8e75c867befbf0dd5b522f", +"databricks": "8e8e75c867befbf0dd5b522f", +"duckdb": "8e8e75c867befbf0dd5b522f", +"generic": "8e8e75c867befbf0dd5b522f", "motherduck": "901c5133ed47d4769286e860", -"mysql": "c4860fb394ecca5e4515f1c9", -"postgres": "c4860fb394ecca5e4515f1c9", -"postgresql": "c4860fb394ecca5e4515f1c9", -"snowflake": "c4860fb394ecca5e4515f1c9", -"tsql": "c4860fb394ecca5e4515f1c9" +"mysql": "8e8e75c867befbf0dd5b522f", +"postgres": "8e8e75c867befbf0dd5b522f", +"postgresql": "8e8e75c867befbf0dd5b522f", +"snowflake": "8e8e75c867befbf0dd5b522f", +"tsql": "8e8e75c867befbf0dd5b522f" } ], [ @@ -5697,18 +5697,18 @@ [ "type", { -"DuckDB": "445eba49a8d5ccbd6775e1b6", -"None": "445eba49a8d5ccbd6775e1b6", -"bigquery": "445eba49a8d5ccbd6775e1b6", -"databricks": "445eba49a8d5ccbd6775e1b6", -"duckdb": "445eba49a8d5ccbd6775e1b6", -"generic": "445eba49a8d5ccbd6775e1b6", +"DuckDB": "a8cae774dd1c333f42d6a630", +"None": "a8cae774dd1c333f42d6a630", +"bigquery": "a8cae774dd1c333f42d6a630", +"databricks": "a8cae774dd1c333f42d6a630", +"duckdb": "a8cae774dd1c333f42d6a630", +"generic": "a8cae774dd1c333f42d6a630", "motherduck": "901c5133ed47d4769286e860", -"mysql": "445eba49a8d5ccbd6775e1b6", -"postgres": "445eba49a8d5ccbd6775e1b6", -"postgresql": "445eba49a8d5ccbd6775e1b6", -"snowflake": "445eba49a8d5ccbd6775e1b6", -"tsql": "445eba49a8d5ccbd6775e1b6" +"mysql": "a8cae774dd1c333f42d6a630", +"postgres": "a8cae774dd1c333f42d6a630", +"postgresql": "a8cae774dd1c333f42d6a630", +"snowflake": "a8cae774dd1c333f42d6a630", +"tsql": "a8cae774dd1c333f42d6a630" } ], [ @@ -6054,18 +6054,18 @@ [ "type", { -"DuckDB": "07aa3fe366bb8a3f270ea3d3", -"None": "07aa3fe366bb8a3f270ea3d3", -"bigquery": "07aa3fe366bb8a3f270ea3d3", -"databricks": "07aa3fe366bb8a3f270ea3d3", -"duckdb": "07aa3fe366bb8a3f270ea3d3", -"generic": "07aa3fe366bb8a3f270ea3d3", +"DuckDB": "7a9cb09cc9d8e6470f5c18fb", +"None": "7a9cb09cc9d8e6470f5c18fb", +"bigquery": "7a9cb09cc9d8e6470f5c18fb", +"databricks": "7a9cb09cc9d8e6470f5c18fb", +"duckdb": "7a9cb09cc9d8e6470f5c18fb", +"generic": "7a9cb09cc9d8e6470f5c18fb", "motherduck": "901c5133ed47d4769286e860", -"mysql": "07aa3fe366bb8a3f270ea3d3", -"postgres": "07aa3fe366bb8a3f270ea3d3", -"postgresql": "07aa3fe366bb8a3f270ea3d3", -"snowflake": "07aa3fe366bb8a3f270ea3d3", -"tsql": "07aa3fe366bb8a3f270ea3d3" +"mysql": "7a9cb09cc9d8e6470f5c18fb", +"postgres": "7a9cb09cc9d8e6470f5c18fb", +"postgresql": "7a9cb09cc9d8e6470f5c18fb", +"snowflake": "7a9cb09cc9d8e6470f5c18fb", +"tsql": "7a9cb09cc9d8e6470f5c18fb" } ], [ @@ -6139,18 +6139,18 @@ [ "type", { -"DuckDB": "c426d93641829fa3d8f49b8d", -"None": "c426d93641829fa3d8f49b8d", -"bigquery": "c426d93641829fa3d8f49b8d", -"databricks": "c426d93641829fa3d8f49b8d", -"duckdb": "c426d93641829fa3d8f49b8d", -"generic": "c426d93641829fa3d8f49b8d", +"DuckDB": "efb799417b64225e2ee93447", +"None": "efb799417b64225e2ee93447", +"bigquery": "efb799417b64225e2ee93447", +"databricks": "efb799417b64225e2ee93447", +"duckdb": "efb799417b64225e2ee93447", +"generic": "efb799417b64225e2ee93447", "motherduck": "901c5133ed47d4769286e860", -"mysql": "c426d93641829fa3d8f49b8d", -"postgres": "c426d93641829fa3d8f49b8d", -"postgresql": "c426d93641829fa3d8f49b8d", -"snowflake": "c426d93641829fa3d8f49b8d", -"tsql": "c426d93641829fa3d8f49b8d" +"mysql": "efb799417b64225e2ee93447", +"postgres": "efb799417b64225e2ee93447", +"postgresql": "efb799417b64225e2ee93447", +"snowflake": "efb799417b64225e2ee93447", +"tsql": "efb799417b64225e2ee93447" } ], [ @@ -6224,18 +6224,18 @@ [ "type", { -"DuckDB": "be15dde1d6e0b4bb3f6da66f", -"None": "be15dde1d6e0b4bb3f6da66f", -"bigquery": "be15dde1d6e0b4bb3f6da66f", -"databricks": "be15dde1d6e0b4bb3f6da66f", -"duckdb": "be15dde1d6e0b4bb3f6da66f", -"generic": "be15dde1d6e0b4bb3f6da66f", +"DuckDB": "400acbf184a17c6b468aab31", +"None": "400acbf184a17c6b468aab31", +"bigquery": "400acbf184a17c6b468aab31", +"databricks": "400acbf184a17c6b468aab31", +"duckdb": "400acbf184a17c6b468aab31", +"generic": "400acbf184a17c6b468aab31", "motherduck": "901c5133ed47d4769286e860", -"mysql": "be15dde1d6e0b4bb3f6da66f", -"postgres": "be15dde1d6e0b4bb3f6da66f", -"postgresql": "be15dde1d6e0b4bb3f6da66f", -"snowflake": "be15dde1d6e0b4bb3f6da66f", -"tsql": "be15dde1d6e0b4bb3f6da66f" +"mysql": "400acbf184a17c6b468aab31", +"postgres": "400acbf184a17c6b468aab31", +"postgresql": "400acbf184a17c6b468aab31", +"snowflake": "400acbf184a17c6b468aab31", +"tsql": "400acbf184a17c6b468aab31" } ], [ @@ -6751,18 +6751,18 @@ [ "type", { -"DuckDB": "182a094bae8a9a33f67845ad", -"None": "182a094bae8a9a33f67845ad", -"bigquery": "182a094bae8a9a33f67845ad", -"databricks": "182a094bae8a9a33f67845ad", -"duckdb": "182a094bae8a9a33f67845ad", -"generic": "182a094bae8a9a33f67845ad", +"DuckDB": "fa13cacc15beaca279323bfb", +"None": "fa13cacc15beaca279323bfb", +"bigquery": "fa13cacc15beaca279323bfb", +"databricks": "fa13cacc15beaca279323bfb", +"duckdb": "fa13cacc15beaca279323bfb", +"generic": "fa13cacc15beaca279323bfb", "motherduck": "901c5133ed47d4769286e860", -"mysql": "182a094bae8a9a33f67845ad", -"postgres": "182a094bae8a9a33f67845ad", -"postgresql": "182a094bae8a9a33f67845ad", -"snowflake": "182a094bae8a9a33f67845ad", -"tsql": "182a094bae8a9a33f67845ad" +"mysql": "fa13cacc15beaca279323bfb", +"postgres": "fa13cacc15beaca279323bfb", +"postgresql": "fa13cacc15beaca279323bfb", +"snowflake": "fa13cacc15beaca279323bfb", +"tsql": "fa13cacc15beaca279323bfb" } ], [ @@ -6955,18 +6955,18 @@ [ "type", { -"DuckDB": "7032aa8254308ba815883957", -"None": "7032aa8254308ba815883957", -"bigquery": "7032aa8254308ba815883957", -"databricks": "7032aa8254308ba815883957", -"duckdb": "7032aa8254308ba815883957", -"generic": "7032aa8254308ba815883957", +"DuckDB": "8e263d7157f195e571a4862b", +"None": "8e263d7157f195e571a4862b", +"bigquery": "8e263d7157f195e571a4862b", +"databricks": "8e263d7157f195e571a4862b", +"duckdb": "8e263d7157f195e571a4862b", +"generic": "8e263d7157f195e571a4862b", "motherduck": "901c5133ed47d4769286e860", -"mysql": "7032aa8254308ba815883957", -"postgres": "7032aa8254308ba815883957", -"postgresql": "7032aa8254308ba815883957", -"snowflake": "7032aa8254308ba815883957", -"tsql": "7032aa8254308ba815883957" +"mysql": "8e263d7157f195e571a4862b", +"postgres": "8e263d7157f195e571a4862b", +"postgresql": "8e263d7157f195e571a4862b", +"snowflake": "8e263d7157f195e571a4862b", +"tsql": "8e263d7157f195e571a4862b" } ], [ @@ -7210,18 +7210,18 @@ [ "type", { -"DuckDB": "8789309b90fc585ddbbe90f9", -"None": "8789309b90fc585ddbbe90f9", -"bigquery": "8789309b90fc585ddbbe90f9", -"databricks": "8789309b90fc585ddbbe90f9", -"duckdb": "8789309b90fc585ddbbe90f9", -"generic": "8789309b90fc585ddbbe90f9", +"DuckDB": "6b4faa1df4db47a526d4ae10", +"None": "6b4faa1df4db47a526d4ae10", +"bigquery": "6b4faa1df4db47a526d4ae10", +"databricks": "6b4faa1df4db47a526d4ae10", +"duckdb": "6b4faa1df4db47a526d4ae10", +"generic": "6b4faa1df4db47a526d4ae10", "motherduck": "901c5133ed47d4769286e860", -"mysql": "8789309b90fc585ddbbe90f9", -"postgres": "8789309b90fc585ddbbe90f9", -"postgresql": "8789309b90fc585ddbbe90f9", -"snowflake": "8789309b90fc585ddbbe90f9", -"tsql": "8789309b90fc585ddbbe90f9" +"mysql": "6b4faa1df4db47a526d4ae10", +"postgres": "6b4faa1df4db47a526d4ae10", +"postgresql": "6b4faa1df4db47a526d4ae10", +"snowflake": "6b4faa1df4db47a526d4ae10", +"tsql": "6b4faa1df4db47a526d4ae10" } ], [ @@ -7330,17 +7330,17 @@ "type", { "DuckDB": "c729c38d2df6efedc6288247", -"None": "eea1002d88cc68fad7424aca", -"bigquery": "eea1002d88cc68fad7424aca", -"databricks": "eea1002d88cc68fad7424aca", +"None": "75f1ffee735aaf2dc58efd48", +"bigquery": "75f1ffee735aaf2dc58efd48", +"databricks": "75f1ffee735aaf2dc58efd48", "duckdb": "c729c38d2df6efedc6288247", -"generic": "eea1002d88cc68fad7424aca", +"generic": "75f1ffee735aaf2dc58efd48", "motherduck": "901c5133ed47d4769286e860", -"mysql": "eea1002d88cc68fad7424aca", -"postgres": "eea1002d88cc68fad7424aca", -"postgresql": "eea1002d88cc68fad7424aca", -"snowflake": "eea1002d88cc68fad7424aca", -"tsql": "eea1002d88cc68fad7424aca" +"mysql": "75f1ffee735aaf2dc58efd48", +"postgres": "75f1ffee735aaf2dc58efd48", +"postgresql": "75f1ffee735aaf2dc58efd48", +"snowflake": "75f1ffee735aaf2dc58efd48", +"tsql": "75f1ffee735aaf2dc58efd48" } ], [ @@ -7465,18 +7465,18 @@ [ "type", { -"DuckDB": "733b04338829c5844270d25e", -"None": "733b04338829c5844270d25e", -"bigquery": "733b04338829c5844270d25e", -"databricks": "733b04338829c5844270d25e", -"duckdb": "733b04338829c5844270d25e", -"generic": "733b04338829c5844270d25e", +"DuckDB": "1bbf30e0166bb61bed8be4d6", +"None": "1bbf30e0166bb61bed8be4d6", +"bigquery": "1bbf30e0166bb61bed8be4d6", +"databricks": "1bbf30e0166bb61bed8be4d6", +"duckdb": "1bbf30e0166bb61bed8be4d6", +"generic": "1bbf30e0166bb61bed8be4d6", "motherduck": "901c5133ed47d4769286e860", -"mysql": "733b04338829c5844270d25e", -"postgres": "733b04338829c5844270d25e", -"postgresql": "733b04338829c5844270d25e", -"snowflake": "733b04338829c5844270d25e", -"tsql": "733b04338829c5844270d25e" +"mysql": "1bbf30e0166bb61bed8be4d6", +"postgres": "1bbf30e0166bb61bed8be4d6", +"postgresql": "1bbf30e0166bb61bed8be4d6", +"snowflake": "1bbf30e0166bb61bed8be4d6", +"tsql": "1bbf30e0166bb61bed8be4d6" } ], [ @@ -8264,18 +8264,18 @@ [ "type", { -"DuckDB": "cc33965a669608ec39e8c246", -"None": "cc33965a669608ec39e8c246", -"bigquery": "cc33965a669608ec39e8c246", -"databricks": "cc33965a669608ec39e8c246", -"duckdb": "cc33965a669608ec39e8c246", -"generic": "cc33965a669608ec39e8c246", +"DuckDB": "30a870a18e4d367086a7d302", +"None": "30a870a18e4d367086a7d302", +"bigquery": "30a870a18e4d367086a7d302", +"databricks": "30a870a18e4d367086a7d302", +"duckdb": "30a870a18e4d367086a7d302", +"generic": "30a870a18e4d367086a7d302", "motherduck": "901c5133ed47d4769286e860", -"mysql": "cc33965a669608ec39e8c246", -"postgres": "cc33965a669608ec39e8c246", -"postgresql": "cc33965a669608ec39e8c246", -"snowflake": "cc33965a669608ec39e8c246", -"tsql": "cc33965a669608ec39e8c246" +"mysql": "30a870a18e4d367086a7d302", +"postgres": "30a870a18e4d367086a7d302", +"postgresql": "30a870a18e4d367086a7d302", +"snowflake": "30a870a18e4d367086a7d302", +"tsql": "30a870a18e4d367086a7d302" } ], [ @@ -8740,18 +8740,18 @@ [ "type", { -"DuckDB": "9626d393c137a37008bb4b89", -"None": "9626d393c137a37008bb4b89", -"bigquery": "9626d393c137a37008bb4b89", -"databricks": "9626d393c137a37008bb4b89", -"duckdb": "9626d393c137a37008bb4b89", -"generic": "9626d393c137a37008bb4b89", +"DuckDB": "386957bbe33b0277f99cac09", +"None": "386957bbe33b0277f99cac09", +"bigquery": "386957bbe33b0277f99cac09", +"databricks": "386957bbe33b0277f99cac09", +"duckdb": "386957bbe33b0277f99cac09", +"generic": "386957bbe33b0277f99cac09", "motherduck": "901c5133ed47d4769286e860", -"mysql": "9626d393c137a37008bb4b89", -"postgres": "9626d393c137a37008bb4b89", -"postgresql": "9626d393c137a37008bb4b89", -"snowflake": "9626d393c137a37008bb4b89", -"tsql": "9626d393c137a37008bb4b89" +"mysql": "386957bbe33b0277f99cac09", +"postgres": "386957bbe33b0277f99cac09", +"postgresql": "386957bbe33b0277f99cac09", +"snowflake": "386957bbe33b0277f99cac09", +"tsql": "386957bbe33b0277f99cac09" } ], [ @@ -8791,18 +8791,18 @@ [ "type", { -"DuckDB": "8fe65454f387031aa6231faf", -"None": "8fe65454f387031aa6231faf", -"bigquery": "8fe65454f387031aa6231faf", -"databricks": "8fe65454f387031aa6231faf", -"duckdb": "8fe65454f387031aa6231faf", -"generic": "8fe65454f387031aa6231faf", +"DuckDB": "9304d412ef114a51a170158d", +"None": "9304d412ef114a51a170158d", +"bigquery": "9304d412ef114a51a170158d", +"databricks": "9304d412ef114a51a170158d", +"duckdb": "9304d412ef114a51a170158d", +"generic": "9304d412ef114a51a170158d", "motherduck": "901c5133ed47d4769286e860", -"mysql": "8fe65454f387031aa6231faf", -"postgres": "8fe65454f387031aa6231faf", -"postgresql": "8fe65454f387031aa6231faf", -"snowflake": "8fe65454f387031aa6231faf", -"tsql": "8fe65454f387031aa6231faf" +"mysql": "9304d412ef114a51a170158d", +"postgres": "9304d412ef114a51a170158d", +"postgresql": "9304d412ef114a51a170158d", +"snowflake": "9304d412ef114a51a170158d", +"tsql": "9304d412ef114a51a170158d" } ], [ @@ -9607,18 +9607,18 @@ [ "type", { -"DuckDB": "c4d601bdc1b1f357b3e9bbdf", -"None": "c4d601bdc1b1f357b3e9bbdf", -"bigquery": "c4d601bdc1b1f357b3e9bbdf", -"databricks": "c4d601bdc1b1f357b3e9bbdf", -"duckdb": "c4d601bdc1b1f357b3e9bbdf", -"generic": "c4d601bdc1b1f357b3e9bbdf", +"DuckDB": "5551318f9b0e622b4b928ca3", +"None": "5551318f9b0e622b4b928ca3", +"bigquery": "5551318f9b0e622b4b928ca3", +"databricks": "5551318f9b0e622b4b928ca3", +"duckdb": "5551318f9b0e622b4b928ca3", +"generic": "5551318f9b0e622b4b928ca3", "motherduck": "901c5133ed47d4769286e860", -"mysql": "c4d601bdc1b1f357b3e9bbdf", -"postgres": "c4d601bdc1b1f357b3e9bbdf", -"postgresql": "c4d601bdc1b1f357b3e9bbdf", -"snowflake": "c4d601bdc1b1f357b3e9bbdf", -"tsql": "c4d601bdc1b1f357b3e9bbdf" +"mysql": "5551318f9b0e622b4b928ca3", +"postgres": "5551318f9b0e622b4b928ca3", +"postgresql": "5551318f9b0e622b4b928ca3", +"snowflake": "5551318f9b0e622b4b928ca3", +"tsql": "5551318f9b0e622b4b928ca3" } ], [ @@ -9828,18 +9828,18 @@ [ "type", { -"DuckDB": "b6b82efd03def9465e4f97b2", -"None": "b6b82efd03def9465e4f97b2", -"bigquery": "b6b82efd03def9465e4f97b2", -"databricks": "b6b82efd03def9465e4f97b2", -"duckdb": "b6b82efd03def9465e4f97b2", -"generic": "b6b82efd03def9465e4f97b2", +"DuckDB": "a8face56f690e623ad76d780", +"None": "a8face56f690e623ad76d780", +"bigquery": "a8face56f690e623ad76d780", +"databricks": "a8face56f690e623ad76d780", +"duckdb": "a8face56f690e623ad76d780", +"generic": "a8face56f690e623ad76d780", "motherduck": "901c5133ed47d4769286e860", -"mysql": "b6b82efd03def9465e4f97b2", -"postgres": "b6b82efd03def9465e4f97b2", -"postgresql": "b6b82efd03def9465e4f97b2", -"snowflake": "b6b82efd03def9465e4f97b2", -"tsql": "b6b82efd03def9465e4f97b2" +"mysql": "a8face56f690e623ad76d780", +"postgres": "a8face56f690e623ad76d780", +"postgresql": "a8face56f690e623ad76d780", +"snowflake": "a8face56f690e623ad76d780", +"tsql": "a8face56f690e623ad76d780" } ], [ @@ -10117,18 +10117,18 @@ [ "type", { -"DuckDB": "4b27937f44f966ff9f3a7c76", -"None": "4b27937f44f966ff9f3a7c76", -"bigquery": "4b27937f44f966ff9f3a7c76", -"databricks": "4b27937f44f966ff9f3a7c76", -"duckdb": "4b27937f44f966ff9f3a7c76", -"generic": "4b27937f44f966ff9f3a7c76", +"DuckDB": "c641dec96086f9f69b467459", +"None": "c641dec96086f9f69b467459", +"bigquery": "c641dec96086f9f69b467459", +"databricks": "c641dec96086f9f69b467459", +"duckdb": "c641dec96086f9f69b467459", +"generic": "c641dec96086f9f69b467459", "motherduck": "901c5133ed47d4769286e860", -"mysql": "4b27937f44f966ff9f3a7c76", -"postgres": "4b27937f44f966ff9f3a7c76", -"postgresql": "4b27937f44f966ff9f3a7c76", -"snowflake": "4b27937f44f966ff9f3a7c76", -"tsql": "4b27937f44f966ff9f3a7c76" +"mysql": "c641dec96086f9f69b467459", +"postgres": "c641dec96086f9f69b467459", +"postgresql": "c641dec96086f9f69b467459", +"snowflake": "c641dec96086f9f69b467459", +"tsql": "c641dec96086f9f69b467459" } ], [ @@ -10236,18 +10236,18 @@ [ "type", { -"DuckDB": "14c32cfb5e5135338d015f42", -"None": "14c32cfb5e5135338d015f42", -"bigquery": "14c32cfb5e5135338d015f42", -"databricks": "14c32cfb5e5135338d015f42", -"duckdb": "14c32cfb5e5135338d015f42", -"generic": "14c32cfb5e5135338d015f42", +"DuckDB": "f8e63510f9a31c4fc8f60cb4", +"None": "f8e63510f9a31c4fc8f60cb4", +"bigquery": "f8e63510f9a31c4fc8f60cb4", +"databricks": "f8e63510f9a31c4fc8f60cb4", +"duckdb": "f8e63510f9a31c4fc8f60cb4", +"generic": "f8e63510f9a31c4fc8f60cb4", "motherduck": "901c5133ed47d4769286e860", -"mysql": "14c32cfb5e5135338d015f42", -"postgres": "14c32cfb5e5135338d015f42", -"postgresql": "14c32cfb5e5135338d015f42", -"snowflake": "14c32cfb5e5135338d015f42", -"tsql": "14c32cfb5e5135338d015f42" +"mysql": "f8e63510f9a31c4fc8f60cb4", +"postgres": "f8e63510f9a31c4fc8f60cb4", +"postgresql": "f8e63510f9a31c4fc8f60cb4", +"snowflake": "f8e63510f9a31c4fc8f60cb4", +"tsql": "f8e63510f9a31c4fc8f60cb4" } ], [ @@ -10865,18 +10865,18 @@ [ "type", { -"DuckDB": "c703e456945a8f06d005c98b", -"None": "c703e456945a8f06d005c98b", -"bigquery": "c703e456945a8f06d005c98b", -"databricks": "c703e456945a8f06d005c98b", -"duckdb": "c703e456945a8f06d005c98b", -"generic": "c703e456945a8f06d005c98b", +"DuckDB": "d1ea13fb2c3d459fff6bf7cd", +"None": "d1ea13fb2c3d459fff6bf7cd", +"bigquery": "d1ea13fb2c3d459fff6bf7cd", +"databricks": "d1ea13fb2c3d459fff6bf7cd", +"duckdb": "d1ea13fb2c3d459fff6bf7cd", +"generic": "d1ea13fb2c3d459fff6bf7cd", "motherduck": "901c5133ed47d4769286e860", -"mysql": "c703e456945a8f06d005c98b", -"postgres": "c703e456945a8f06d005c98b", -"postgresql": "c703e456945a8f06d005c98b", -"snowflake": "c703e456945a8f06d005c98b", -"tsql": "c703e456945a8f06d005c98b" +"mysql": "d1ea13fb2c3d459fff6bf7cd", +"postgres": "d1ea13fb2c3d459fff6bf7cd", +"postgresql": "d1ea13fb2c3d459fff6bf7cd", +"snowflake": "d1ea13fb2c3d459fff6bf7cd", +"tsql": "d1ea13fb2c3d459fff6bf7cd" } ], [ @@ -11307,18 +11307,18 @@ [ "type", { -"DuckDB": "aea8a680fa18770e56bf6220", -"None": "aea8a680fa18770e56bf6220", -"bigquery": "aea8a680fa18770e56bf6220", -"databricks": "aea8a680fa18770e56bf6220", -"duckdb": "aea8a680fa18770e56bf6220", -"generic": "aea8a680fa18770e56bf6220", +"DuckDB": "6fa5d7254a7ed25d37629163", +"None": "6fa5d7254a7ed25d37629163", +"bigquery": "6fa5d7254a7ed25d37629163", +"databricks": "6fa5d7254a7ed25d37629163", +"duckdb": "6fa5d7254a7ed25d37629163", +"generic": "6fa5d7254a7ed25d37629163", "motherduck": "901c5133ed47d4769286e860", -"mysql": "aea8a680fa18770e56bf6220", -"postgres": "aea8a680fa18770e56bf6220", -"postgresql": "aea8a680fa18770e56bf6220", -"snowflake": "aea8a680fa18770e56bf6220", -"tsql": "aea8a680fa18770e56bf6220" +"mysql": "6fa5d7254a7ed25d37629163", +"postgres": "6fa5d7254a7ed25d37629163", +"postgresql": "6fa5d7254a7ed25d37629163", +"snowflake": "6fa5d7254a7ed25d37629163", +"tsql": "6fa5d7254a7ed25d37629163" } ], [ @@ -13534,18 +13534,18 @@ [ "type", { -"DuckDB": "7c11e0be7630905d93bd9ce1", -"None": "7c11e0be7630905d93bd9ce1", -"bigquery": "7c11e0be7630905d93bd9ce1", -"databricks": "7c11e0be7630905d93bd9ce1", -"duckdb": "7c11e0be7630905d93bd9ce1", -"generic": "7c11e0be7630905d93bd9ce1", +"DuckDB": "86825c198b081bf6e208eaa7", +"None": "86825c198b081bf6e208eaa7", +"bigquery": "86825c198b081bf6e208eaa7", +"databricks": "86825c198b081bf6e208eaa7", +"duckdb": "86825c198b081bf6e208eaa7", +"generic": "86825c198b081bf6e208eaa7", "motherduck": "901c5133ed47d4769286e860", -"mysql": "7c11e0be7630905d93bd9ce1", -"postgres": "7c11e0be7630905d93bd9ce1", -"postgresql": "7c11e0be7630905d93bd9ce1", -"snowflake": "7c11e0be7630905d93bd9ce1", -"tsql": "7c11e0be7630905d93bd9ce1" +"mysql": "86825c198b081bf6e208eaa7", +"postgres": "86825c198b081bf6e208eaa7", +"postgresql": "86825c198b081bf6e208eaa7", +"snowflake": "86825c198b081bf6e208eaa7", +"tsql": "86825c198b081bf6e208eaa7" } ], [ @@ -14112,18 +14112,18 @@ [ "type", { -"DuckDB": "b55b02912e30e27dcb40ad93", -"None": "b55b02912e30e27dcb40ad93", -"bigquery": "b55b02912e30e27dcb40ad93", -"databricks": "b55b02912e30e27dcb40ad93", -"duckdb": "b55b02912e30e27dcb40ad93", -"generic": "b55b02912e30e27dcb40ad93", +"DuckDB": "f923673fc4041d12a88250a4", +"None": "f923673fc4041d12a88250a4", +"bigquery": "f923673fc4041d12a88250a4", +"databricks": "f923673fc4041d12a88250a4", +"duckdb": "f923673fc4041d12a88250a4", +"generic": "f923673fc4041d12a88250a4", "motherduck": "901c5133ed47d4769286e860", -"mysql": "b55b02912e30e27dcb40ad93", -"postgres": "b55b02912e30e27dcb40ad93", -"postgresql": "b55b02912e30e27dcb40ad93", -"snowflake": "b55b02912e30e27dcb40ad93", -"tsql": "b55b02912e30e27dcb40ad93" +"mysql": "f923673fc4041d12a88250a4", +"postgres": "f923673fc4041d12a88250a4", +"postgresql": "f923673fc4041d12a88250a4", +"snowflake": "f923673fc4041d12a88250a4", +"tsql": "f923673fc4041d12a88250a4" } ], [ @@ -15234,35 +15234,35 @@ [ "type", { -"DuckDB": "8917b70db141a02ac2e2bb1e", -"None": "8917b70db141a02ac2e2bb1e", -"bigquery": "8917b70db141a02ac2e2bb1e", -"databricks": "8917b70db141a02ac2e2bb1e", -"duckdb": "8917b70db141a02ac2e2bb1e", -"generic": "8917b70db141a02ac2e2bb1e", +"DuckDB": "9122e0d12e93512217da87fa", +"None": "9122e0d12e93512217da87fa", +"bigquery": "9122e0d12e93512217da87fa", +"databricks": "9122e0d12e93512217da87fa", +"duckdb": "9122e0d12e93512217da87fa", +"generic": "9122e0d12e93512217da87fa", "motherduck": "901c5133ed47d4769286e860", -"mysql": "8917b70db141a02ac2e2bb1e", -"postgres": "8917b70db141a02ac2e2bb1e", -"postgresql": "8917b70db141a02ac2e2bb1e", -"snowflake": "8917b70db141a02ac2e2bb1e", -"tsql": "8917b70db141a02ac2e2bb1e" +"mysql": "9122e0d12e93512217da87fa", +"postgres": "9122e0d12e93512217da87fa", +"postgresql": "9122e0d12e93512217da87fa", +"snowflake": "9122e0d12e93512217da87fa", +"tsql": "9122e0d12e93512217da87fa" } ], [ "type", { -"DuckDB": "ef480cafe4d260b70ce5214c", -"None": "ef480cafe4d260b70ce5214c", -"bigquery": "ef480cafe4d260b70ce5214c", -"databricks": "ef480cafe4d260b70ce5214c", -"duckdb": "ef480cafe4d260b70ce5214c", -"generic": "ef480cafe4d260b70ce5214c", +"DuckDB": "b89338179329ef976f911a46", +"None": "b89338179329ef976f911a46", +"bigquery": "b89338179329ef976f911a46", +"databricks": "b89338179329ef976f911a46", +"duckdb": "b89338179329ef976f911a46", +"generic": "b89338179329ef976f911a46", "motherduck": "901c5133ed47d4769286e860", -"mysql": "ef480cafe4d260b70ce5214c", -"postgres": "ef480cafe4d260b70ce5214c", -"postgresql": "ef480cafe4d260b70ce5214c", -"snowflake": "ef480cafe4d260b70ce5214c", -"tsql": "ef480cafe4d260b70ce5214c" +"mysql": "b89338179329ef976f911a46", +"postgres": "b89338179329ef976f911a46", +"postgresql": "b89338179329ef976f911a46", +"snowflake": "b89338179329ef976f911a46", +"tsql": "b89338179329ef976f911a46" } ], [ @@ -15846,18 +15846,18 @@ [ "type", { -"DuckDB": "d44c023c720f385769f2ab81", -"None": "d44c023c720f385769f2ab81", -"bigquery": "d44c023c720f385769f2ab81", -"databricks": "d44c023c720f385769f2ab81", -"duckdb": "d44c023c720f385769f2ab81", -"generic": "d44c023c720f385769f2ab81", +"DuckDB": "63904421f84671303411fe93", +"None": "63904421f84671303411fe93", +"bigquery": "63904421f84671303411fe93", +"databricks": "63904421f84671303411fe93", +"duckdb": "63904421f84671303411fe93", +"generic": "63904421f84671303411fe93", "motherduck": "901c5133ed47d4769286e860", -"mysql": "d44c023c720f385769f2ab81", -"postgres": "d44c023c720f385769f2ab81", -"postgresql": "d44c023c720f385769f2ab81", -"snowflake": "d44c023c720f385769f2ab81", -"tsql": "d44c023c720f385769f2ab81" +"mysql": "63904421f84671303411fe93", +"postgres": "63904421f84671303411fe93", +"postgresql": "63904421f84671303411fe93", +"snowflake": "63904421f84671303411fe93", +"tsql": "63904421f84671303411fe93" } ], [ @@ -16288,35 +16288,35 @@ [ "type", { -"DuckDB": "24b677b214ecdacdb007642e", -"None": "24b677b214ecdacdb007642e", -"bigquery": "24b677b214ecdacdb007642e", -"databricks": "24b677b214ecdacdb007642e", -"duckdb": "24b677b214ecdacdb007642e", -"generic": "24b677b214ecdacdb007642e", +"DuckDB": "15897ece5a8fa1f12e32bed8", +"None": "15897ece5a8fa1f12e32bed8", +"bigquery": "15897ece5a8fa1f12e32bed8", +"databricks": "15897ece5a8fa1f12e32bed8", +"duckdb": "15897ece5a8fa1f12e32bed8", +"generic": "15897ece5a8fa1f12e32bed8", "motherduck": "901c5133ed47d4769286e860", -"mysql": "24b677b214ecdacdb007642e", -"postgres": "24b677b214ecdacdb007642e", -"postgresql": "24b677b214ecdacdb007642e", -"snowflake": "24b677b214ecdacdb007642e", -"tsql": "24b677b214ecdacdb007642e" +"mysql": "15897ece5a8fa1f12e32bed8", +"postgres": "15897ece5a8fa1f12e32bed8", +"postgresql": "15897ece5a8fa1f12e32bed8", +"snowflake": "15897ece5a8fa1f12e32bed8", +"tsql": "15897ece5a8fa1f12e32bed8" } ], [ "type", { -"DuckDB": "4c2128e6078118fd0d3856fa", -"None": "4c2128e6078118fd0d3856fa", -"bigquery": "4c2128e6078118fd0d3856fa", -"databricks": "4c2128e6078118fd0d3856fa", -"duckdb": "4c2128e6078118fd0d3856fa", -"generic": "4c2128e6078118fd0d3856fa", +"DuckDB": "2377bd0302f3eba296b44fee", +"None": "2377bd0302f3eba296b44fee", +"bigquery": "2377bd0302f3eba296b44fee", +"databricks": "2377bd0302f3eba296b44fee", +"duckdb": "2377bd0302f3eba296b44fee", +"generic": "2377bd0302f3eba296b44fee", "motherduck": "901c5133ed47d4769286e860", -"mysql": "4c2128e6078118fd0d3856fa", -"postgres": "4c2128e6078118fd0d3856fa", -"postgresql": "4c2128e6078118fd0d3856fa", -"snowflake": "4c2128e6078118fd0d3856fa", -"tsql": "4c2128e6078118fd0d3856fa" +"mysql": "2377bd0302f3eba296b44fee", +"postgres": "2377bd0302f3eba296b44fee", +"postgresql": "2377bd0302f3eba296b44fee", +"snowflake": "2377bd0302f3eba296b44fee", +"tsql": "2377bd0302f3eba296b44fee" } ], [ @@ -16469,7 +16469,7 @@ "postgres": "6f851103102e7b986535376f", "postgresql": "6f851103102e7b986535376f", "snowflake": "87800133cac38d9119180c0a", -"tsql": "0e9ef5373e5d52970e8e1656" +"tsql": "6ec12741f1435c1a4011e935" } ], [ @@ -16543,18 +16543,18 @@ [ "type", { -"DuckDB": "3009be83c84cdfc92bbb075e", -"None": "3009be83c84cdfc92bbb075e", -"bigquery": "3009be83c84cdfc92bbb075e", -"databricks": "3009be83c84cdfc92bbb075e", -"duckdb": "3009be83c84cdfc92bbb075e", -"generic": "3009be83c84cdfc92bbb075e", +"DuckDB": "f9afa4d07d3af890563ba291", +"None": "f9afa4d07d3af890563ba291", +"bigquery": "f9afa4d07d3af890563ba291", +"databricks": "f9afa4d07d3af890563ba291", +"duckdb": "f9afa4d07d3af890563ba291", +"generic": "f9afa4d07d3af890563ba291", "motherduck": "901c5133ed47d4769286e860", -"mysql": "3009be83c84cdfc92bbb075e", -"postgres": "3009be83c84cdfc92bbb075e", -"postgresql": "3009be83c84cdfc92bbb075e", -"snowflake": "3009be83c84cdfc92bbb075e", -"tsql": "3009be83c84cdfc92bbb075e" +"mysql": "f9afa4d07d3af890563ba291", +"postgres": "f9afa4d07d3af890563ba291", +"postgresql": "f9afa4d07d3af890563ba291", +"snowflake": "f9afa4d07d3af890563ba291", +"tsql": "f9afa4d07d3af890563ba291" } ], [ @@ -16747,18 +16747,18 @@ [ "type", { -"DuckDB": "07f8200cee37aa0f6990da3d", -"None": "07f8200cee37aa0f6990da3d", -"bigquery": "07f8200cee37aa0f6990da3d", -"databricks": "07f8200cee37aa0f6990da3d", -"duckdb": "07f8200cee37aa0f6990da3d", -"generic": "07f8200cee37aa0f6990da3d", +"DuckDB": "293b92524b77d21b6865addc", +"None": "293b92524b77d21b6865addc", +"bigquery": "293b92524b77d21b6865addc", +"databricks": "293b92524b77d21b6865addc", +"duckdb": "293b92524b77d21b6865addc", +"generic": "293b92524b77d21b6865addc", "motherduck": "901c5133ed47d4769286e860", -"mysql": "07f8200cee37aa0f6990da3d", -"postgres": "07f8200cee37aa0f6990da3d", -"postgresql": "07f8200cee37aa0f6990da3d", -"snowflake": "07f8200cee37aa0f6990da3d", -"tsql": "07f8200cee37aa0f6990da3d" +"mysql": "293b92524b77d21b6865addc", +"postgres": "293b92524b77d21b6865addc", +"postgresql": "293b92524b77d21b6865addc", +"snowflake": "293b92524b77d21b6865addc", +"tsql": "293b92524b77d21b6865addc" } ], [ @@ -17342,18 +17342,18 @@ [ "type", { -"DuckDB": "d03c5e5797a8cbc0c6a81112", -"None": "d03c5e5797a8cbc0c6a81112", -"bigquery": "d03c5e5797a8cbc0c6a81112", -"databricks": "d03c5e5797a8cbc0c6a81112", -"duckdb": "d03c5e5797a8cbc0c6a81112", -"generic": "d03c5e5797a8cbc0c6a81112", +"DuckDB": "02a19edb16071249930beb5e", +"None": "02a19edb16071249930beb5e", +"bigquery": "02a19edb16071249930beb5e", +"databricks": "02a19edb16071249930beb5e", +"duckdb": "02a19edb16071249930beb5e", +"generic": "02a19edb16071249930beb5e", "motherduck": "901c5133ed47d4769286e860", -"mysql": "d03c5e5797a8cbc0c6a81112", -"postgres": "d03c5e5797a8cbc0c6a81112", -"postgresql": "d03c5e5797a8cbc0c6a81112", -"snowflake": "d03c5e5797a8cbc0c6a81112", -"tsql": "d03c5e5797a8cbc0c6a81112" +"mysql": "02a19edb16071249930beb5e", +"postgres": "02a19edb16071249930beb5e", +"postgresql": "02a19edb16071249930beb5e", +"snowflake": "02a19edb16071249930beb5e", +"tsql": "02a19edb16071249930beb5e" } ], [ @@ -18651,18 +18651,18 @@ [ "type", { -"DuckDB": "d043825e95555167dc033b5f", -"None": "d043825e95555167dc033b5f", -"bigquery": "d043825e95555167dc033b5f", -"databricks": "d043825e95555167dc033b5f", -"duckdb": "d043825e95555167dc033b5f", -"generic": "d043825e95555167dc033b5f", +"DuckDB": "2726169407c1e410edb2f532", +"None": "2726169407c1e410edb2f532", +"bigquery": "2726169407c1e410edb2f532", +"databricks": "2726169407c1e410edb2f532", +"duckdb": "2726169407c1e410edb2f532", +"generic": "2726169407c1e410edb2f532", "motherduck": "901c5133ed47d4769286e860", -"mysql": "d043825e95555167dc033b5f", -"postgres": "d043825e95555167dc033b5f", -"postgresql": "d043825e95555167dc033b5f", -"snowflake": "d043825e95555167dc033b5f", -"tsql": "d043825e95555167dc033b5f" +"mysql": "2726169407c1e410edb2f532", +"postgres": "2726169407c1e410edb2f532", +"postgresql": "2726169407c1e410edb2f532", +"snowflake": "2726169407c1e410edb2f532", +"tsql": "2726169407c1e410edb2f532" } ], [ @@ -18906,18 +18906,18 @@ [ "type", { -"DuckDB": "200c00626c910f6fd6177bb5", -"None": "200c00626c910f6fd6177bb5", -"bigquery": "200c00626c910f6fd6177bb5", -"databricks": "200c00626c910f6fd6177bb5", -"duckdb": "200c00626c910f6fd6177bb5", -"generic": "200c00626c910f6fd6177bb5", +"DuckDB": "e8e44701a29d5f80cd954213", +"None": "e8e44701a29d5f80cd954213", +"bigquery": "e8e44701a29d5f80cd954213", +"databricks": "e8e44701a29d5f80cd954213", +"duckdb": "e8e44701a29d5f80cd954213", +"generic": "e8e44701a29d5f80cd954213", "motherduck": "901c5133ed47d4769286e860", -"mysql": "200c00626c910f6fd6177bb5", -"postgres": "200c00626c910f6fd6177bb5", -"postgresql": "200c00626c910f6fd6177bb5", -"snowflake": "200c00626c910f6fd6177bb5", -"tsql": "200c00626c910f6fd6177bb5" +"mysql": "e8e44701a29d5f80cd954213", +"postgres": "e8e44701a29d5f80cd954213", +"postgresql": "e8e44701a29d5f80cd954213", +"snowflake": "e8e44701a29d5f80cd954213", +"tsql": "e8e44701a29d5f80cd954213" } ], [ @@ -19008,18 +19008,18 @@ [ "type", { -"DuckDB": "55a086bc8305ad303017ae10", -"None": "55a086bc8305ad303017ae10", -"bigquery": "55a086bc8305ad303017ae10", -"databricks": "55a086bc8305ad303017ae10", -"duckdb": "55a086bc8305ad303017ae10", -"generic": "55a086bc8305ad303017ae10", +"DuckDB": "d077d7ce7054943437bc495a", +"None": "d077d7ce7054943437bc495a", +"bigquery": "d077d7ce7054943437bc495a", +"databricks": "d077d7ce7054943437bc495a", +"duckdb": "d077d7ce7054943437bc495a", +"generic": "d077d7ce7054943437bc495a", "motherduck": "901c5133ed47d4769286e860", -"mysql": "55a086bc8305ad303017ae10", -"postgres": "55a086bc8305ad303017ae10", -"postgresql": "55a086bc8305ad303017ae10", -"snowflake": "55a086bc8305ad303017ae10", -"tsql": "55a086bc8305ad303017ae10" +"mysql": "d077d7ce7054943437bc495a", +"postgres": "d077d7ce7054943437bc495a", +"postgresql": "d077d7ce7054943437bc495a", +"snowflake": "d077d7ce7054943437bc495a", +"tsql": "d077d7ce7054943437bc495a" } ], [ @@ -19042,18 +19042,18 @@ [ "type", { -"DuckDB": "d3b6a6835220f9cedfa363c7", -"None": "d3b6a6835220f9cedfa363c7", -"bigquery": "d3b6a6835220f9cedfa363c7", -"databricks": "d3b6a6835220f9cedfa363c7", -"duckdb": "d3b6a6835220f9cedfa363c7", -"generic": "d3b6a6835220f9cedfa363c7", +"DuckDB": "9d9ab720ece92a330f5bf049", +"None": "9d9ab720ece92a330f5bf049", +"bigquery": "9d9ab720ece92a330f5bf049", +"databricks": "9d9ab720ece92a330f5bf049", +"duckdb": "9d9ab720ece92a330f5bf049", +"generic": "9d9ab720ece92a330f5bf049", "motherduck": "901c5133ed47d4769286e860", -"mysql": "d3b6a6835220f9cedfa363c7", -"postgres": "d3b6a6835220f9cedfa363c7", -"postgresql": "d3b6a6835220f9cedfa363c7", -"snowflake": "d3b6a6835220f9cedfa363c7", -"tsql": "d3b6a6835220f9cedfa363c7" +"mysql": "9d9ab720ece92a330f5bf049", +"postgres": "9d9ab720ece92a330f5bf049", +"postgresql": "9d9ab720ece92a330f5bf049", +"snowflake": "9d9ab720ece92a330f5bf049", +"tsql": "9d9ab720ece92a330f5bf049" } ], [ @@ -19246,18 +19246,18 @@ [ "type", { -"DuckDB": "e5199d22ebbd1383c1e821df", -"None": "e5199d22ebbd1383c1e821df", -"bigquery": "e5199d22ebbd1383c1e821df", -"databricks": "e5199d22ebbd1383c1e821df", -"duckdb": "e5199d22ebbd1383c1e821df", -"generic": "e5199d22ebbd1383c1e821df", +"DuckDB": "5a46370a4920d34e81123c9a", +"None": "5a46370a4920d34e81123c9a", +"bigquery": "5a46370a4920d34e81123c9a", +"databricks": "5a46370a4920d34e81123c9a", +"duckdb": "5a46370a4920d34e81123c9a", +"generic": "5a46370a4920d34e81123c9a", "motherduck": "901c5133ed47d4769286e860", -"mysql": "e5199d22ebbd1383c1e821df", -"postgres": "e5199d22ebbd1383c1e821df", -"postgresql": "e5199d22ebbd1383c1e821df", -"snowflake": "e5199d22ebbd1383c1e821df", -"tsql": "e5199d22ebbd1383c1e821df" +"mysql": "5a46370a4920d34e81123c9a", +"postgres": "5a46370a4920d34e81123c9a", +"postgresql": "5a46370a4920d34e81123c9a", +"snowflake": "5a46370a4920d34e81123c9a", +"tsql": "5a46370a4920d34e81123c9a" } ], [ @@ -19314,18 +19314,18 @@ [ "type", { -"DuckDB": "d91c8f17eb347629690400c7", -"None": "d91c8f17eb347629690400c7", -"bigquery": "d91c8f17eb347629690400c7", -"databricks": "d91c8f17eb347629690400c7", -"duckdb": "d91c8f17eb347629690400c7", -"generic": "d91c8f17eb347629690400c7", +"DuckDB": "a908355fb520168da033bacc", +"None": "a908355fb520168da033bacc", +"bigquery": "a908355fb520168da033bacc", +"databricks": "a908355fb520168da033bacc", +"duckdb": "a908355fb520168da033bacc", +"generic": "a908355fb520168da033bacc", "motherduck": "901c5133ed47d4769286e860", -"mysql": "d91c8f17eb347629690400c7", -"postgres": "d91c8f17eb347629690400c7", -"postgresql": "d91c8f17eb347629690400c7", -"snowflake": "d91c8f17eb347629690400c7", -"tsql": "d91c8f17eb347629690400c7" +"mysql": "a908355fb520168da033bacc", +"postgres": "a908355fb520168da033bacc", +"postgresql": "a908355fb520168da033bacc", +"snowflake": "a908355fb520168da033bacc", +"tsql": "a908355fb520168da033bacc" } ], [ @@ -19467,18 +19467,18 @@ [ "type", { -"DuckDB": "fd31c8e468772027804119ec", -"None": "fd31c8e468772027804119ec", -"bigquery": "fd31c8e468772027804119ec", -"databricks": "fd31c8e468772027804119ec", -"duckdb": "fd31c8e468772027804119ec", -"generic": "fd31c8e468772027804119ec", +"DuckDB": "7fe23b4f83ff4afac4eae8da", +"None": "7fe23b4f83ff4afac4eae8da", +"bigquery": "7fe23b4f83ff4afac4eae8da", +"databricks": "7fe23b4f83ff4afac4eae8da", +"duckdb": "7fe23b4f83ff4afac4eae8da", +"generic": "7fe23b4f83ff4afac4eae8da", "motherduck": "901c5133ed47d4769286e860", -"mysql": "fd31c8e468772027804119ec", -"postgres": "fd31c8e468772027804119ec", -"postgresql": "fd31c8e468772027804119ec", -"snowflake": "fd31c8e468772027804119ec", -"tsql": "fd31c8e468772027804119ec" +"mysql": "7fe23b4f83ff4afac4eae8da", +"postgres": "7fe23b4f83ff4afac4eae8da", +"postgresql": "7fe23b4f83ff4afac4eae8da", +"snowflake": "7fe23b4f83ff4afac4eae8da", +"tsql": "7fe23b4f83ff4afac4eae8da" } ], [ @@ -19790,18 +19790,18 @@ [ "type", { -"DuckDB": "4d90ccac34ef123bb0b2a15c", -"None": "4d90ccac34ef123bb0b2a15c", -"bigquery": "4d90ccac34ef123bb0b2a15c", -"databricks": "4d90ccac34ef123bb0b2a15c", -"duckdb": "4d90ccac34ef123bb0b2a15c", -"generic": "4d90ccac34ef123bb0b2a15c", +"DuckDB": "f1fdef11fa8bdd4185ea7944", +"None": "f1fdef11fa8bdd4185ea7944", +"bigquery": "f1fdef11fa8bdd4185ea7944", +"databricks": "f1fdef11fa8bdd4185ea7944", +"duckdb": "f1fdef11fa8bdd4185ea7944", +"generic": "f1fdef11fa8bdd4185ea7944", "motherduck": "901c5133ed47d4769286e860", -"mysql": "4d90ccac34ef123bb0b2a15c", -"postgres": "4d90ccac34ef123bb0b2a15c", -"postgresql": "4d90ccac34ef123bb0b2a15c", -"snowflake": "4d90ccac34ef123bb0b2a15c", -"tsql": "4d90ccac34ef123bb0b2a15c" +"mysql": "f1fdef11fa8bdd4185ea7944", +"postgres": "f1fdef11fa8bdd4185ea7944", +"postgresql": "f1fdef11fa8bdd4185ea7944", +"snowflake": "f1fdef11fa8bdd4185ea7944", +"tsql": "f1fdef11fa8bdd4185ea7944" } ], [ @@ -19960,18 +19960,18 @@ [ "type", { -"DuckDB": "2e5b1ffe62d65d4d8c389ecd", -"None": "2e5b1ffe62d65d4d8c389ecd", -"bigquery": "2e5b1ffe62d65d4d8c389ecd", -"databricks": "2e5b1ffe62d65d4d8c389ecd", -"duckdb": "2e5b1ffe62d65d4d8c389ecd", -"generic": "2e5b1ffe62d65d4d8c389ecd", +"DuckDB": "769fe9b23909b49818dba3fc", +"None": "769fe9b23909b49818dba3fc", +"bigquery": "769fe9b23909b49818dba3fc", +"databricks": "769fe9b23909b49818dba3fc", +"duckdb": "769fe9b23909b49818dba3fc", +"generic": "769fe9b23909b49818dba3fc", "motherduck": "901c5133ed47d4769286e860", -"mysql": "2e5b1ffe62d65d4d8c389ecd", -"postgres": "2e5b1ffe62d65d4d8c389ecd", -"postgresql": "2e5b1ffe62d65d4d8c389ecd", -"snowflake": "2e5b1ffe62d65d4d8c389ecd", -"tsql": "2e5b1ffe62d65d4d8c389ecd" +"mysql": "769fe9b23909b49818dba3fc", +"postgres": "769fe9b23909b49818dba3fc", +"postgresql": "769fe9b23909b49818dba3fc", +"snowflake": "769fe9b23909b49818dba3fc", +"tsql": "769fe9b23909b49818dba3fc" } ], [ @@ -20776,18 +20776,18 @@ [ "type", { -"DuckDB": "78709b3317abd96cc20340f7", -"None": "78709b3317abd96cc20340f7", -"bigquery": "78709b3317abd96cc20340f7", -"databricks": "78709b3317abd96cc20340f7", -"duckdb": "78709b3317abd96cc20340f7", -"generic": "78709b3317abd96cc20340f7", +"DuckDB": "7bd88d7cc8f9a92e789a9f44", +"None": "7bd88d7cc8f9a92e789a9f44", +"bigquery": "7bd88d7cc8f9a92e789a9f44", +"databricks": "7bd88d7cc8f9a92e789a9f44", +"duckdb": "7bd88d7cc8f9a92e789a9f44", +"generic": "7bd88d7cc8f9a92e789a9f44", "motherduck": "901c5133ed47d4769286e860", -"mysql": "78709b3317abd96cc20340f7", -"postgres": "78709b3317abd96cc20340f7", -"postgresql": "78709b3317abd96cc20340f7", -"snowflake": "78709b3317abd96cc20340f7", -"tsql": "78709b3317abd96cc20340f7" +"mysql": "7bd88d7cc8f9a92e789a9f44", +"postgres": "7bd88d7cc8f9a92e789a9f44", +"postgresql": "7bd88d7cc8f9a92e789a9f44", +"snowflake": "7bd88d7cc8f9a92e789a9f44", +"tsql": "7bd88d7cc8f9a92e789a9f44" } ], [ @@ -20929,18 +20929,18 @@ [ "type", { -"DuckDB": "dcbfb83ead5b0e32e500635b", -"None": "dcbfb83ead5b0e32e500635b", -"bigquery": "dcbfb83ead5b0e32e500635b", -"databricks": "dcbfb83ead5b0e32e500635b", -"duckdb": "dcbfb83ead5b0e32e500635b", -"generic": "dcbfb83ead5b0e32e500635b", +"DuckDB": "6672095306335323ebbfa61b", +"None": "6672095306335323ebbfa61b", +"bigquery": "6672095306335323ebbfa61b", +"databricks": "6672095306335323ebbfa61b", +"duckdb": "6672095306335323ebbfa61b", +"generic": "6672095306335323ebbfa61b", "motherduck": "901c5133ed47d4769286e860", -"mysql": "dcbfb83ead5b0e32e500635b", -"postgres": "dcbfb83ead5b0e32e500635b", -"postgresql": "dcbfb83ead5b0e32e500635b", -"snowflake": "dcbfb83ead5b0e32e500635b", -"tsql": "dcbfb83ead5b0e32e500635b" +"mysql": "6672095306335323ebbfa61b", +"postgres": "6672095306335323ebbfa61b", +"postgresql": "6672095306335323ebbfa61b", +"snowflake": "6672095306335323ebbfa61b", +"tsql": "6672095306335323ebbfa61b" } ], [ @@ -21490,18 +21490,18 @@ [ "type", { -"DuckDB": "277693b7c19b2696f01403ba", -"None": "277693b7c19b2696f01403ba", -"bigquery": "277693b7c19b2696f01403ba", -"databricks": "277693b7c19b2696f01403ba", -"duckdb": "277693b7c19b2696f01403ba", -"generic": "277693b7c19b2696f01403ba", +"DuckDB": "1a590fb9ec69f70166bed628", +"None": "1a590fb9ec69f70166bed628", +"bigquery": "1a590fb9ec69f70166bed628", +"databricks": "1a590fb9ec69f70166bed628", +"duckdb": "1a590fb9ec69f70166bed628", +"generic": "1a590fb9ec69f70166bed628", "motherduck": "901c5133ed47d4769286e860", -"mysql": "277693b7c19b2696f01403ba", -"postgres": "277693b7c19b2696f01403ba", -"postgresql": "277693b7c19b2696f01403ba", -"snowflake": "277693b7c19b2696f01403ba", -"tsql": "277693b7c19b2696f01403ba" +"mysql": "1a590fb9ec69f70166bed628", +"postgres": "1a590fb9ec69f70166bed628", +"postgresql": "1a590fb9ec69f70166bed628", +"snowflake": "1a590fb9ec69f70166bed628", +"tsql": "1a590fb9ec69f70166bed628" } ], [ @@ -22085,18 +22085,18 @@ [ "type", { -"DuckDB": "126765943545b2608ebe7128", -"None": "126765943545b2608ebe7128", -"bigquery": "126765943545b2608ebe7128", -"databricks": "126765943545b2608ebe7128", -"duckdb": "126765943545b2608ebe7128", -"generic": "126765943545b2608ebe7128", +"DuckDB": "fd0efa34f8363aad442b368c", +"None": "fd0efa34f8363aad442b368c", +"bigquery": "fd0efa34f8363aad442b368c", +"databricks": "fd0efa34f8363aad442b368c", +"duckdb": "fd0efa34f8363aad442b368c", +"generic": "fd0efa34f8363aad442b368c", "motherduck": "901c5133ed47d4769286e860", -"mysql": "126765943545b2608ebe7128", -"postgres": "126765943545b2608ebe7128", -"postgresql": "126765943545b2608ebe7128", -"snowflake": "126765943545b2608ebe7128", -"tsql": "126765943545b2608ebe7128" +"mysql": "fd0efa34f8363aad442b368c", +"postgres": "fd0efa34f8363aad442b368c", +"postgresql": "fd0efa34f8363aad442b368c", +"snowflake": "fd0efa34f8363aad442b368c", +"tsql": "fd0efa34f8363aad442b368c" } ], [ @@ -22442,18 +22442,18 @@ [ "type", { -"DuckDB": "cb06a5553de70a4924ed6600", -"None": "cb06a5553de70a4924ed6600", -"bigquery": "cb06a5553de70a4924ed6600", -"databricks": "cb06a5553de70a4924ed6600", -"duckdb": "cb06a5553de70a4924ed6600", -"generic": "cb06a5553de70a4924ed6600", +"DuckDB": "5aa2b9544add6f15c3e51f3e", +"None": "5aa2b9544add6f15c3e51f3e", +"bigquery": "5aa2b9544add6f15c3e51f3e", +"databricks": "5aa2b9544add6f15c3e51f3e", +"duckdb": "5aa2b9544add6f15c3e51f3e", +"generic": "5aa2b9544add6f15c3e51f3e", "motherduck": "901c5133ed47d4769286e860", -"mysql": "cb06a5553de70a4924ed6600", -"postgres": "cb06a5553de70a4924ed6600", -"postgresql": "cb06a5553de70a4924ed6600", -"snowflake": "cb06a5553de70a4924ed6600", -"tsql": "cb06a5553de70a4924ed6600" +"mysql": "5aa2b9544add6f15c3e51f3e", +"postgres": "5aa2b9544add6f15c3e51f3e", +"postgresql": "5aa2b9544add6f15c3e51f3e", +"snowflake": "5aa2b9544add6f15c3e51f3e", +"tsql": "5aa2b9544add6f15c3e51f3e" } ], [ @@ -22731,18 +22731,18 @@ [ "type", { -"DuckDB": "e2eee356d05f550b28bf85ad", -"None": "e2eee356d05f550b28bf85ad", -"bigquery": "e2eee356d05f550b28bf85ad", -"databricks": "e2eee356d05f550b28bf85ad", -"duckdb": "e2eee356d05f550b28bf85ad", -"generic": "e2eee356d05f550b28bf85ad", +"DuckDB": "6f1f2ad566432a6140ba8ac3", +"None": "6f1f2ad566432a6140ba8ac3", +"bigquery": "6f1f2ad566432a6140ba8ac3", +"databricks": "6f1f2ad566432a6140ba8ac3", +"duckdb": "6f1f2ad566432a6140ba8ac3", +"generic": "6f1f2ad566432a6140ba8ac3", "motherduck": "901c5133ed47d4769286e860", -"mysql": "e2eee356d05f550b28bf85ad", -"postgres": "e2eee356d05f550b28bf85ad", -"postgresql": "e2eee356d05f550b28bf85ad", -"snowflake": "e2eee356d05f550b28bf85ad", -"tsql": "e2eee356d05f550b28bf85ad" +"mysql": "6f1f2ad566432a6140ba8ac3", +"postgres": "6f1f2ad566432a6140ba8ac3", +"postgresql": "6f1f2ad566432a6140ba8ac3", +"snowflake": "6f1f2ad566432a6140ba8ac3", +"tsql": "6f1f2ad566432a6140ba8ac3" } ], [ @@ -22816,18 +22816,18 @@ [ "type", { -"DuckDB": "cd9c38fc2cf84c8551e29278", -"None": "cd9c38fc2cf84c8551e29278", -"bigquery": "cd9c38fc2cf84c8551e29278", -"databricks": "cd9c38fc2cf84c8551e29278", -"duckdb": "cd9c38fc2cf84c8551e29278", -"generic": "cd9c38fc2cf84c8551e29278", +"DuckDB": "17c9ac03f8051f8cd6cc398f", +"None": "17c9ac03f8051f8cd6cc398f", +"bigquery": "17c9ac03f8051f8cd6cc398f", +"databricks": "17c9ac03f8051f8cd6cc398f", +"duckdb": "17c9ac03f8051f8cd6cc398f", +"generic": "17c9ac03f8051f8cd6cc398f", "motherduck": "901c5133ed47d4769286e860", -"mysql": "cd9c38fc2cf84c8551e29278", -"postgres": "cd9c38fc2cf84c8551e29278", -"postgresql": "cd9c38fc2cf84c8551e29278", -"snowflake": "cd9c38fc2cf84c8551e29278", -"tsql": "cd9c38fc2cf84c8551e29278" +"mysql": "17c9ac03f8051f8cd6cc398f", +"postgres": "17c9ac03f8051f8cd6cc398f", +"postgresql": "17c9ac03f8051f8cd6cc398f", +"snowflake": "17c9ac03f8051f8cd6cc398f", +"tsql": "17c9ac03f8051f8cd6cc398f" } ], [ @@ -22918,18 +22918,18 @@ [ "type", { -"DuckDB": "1fdc0b9f28b115c175e88d46", -"None": "1fdc0b9f28b115c175e88d46", -"bigquery": "1fdc0b9f28b115c175e88d46", -"databricks": "1fdc0b9f28b115c175e88d46", -"duckdb": "1fdc0b9f28b115c175e88d46", -"generic": "1fdc0b9f28b115c175e88d46", +"DuckDB": "c748aee20c9eae68e05429a3", +"None": "c748aee20c9eae68e05429a3", +"bigquery": "c748aee20c9eae68e05429a3", +"databricks": "c748aee20c9eae68e05429a3", +"duckdb": "c748aee20c9eae68e05429a3", +"generic": "c748aee20c9eae68e05429a3", "motherduck": "901c5133ed47d4769286e860", -"mysql": "1fdc0b9f28b115c175e88d46", -"postgres": "1fdc0b9f28b115c175e88d46", -"postgresql": "1fdc0b9f28b115c175e88d46", -"snowflake": "1fdc0b9f28b115c175e88d46", -"tsql": "1fdc0b9f28b115c175e88d46" +"mysql": "c748aee20c9eae68e05429a3", +"postgres": "c748aee20c9eae68e05429a3", +"postgresql": "c748aee20c9eae68e05429a3", +"snowflake": "c748aee20c9eae68e05429a3", +"tsql": "c748aee20c9eae68e05429a3" } ], [ @@ -22969,18 +22969,18 @@ [ "type", { -"DuckDB": "c0d59e65280108f563dbb04b", -"None": "c0d59e65280108f563dbb04b", -"bigquery": "c0d59e65280108f563dbb04b", -"databricks": "c0d59e65280108f563dbb04b", -"duckdb": "c0d59e65280108f563dbb04b", -"generic": "c0d59e65280108f563dbb04b", +"DuckDB": "ac2d267e58647dba68e3f231", +"None": "ac2d267e58647dba68e3f231", +"bigquery": "ac2d267e58647dba68e3f231", +"databricks": "ac2d267e58647dba68e3f231", +"duckdb": "ac2d267e58647dba68e3f231", +"generic": "ac2d267e58647dba68e3f231", "motherduck": "901c5133ed47d4769286e860", -"mysql": "c0d59e65280108f563dbb04b", -"postgres": "c0d59e65280108f563dbb04b", -"postgresql": "c0d59e65280108f563dbb04b", -"snowflake": "c0d59e65280108f563dbb04b", -"tsql": "c0d59e65280108f563dbb04b" +"mysql": "ac2d267e58647dba68e3f231", +"postgres": "ac2d267e58647dba68e3f231", +"postgresql": "ac2d267e58647dba68e3f231", +"snowflake": "ac2d267e58647dba68e3f231", +"tsql": "ac2d267e58647dba68e3f231" } ], [ @@ -23904,18 +23904,18 @@ [ "type", { -"DuckDB": "7a00586f9fdb1d97256bfff8", -"None": "7a00586f9fdb1d97256bfff8", -"bigquery": "7a00586f9fdb1d97256bfff8", -"databricks": "7a00586f9fdb1d97256bfff8", -"duckdb": "7a00586f9fdb1d97256bfff8", -"generic": "7a00586f9fdb1d97256bfff8", +"DuckDB": "c2ad40bc8b050424142cf287", +"None": "c2ad40bc8b050424142cf287", +"bigquery": "c2ad40bc8b050424142cf287", +"databricks": "c2ad40bc8b050424142cf287", +"duckdb": "c2ad40bc8b050424142cf287", +"generic": "c2ad40bc8b050424142cf287", "motherduck": "901c5133ed47d4769286e860", -"mysql": "7a00586f9fdb1d97256bfff8", -"postgres": "7a00586f9fdb1d97256bfff8", -"postgresql": "7a00586f9fdb1d97256bfff8", -"snowflake": "7a00586f9fdb1d97256bfff8", -"tsql": "7a00586f9fdb1d97256bfff8" +"mysql": "c2ad40bc8b050424142cf287", +"postgres": "c2ad40bc8b050424142cf287", +"postgresql": "c2ad40bc8b050424142cf287", +"snowflake": "c2ad40bc8b050424142cf287", +"tsql": "c2ad40bc8b050424142cf287" } ], [ @@ -24057,18 +24057,18 @@ [ "type", { -"DuckDB": "ef2689646fd9e772e27d2727", -"None": "ef2689646fd9e772e27d2727", -"bigquery": "ef2689646fd9e772e27d2727", +"DuckDB": "da9c22183f7aed4dcccc3e86", +"None": "da9c22183f7aed4dcccc3e86", +"bigquery": "da9c22183f7aed4dcccc3e86", "databricks": "e63b3aab09c82955e2560928", -"duckdb": "ef2689646fd9e772e27d2727", -"generic": "ef2689646fd9e772e27d2727", +"duckdb": "da9c22183f7aed4dcccc3e86", +"generic": "da9c22183f7aed4dcccc3e86", "motherduck": "901c5133ed47d4769286e860", "mysql": "6c03a3450ea4742ee7396e3c", -"postgres": "ef2689646fd9e772e27d2727", -"postgresql": "ef2689646fd9e772e27d2727", -"snowflake": "ef2689646fd9e772e27d2727", -"tsql": "ef2689646fd9e772e27d2727" +"postgres": "da9c22183f7aed4dcccc3e86", +"postgresql": "da9c22183f7aed4dcccc3e86", +"snowflake": "da9c22183f7aed4dcccc3e86", +"tsql": "da9c22183f7aed4dcccc3e86" } ], [ diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py b/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py index 350086450d..3d3515d7c3 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py @@ -88,7 +88,7 @@ class CteFactRecoveryParityTestCase: @dataclass(frozen=True) class AdapterRuleCallbackTestCase: - """A project-local adapter rule the native session calls back into, and what it raises.""" + """A project-local adapter rule the Polyglot route calls back into, and what it raises.""" description: str seed: int diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index a4f2c1993f..7ffe4d1f00 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -757,7 +757,10 @@ def __call__(self, arguments: tuple[InferredNullability, ...]) -> InferredNullab def adapter_rule_project( *, test_case: AdapterRuleCallbackTestCase, rule: RecordingNullabilityRule, tmp_path: Path ) -> tuple[CompileInputsView, ExpressionInferenceProfile]: - """A generated project with the adapter-rule models, and a profile calling `rule`.""" + """A generated project with the adapter-rule models, and a Polyglot profile calling `rule`. + + A native route does not call an adapter's own rules; compile warns instead. + """ return ( compile_inputs( @@ -770,7 +773,7 @@ def adapter_rule_project( }, ), ExpressionInferenceProfile( - sql_analysis_dialect="duckdb", + sql_analysis_dialect=None, function_nullability_rules={"UPPER": rule, "IFF": rule}, ), ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py b/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py index ded903710f..f55a9bd700 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py @@ -390,3 +390,13 @@ class StarShapeOrderTestCase: type PreparedCompile = tuple[ CompileOutcome, tuple[str, ...], dict[str, object | None], tuple[bytes, ...] ] + + +@dataclass(frozen=True) +class AdapterRecoveryCase: + description: str + adapter: str + expected_findings: tuple[tuple[str, str], ...] = ( + ("B002", "downstream"), + ("B212", "upstream"), + ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py index 6bc4077592..ad9eeb940c 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py @@ -18,6 +18,7 @@ from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( + AdapterRecoveryCase, FunctionArgumentTypeCase, SemanticTriageCase, ) @@ -239,6 +240,47 @@ def test_given_cached_validation_when_warm_or_schema_changed_then_reuses_only_ma assert tuple(item["code"] for item in result["diagnostics"]) == test_case.expected_codes +@pytest.mark.parametrize( + "test_case", + [ + AdapterRecoveryCase("bigquery", adapter="bigquery"), + AdapterRecoveryCase("duckdb", adapter="duckdb"), + AdapterRecoveryCase("postgres", adapter="postgres"), + AdapterRecoveryCase("snowflake", adapter="snowflake"), + ], + ids=lambda case: case.description, +) +def test_given_a_type_error_upstream_of_a_model_with_its_own_error_when_compiling_then_both_are_reported( + test_case: AdapterRecoveryCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + f'name = "orders"\nadapter = "{test_case.adapter}"\n[rules]\nselect = []\n' + ) + (tmp_path / "models").mkdir() + header: str = ( + "MODEL (description 'Test model.', materialized view, database warehouse, schema analytics" + ) + files: dict[str, str] = { + "orders": f"{header});\n" + "SELECT CAST(1 AS INTEGER) AS quantity, CAST('2026-04-01' AS TIMESTAMP) AS ordered_at", + "upstream": f'{header});\nSELECT quantity, ordered_at, quantity + ordered_at AS bad FROM __ref("orders")', + "downstream": f'{header});\nSELECT bad > ordered_at AS compared, missing FROM __ref("upstream")', + } + for name, sql in files.items(): + (tmp_path / "models" / f"{name}.sql").write_text(sql) + assert main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) == 1 + result: dict[str, Any] = json.loads(capsys.readouterr().out) + found: dict[tuple[str, str], list[str]] = { + (item["code"], item["resource_name"]): item.get("notes", []) + for item in result["diagnostics"] + } + assert tuple(sorted(found)) == test_case.expected_findings, result["diagnostics"] + assert ( + "1 downstream output uses were not type-checked because of this error" + in found[("B212", "upstream")] + ) + + @pytest.mark.parametrize( "test_case", [SemanticTriageCase("distinct spans with identical messages", expected_codes=("B212", "B212"))], diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py index a0488f1fa2..80cd8c7dda 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py @@ -13,6 +13,7 @@ import sqlbuild._native as native_module from sqlbuild._native import ( CompilerDiagnostic, + NativeCompilerError, project_with, ) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile @@ -26,6 +27,7 @@ if TYPE_CHECKING: from sqlbuild._native import DiagnosticRow +_NATIVE_SESSION_REQUIRED: str = "revalidating a native route without its session" _RELATIONS: dict[str, tuple[str, ...]] = { '__source("raw_orders")': ( "order_id", @@ -262,6 +264,17 @@ def without_session(inputs: SemanticInputs) -> SemanticInputs: return replace(inputs, session=None) +def payload_completion_view(inputs: SemanticInputs) -> tuple[object, ...] | None: + """The completion from payload facts alone; `None` where a native route requires the session.""" + + try: + completed: CompiledProjectView = completed_natively(without_session(inputs)) + except NativeCompilerError as error: + assert _NATIVE_SESSION_REQUIRED in str(error) + return None + return completion_view(completed) + + def session_fact_models(inputs: SemanticInputs) -> frozenset[str]: """Models whose output names and lineage the finished session supplies to the stage.""" diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py index 1f2c4f7dcb..da2259b155 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py @@ -38,12 +38,12 @@ native_completion, note_kinds, outcome_label, + payload_completion_view, proven_output_count, semantic_outcome, session_corpus_files, session_fact_models, with_dialect, - without_session, ) _NOTE_KINDS: tuple[str, ...] = ("downstream output uses", "downstream uses of", " has: ") @@ -261,14 +261,14 @@ def test_given_compiles_when_completing_from_the_session_then_matches_payload_an selections: list[frozenset[str]] = [session_fact_models(inputs) for inputs in captured] codes: Counter[str] = Counter() golden: list[GoldenEntry] = [] - payload_views: list[tuple[object, ...]] = [] + payload_views: list[tuple[object, ...] | None] = [] session_views: list[tuple[object, ...]] = [] for index, inputs in enumerate(captured): completed: CompiledProjectView = completed_natively(inputs) codes.update(item.code for item in completed.diagnostics) codes.update(note_kinds(completed, _NOTE_KINDS)) session_views.append(completion_view(completed)) - payload_views.append(completion_view(completed_natively(without_session(inputs)))) + payload_views.append(payload_completion_view(inputs)) golden.append( golden_entry( "completion", session_views[-1], masked=(str(tmp_path / f"project_{index}"),) @@ -284,7 +284,10 @@ def test_given_compiles_when_completing_from_the_session_then_matches_payload_an ) == [] ) - assert payload_views == session_views + assert all( + payload in (None, session) + for payload, session in zip(payload_views, session_views, strict=True) + ) assert ( session_models, model_count - session_models,