diff --git a/Cargo.lock b/Cargo.lock index 0d8fc7aca..670a81fb5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1470,6 +1470,8 @@ dependencies = [ "sqlquack-snowflake", "sqlquack-syntax", "sqlquack-syntax-postgres", + "sqlquack-syntax-sqlite", + "stacker", ] [[package]] @@ -1532,6 +1534,15 @@ dependencies = [ "sqlquack-syntax", ] +[[package]] +name = "sqlquack-syntax-sqlite" +version = "0.2.0" +dependencies = [ + "serde_json", + "sqlquack-syntax", + "stacker", +] + [[package]] name = "stacker" version = "0.1.25" diff --git a/Cargo.toml b/Cargo.toml index 9b67fd5d9..f9e65c977 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ members = [ "crates/sqlquack-syntax-coverage", "crates/sqlquack-syntax-postgres", "crates/sqlquack-syntax-bigquery", + "crates/sqlquack-syntax-sqlite", "crates/sqlquack-python", ] exclude = ["fuzz"] @@ -66,6 +67,7 @@ sqlquack-syntax = { path = "crates/sqlquack-syntax" } sqlquack-syntax-derive = { path = "crates/sqlquack-syntax-derive" } sqlquack-syntax-postgres = { path = "crates/sqlquack-syntax-postgres" } sqlquack-syntax-bigquery = { path = "crates/sqlquack-syntax-bigquery" } +sqlquack-syntax-sqlite = { path = "crates/sqlquack-syntax-sqlite" } stacker = "0.1" syn = { version = "2", features = ["full"] } tempfile = "3" diff --git a/crates/sqlquack-semantics/Cargo.toml b/crates/sqlquack-semantics/Cargo.toml index f900c9183..9cd6b48b0 100644 --- a/crates/sqlquack-semantics/Cargo.toml +++ b/crates/sqlquack-semantics/Cargo.toml @@ -15,9 +15,11 @@ serde_json.workspace = true sqlquack-syntax.workspace = true sqlquack-core.workspace = true sqlquack-snowflake.workspace = true +stacker.workspace = true [dev-dependencies] sqlquack-syntax-postgres.workspace = true +sqlquack-syntax-sqlite.workspace = true [lints] workspace = true diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/checks/coercion.rs b/crates/sqlquack-semantics/src/lineage/_helpers/checks/coercion.rs index 28be18ebb..d4ee13af1 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/checks/coercion.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/checks/coercion.rs @@ -4,7 +4,7 @@ use sqlquack_syntax::Expression; use sqlquack_syntax::expressions::names::models::Literal; use crate::lineage::_helpers::checks::families::Family; -use crate::lineage::constants::{DUCKDB_PROFILE, POSTGRES_PROFILE}; +use crate::lineage::constants::{DUCKDB_PROFILE, POSTGRES_PROFILE, SQLITE_PROFILE}; use crate::lineage::models::LineageContext; /// The length of an ISO date, `YYYY-MM-DD`, and where its dashes and parts are. @@ -30,6 +30,8 @@ pub(crate) enum Engine { Snowflake, /// PostgreSQL: no implicit predicate, comparison or string set-operation coercions. Postgres, + /// SQLite, whose dynamic typing is not recorded: no type findings, only scope and name rules. + Sqlite, } impl Engine { @@ -37,6 +39,7 @@ impl Engine { match context.profile.name() { DUCKDB_PROFILE => Self::DuckDB, POSTGRES_PROFILE => Self::Postgres, + SQLITE_PROFILE => Self::Sqlite, _ => Self::Snowflake, } } @@ -46,7 +49,7 @@ impl Engine { match self { Self::DuckDB => &[Family::Integer, Family::Numeric], Self::Snowflake => &[Family::Integer, Family::Numeric, Family::String], - Self::Postgres => &[], + Self::Postgres | Self::Sqlite => &[], } } @@ -67,7 +70,7 @@ impl Engine { (Family::Integer, Family::Boolean), (Family::Numeric, Family::Boolean), ], - Self::Postgres => &[], + Self::Postgres | Self::Sqlite => &[], } } } diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/checks/node_checks.rs b/crates/sqlquack-semantics/src/lineage/_helpers/checks/node_checks.rs index 9aa17e483..8d99b6a7b 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/checks/node_checks.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/checks/node_checks.rs @@ -48,8 +48,9 @@ fn finding(code: &'static str, message: String, node: &Expression) -> NameFindin /// The type findings of `node`, as polyglot's type checks report them, in its order. pub(crate) fn node_findings(node: &Expression, level: NodeLevel<'_, '_>) -> Vec { + let sqlite: bool = level.engine() == Engine::Sqlite; let mut findings: Vec = match node { - Expression::Select(select) => clause_predicates(select, node, level), + Expression::Select(select) if !sqlite => clause_predicates(select, node, level), _ => Vec::new(), }; if let Expression::Function(function) = node @@ -67,6 +68,10 @@ pub(crate) fn node_findings(node: &Expression, level: NodeLevel<'_, '_>) -> Vec< } _ => Vec::new(), }; + if sqlite { + findings.extend(located(own, node)); + return findings; + } let checked = check_call(node, level.engine(), &family); let signed: bool = checked.as_ref().is_some_and(|checked| checked.signed); own.extend(checked.map(|checked| checked.findings).unwrap_or_default()); @@ -166,7 +171,7 @@ fn clause_predicates( fn predicate_code(engine: Engine) -> &'static str { match engine { Engine::DuckDB => "W215", - Engine::Snowflake | Engine::Postgres => "E211", + Engine::Snowflake | Engine::Postgres | Engine::Sqlite => "E211", } } @@ -233,7 +238,7 @@ fn expression_checks(node: &Expression, level: NodeLevel<'_, '_>) -> Vec = match engine { Engine::Snowflake => Some("E218"), Engine::DuckDB => Some("W213"), - Engine::Postgres => None, + Engine::Postgres | Engine::Sqlite => None, }; findings.extend( code.map(|code| finding(code, "Invalid interval literal".to_owned(), node)), @@ -473,7 +478,7 @@ fn impossible_cast( impossible.then(|| { finding( match engine { - Engine::Snowflake | Engine::Postgres => "E218", + Engine::Snowflake | Engine::Postgres | Engine::Sqlite => "E218", Engine::DuckDB => "W213", }, format!("Cannot cast {} to {}", source.name(), target.name()), diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/checks/relational.rs b/crates/sqlquack-semantics/src/lineage/_helpers/checks/relational.rs index 8620a8367..687726aee 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/checks/relational.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/checks/relational.rs @@ -185,6 +185,7 @@ pub(crate) fn recursive_findings( let literal: Option<&Expression> = projection_literal(&union.right, index); let invalid: bool = match engine { Engine::Snowflake => !setop_compatible(target, source), + Engine::Sqlite => false, Engine::DuckDB | Engine::Postgres => { literal.is_some_and(|value| invalid_literal_for(value, target)) || (source == Family::String @@ -197,7 +198,7 @@ pub(crate) fn recursive_findings( findings.push(finding( match engine { Engine::Snowflake => "E215", - Engine::DuckDB | Engine::Postgres => "W214", + Engine::DuckDB | Engine::Postgres | Engine::Sqlite => "W214", }, "Recursive branch cannot convert to the anchor column type".to_owned(), node, diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/checks/scope_rules.rs b/crates/sqlquack-semantics/src/lineage/_helpers/checks/scope_rules.rs index 7cf7194bb..91b3a3ecb 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/checks/scope_rules.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/checks/scope_rules.rs @@ -12,7 +12,7 @@ use crate::lineage::_helpers::checks::issues::{in_scope, issue, rule_location}; use crate::lineage::_helpers::tracing::expressions::is_aggregate; use crate::lineage::_helpers::tracing::resolution::key; use crate::lineage::_helpers::typing::postgres_typing::postgres_window_only; -use crate::lineage::constants::{POSTGRES_PROFILE, SCOPE_PHASE, SNOWFLAKE_PROFILE}; +use crate::lineage::constants::{POSTGRES_PROFILE, SCOPE_PHASE, SNOWFLAKE_PROFILE, SQLITE_PROFILE}; use crate::lineage::models::{LineageContext, LineageScope, NameFinding, ScopeSource}; const COLUMNS_FUNCTION: &str = "columns"; @@ -253,7 +253,7 @@ fn structure_rules( for cte in &with.ctes { let profile: &str = level.context.profile.name(); let mismatched = |width: usize| match profile { - SNOWFLAKE_PROFILE => width != cte.columns.len(), + SNOWFLAKE_PROFILE | SQLITE_PROFILE => width != cte.columns.len(), POSTGRES_PROFILE => width < cte.columns.len(), _ => false, }; @@ -266,7 +266,7 @@ fn structure_rules( )); } if !names.insert(level.key(&cte.alias)) - && matches!(engine, Engine::DuckDB | Engine::Postgres) + && matches!(engine, Engine::DuckDB | Engine::Postgres | Engine::Sqlite) { findings.push(rule( "E233", @@ -278,7 +278,10 @@ fn structure_rules( } } let mut names: HashSet = HashSet::new(); - for item in from_items(select) { + for item in from_items(select) + .into_iter() + .filter(|_| engine != Engine::Sqlite) + { if let Some(name) = item_name(item) && !names.insert(level.key(name)) { @@ -312,7 +315,7 @@ fn structure_rules( } } } - if matches!(engine, Engine::Postgres) { + if matches!(engine, Engine::Postgres | Engine::Sqlite) { findings.extend(unwindowed_calls(node)); } if select.qualify.is_some() && !has_window { @@ -330,7 +333,11 @@ fn structure_rules( .map(|limit| &limit.this) .chain(select.offset.iter().map(|offset| &offset.this)); for value in bounds { - if invalid_bound(value) { + let invalid: bool = match engine { + Engine::Sqlite => matches!(value, Expression::Column(_)), + _ => invalid_bound(value), + }; + if invalid { findings.push(rule( "E234", "LIMIT/OFFSET must be a non-negative constant numeric expression".to_owned(), @@ -739,12 +746,15 @@ fn offers(source: &ScopeSource, name: &str, level: RuleLevel<'_, '_>) -> bool { .any(|offered| level.context.profile.same_identifier(&offered.key, name)) } -/// Columns an aggregate query reads that are neither grouped nor aggregated. +/// Ungrouped, unaggregated columns an aggregate query reads; none in SQLite, which reads any row. fn grouping( (node, select): (&Expression, &Select), aliases: &HashMap, level: RuleLevel<'_, '_>, ) -> Vec { + if Engine::of(level.context) == Engine::Sqlite { + return Vec::new(); + } let aggregate_query: bool = select.group_by.is_some() || select.expressions.iter().any(ordinary_aggregate) || select @@ -765,13 +775,24 @@ fn grouping( } let groups: Vec<&Expression> = group_keys(select, aliases, level); let unmarked_groups: Vec<&Expression> = groups.iter().map(|group| unmarked(group)).collect(); - let grouped_columns: HashSet<(Option, String)> = groups + let grouped_columns: Vec<(Option, String)> = groups .iter() .filter_map(|group| match group { Expression::Column(column) => Some(column_key(column, level)), _ => None, }) .collect(); + let same = |left: &str, right: &str| level.context.profile.same_identifier(left, right); + let grouped = |(table, name): &(Option, String)| { + grouped_columns.iter().any(|(grouped_table, grouped_name)| { + same(grouped_name, name) + && match (grouped_table, table) { + (Some(left), Some(right)) => same(left, right), + (None, None) => true, + _ => false, + } + }) + }; let aliases_outside_order: bool = Engine::of(level.context) != Engine::Postgres; let mut read: Vec<(&Expression, bool)> = select .expressions @@ -817,7 +838,7 @@ fn grouping( continue; } if let Expression::Column(column) = current { - if grouped_columns.contains(&column_key(column, level)) { + if grouped(&column_key(column, level)) { continue; } if column.table.is_none() && named { diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/checks/sets.rs b/crates/sqlquack-semantics/src/lineage/_helpers/checks/sets.rs index 6c450053c..0784f06fd 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/checks/sets.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/checks/sets.rs @@ -166,6 +166,9 @@ pub(crate) fn set_findings( }; let findings: Vec = match Engine::of(context) { Engine::Snowflake => snowflake_findings(query, branches, context), + Engine::Sqlite => width_findings((operator, query), (left, right), branches, context) + .into_iter() + .collect(), engine @ (Engine::DuckDB | Engine::Postgres) => { binder_findings((engine, query, operator), (left, right), branches, context) } @@ -177,6 +180,38 @@ pub(crate) fn set_findings( .collect() } +/// The finding of branches whose column counts differ, which every engine refuses. +fn width_finding( + operator: &str, + (left, right): (&[Family], &[Family]), + query: &Expression, +) -> Option { + (left.len() != right.len()).then(|| { + issue( + ("E216", TYPE_PHASE), + format!( + "{operator} operands return different column counts: left {}, right {}", + left.len(), + right.len() + ), + None, + query, + ) + }) +} + +/// SQLite's findings: column counts only, as its coercions are not recorded. +fn width_findings( + (operator, query): (&str, &Expression), + (left, right): (&Expression, &Expression), + branches: Branches<'_, '_>, + context: LineageContext<'_>, +) -> Option { + let left_families: Vec = families(left, branches, context)?; + let right_families: Vec = families(right, branches, context)?; + width_finding(operator, (&left_families, &right_families), query) +} + /// DuckDB's and PostgreSQL's findings: column counts, then each column's families. fn binder_findings( (engine, query, operator): (Engine, &Expression, &str), @@ -190,17 +225,8 @@ fn binder_findings( ) else { return Vec::new(); }; - if left_families.len() != right_families.len() { - return vec![issue( - ("E216", TYPE_PHASE), - format!( - "{operator} operands return different column counts: left {}, right {}", - left_families.len(), - right_families.len() - ), - None, - query, - )]; + if let Some(finding) = width_finding(operator, (&left_families, &right_families), query) { + return vec![finding]; } let literal = |branch: &Expression, index: usize, target: Family| { projection_literal(branch, index).is_some_and(|value| literal_coerces(value, target)) diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/findings/levels.rs b/crates/sqlquack-semantics/src/lineage/_helpers/findings/levels.rs index c6299b387..2715a77fb 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/findings/levels.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/findings/levels.rs @@ -27,6 +27,16 @@ pub(crate) fn query_findings( query: &Expression, (ctes, outer, aliases): (&[NamedQuery], Option<&LineageScope<'_>>, &[String]), context: LineageContext<'_>, +) -> Vec { + crate::lineage::_helpers::tracing::stack::deeper(|| { + query_findings_level(query, (ctes, outer, aliases), context) + }) +} + +fn query_findings_level( + query: &Expression, + (ctes, outer, aliases): (&[NamedQuery], Option<&LineageScope<'_>>, &[String]), + context: LineageContext<'_>, ) -> Vec { let branches = |with: Option<&With>, left: &Expression, right: &Expression| { let (ctes, mut found) = visited_ctes(with, (ctes, outer, aliases), context); diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/expressions.rs b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/expressions.rs index 0696514a6..eda7176cf 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/expressions.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/expressions.rs @@ -16,9 +16,11 @@ use crate::lineage::_helpers::typing::postgres_typing::postgres_cast_spelling; use crate::lineage::_helpers::typing::rules::{ ValueType, computed_type, literal_type, mapped_timestamp, rendered_type, required_non_null, }; +use crate::lineage::_helpers::typing::sqlite_typing::sqlite_cast_spelling; use crate::lineage::constants::{ AGGREGATE_FUNCTIONS, BOOLEAN_TYPE, DATE_PART_FUNCTIONS, DUCKDB_PROFILE, POSITIONAL_COLUMN_BASE, POSTGRES_PROFILE, SNOWFLAKE_JSON_HOLDERS, SNOWFLAKE_PROFILE, SNOWFLAKE_VARIANT_TYPE, + SQLITE_PROFILE, }; use crate::lineage::models::{ Lineage, LineageContext, LineageScope, Nullability, ResolvedQuery, RowReads, TransformKind, @@ -29,6 +31,16 @@ pub(crate) fn expression_lineage( expression: &Expression, scope: &LineageScope<'_>, context: LineageContext<'_>, +) -> Lineage { + crate::lineage::_helpers::tracing::stack::deeper(|| { + expression_lineage_level(expression, scope, context) + }) +} + +fn expression_lineage_level( + expression: &Expression, + scope: &LineageScope<'_>, + context: LineageContext<'_>, ) -> Lineage { match expression { Expression::Column(column) => { @@ -81,6 +93,7 @@ pub(crate) fn expression_lineage( let target: Option = match context.profile.name() { DUCKDB_PROFILE => cast_spelling(&cast.to), POSTGRES_PROFILE => postgres_cast_spelling(&cast.to), + SQLITE_PROFILE => sqlite_cast_spelling(&cast.to), _ => rendered_type(&cast.to), }; let target: Option> = diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/mod.rs b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/mod.rs index ddc0955d4..6a6d93339 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/mod.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/mod.rs @@ -6,3 +6,4 @@ pub(crate) mod postgres_names; pub(crate) mod queries; pub(crate) mod resolution; pub(crate) mod sources; +pub(crate) mod stack; diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/queries.rs b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/queries.rs index 0282a0dc0..a32c8e8d5 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/queries.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/queries.rs @@ -1,6 +1,7 @@ //! The outputs of a query, set operation or CTE, with their lineage. use std::borrow::Cow; +use std::rc::Rc; use sqlquack_syntax::Expression; use sqlquack_syntax::ExpressionWalk; @@ -14,8 +15,8 @@ use crate::lineage::_helpers::tracing::resolution::{key, plain_key}; use crate::lineage::_helpers::tracing::sources::{from_sources, listed, values_query}; use crate::lineage::_helpers::typing::rules::filtered_non_null; use crate::lineage::constants::{ - DUCKDB_PROFILE, ORDINAL_BASE, POSTGRES_PROFILE, RECURSION_LIMIT, SQL_STAR, SUBTOTAL_GROUPINGS, - UNNAMED_CHARACTERS, + DUCKDB_PROFILE, ORDINAL_BASE, POSTGRES_PROFILE, RECURSION_LIMIT, SQL_STAR, SQLITE_PROFILE, + SUBTOTAL_GROUPINGS, UNNAMED_CHARACTERS, }; use crate::lineage::models::{ FromItems, Lineage, LineageContext, LineageScope, NamedQuery, Nullability, ResolvedQuery, @@ -28,6 +29,17 @@ pub(crate) fn resolved_query( ctes: &[NamedQuery], outer: Option<&LineageScope<'_>>, context: LineageContext<'_>, +) -> ResolvedQuery { + crate::lineage::_helpers::tracing::stack::deeper(|| { + resolved_query_level(query, ctes, outer, context) + }) +} + +fn resolved_query_level( + query: &Expression, + ctes: &[NamedQuery], + outer: Option<&LineageScope<'_>>, + context: LineageContext<'_>, ) -> ResolvedQuery { match query { Expression::Select(select) => select_query(select, ctes, outer, context), @@ -320,7 +332,7 @@ fn reads_itself(cte: &Cte, name: &str, context: LineageContext<'_>) -> bool { }) } -/// A recursive CTE: its anchor, then each iteration merged in until nothing changes. +/// A recursive CTE: its anchor, then each iteration merged in; DuckDB keeps the anchor's types. fn recursive_query( cte: &Cte, (name, reach): (&str, &[NamedQuery]), @@ -339,6 +351,11 @@ fn recursive_query( &cte.columns, context, ); + let anchor_types: Vec>> = current + .columns + .iter() + .map(|column| column.lineage.data_type.clone()) + .collect(); for _ in 0..RECURSION_LIMIT { let mut iteration: Vec = reach.to_vec(); iteration.push(NamedQuery { @@ -350,7 +367,12 @@ fn recursive_query( &cte.columns, context, ); - let joined: ResolvedQuery = positional(vec![current.clone(), next], context); + let mut joined: ResolvedQuery = positional(vec![current.clone(), next], context); + if context.profile.name() == DUCKDB_PROFILE { + for (column, data_type) in joined.columns.iter_mut().zip(&anchor_types) { + column.lineage.data_type.clone_from(data_type); + } + } if same_lineage(&joined, ¤t) { return joined; } @@ -758,7 +780,7 @@ fn output_name(projection: &Expression, context: LineageContext<'_>) -> String { _ => String::new(), } } - _ if context.profile.name() == DUCKDB_PROFILE => String::new(), + _ if matches!(context.profile.name(), DUCKDB_PROFILE | SQLITE_PROFILE) => String::new(), _ if context.profile.name() == POSTGRES_PROFILE => postgres_name(projection), _ => snowflake_name(projection, context).unwrap_or_default(), } diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/sources.rs b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/sources.rs index f2f4980d5..1ea400a28 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/sources.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/sources.rs @@ -16,8 +16,8 @@ use crate::lineage::_helpers::tracing::queries::{empty_scope, renamed, resolved_ use crate::lineage::_helpers::tracing::resolution::{key, plain_key}; use crate::lineage::constants::{ DUCKDB_PROFILE, DUCKDB_VALUES_COLUMN, IDENTIFIER_FUNCTION, ONE_AGGREGATE, OUTER_LEFT_JOINS, - OUTER_RIGHT_JOINS, SNOWFLAKE_PROFILE, SNOWFLAKE_VALUES_COLUMN, TABLE_FUNCTION_COLUMNS, - TABLE_WRAPPER, + OUTER_RIGHT_JOINS, SNOWFLAKE_PROFILE, SNOWFLAKE_VALUES_COLUMN, SQLITE_PROFILE, + SQLITE_VALUES_COLUMN, TABLE_FUNCTION_COLUMNS, TABLE_WRAPPER, }; use crate::lineage::models::{ FromItems, Lineage, LineageContext, LineageScope, NamedQuery, Nullability, ResolvedQuery, @@ -215,6 +215,14 @@ pub(crate) fn item_sources( item: &Expression, scope: &LineageScope<'_>, context: LineageContext<'_>, +) -> Vec { + crate::lineage::_helpers::tracing::stack::deeper(|| item_sources_level(item, scope, context)) +} + +fn item_sources_level( + item: &Expression, + scope: &LineageScope<'_>, + context: LineageContext<'_>, ) -> Vec { match item { Expression::Table(table) => vec![table_source(table, scope, context)], @@ -514,6 +522,7 @@ pub(crate) fn values_query( .unwrap_or(0); let generated = |position: usize| match context.profile.name() { DUCKDB_PROFILE => format!("{DUCKDB_VALUES_COLUMN}{position}"), + SQLITE_PROFILE => format!("{SQLITE_VALUES_COLUMN}{}", position + 1), _ => format!("{SNOWFLAKE_VALUES_COLUMN}{}", position + 1), }; let columns: Vec = (0..width) diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/stack.rs b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/stack.rs new file mode 100644 index 000000000..d02a5a2d3 --- /dev/null +++ b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/stack.rs @@ -0,0 +1,8 @@ +//! Stack-safe recursion: the walks over a tree run on added stack segments when it nests deeply. + +use crate::lineage::constants::{STACK_RED_ZONE, STACK_SEGMENT}; + +/// Runs one level of a recursive walk, on a new stack segment when the current one runs low. +pub(crate) fn deeper(level: impl FnOnce() -> R) -> R { + stacker::maybe_grow(STACK_RED_ZONE, STACK_SEGMENT, level) +} diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/typing/mod.rs b/crates/sqlquack-semantics/src/lineage/_helpers/typing/mod.rs index de1c55bfd..3a5e5e200 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/typing/mod.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/typing/mod.rs @@ -6,3 +6,4 @@ pub(crate) mod postgres_typing; pub(crate) mod rules; pub(crate) mod snowflake_sets; pub(crate) mod snowflake_typing; +pub(crate) mod sqlite_typing; diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/typing/rules.rs b/crates/sqlquack-semantics/src/lineage/_helpers/typing/rules.rs index 672b838d1..2dcb6788b 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/typing/rules.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/typing/rules.rs @@ -14,12 +14,13 @@ use crate::lineage::_helpers::typing::postgres_typing::{postgres_literal_type, p use crate::lineage::_helpers::typing::snowflake_typing::{ snowflake_literal_type, snowflake_type, snowflake_unified, }; +use crate::lineage::_helpers::typing::sqlite_typing::sqlite_literal_type; use crate::lineage::constants::{ ARGUMENT_TYPED_FUNCTIONS, BOOLEAN_TYPE, COALESCING_FUNCTIONS, CONDITIONAL_ARGUMENTS, COUNT_TYPE, COUNTING_FUNCTIONS, DECIMAL_LITERAL_TYPE, DUCKDB_PROFILE, FIRST_ARGUMENT_NULLABILITY, FLOAT_TYPES, FUNCTION_RESULT_TYPES, INTEGER_LITERAL_TYPE, - INTEGER_TYPES, OPERAND_TYPED_FUNCTIONS, POSTGRES_PROFILE, SNOWFLAKE_PROFILE, TEXT_TYPE, - UNMAPPED_TIMESTAMP, + INTEGER_TYPES, OPERAND_TYPED_FUNCTIONS, POSTGRES_PROFILE, SNOWFLAKE_PROFILE, SQLITE_PROFILE, + TEXT_TYPE, UNMAPPED_TIMESTAMP, }; use crate::lineage::models::{Lineage, LineageContext, Nullability, NullabilityRule}; @@ -77,6 +78,10 @@ pub(crate) fn literal_type(expression: &Expression, context: LineageContext<'_>) postgres_literal_type(literal).map(Rc::from).or(data_type), nullability, ), + Expression::Literal(literal) if context.profile.name() == SQLITE_PROFILE => { + (sqlite_literal_type(literal).map(Rc::from), nullability) + } + _ if context.profile.name() == SQLITE_PROFILE => (None, nullability), Expression::Literal(literal) if context.profile.name() == SNOWFLAKE_PROFILE => { match snowflake_literal_type(literal) { Some(recorded) => (recorded, nullability), @@ -105,7 +110,7 @@ fn generic_literal_type(expression: &Expression) -> ValueType { (data_type.map(Rc::from), nullability) } -/// The type and nullability of `expression`, its children's lineage given in `parts`. +/// The type and nullability of `expression` from its children's `parts`; untyped in SQLite. pub(crate) fn computed_type( expression: &Expression, parts: &[(&Expression, Lineage)], @@ -116,6 +121,13 @@ pub(crate) fn computed_type( DUCKDB_PROFILE => duckdb_type(expression, parts), SNOWFLAKE_PROFILE => snowflake_type(expression, parts), POSTGRES_PROFILE => postgres_type(expression, parts), + SQLITE_PROFILE => { + let nullability: Nullability = match nullability { + Nullability::Nullable => Nullability::Nullable, + _ => Nullability::Unknown, + }; + return (None, nullability); + } _ => None, }; let data_type: Option> = ruled.unwrap_or(data_type); diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/typing/sqlite_typing.rs b/crates/sqlquack-semantics/src/lineage/_helpers/typing/sqlite_typing.rs new file mode 100644 index 000000000..aafc6f868 --- /dev/null +++ b/crates/sqlquack-semantics/src/lineage/_helpers/typing/sqlite_typing.rs @@ -0,0 +1,51 @@ +//! SQLite's types: only a literal's or a fixed-affinity cast's storage class, as values vary by row. + +use sqlquack_syntax::DataType; +use sqlquack_syntax::expressions::names::models::Literal; + +use crate::lineage::_helpers::typing::rules::rendered_type; + +const INTEGER: &str = "INTEGER"; +const REAL: &str = "REAL"; +const TEXT: &str = "TEXT"; +const BLOB: &str = "BLOB"; +/// The longest decimal integer literal that always fits SQLite's 64-bit INTEGER. +const SAFE_INTEGER_DIGITS: usize = 18; + +/// The storage class SQLite gives a literal, when it has one. +pub(crate) fn sqlite_literal_type(literal: &Literal) -> Option<&'static str> { + match literal { + Literal::String(_) => Some(TEXT), + Literal::HexString(_) => Some(BLOB), + Literal::Number(text) if text.contains(['.', 'e', 'E']) => Some(REAL), + Literal::Number(text) + if text.len() <= SAFE_INTEGER_DIGITS && text.bytes().all(|b| b.is_ascii_digit()) => + { + Some(INTEGER) + } + _ => None, + } +} + +/// A cast's storage class by affinity; none for NUMERIC affinity, INTEGER or REAL by value. +pub(crate) fn sqlite_cast_spelling(data_type: &DataType) -> Option { + let written: String = rendered_type(data_type)?.to_ascii_uppercase(); + let class: &str = if written.contains("INT") { + INTEGER + } else if ["CHAR", "CLOB", "TEXT"] + .iter() + .any(|part| written.contains(part)) + { + TEXT + } else if written.contains(BLOB) { + BLOB + } else if ["REAL", "FLOA", "DOUB"] + .iter() + .any(|part| written.contains(part)) + { + REAL + } else { + return None; + }; + Some(class.to_owned()) +} diff --git a/crates/sqlquack-semantics/src/lineage/constants.rs b/crates/sqlquack-semantics/src/lineage/constants.rs index c8f35f749..e39c223c8 100644 --- a/crates/sqlquack-semantics/src/lineage/constants.rs +++ b/crates/sqlquack-semantics/src/lineage/constants.rs @@ -86,6 +86,10 @@ pub(crate) const AGGREGATE_FUNCTIONS: &[&str] = &[ pub(crate) const POSITIONAL_COLUMN_BASE: usize = 1; /// Snowflake's `IDENTIFIER(...)`: a relation named at run time, not traced. pub(crate) const IDENTIFIER_FUNCTION: &str = "IDENTIFIER"; +/// Stack a recursive walk keeps free before it continues on a new segment. +pub(crate) const STACK_RED_ZONE: usize = 256 * 1024; +/// The size of each stack segment a deep walk adds. +pub(crate) const STACK_SEGMENT: usize = 4 * 1024 * 1024; /// Iterations of a recursive CTE's lineage before it is left untraced. pub(crate) const RECURSION_LIMIT: usize = 32; /// Characters in an unaliased expression whose Snowflake output name lineage does not model. @@ -256,6 +260,10 @@ pub(crate) const FIRST_ARGUMENT_NULLABILITY: [&str; 2] = ["UPPER", "LOWER"]; pub(crate) const SNOWFLAKE_PROFILE: &str = "Snowflake"; /// The PostgreSQL profile's name. pub(crate) const POSTGRES_PROFILE: &str = "PostgreSQL"; +/// The SQLite profile's name. +pub(crate) const SQLITE_PROFILE: &str = "SQLite"; +/// SQLite's name for the n-th column of a VALUES list, numbered from 1. +pub(crate) const SQLITE_VALUES_COLUMN: &str = "column"; /// PostgreSQL's system columns, which every stored table has and no star expands. pub(crate) const POSTGRES_SYSTEM_COLUMNS: [&str; 6] = ["tableoid", "xmin", "cmin", "xmax", "cmax", "ctid"]; diff --git a/crates/sqlquack-semantics/src/lineage/tests.rs b/crates/sqlquack-semantics/src/lineage/tests.rs index 34a2c68a6..f14e7c7c0 100644 --- a/crates/sqlquack-semantics/src/lineage/tests.rs +++ b/crates/sqlquack-semantics/src/lineage/tests.rs @@ -12,6 +12,8 @@ mod test_output_types; mod test_query_lineage; #[path = "tests/test_recorded_types.rs"] mod test_recorded_types; +#[path = "tests/test_sqlite.rs"] +mod test_sqlite; #[path = "tests/test_statement_findings.rs"] mod test_statement_findings; #[path = "tests/test_stored_columns.rs"] diff --git a/crates/sqlquack-semantics/src/lineage/tests/helpers.rs b/crates/sqlquack-semantics/src/lineage/tests/helpers.rs index 7ffa31235..8da7cd68c 100644 --- a/crates/sqlquack-semantics/src/lineage/tests/helpers.rs +++ b/crates/sqlquack-semantics/src/lineage/tests/helpers.rs @@ -4,7 +4,7 @@ use crate::lineage::models::NullabilityRule; use crate::lineage::models::{Nullability, SchemaColumn, SourceColumn}; use sqlquack_syntax::{CallMode, Dialect, ParseOptions, parse_statements_with}; -use crate::dialects::constants::{DUCKDB, POSTGRES}; +use crate::dialects::constants::{DUCKDB, POSTGRES, SQLITE}; use crate::dialects::types::DialectProfile; use crate::lineage::_helpers::findings::statement::statement_findings; use crate::lineage::main::query_lineage::query_lineage; @@ -498,3 +498,111 @@ const CANONICAL_DUCKDB_NAMES: [(&[&str], &str); 19] = [ (&["TIMETZ", "TIME WITH TIME ZONE"], "TIMETZ"), (&["BLOB", "BYTEA", "VARBINARY"], "BLOB"), ]; + +/// A SQLite shop, with the declared types its tables are created with. +pub(crate) fn sqlite_shop_schema() -> TypedShopSchema { + TypedShopSchema { + closed: HashMap::from([ + ( + "orders".to_owned(), + vec![ + typed_column("id", "INTEGER"), + typed_column("quantity", "INTEGER"), + typed_column("price", "REAL"), + typed_column("status", "TEXT"), + ], + ), + ( + "customers".to_owned(), + vec![typed_column("id", "INTEGER"), typed_column("name", "TEXT")], + ), + ]), + ..TypedShopSchema::default() + } +} + +/// Each SQLite output's name, type and nullability. +pub(crate) fn sqlite_outputs( + schema: &TypedShopSchema, + sql: &str, +) -> Vec<(String, Option, Nullability)> { + let context = LineageContext { + profile: &SQLITE, + schema, + sql: Some(sql), + }; + query_lineage(sql, Dialect::SQLite, &ParseOptions::default(), context) + .unwrap_or_else(|error| panic!("{sql}: {error}")) + .outputs + .into_iter() + .map(|output| (output.name, output.data_type, output.nullability)) + .collect() +} + +/// `sql`'s statement with its first output's call kept generic, as the native SQLite parser writes calls. +pub(crate) fn generic_calls(sql: &str) -> sqlquack_syntax::Expression { + let written: String = sql.replacen("SELECT ", "SELECT zz_", 1); + let mut statements = parse_statements_with( + &written, + Dialect::SQLite, + &ParseOptions::default(), + CallMode::FaithfulLocated, + ) + .unwrap_or_else(|error| panic!("{sql}: {error}")); + let mut statement = statements.remove(0); + let sqlquack_syntax::Expression::Select(select) = &mut statement else { + panic!("{sql}: not a SELECT"); + }; + let Some(sqlquack_syntax::Expression::Alias(alias)) = select.expressions.first_mut() else { + panic!("{sql}: no aliased first output"); + }; + let sqlquack_syntax::Expression::Function(function) = &mut alias.this else { + panic!("{sql}: the first output is not a call"); + }; + function.name = function.name.trim_start_matches("zz_").to_owned(); + statement +} + +/// A SELECT of `orders` wrapped in `depth` derived tables. +pub(crate) fn nested_derived_tables(depth: usize) -> String { + let mut sql: String = "SELECT id FROM orders".to_owned(); + for _ in 0..depth { + sql = format!("SELECT * FROM ({sql}) AS t"); + } + sql +} + +/// `sql`'s tree depth, outputs and findings, analysed on a 2 MiB thread, or the parser's refusal. +pub(crate) fn deep_sqlite_analysis(sql: String) -> Result<(usize, usize, usize), String> { + std::thread::Builder::new() + .stack_size(2 * 1024 * 1024) + .spawn(move || { + let parsed = sqlquack_syntax_sqlite::parse(&sql).map_err(|error| error.message)?; + let schema: TypedShopSchema = sqlite_shop_schema(); + let context = LineageContext { + profile: &SQLITE, + schema: &schema, + sql: Some(&sql), + }; + let statement: &sqlquack_syntax::Expression = &parsed.statements[0]; + Ok(( + tree_depth(statement), + crate::statement_lineage(statement, context).outputs.len(), + crate::statement_name_findings(statement, context).len(), + )) + }) + .unwrap_or_else(|error| panic!("{error}")) + .join() + .unwrap_or_else(|_| panic!("the analysis thread panicked")) +} + +/// The number of levels in the tree under `root`, counted level by level without recursion. +fn tree_depth(root: &sqlquack_syntax::Expression) -> usize { + use sqlquack_syntax::ExpressionWalk; + std::iter::successors(Some(vec![root]), |level| { + let next: Vec<&sqlquack_syntax::Expression> = + level.iter().flat_map(|node| node.children()).collect(); + (!next.is_empty()).then_some(next) + }) + .count() +} diff --git a/crates/sqlquack-semantics/src/lineage/tests/test_checks.rs b/crates/sqlquack-semantics/src/lineage/tests/test_checks.rs index e1be90b71..a0a228e4b 100644 --- a/crates/sqlquack-semantics/src/lineage/tests/test_checks.rs +++ b/crates/sqlquack-semantics/src/lineage/tests/test_checks.rs @@ -61,6 +61,16 @@ fn given_a_duckdb_select_when_checking_its_scope_rules_then_broken_rules_are_fou sql: "SELECT status, COUNT(*) AS n FROM orders GROUP BY 1 HAVING COUNT(*) > 1 ORDER BY n", expected_findings: &[], }, + QueryFindingsTestCase { + description: "a quoted read of a grouped column in another case, which DuckDB matches", + sql: "SELECT \"STATUS\", COUNT(*) AS n FROM orders GROUP BY status", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "a qualified grouped column read in another case", + sql: "SELECT O.Status, COUNT(*) AS n FROM orders AS o GROUP BY o.status", + expected_findings: &[], + }, ]; let schema = typed_duckdb_shop_schema(); for test_case in test_cases { diff --git a/crates/sqlquack-semantics/src/lineage/tests/test_duckdb_types.rs b/crates/sqlquack-semantics/src/lineage/tests/test_duckdb_types.rs index c89342014..6695d3112 100644 --- a/crates/sqlquack-semantics/src/lineage/tests/test_duckdb_types.rs +++ b/crates/sqlquack-semantics/src/lineage/tests/test_duckdb_types.rs @@ -49,6 +49,29 @@ fn given_duckdb_functions_when_typing_outputs_then_each_has_the_engine_type() { sql: "SELECT LOWER(CAST(quantity AS VARCHAR)) AS v FROM t", expected_type: Some("VARCHAR"), }, + DuckdbTypeTestCase { + description: "a recursive CTE keeps its anchor's type when its recursive branch is text", + sql: "WITH RECURSIVE r(n) AS (SELECT 1 UNION ALL SELECT 'x' FROM r WHERE n < 3) \ + SELECT n AS v FROM r", + expected_type: Some("INTEGER"), + }, + DuckdbTypeTestCase { + description: "a recursive CTE keeps its anchor's text type over an integer branch", + sql: "WITH RECURSIVE r(n) AS (SELECT 'a' UNION ALL SELECT 1 FROM r WHERE length(n) < 3) \ + SELECT n AS v FROM r", + expected_type: Some("VARCHAR"), + }, + DuckdbTypeTestCase { + description: "a recursive CTE keeps an INTEGER anchor over a DECIMAL step", + sql: "WITH RECURSIVE r(n) AS (SELECT 1 UNION SELECT n + 1.5 FROM r WHERE n < 3) \ + SELECT n AS v FROM r", + expected_type: Some("INTEGER"), + }, + DuckdbTypeTestCase { + description: "a plain UNION ALL of an integer and a text literal unifies to text", + sql: "SELECT 1 AS v UNION ALL SELECT 'x'", + expected_type: Some("VARCHAR"), + }, DuckdbTypeTestCase { description: "ROW_NUMBER is BIGINT", sql: "SELECT ROW_NUMBER() OVER (ORDER BY quantity) AS v FROM t", diff --git a/crates/sqlquack-semantics/src/lineage/tests/test_sqlite.rs b/crates/sqlquack-semantics/src/lineage/tests/test_sqlite.rs new file mode 100644 index 000000000..4b71da7af --- /dev/null +++ b/crates/sqlquack-semantics/src/lineage/tests/test_sqlite.rs @@ -0,0 +1,308 @@ +//! SQLite's scope rules, and the types and names it records, as sqlite3 3.53 prepares queries. + +use sqlquack_syntax::Dialect; + +use crate::dialects::constants::SQLITE; +use crate::lineage::_helpers::findings::statement::statement_findings; +use crate::lineage::models::{LineageContext, Nullability}; +use crate::lineage::tests::helpers::{ + deep_sqlite_analysis, found_findings, generic_calls, nested_derived_tables, owned_findings, + sqlite_outputs, sqlite_shop_schema, +}; +use crate::lineage::tests::test_types::{ + DeepSqliteTestCase, QueryFindingsTestCase, SqliteOutputsTestCase, +}; + +#[test] +fn given_a_sqlite_query_sqlite_rejects_when_checking_then_the_broken_rule_is_found() { + let test_cases = [ + QueryFindingsTestCase { + description: "two CTEs under one name", + sql: "WITH x AS (SELECT 1 AS a), x AS (SELECT 2 AS a) SELECT a FROM x", + expected_findings: &[("E233", "Duplicate CTE name 'x'", 55)], + }, + QueryFindingsTestCase { + description: "fewer CTE column aliases than its outputs", + sql: "WITH x(a) AS (SELECT 1, 2) SELECT a FROM x", + expected_findings: &[( + "E223", + "CTE column alias count does not match its output column count", + 0, + )], + }, + QueryFindingsTestCase { + description: "a ranking function without OVER", + sql: "SELECT ROW_NUMBER() AS r FROM orders", + expected_findings: &[("E232", "Window function requires an OVER clause", 0)], + }, + QueryFindingsTestCase { + description: "a column as the LIMIT", + sql: "SELECT id FROM orders LIMIT quantity", + expected_findings: &[( + "E234", + "LIMIT/OFFSET must be a non-negative constant numeric expression", + 28, + )], + }, + QueryFindingsTestCase { + description: "UNION branches of different widths", + sql: "SELECT id, status FROM orders UNION ALL SELECT id FROM orders", + expected_findings: &[( + "E216", + "UNION operands return different column counts: left 2, right 1", + 0, + )], + }, + QueryFindingsTestCase { + description: "an unknown column", + sql: "SELECT nope FROM orders", + expected_findings: &[("E201", "Unknown column 'nope' in table 'orders'", 7)], + }, + ]; + let schema = sqlite_shop_schema(); + for test_case in test_cases { + assert_eq!( + found_findings(test_case.sql, (Dialect::SQLite, &SQLITE, &schema)), + owned_findings(test_case.expected_findings), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_sqlite_query_sqlite_accepts_when_checking_then_nothing_is_found() { + let test_cases = [ + QueryFindingsTestCase { + description: "a bare column in an aggregate query, read from any row of its group", + sql: "SELECT id, status, SUM(price) AS total FROM orders GROUP BY id", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "HAVING reads a column neither grouped nor aggregated", + sql: "SELECT status FROM orders GROUP BY id HAVING price > 1", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "two FROM items under one alias, no column read through it", + sql: "SELECT 1 AS one FROM orders AS a, customers AS a", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "a negative LIMIT, which SQLite reads as no limit", + sql: "SELECT id FROM orders LIMIT -1", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "a text column as a predicate and text plus a number", + sql: "SELECT status + 1 AS bumped FROM orders WHERE status", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "a quoted name in another case, which SQLite matches", + sql: "SELECT \"STATUS\", COUNT(*) AS n FROM orders GROUP BY status", + expected_findings: &[], + }, + ]; + let schema = sqlite_shop_schema(); + for test_case in test_cases { + assert_eq!( + found_findings(test_case.sql, (Dialect::SQLite, &SQLITE, &schema)), + owned_findings(test_case.expected_findings), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_sqlite_outputs_when_typing_then_only_one_storage_class_values_are_typed() { + let test_cases = [ + SqliteOutputsTestCase { + description: "literals have the storage class typeof() reports", + sql: "SELECT 1 AS i, 1.5 AS r, 1e3 AS e, 'a' AS t, NULL AS n", + expected_outputs: &[ + ("i", Some("INTEGER"), Nullability::NonNull), + ("r", Some("REAL"), Nullability::NonNull), + ("e", Some("REAL"), Nullability::NonNull), + ("t", Some("TEXT"), Nullability::NonNull), + ("n", None, Nullability::Nullable), + ], + }, + SqliteOutputsTestCase { + description: "casts type by affinity; NUMERIC affinity is INTEGER or REAL by value", + sql: "SELECT CAST(price AS VARCHAR(10)) AS t, CAST(status AS BIGINT) AS i, \ + CAST(id AS DOUBLE) AS r, CAST(price AS DECIMAL(10,2)) AS d, \ + CAST(status AS DATE) AS dt FROM orders", + expected_outputs: &[ + ("t", Some("TEXT"), Nullability::Unknown), + ("i", Some("INTEGER"), Nullability::Unknown), + ("r", Some("REAL"), Nullability::Unknown), + ("d", None, Nullability::Unknown), + ("dt", None, Nullability::Unknown), + ], + }, + SqliteOutputsTestCase { + description: "computed values have no one type, and a division can be NULL", + sql: "SELECT SUM(quantity) AS s, quantity / 0 AS q, UPPER(status) AS u FROM orders", + expected_outputs: &[ + ("s", None, Nullability::Unknown), + ("q", None, Nullability::Unknown), + ("u", None, Nullability::Unknown), + ], + }, + SqliteOutputsTestCase { + description: "a column keeps its declared type; an unaliased expression is unnamed", + sql: "SELECT price, quantity * 2 FROM orders", + expected_outputs: &[ + ("price", Some("REAL"), Nullability::Unknown), + ("", None, Nullability::Unknown), + ], + }, + SqliteOutputsTestCase { + description: "VALUES columns are named column1, column2", + sql: "SELECT * FROM (VALUES (1, 'a'))", + expected_outputs: &[ + ("column1", Some("INTEGER"), Nullability::NonNull), + ("column2", Some("TEXT"), Nullability::NonNull), + ], + }, + ]; + let schema = sqlite_shop_schema(); + for test_case in test_cases { + assert_eq!( + sqlite_outputs(&schema, test_case.sql), + test_case + .expected_outputs + .iter() + .map(|(name, data_type, nullability)| { + ( + (*name).to_owned(), + data_type.map(str::to_owned), + *nullability, + ) + }) + .collect::, Nullability)>>(), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_sqlite_window_only_calls_by_name_when_checking_then_each_without_over_is_found() { + let test_cases = [ + QueryFindingsTestCase { + description: "row_number without OVER, as sqlite3 refuses it", + sql: "SELECT row_number() AS r FROM orders", + expected_findings: &[("E232", "Window function requires an OVER clause", 7)], + }, + QueryFindingsTestCase { + description: "rank without OVER, as sqlite3 refuses it", + sql: "SELECT rank() AS r FROM orders", + expected_findings: &[("E232", "Window function requires an OVER clause", 7)], + }, + QueryFindingsTestCase { + description: "dense_rank without OVER, as sqlite3 refuses it", + sql: "SELECT dense_rank() AS r FROM orders", + expected_findings: &[("E232", "Window function requires an OVER clause", 7)], + }, + QueryFindingsTestCase { + description: "percent_rank without OVER, as sqlite3 refuses it", + sql: "SELECT percent_rank() AS r FROM orders", + expected_findings: &[("E232", "Window function requires an OVER clause", 7)], + }, + QueryFindingsTestCase { + description: "cume_dist without OVER, as sqlite3 refuses it", + sql: "SELECT cume_dist() AS r FROM orders", + expected_findings: &[("E232", "Window function requires an OVER clause", 7)], + }, + QueryFindingsTestCase { + description: "ntile without OVER, as sqlite3 refuses it", + sql: "SELECT ntile(2) AS r FROM orders", + expected_findings: &[("E232", "Window function requires an OVER clause", 7)], + }, + QueryFindingsTestCase { + description: "lag without OVER, as sqlite3 refuses it", + sql: "SELECT lag(id) AS r FROM orders", + expected_findings: &[("E232", "Window function requires an OVER clause", 7)], + }, + QueryFindingsTestCase { + description: "lead without OVER, as sqlite3 refuses it", + sql: "SELECT lead(id) AS r FROM orders", + expected_findings: &[("E232", "Window function requires an OVER clause", 7)], + }, + QueryFindingsTestCase { + description: "first_value without OVER, as sqlite3 refuses it", + sql: "SELECT first_value(id) AS r FROM orders", + expected_findings: &[("E232", "Window function requires an OVER clause", 7)], + }, + QueryFindingsTestCase { + description: "last_value without OVER, as sqlite3 refuses it", + sql: "SELECT last_value(id) AS r FROM orders", + expected_findings: &[("E232", "Window function requires an OVER clause", 7)], + }, + QueryFindingsTestCase { + description: "nth_value without OVER, as sqlite3 refuses it", + sql: "SELECT nth_value(id, 1) AS r FROM orders", + expected_findings: &[("E232", "Window function requires an OVER clause", 7)], + }, + QueryFindingsTestCase { + description: "an ordinary call by name", + sql: "SELECT upper(status) AS r FROM orders", + expected_findings: &[], + }, + ]; + let schema = sqlite_shop_schema(); + for test_case in test_cases { + let context = LineageContext { + profile: &SQLITE, + schema: &schema, + sql: Some(test_case.sql), + }; + let statement = generic_calls(test_case.sql); + assert_eq!( + statement_findings(&statement, context) + .into_iter() + .map(|finding| ( + finding.code, + finding.message, + finding.span.map_or(0, |span| span.start) + )) + .collect::>(), + owned_findings(test_case.expected_findings), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_sqlite_tree_nested_to_the_parser_limit_when_analysed_on_a_small_stack_then_it_completes() +{ + let test_cases = [ + DeepSqliteTestCase { + description: "998 nested derived tables, a tree about 2,000 levels deep", + sql: nested_derived_tables(998), + expected_result: Ok((1998, 1, 0)), + }, + DeepSqliteTestCase { + description: "990 chained additions in one output", + sql: format!("SELECT 1{} AS a FROM orders", " + 1".repeat(990)), + expected_result: Ok((993, 1, 0)), + }, + DeepSqliteTestCase { + description: "derived tables nested past the parser limit are refused", + sql: nested_derived_tables(1100), + expected_result: Err("the statement nests too deeply"), + }, + ]; + for test_case in test_cases { + assert_eq!( + deep_sqlite_analysis(test_case.sql), + test_case.expected_result.map_err(str::to_owned), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack-semantics/src/lineage/tests/test_types.rs b/crates/sqlquack-semantics/src/lineage/tests/test_types.rs index 07d22d229..07b6d3982 100644 --- a/crates/sqlquack-semantics/src/lineage/tests/test_types.rs +++ b/crates/sqlquack-semantics/src/lineage/tests/test_types.rs @@ -157,3 +157,17 @@ pub(crate) struct QueryFindingsTestCase { pub(crate) sql: &'static str, pub(crate) expected_findings: &'static [(&'static str, &'static str, usize)], } + +/// SQLite outputs: the query and each output's name, type and nullability. +pub(crate) struct SqliteOutputsTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_outputs: &'static [(&'static str, Option<&'static str>, Nullability)], +} + +/// A deeply nested SQLite query: its tree depth, outputs and findings, or the parser's refusal. +pub(crate) struct DeepSqliteTestCase { + pub(crate) description: &'static str, + pub(crate) sql: String, + pub(crate) expected_result: Result<(usize, usize, usize), &'static str>, +} diff --git a/crates/sqlquack-syntax-sqlite/Cargo.toml b/crates/sqlquack-syntax-sqlite/Cargo.toml new file mode 100644 index 000000000..b8cd94e58 --- /dev/null +++ b/crates/sqlquack-syntax-sqlite/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "sqlquack-syntax-sqlite" +version.workspace = true +description = "SQLQuack's SQLite 3.53 scanner and parser into the shared sqlquack-syntax tree." +edition.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +publish = false +rust-version.workspace = true + +[dependencies] +sqlquack-syntax.workspace = true +stacker.workspace = true + +[dev-dependencies] +serde_json.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlquack-syntax-sqlite/README.md b/crates/sqlquack-syntax-sqlite/README.md new file mode 100644 index 000000000..fd13e4282 --- /dev/null +++ b/crates/sqlquack-syntax-sqlite/README.md @@ -0,0 +1,102 @@ +# sqlquack-syntax-sqlite + +**Work in progress.** A SQLite 3.53.4 scanner and parser that builds the shared +`sqlquack-syntax` tree. It follows SQLite's `tokenize.c`, `parse.y` and the checks the grammar's +actions make, and is checked against the 3.53.4 engine's `sqlite3_prepare_v2`. The crate's layout +follows `sqlquack-syntax-duckdb`. + +## Done + +- `scan`: `sqlite3GetToken` rule for rule. It covers the four identifier quotes (`"`, `` ` ``, + `[ ]`, and a string where a name may stand), strings, `X'..'` blobs, decimal, hex and + digit-separated numbers, the five parameter forms, `||`, `->` and `->>`. `WINDOW`, `OVER` and + `FILTER` are keywords only where SQLite's lookahead makes them so. The scanner stops at a NUL + byte, as SQLite does. `uncovered` reports a byte that no token, blank or comment covers. +- `keywords`: SQLite's 147 keywords and the grammar's `%fallback ID` list. +- `parse`: `SELECT`, `VALUES` and `WITH` queries by recursive descent over `parse.y`, with + SQLite's precedence tiers: + - Compounds and the result modifiers: `ORDER BY`, and `LIMIT` with `OFFSET` or a comma. + - CTEs, with `[NOT] MATERIALIZED`, and named windows. + - Joins, with the join words read as `sqlite3JoinType` reads them, nested table lists and + table functions. + - `CAST`, `CASE`, `EXISTS`, `IN` with a list or a subquery, `BETWEEN`, `LIKE`/`GLOB`/`REGEXP`/ + `MATCH` with `ESCAPE`, `IS [NOT] [DISTINCT FROM]`, `COLLATE`. + - Calls with `DISTINCT`, an argument `ORDER BY`, `FILTER`, and `OVER` with a frame. +- The checks of the grammar's actions keep SQLite's messages. They are: + - a duplicate CTE name; + - `ORDER BY` or `LIMIT` before a compound operator; + - more than 500 compound terms; + - an unknown join type; + - `ON` or `USING` without a join; + - an unsupported frame; + - `DISTINCT` in a window function; + - an argument `ORDER BY` in a window function; + - too many arguments; + - a CTE column with `COLLATE` or a sort order; + - a window chained to a missing or misused base; + - parameter numbers out of range. + + As in SQLite, these checks report their error only once the parser takes the next token. So a + later syntax error wins over them when it falls on that token. + +## Local parser limits + +Past these limits, parsing fails with "the statement nests too deeply" (`syntax: false`) instead of +overflowing the stack. Some inputs that SQLite 3.53.4 accepts fall past them. + +| Shape, nested or chained | Parser accepts up to | sqlite3 3.53.4 accepts up to | +|---|---|---| +| Derived tables, `(SELECT * FROM (...))` | 999 | 415 | +| Scalar subqueries, `(SELECT (SELECT ...))` | 499 | 498 | +| `EXISTS` subqueries | 499 | 415 | +| `IN` subqueries | 499 | 355 | +| Subqueries holding a `UNION` | 499 | 355 | +| `IN` lists, `1 IN (1 IN (...))` | 998 | 499 | +| `CASE WHEN 1 THEN (CASE ...) END` | 998 | 498 | +| Nested `WITH`, `WITH a AS (WITH a AS (...) ...)` | 999 | 498 | +| `FILTER (WHERE count(*) FILTER (...))` | 998 | 356 | +| Window `ORDER BY`, `count(*) OVER (ORDER BY count(*) OVER (...))` | 998 | 311 | +| Flat chains of `AND`, `+` and similar | 998 operators | 999 operators | +| Parentheses around one value | 998 | 2,493 | +| Parentheses around one table in `FROM` | 999 | 1,245 | +| Terms of a compound `SELECT` | 500 (SQLite's own check) | 500 | + +The parser's figures come from two limits: 1,000 levels of recursion and chained operators, as +sqlite3's 1,000-level expression depth (`SQLITE_MAX_EXPR_DEPTH`), and a tree depth of 8,000, the +depth the shared tree's clone, comparison, serialization and drop are tested to on a 2 MiB stack. +Within the recursion limit no shape builds a tree deeper than about 2,000 levels, so the tree-depth +limit is a backstop. + +sqlite3's lower figures for subqueries, `IN` lists, `CASE`, `WITH`, `FILTER` and window `ORDER BY` +come from its resource limits: its parser's recursion limit ("Recursion limit", the depth of its +LALR stack) and "Expression tree is too large". The parser does not model them, so it accepts these +shapes past sqlite3's figures. These are known false accepts at depth. Its higher figures for +parentheses come from the same LALR stack, which does not count a parenthesis as expression depth; +there the parser refuses earlier. Flat chains stop one operator short of sqlite3. + +A depth probe checks the limits: 31 shapes at depths from 50 to 100,000, on 8 MiB and 2 MiB stacks, +in debug and release builds. Each case is parsed and serialized in its own process. It found no +crashes and no case slower than 1.6 s. + +## Gaps + +Each gap turns the whole statement into a `Command` node and records a named gap. Nothing is +accepted silently. + +- Every statement that is not a query: DML, upsert, DDL, `PRAGMA`, transactions and the rest. The + gap is named after the statement's first keyword in lower case, such as `insert` or `pragma`. + The parser does not check these statements past that keyword. +- `IN table` and `IN table-function(...)` (`in_table`). +- `INDEXED BY` and `NOT INDEXED` (`indexed_by`). +- Tcl-style `$a::b(c)` and `#name` parameters (`parameter`). +- `RAISE` (`raise`), which only appears in trigger bodies. + +## Lossy mappings + +- Calls are generic `Function` and `AggregateFunction` nodes, named as written. Later stages + classify them by name, as they do for the DuckDB parser. +- An unquoted, one-part `true` or `false` is a `Boolean`. SQLite resolves such a name to a boolean + only when no column has that name. +- A type name in `CAST` maps to a `DataType` when SQLite's affinity words name one. Otherwise it is + `Custom` with its modifiers written out. +- `0x` numbers are `HexNumber` and blobs are `HexString`. diff --git a/crates/sqlquack-syntax-sqlite/src/keywords/constants.rs b/crates/sqlquack-syntax-sqlite/src/keywords/constants.rs new file mode 100644 index 000000000..0032ba7f8 --- /dev/null +++ b/crates/sqlquack-syntax-sqlite/src/keywords/constants.rs @@ -0,0 +1,229 @@ +//! The keyword table and the `%fallback ID` list of SQLite 3.53.4. + +/// Every keyword `mkkeywordhash.c` defines in the release build (no `WITHIN`), and its token. +pub const KEYWORDS: [(&str, &str); 147] = [ + ("ABORT", "ABORT"), + ("ACTION", "ACTION"), + ("ADD", "ADD"), + ("AFTER", "AFTER"), + ("ALL", "ALL"), + ("ALTER", "ALTER"), + ("ALWAYS", "ALWAYS"), + ("ANALYZE", "ANALYZE"), + ("AND", "AND"), + ("AS", "AS"), + ("ASC", "ASC"), + ("ATTACH", "ATTACH"), + ("AUTOINCREMENT", "AUTOINCR"), + ("BEFORE", "BEFORE"), + ("BEGIN", "BEGIN"), + ("BETWEEN", "BETWEEN"), + ("BY", "BY"), + ("CASCADE", "CASCADE"), + ("CASE", "CASE"), + ("CAST", "CAST"), + ("CHECK", "CHECK"), + ("COLLATE", "COLLATE"), + ("COLUMN", "COLUMNKW"), + ("COMMIT", "COMMIT"), + ("CONFLICT", "CONFLICT"), + ("CONSTRAINT", "CONSTRAINT"), + ("CREATE", "CREATE"), + ("CROSS", "JOIN_KW"), + ("CURRENT", "CURRENT"), + ("CURRENT_DATE", "CTIME_KW"), + ("CURRENT_TIME", "CTIME_KW"), + ("CURRENT_TIMESTAMP", "CTIME_KW"), + ("DATABASE", "DATABASE"), + ("DEFAULT", "DEFAULT"), + ("DEFERRED", "DEFERRED"), + ("DEFERRABLE", "DEFERRABLE"), + ("DELETE", "DELETE"), + ("DESC", "DESC"), + ("DETACH", "DETACH"), + ("DISTINCT", "DISTINCT"), + ("DO", "DO"), + ("DROP", "DROP"), + ("END", "END"), + ("EACH", "EACH"), + ("ELSE", "ELSE"), + ("ESCAPE", "ESCAPE"), + ("EXCEPT", "EXCEPT"), + ("EXCLUSIVE", "EXCLUSIVE"), + ("EXCLUDE", "EXCLUDE"), + ("EXISTS", "EXISTS"), + ("EXPLAIN", "EXPLAIN"), + ("FAIL", "FAIL"), + ("FILTER", "FILTER"), + ("FIRST", "FIRST"), + ("FOLLOWING", "FOLLOWING"), + ("FOR", "FOR"), + ("FOREIGN", "FOREIGN"), + ("FROM", "FROM"), + ("FULL", "JOIN_KW"), + ("GENERATED", "GENERATED"), + ("GLOB", "LIKE_KW"), + ("GROUP", "GROUP"), + ("GROUPS", "GROUPS"), + ("HAVING", "HAVING"), + ("IF", "IF"), + ("IGNORE", "IGNORE"), + ("IMMEDIATE", "IMMEDIATE"), + ("IN", "IN"), + ("INDEX", "INDEX"), + ("INDEXED", "INDEXED"), + ("INITIALLY", "INITIALLY"), + ("INNER", "JOIN_KW"), + ("INSERT", "INSERT"), + ("INSTEAD", "INSTEAD"), + ("INTERSECT", "INTERSECT"), + ("INTO", "INTO"), + ("IS", "IS"), + ("ISNULL", "ISNULL"), + ("JOIN", "JOIN"), + ("KEY", "KEY"), + ("LAST", "LAST"), + ("LEFT", "JOIN_KW"), + ("LIKE", "LIKE_KW"), + ("LIMIT", "LIMIT"), + ("MATCH", "MATCH"), + ("MATERIALIZED", "MATERIALIZED"), + ("NATURAL", "JOIN_KW"), + ("NO", "NO"), + ("NOT", "NOT"), + ("NOTHING", "NOTHING"), + ("NOTNULL", "NOTNULL"), + ("NULL", "NULL"), + ("NULLS", "NULLS"), + ("OF", "OF"), + ("OFFSET", "OFFSET"), + ("ON", "ON"), + ("OR", "OR"), + ("ORDER", "ORDER"), + ("OTHERS", "OTHERS"), + ("OUTER", "JOIN_KW"), + ("OVER", "OVER"), + ("PARTITION", "PARTITION"), + ("PLAN", "PLAN"), + ("PRAGMA", "PRAGMA"), + ("PRECEDING", "PRECEDING"), + ("PRIMARY", "PRIMARY"), + ("QUERY", "QUERY"), + ("RAISE", "RAISE"), + ("RANGE", "RANGE"), + ("RECURSIVE", "RECURSIVE"), + ("REFERENCES", "REFERENCES"), + ("REGEXP", "LIKE_KW"), + ("REINDEX", "REINDEX"), + ("RELEASE", "RELEASE"), + ("RENAME", "RENAME"), + ("REPLACE", "REPLACE"), + ("RESTRICT", "RESTRICT"), + ("RETURNING", "RETURNING"), + ("RIGHT", "JOIN_KW"), + ("ROLLBACK", "ROLLBACK"), + ("ROW", "ROW"), + ("ROWS", "ROWS"), + ("SAVEPOINT", "SAVEPOINT"), + ("SELECT", "SELECT"), + ("SET", "SET"), + ("TABLE", "TABLE"), + ("TEMP", "TEMP"), + ("TEMPORARY", "TEMP"), + ("THEN", "THEN"), + ("TIES", "TIES"), + ("TO", "TO"), + ("TRANSACTION", "TRANSACTION"), + ("TRIGGER", "TRIGGER"), + ("UNBOUNDED", "UNBOUNDED"), + ("UNION", "UNION"), + ("UNIQUE", "UNIQUE"), + ("UPDATE", "UPDATE"), + ("USING", "USING"), + ("VACUUM", "VACUUM"), + ("VALUES", "VALUES"), + ("VIEW", "VIEW"), + ("VIRTUAL", "VIRTUAL"), + ("WHEN", "WHEN"), + ("WHERE", "WHERE"), + ("WINDOW", "WINDOW"), + ("WITH", "WITH"), + ("WITHOUT", "WITHOUT"), +]; + +/// The tokens parse.y's `%fallback ID` reads as `ID` where the keyword itself has no action. +pub const FALLBACK_TOKENS: [&str; 73] = [ + "ABORT", + "ACTION", + "AFTER", + "ANALYZE", + "ASC", + "ATTACH", + "BEFORE", + "BEGIN", + "BY", + "CASCADE", + "CAST", + "COLUMNKW", + "CONFLICT", + "DATABASE", + "DEFERRED", + "DESC", + "DETACH", + "DO", + "EACH", + "END", + "EXCLUSIVE", + "EXPLAIN", + "FAIL", + "FOR", + "IGNORE", + "IMMEDIATE", + "INITIALLY", + "INSTEAD", + "LIKE_KW", + "MATCH", + "NO", + "PLAN", + "QUERY", + "KEY", + "OF", + "OFFSET", + "PRAGMA", + "RAISE", + "RECURSIVE", + "RELEASE", + "REPLACE", + "RESTRICT", + "ROW", + "ROWS", + "ROLLBACK", + "SAVEPOINT", + "TEMP", + "TRIGGER", + "VACUUM", + "VIEW", + "VIRTUAL", + "WITH", + "WITHOUT", + "NULLS", + "FIRST", + "LAST", + "CURRENT", + "FOLLOWING", + "PARTITION", + "PRECEDING", + "RANGE", + "UNBOUNDED", + "EXCLUDE", + "GROUPS", + "OTHERS", + "TIES", + "GENERATED", + "ALWAYS", + "MATERIALIZED", + "REINDEX", + "RENAME", + "CTIME_KW", + "IF", +]; diff --git a/crates/sqlquack-syntax-sqlite/src/keywords/main/keyword.rs b/crates/sqlquack-syntax-sqlite/src/keywords/main/keyword.rs new file mode 100644 index 000000000..6b22da1b8 --- /dev/null +++ b/crates/sqlquack-syntax-sqlite/src/keywords/main/keyword.rs @@ -0,0 +1,15 @@ +//! `keyword`: the keyword a word is, as `keywordCode` finds it. + +use crate::keywords::constants::{FALLBACK_TOKENS, KEYWORDS}; +use crate::keywords::models::Keyword; + +/// The keyword `word` spells, compared without case, or none when it is an identifier. +pub fn keyword(word: &str) -> Option { + KEYWORDS + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(word)) + .map(|(_, token)| Keyword { + token, + fallback: FALLBACK_TOKENS.contains(token), + }) +} diff --git a/crates/sqlquack-syntax-sqlite/src/keywords/main/mod.rs b/crates/sqlquack-syntax-sqlite/src/keywords/main/mod.rs new file mode 100644 index 000000000..8dc448961 --- /dev/null +++ b/crates/sqlquack-syntax-sqlite/src/keywords/main/mod.rs @@ -0,0 +1,3 @@ +//! Entry points. + +pub mod keyword; diff --git a/crates/sqlquack-syntax-sqlite/src/keywords/mod.rs b/crates/sqlquack-syntax-sqlite/src/keywords/mod.rs new file mode 100644 index 000000000..e43b18e53 --- /dev/null +++ b/crates/sqlquack-syntax-sqlite/src/keywords/mod.rs @@ -0,0 +1,5 @@ +//! SQLite 3.53's keywords (`tool/mkkeywordhash.c`) and which fall back to identifiers (`parse.y`). + +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlquack-syntax-sqlite/src/keywords/models.rs b/crates/sqlquack-syntax-sqlite/src/keywords/models.rs new file mode 100644 index 000000000..398388229 --- /dev/null +++ b/crates/sqlquack-syntax-sqlite/src/keywords/models.rs @@ -0,0 +1,10 @@ +//! A keyword as the tokenizer hands it to the grammar. + +/// A keyword: the grammar token it is, and whether `%fallback ID` lets it stand for a name. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Keyword { + /// The grammar token without its `TK_` prefix, such as `SELECT`, `JOIN_KW` or `CTIME_KW`. + pub token: &'static str, + /// Whether the parser reads it as `ID` where the keyword itself has no action. + pub fallback: bool, +} diff --git a/crates/sqlquack-syntax-sqlite/src/lib.rs b/crates/sqlquack-syntax-sqlite/src/lib.rs new file mode 100644 index 000000000..783de4015 --- /dev/null +++ b/crates/sqlquack-syntax-sqlite/src/lib.rs @@ -0,0 +1,16 @@ +//! SQLQuack's SQLite 3.53 scanner and parser into the shared sqlquack-syntax tree. + +#![forbid(unsafe_code)] + +pub mod keywords; +pub mod parse; +pub mod scan; + +pub use crate::keywords::main::keyword::keyword; +pub use crate::keywords::models::Keyword; +pub use crate::parse::errors::{ParseError, StrictFailure}; +pub use crate::parse::main::parse::parse; +pub use crate::parse::main::parse_strict::parse_strict; +pub use crate::scan::main::scan::scan; +pub use crate::scan::main::uncovered::uncovered; +pub use sqlquack_syntax::{Gap, Parsed}; diff --git a/crates/sqlquack-syntax-sqlite/src/parse/_helpers/commands/from.rs b/crates/sqlquack-syntax-sqlite/src/parse/_helpers/commands/from.rs new file mode 100644 index 000000000..341d31b0e --- /dev/null +++ b/crates/sqlquack-syntax-sqlite/src/parse/_helpers/commands/from.rs @@ -0,0 +1,283 @@ +//! `seltablist`, `joinop` and `on_using` (parse.y): the `FROM` clause. + +use sqlquack_syntax::Expression; +use sqlquack_syntax::expressions::names::models::{Identifier, TableRef}; +use sqlquack_syntax::expressions::operators::models::Alias; +use sqlquack_syntax::expressions::queries::models::{From, Join, JoinKind, Select}; + +use crate::parse::_helpers::tree::cursor::{Parser, Parsing, is_ids, is_nm}; +use crate::parse::_helpers::tree::nodes::{ + JoinConstraint, TableAlias, function, join, joined_table, subquery, table, +}; +use crate::parse::constants::{ + JOIN_WORDS, JT_CROSS, JT_INNER, JT_LEFT, JT_NATURAL, JT_OUTER, JT_RIGHT, MAX_JOIN_WORDS, +}; +use crate::scan::models::Token; + +/// How an item joins the items before it: by a comma, or by a join of a kind. +#[derive(Clone, Copy)] +enum Joiner { + Comma, + Join { + kind: JoinKind, + inner: bool, + outer: bool, + }, +} + +/// One `seltablist` item: how it joins, the item, and its `ON` or `USING`. +struct Item { + joiner: Joiner, + this: Expression, + constraint: JoinConstraint, +} + +/// The shared tree's join kind for `sqlite3JoinType`'s bits. +fn join_kind(bits: u8) -> JoinKind { + let natural: bool = bits & JT_NATURAL != 0; + let (left, right) = (bits & JT_LEFT != 0, bits & JT_RIGHT != 0); + match (natural, left, right) { + (true, true, true) => JoinKind::NaturalFull, + (true, true, false) => JoinKind::NaturalLeft, + (true, false, true) => JoinKind::NaturalRight, + (true, false, false) => JoinKind::Natural, + (false, true, true) => JoinKind::Full, + (false, true, false) => JoinKind::Left, + (false, false, true) => JoinKind::Right, + (false, false, false) if bits & JT_CROSS != 0 => JoinKind::Cross, + (false, false, false) => JoinKind::Inner, + } +} + +/// `item` under `alias`, as the shared tree keeps an aliased item of each kind. +fn aliased(item: Expression, alias: Option) -> Expression { + let Some(alias) = alias else { + return item; + }; + match item { + Expression::Table(mut relation) => { + relation.alias = Some(alias.name); + relation.alias_explicit_as = alias.explicit; + Expression::Table(relation) + } + Expression::Subquery(mut node) => { + node.alias = Some(alias.name); + node.alias_explicit_as = alias.explicit; + node.alias_keyword = alias.explicit.then(|| "AS".to_owned()); + Expression::Subquery(node) + } + Expression::JoinedTable(mut node) => { + node.alias = Some(alias.name); + Expression::JoinedTable(node) + } + Expression::Alias(mut node) => { + node.alias = alias.name; + node.alias_explicit_as = alias.explicit; + node.alias_keyword = alias.explicit.then(|| "AS".to_owned()); + Expression::Alias(node) + } + other => { + let mut node: Alias = Alias::new(other, alias.name); + node.alias_explicit_as = alias.explicit; + node.alias_keyword = alias.explicit.then(|| "AS".to_owned()); + Expression::Alias(Box::new(node)) + } + } +} + +impl Parser<'_> { + /// `FROM seltablist`: comma-joined items before any join are sources, the rest are joins. + pub(crate) fn select_from(&mut self, mut select: Box { + Box::new(Select::new()) +} + +/// `body` with `with` as its common table expressions. +fn with_ctes(mut body: Expression, with: Option) -> Expression { + match &mut body { + Expression::Select(select) => select.with = with, + Expression::Union(node) => node.with = with, + Expression::Intersect(node) => node.with = with, + Expression::Except(node) => node.with = with, + _ => {} + } + body +} + +/// `sqlite3SelectOpName` of a compound operator, as the misplaced-clause message names it. +fn operator_name(operator: &str, all: bool) -> &'static str { + match (operator, all) { + ("UNION", true) => "UNION ALL", + ("UNION", false) => "UNION", + ("INTERSECT", _) => "INTERSECT", + _ => "EXCEPT", + } +} + +impl Parser<'_> { + /// `select`: `[WITH [RECURSIVE] wqlist] selectnowith`. + pub(crate) fn select(&mut self) -> Parsing { + self.deeper(|parser| { + let with: Option = match parser.kw("WITH") { + true => Some(parser.with_clause()?), + false => None, + }; + parser.select_with(with) + }) + } + + /// `selectnowith` after its `WITH` clause, if it has one. + pub(crate) fn select_with(&mut self, with: Option) -> Parsing { + let first: usize = self.at; + let start: usize = self.offset(); + let mut term: Term = self.one_select()?; + let mut terms: usize = 1; + let mut body: Expression = term.body; + let mut last_values: bool = term.values; + let mut misplaced: Option = None; + while let Some((operator, all)) = self.compound_operator() { + if term.order_by.is_some() || term.limit.is_some() { + let clause: &str = match term.order_by.is_some() { + true => "ORDER BY", + false => "LIMIT", + }; + misplaced = Some(format!( + "{clause} clause should come after {} not before", + operator_name(operator, all) + )); + } + term = self.one_select()?; + terms += 1; + last_values = term.values; + body = set_operation(operator, body, term.body, (all, false)); + } + if terms > MAX_COMPOUND_SELECT && !last_values { + self.defer(self.error_with(COMPOUND_TERMS, start, false)); + } else if let Some(message) = misplaced { + self.defer(self.error_with(&message, start, true)); + } + let body: Expression = finished(body, term.order_by, term.limit); + Ok(self.respanned(first, with_ctes(body, with))) + } + + /// `multiselect_op`: `UNION [ALL]`, `INTERSECT` or `EXCEPT`, consumed. + fn compound_operator(&mut self) -> Option<(&'static str, bool)> { + let operator: &'static str = match self.kind_at(0)? { + TokenKind::Keyword("UNION") => "UNION", + TokenKind::Keyword("INTERSECT") => "INTERSECT", + TokenKind::Keyword("EXCEPT") => "EXCEPT", + _ => return None, + }; + self.at += 1; + let all: bool = matches!(operator, "UNION") && self.eat_kw("ALL"); + Some((operator, all)) + } + + /// `oneselect`: a `SELECT` with its clauses, or `VALUES` rows. + fn one_select(&mut self) -> Parsing { + let first: usize = self.at; + if self.eat_kw("VALUES") { + let mut rows: Vec> = vec![self.values_row()?]; + while self.eat_op("COMMA") { + rows.push(self.values_row()?); + } + return Ok(Term { + body: self.respanned(first, values(rows, None)), + order_by: None, + limit: None, + values: true, + }); + } + self.expect_kw("SELECT")?; + let mut select: Box