From 5fbcde1bd9e9ef868ca2283737481e658af63586 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 16:33:30 +0000 Subject: [PATCH 01/10] WIP (unreviewed): sqlquack-semantics: SQLite checks and types abstain where not recorded; DuckDB group keys and recursive types SQLite fell through to Snowflake's engine. It now has its own: no type findings, as its coercions are not recorded; no grouping rule, as SQLite reads a bare column from any row of its group; duplicate table aliases and negative LIMITs pass, as sqlite3 accepts them. Duplicate CTE names, CTE column alias counts, set-operation widths, window calls without OVER and catalogue arities still raise. Only literals and casts to a fixed affinity are typed; computed values stay untyped, and unaliased expressions unnamed. A quoted read of a grouped column in another case no longer raises E230 in DuckDB, which compares names without regard to case. A DuckDB recursive CTE keeps its anchor's column types, as DuckDB casts the recursive branch. Snowflake and PostgreSQL output is unchanged. --- .../src/lineage/_helpers/checks/coercion.rs | 9 +- .../lineage/_helpers/checks/node_checks.rs | 13 +- .../src/lineage/_helpers/checks/relational.rs | 3 +- .../lineage/_helpers/checks/scope_rules.rs | 39 +++- .../src/lineage/_helpers/checks/sets.rs | 48 +++-- .../lineage/_helpers/tracing/expressions.rs | 3 + .../src/lineage/_helpers/tracing/queries.rs | 21 +- .../src/lineage/_helpers/tracing/sources.rs | 5 +- .../src/lineage/_helpers/typing/mod.rs | 1 + .../src/lineage/_helpers/typing/rules.rs | 18 +- .../lineage/_helpers/typing/sqlite_typing.rs | 51 +++++ .../src/lineage/constants.rs | 4 + .../sqlquack-semantics/src/lineage/tests.rs | 2 + .../src/lineage/tests/helpers.rs | 42 +++- .../src/lineage/tests/test_checks.rs | 10 + .../src/lineage/tests/test_duckdb_types.rs | 23 +++ .../src/lineage/tests/test_sqlite.rs | 186 ++++++++++++++++++ .../src/lineage/tests/test_types.rs | 7 + 18 files changed, 446 insertions(+), 39 deletions(-) create mode 100644 crates/sqlquack-semantics/src/lineage/_helpers/typing/sqlite_typing.rs create mode 100644 crates/sqlquack-semantics/src/lineage/tests/test_sqlite.rs diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/checks/coercion.rs b/crates/sqlquack-semantics/src/lineage/_helpers/checks/coercion.rs index 28be18ebb..d4ee13af1 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/checks/coercion.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/checks/coercion.rs @@ -4,7 +4,7 @@ use sqlquack_syntax::Expression; use sqlquack_syntax::expressions::names::models::Literal; use crate::lineage::_helpers::checks::families::Family; -use crate::lineage::constants::{DUCKDB_PROFILE, POSTGRES_PROFILE}; +use crate::lineage::constants::{DUCKDB_PROFILE, POSTGRES_PROFILE, SQLITE_PROFILE}; use crate::lineage::models::LineageContext; /// The length of an ISO date, `YYYY-MM-DD`, and where its dashes and parts are. @@ -30,6 +30,8 @@ pub(crate) enum Engine { Snowflake, /// PostgreSQL: no implicit predicate, comparison or string set-operation coercions. Postgres, + /// SQLite, whose dynamic typing is not recorded: no type findings, only scope and name rules. + Sqlite, } impl Engine { @@ -37,6 +39,7 @@ impl Engine { match context.profile.name() { DUCKDB_PROFILE => Self::DuckDB, POSTGRES_PROFILE => Self::Postgres, + SQLITE_PROFILE => Self::Sqlite, _ => Self::Snowflake, } } @@ -46,7 +49,7 @@ impl Engine { match self { Self::DuckDB => &[Family::Integer, Family::Numeric], Self::Snowflake => &[Family::Integer, Family::Numeric, Family::String], - Self::Postgres => &[], + Self::Postgres | Self::Sqlite => &[], } } @@ -67,7 +70,7 @@ impl Engine { (Family::Integer, Family::Boolean), (Family::Numeric, Family::Boolean), ], - Self::Postgres => &[], + Self::Postgres | Self::Sqlite => &[], } } } diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/checks/node_checks.rs b/crates/sqlquack-semantics/src/lineage/_helpers/checks/node_checks.rs index 9aa17e483..8d99b6a7b 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/checks/node_checks.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/checks/node_checks.rs @@ -48,8 +48,9 @@ fn finding(code: &'static str, message: String, node: &Expression) -> NameFindin /// The type findings of `node`, as polyglot's type checks report them, in its order. pub(crate) fn node_findings(node: &Expression, level: NodeLevel<'_, '_>) -> Vec { + let sqlite: bool = level.engine() == Engine::Sqlite; let mut findings: Vec = match node { - Expression::Select(select) => clause_predicates(select, node, level), + Expression::Select(select) if !sqlite => clause_predicates(select, node, level), _ => Vec::new(), }; if let Expression::Function(function) = node @@ -67,6 +68,10 @@ pub(crate) fn node_findings(node: &Expression, level: NodeLevel<'_, '_>) -> Vec< } _ => Vec::new(), }; + if sqlite { + findings.extend(located(own, node)); + return findings; + } let checked = check_call(node, level.engine(), &family); let signed: bool = checked.as_ref().is_some_and(|checked| checked.signed); own.extend(checked.map(|checked| checked.findings).unwrap_or_default()); @@ -166,7 +171,7 @@ fn clause_predicates( fn predicate_code(engine: Engine) -> &'static str { match engine { Engine::DuckDB => "W215", - Engine::Snowflake | Engine::Postgres => "E211", + Engine::Snowflake | Engine::Postgres | Engine::Sqlite => "E211", } } @@ -233,7 +238,7 @@ fn expression_checks(node: &Expression, level: NodeLevel<'_, '_>) -> Vec = match engine { Engine::Snowflake => Some("E218"), Engine::DuckDB => Some("W213"), - Engine::Postgres => None, + Engine::Postgres | Engine::Sqlite => None, }; findings.extend( code.map(|code| finding(code, "Invalid interval literal".to_owned(), node)), @@ -473,7 +478,7 @@ fn impossible_cast( impossible.then(|| { finding( match engine { - Engine::Snowflake | Engine::Postgres => "E218", + Engine::Snowflake | Engine::Postgres | Engine::Sqlite => "E218", Engine::DuckDB => "W213", }, format!("Cannot cast {} to {}", source.name(), target.name()), diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/checks/relational.rs b/crates/sqlquack-semantics/src/lineage/_helpers/checks/relational.rs index 8620a8367..687726aee 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/checks/relational.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/checks/relational.rs @@ -185,6 +185,7 @@ pub(crate) fn recursive_findings( let literal: Option<&Expression> = projection_literal(&union.right, index); let invalid: bool = match engine { Engine::Snowflake => !setop_compatible(target, source), + Engine::Sqlite => false, Engine::DuckDB | Engine::Postgres => { literal.is_some_and(|value| invalid_literal_for(value, target)) || (source == Family::String @@ -197,7 +198,7 @@ pub(crate) fn recursive_findings( findings.push(finding( match engine { Engine::Snowflake => "E215", - Engine::DuckDB | Engine::Postgres => "W214", + Engine::DuckDB | Engine::Postgres | Engine::Sqlite => "W214", }, "Recursive branch cannot convert to the anchor column type".to_owned(), node, diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/checks/scope_rules.rs b/crates/sqlquack-semantics/src/lineage/_helpers/checks/scope_rules.rs index 7cf7194bb..91b3a3ecb 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/checks/scope_rules.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/checks/scope_rules.rs @@ -12,7 +12,7 @@ use crate::lineage::_helpers::checks::issues::{in_scope, issue, rule_location}; use crate::lineage::_helpers::tracing::expressions::is_aggregate; use crate::lineage::_helpers::tracing::resolution::key; use crate::lineage::_helpers::typing::postgres_typing::postgres_window_only; -use crate::lineage::constants::{POSTGRES_PROFILE, SCOPE_PHASE, SNOWFLAKE_PROFILE}; +use crate::lineage::constants::{POSTGRES_PROFILE, SCOPE_PHASE, SNOWFLAKE_PROFILE, SQLITE_PROFILE}; use crate::lineage::models::{LineageContext, LineageScope, NameFinding, ScopeSource}; const COLUMNS_FUNCTION: &str = "columns"; @@ -253,7 +253,7 @@ fn structure_rules( for cte in &with.ctes { let profile: &str = level.context.profile.name(); let mismatched = |width: usize| match profile { - SNOWFLAKE_PROFILE => width != cte.columns.len(), + SNOWFLAKE_PROFILE | SQLITE_PROFILE => width != cte.columns.len(), POSTGRES_PROFILE => width < cte.columns.len(), _ => false, }; @@ -266,7 +266,7 @@ fn structure_rules( )); } if !names.insert(level.key(&cte.alias)) - && matches!(engine, Engine::DuckDB | Engine::Postgres) + && matches!(engine, Engine::DuckDB | Engine::Postgres | Engine::Sqlite) { findings.push(rule( "E233", @@ -278,7 +278,10 @@ fn structure_rules( } } let mut names: HashSet = HashSet::new(); - for item in from_items(select) { + for item in from_items(select) + .into_iter() + .filter(|_| engine != Engine::Sqlite) + { if let Some(name) = item_name(item) && !names.insert(level.key(name)) { @@ -312,7 +315,7 @@ fn structure_rules( } } } - if matches!(engine, Engine::Postgres) { + if matches!(engine, Engine::Postgres | Engine::Sqlite) { findings.extend(unwindowed_calls(node)); } if select.qualify.is_some() && !has_window { @@ -330,7 +333,11 @@ fn structure_rules( .map(|limit| &limit.this) .chain(select.offset.iter().map(|offset| &offset.this)); for value in bounds { - if invalid_bound(value) { + let invalid: bool = match engine { + Engine::Sqlite => matches!(value, Expression::Column(_)), + _ => invalid_bound(value), + }; + if invalid { findings.push(rule( "E234", "LIMIT/OFFSET must be a non-negative constant numeric expression".to_owned(), @@ -739,12 +746,15 @@ fn offers(source: &ScopeSource, name: &str, level: RuleLevel<'_, '_>) -> bool { .any(|offered| level.context.profile.same_identifier(&offered.key, name)) } -/// Columns an aggregate query reads that are neither grouped nor aggregated. +/// Ungrouped, unaggregated columns an aggregate query reads; none in SQLite, which reads any row. fn grouping( (node, select): (&Expression, &Select), aliases: &HashMap, level: RuleLevel<'_, '_>, ) -> Vec { + if Engine::of(level.context) == Engine::Sqlite { + return Vec::new(); + } let aggregate_query: bool = select.group_by.is_some() || select.expressions.iter().any(ordinary_aggregate) || select @@ -765,13 +775,24 @@ fn grouping( } let groups: Vec<&Expression> = group_keys(select, aliases, level); let unmarked_groups: Vec<&Expression> = groups.iter().map(|group| unmarked(group)).collect(); - let grouped_columns: HashSet<(Option, String)> = groups + let grouped_columns: Vec<(Option, String)> = groups .iter() .filter_map(|group| match group { Expression::Column(column) => Some(column_key(column, level)), _ => None, }) .collect(); + let same = |left: &str, right: &str| level.context.profile.same_identifier(left, right); + let grouped = |(table, name): &(Option, String)| { + grouped_columns.iter().any(|(grouped_table, grouped_name)| { + same(grouped_name, name) + && match (grouped_table, table) { + (Some(left), Some(right)) => same(left, right), + (None, None) => true, + _ => false, + } + }) + }; let aliases_outside_order: bool = Engine::of(level.context) != Engine::Postgres; let mut read: Vec<(&Expression, bool)> = select .expressions @@ -817,7 +838,7 @@ fn grouping( continue; } if let Expression::Column(column) = current { - if grouped_columns.contains(&column_key(column, level)) { + if grouped(&column_key(column, level)) { continue; } if column.table.is_none() && named { diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/checks/sets.rs b/crates/sqlquack-semantics/src/lineage/_helpers/checks/sets.rs index 6c450053c..0784f06fd 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/checks/sets.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/checks/sets.rs @@ -166,6 +166,9 @@ pub(crate) fn set_findings( }; let findings: Vec = match Engine::of(context) { Engine::Snowflake => snowflake_findings(query, branches, context), + Engine::Sqlite => width_findings((operator, query), (left, right), branches, context) + .into_iter() + .collect(), engine @ (Engine::DuckDB | Engine::Postgres) => { binder_findings((engine, query, operator), (left, right), branches, context) } @@ -177,6 +180,38 @@ pub(crate) fn set_findings( .collect() } +/// The finding of branches whose column counts differ, which every engine refuses. +fn width_finding( + operator: &str, + (left, right): (&[Family], &[Family]), + query: &Expression, +) -> Option { + (left.len() != right.len()).then(|| { + issue( + ("E216", TYPE_PHASE), + format!( + "{operator} operands return different column counts: left {}, right {}", + left.len(), + right.len() + ), + None, + query, + ) + }) +} + +/// SQLite's findings: column counts only, as its coercions are not recorded. +fn width_findings( + (operator, query): (&str, &Expression), + (left, right): (&Expression, &Expression), + branches: Branches<'_, '_>, + context: LineageContext<'_>, +) -> Option { + let left_families: Vec = families(left, branches, context)?; + let right_families: Vec = families(right, branches, context)?; + width_finding(operator, (&left_families, &right_families), query) +} + /// DuckDB's and PostgreSQL's findings: column counts, then each column's families. fn binder_findings( (engine, query, operator): (Engine, &Expression, &str), @@ -190,17 +225,8 @@ fn binder_findings( ) else { return Vec::new(); }; - if left_families.len() != right_families.len() { - return vec![issue( - ("E216", TYPE_PHASE), - format!( - "{operator} operands return different column counts: left {}, right {}", - left_families.len(), - right_families.len() - ), - None, - query, - )]; + if let Some(finding) = width_finding(operator, (&left_families, &right_families), query) { + return vec![finding]; } let literal = |branch: &Expression, index: usize, target: Family| { projection_literal(branch, index).is_some_and(|value| literal_coerces(value, target)) diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/expressions.rs b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/expressions.rs index 0696514a6..49329f340 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/expressions.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/expressions.rs @@ -16,9 +16,11 @@ use crate::lineage::_helpers::typing::postgres_typing::postgres_cast_spelling; use crate::lineage::_helpers::typing::rules::{ ValueType, computed_type, literal_type, mapped_timestamp, rendered_type, required_non_null, }; +use crate::lineage::_helpers::typing::sqlite_typing::sqlite_cast_spelling; use crate::lineage::constants::{ AGGREGATE_FUNCTIONS, BOOLEAN_TYPE, DATE_PART_FUNCTIONS, DUCKDB_PROFILE, POSITIONAL_COLUMN_BASE, POSTGRES_PROFILE, SNOWFLAKE_JSON_HOLDERS, SNOWFLAKE_PROFILE, SNOWFLAKE_VARIANT_TYPE, + SQLITE_PROFILE, }; use crate::lineage::models::{ Lineage, LineageContext, LineageScope, Nullability, ResolvedQuery, RowReads, TransformKind, @@ -81,6 +83,7 @@ pub(crate) fn expression_lineage( let target: Option = match context.profile.name() { DUCKDB_PROFILE => cast_spelling(&cast.to), POSTGRES_PROFILE => postgres_cast_spelling(&cast.to), + SQLITE_PROFILE => sqlite_cast_spelling(&cast.to), _ => rendered_type(&cast.to), }; let target: Option> = diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/queries.rs b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/queries.rs index 6538e0c76..d5459fca1 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/queries.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/queries.rs @@ -1,6 +1,7 @@ //! The outputs of a query, set operation or CTE, with their lineage. use std::borrow::Cow; +use std::rc::Rc; use sqlquack_syntax::Expression; use sqlquack_syntax::ExpressionWalk; @@ -14,8 +15,8 @@ use crate::lineage::_helpers::tracing::resolution::{key, plain_key}; use crate::lineage::_helpers::tracing::sources::{from_sources, listed, values_query}; use crate::lineage::_helpers::typing::rules::filtered_non_null; use crate::lineage::constants::{ - DUCKDB_PROFILE, ORDINAL_BASE, POSTGRES_PROFILE, RECURSION_LIMIT, SQL_STAR, SUBTOTAL_GROUPINGS, - UNNAMED_CHARACTERS, + DUCKDB_PROFILE, ORDINAL_BASE, POSTGRES_PROFILE, RECURSION_LIMIT, SQL_STAR, SQLITE_PROFILE, + SUBTOTAL_GROUPINGS, UNNAMED_CHARACTERS, }; use crate::lineage::models::{ FromItems, Lineage, LineageContext, LineageScope, NamedQuery, Nullability, ResolvedQuery, @@ -319,7 +320,7 @@ fn reads_itself(cte: &Cte, name: &str, context: LineageContext<'_>) -> bool { }) } -/// A recursive CTE: its anchor, then each iteration merged in until nothing changes. +/// A recursive CTE: its anchor, then each iteration merged in; DuckDB keeps the anchor's types. fn recursive_query( cte: &Cte, (name, reach): (&str, &[NamedQuery]), @@ -338,6 +339,11 @@ fn recursive_query( &cte.columns, context, ); + let anchor_types: Vec>> = current + .columns + .iter() + .map(|column| column.lineage.data_type.clone()) + .collect(); for _ in 0..RECURSION_LIMIT { let mut iteration: Vec = reach.to_vec(); iteration.push(NamedQuery { @@ -349,7 +355,12 @@ fn recursive_query( &cte.columns, context, ); - let joined: ResolvedQuery = positional(vec![current.clone(), next], context); + let mut joined: ResolvedQuery = positional(vec![current.clone(), next], context); + if context.profile.name() == DUCKDB_PROFILE { + for (column, data_type) in joined.columns.iter_mut().zip(&anchor_types) { + column.lineage.data_type.clone_from(data_type); + } + } if same_lineage(&joined, ¤t) { return joined; } @@ -756,7 +767,7 @@ fn output_name(projection: &Expression, context: LineageContext<'_>) -> String { _ => String::new(), } } - _ if context.profile.name() == DUCKDB_PROFILE => String::new(), + _ if matches!(context.profile.name(), DUCKDB_PROFILE | SQLITE_PROFILE) => String::new(), _ if context.profile.name() == POSTGRES_PROFILE => postgres_name(projection), _ => snowflake_name(projection, context).unwrap_or_default(), } diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/sources.rs b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/sources.rs index f2f4980d5..2d766be8a 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/sources.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/sources.rs @@ -16,8 +16,8 @@ use crate::lineage::_helpers::tracing::queries::{empty_scope, renamed, resolved_ use crate::lineage::_helpers::tracing::resolution::{key, plain_key}; use crate::lineage::constants::{ DUCKDB_PROFILE, DUCKDB_VALUES_COLUMN, IDENTIFIER_FUNCTION, ONE_AGGREGATE, OUTER_LEFT_JOINS, - OUTER_RIGHT_JOINS, SNOWFLAKE_PROFILE, SNOWFLAKE_VALUES_COLUMN, TABLE_FUNCTION_COLUMNS, - TABLE_WRAPPER, + OUTER_RIGHT_JOINS, SNOWFLAKE_PROFILE, SNOWFLAKE_VALUES_COLUMN, SQLITE_PROFILE, + SQLITE_VALUES_COLUMN, TABLE_FUNCTION_COLUMNS, TABLE_WRAPPER, }; use crate::lineage::models::{ FromItems, Lineage, LineageContext, LineageScope, NamedQuery, Nullability, ResolvedQuery, @@ -514,6 +514,7 @@ pub(crate) fn values_query( .unwrap_or(0); let generated = |position: usize| match context.profile.name() { DUCKDB_PROFILE => format!("{DUCKDB_VALUES_COLUMN}{position}"), + SQLITE_PROFILE => format!("{SQLITE_VALUES_COLUMN}{}", position + 1), _ => format!("{SNOWFLAKE_VALUES_COLUMN}{}", position + 1), }; let columns: Vec = (0..width) diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/typing/mod.rs b/crates/sqlquack-semantics/src/lineage/_helpers/typing/mod.rs index de1c55bfd..3a5e5e200 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/typing/mod.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/typing/mod.rs @@ -6,3 +6,4 @@ pub(crate) mod postgres_typing; pub(crate) mod rules; pub(crate) mod snowflake_sets; pub(crate) mod snowflake_typing; +pub(crate) mod sqlite_typing; diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/typing/rules.rs b/crates/sqlquack-semantics/src/lineage/_helpers/typing/rules.rs index 672b838d1..2dcb6788b 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/typing/rules.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/typing/rules.rs @@ -14,12 +14,13 @@ use crate::lineage::_helpers::typing::postgres_typing::{postgres_literal_type, p use crate::lineage::_helpers::typing::snowflake_typing::{ snowflake_literal_type, snowflake_type, snowflake_unified, }; +use crate::lineage::_helpers::typing::sqlite_typing::sqlite_literal_type; use crate::lineage::constants::{ ARGUMENT_TYPED_FUNCTIONS, BOOLEAN_TYPE, COALESCING_FUNCTIONS, CONDITIONAL_ARGUMENTS, COUNT_TYPE, COUNTING_FUNCTIONS, DECIMAL_LITERAL_TYPE, DUCKDB_PROFILE, FIRST_ARGUMENT_NULLABILITY, FLOAT_TYPES, FUNCTION_RESULT_TYPES, INTEGER_LITERAL_TYPE, - INTEGER_TYPES, OPERAND_TYPED_FUNCTIONS, POSTGRES_PROFILE, SNOWFLAKE_PROFILE, TEXT_TYPE, - UNMAPPED_TIMESTAMP, + INTEGER_TYPES, OPERAND_TYPED_FUNCTIONS, POSTGRES_PROFILE, SNOWFLAKE_PROFILE, SQLITE_PROFILE, + TEXT_TYPE, UNMAPPED_TIMESTAMP, }; use crate::lineage::models::{Lineage, LineageContext, Nullability, NullabilityRule}; @@ -77,6 +78,10 @@ pub(crate) fn literal_type(expression: &Expression, context: LineageContext<'_>) postgres_literal_type(literal).map(Rc::from).or(data_type), nullability, ), + Expression::Literal(literal) if context.profile.name() == SQLITE_PROFILE => { + (sqlite_literal_type(literal).map(Rc::from), nullability) + } + _ if context.profile.name() == SQLITE_PROFILE => (None, nullability), Expression::Literal(literal) if context.profile.name() == SNOWFLAKE_PROFILE => { match snowflake_literal_type(literal) { Some(recorded) => (recorded, nullability), @@ -105,7 +110,7 @@ fn generic_literal_type(expression: &Expression) -> ValueType { (data_type.map(Rc::from), nullability) } -/// The type and nullability of `expression`, its children's lineage given in `parts`. +/// The type and nullability of `expression` from its children's `parts`; untyped in SQLite. pub(crate) fn computed_type( expression: &Expression, parts: &[(&Expression, Lineage)], @@ -116,6 +121,13 @@ pub(crate) fn computed_type( DUCKDB_PROFILE => duckdb_type(expression, parts), SNOWFLAKE_PROFILE => snowflake_type(expression, parts), POSTGRES_PROFILE => postgres_type(expression, parts), + SQLITE_PROFILE => { + let nullability: Nullability = match nullability { + Nullability::Nullable => Nullability::Nullable, + _ => Nullability::Unknown, + }; + return (None, nullability); + } _ => None, }; let data_type: Option> = ruled.unwrap_or(data_type); diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/typing/sqlite_typing.rs b/crates/sqlquack-semantics/src/lineage/_helpers/typing/sqlite_typing.rs new file mode 100644 index 000000000..aafc6f868 --- /dev/null +++ b/crates/sqlquack-semantics/src/lineage/_helpers/typing/sqlite_typing.rs @@ -0,0 +1,51 @@ +//! SQLite's types: only a literal's or a fixed-affinity cast's storage class, as values vary by row. + +use sqlquack_syntax::DataType; +use sqlquack_syntax::expressions::names::models::Literal; + +use crate::lineage::_helpers::typing::rules::rendered_type; + +const INTEGER: &str = "INTEGER"; +const REAL: &str = "REAL"; +const TEXT: &str = "TEXT"; +const BLOB: &str = "BLOB"; +/// The longest decimal integer literal that always fits SQLite's 64-bit INTEGER. +const SAFE_INTEGER_DIGITS: usize = 18; + +/// The storage class SQLite gives a literal, when it has one. +pub(crate) fn sqlite_literal_type(literal: &Literal) -> Option<&'static str> { + match literal { + Literal::String(_) => Some(TEXT), + Literal::HexString(_) => Some(BLOB), + Literal::Number(text) if text.contains(['.', 'e', 'E']) => Some(REAL), + Literal::Number(text) + if text.len() <= SAFE_INTEGER_DIGITS && text.bytes().all(|b| b.is_ascii_digit()) => + { + Some(INTEGER) + } + _ => None, + } +} + +/// A cast's storage class by affinity; none for NUMERIC affinity, INTEGER or REAL by value. +pub(crate) fn sqlite_cast_spelling(data_type: &DataType) -> Option { + let written: String = rendered_type(data_type)?.to_ascii_uppercase(); + let class: &str = if written.contains("INT") { + INTEGER + } else if ["CHAR", "CLOB", "TEXT"] + .iter() + .any(|part| written.contains(part)) + { + TEXT + } else if written.contains(BLOB) { + BLOB + } else if ["REAL", "FLOA", "DOUB"] + .iter() + .any(|part| written.contains(part)) + { + REAL + } else { + return None; + }; + Some(class.to_owned()) +} diff --git a/crates/sqlquack-semantics/src/lineage/constants.rs b/crates/sqlquack-semantics/src/lineage/constants.rs index c8f35f749..f0020dba0 100644 --- a/crates/sqlquack-semantics/src/lineage/constants.rs +++ b/crates/sqlquack-semantics/src/lineage/constants.rs @@ -256,6 +256,10 @@ pub(crate) const FIRST_ARGUMENT_NULLABILITY: [&str; 2] = ["UPPER", "LOWER"]; pub(crate) const SNOWFLAKE_PROFILE: &str = "Snowflake"; /// The PostgreSQL profile's name. pub(crate) const POSTGRES_PROFILE: &str = "PostgreSQL"; +/// The SQLite profile's name. +pub(crate) const SQLITE_PROFILE: &str = "SQLite"; +/// SQLite's name for the n-th column of a VALUES list, numbered from 1. +pub(crate) const SQLITE_VALUES_COLUMN: &str = "column"; /// PostgreSQL's system columns, which every stored table has and no star expands. pub(crate) const POSTGRES_SYSTEM_COLUMNS: [&str; 6] = ["tableoid", "xmin", "cmin", "xmax", "cmax", "ctid"]; diff --git a/crates/sqlquack-semantics/src/lineage/tests.rs b/crates/sqlquack-semantics/src/lineage/tests.rs index 34a2c68a6..f14e7c7c0 100644 --- a/crates/sqlquack-semantics/src/lineage/tests.rs +++ b/crates/sqlquack-semantics/src/lineage/tests.rs @@ -12,6 +12,8 @@ mod test_output_types; mod test_query_lineage; #[path = "tests/test_recorded_types.rs"] mod test_recorded_types; +#[path = "tests/test_sqlite.rs"] +mod test_sqlite; #[path = "tests/test_statement_findings.rs"] mod test_statement_findings; #[path = "tests/test_stored_columns.rs"] diff --git a/crates/sqlquack-semantics/src/lineage/tests/helpers.rs b/crates/sqlquack-semantics/src/lineage/tests/helpers.rs index 0eabc4d0e..d2614b841 100644 --- a/crates/sqlquack-semantics/src/lineage/tests/helpers.rs +++ b/crates/sqlquack-semantics/src/lineage/tests/helpers.rs @@ -4,7 +4,7 @@ use crate::lineage::models::NullabilityRule; use crate::lineage::models::{Nullability, SchemaColumn, SourceColumn}; use sqlquack_syntax::{CallMode, Dialect, ParseOptions, parse_statements_with}; -use crate::dialects::constants::{DUCKDB, POSTGRES}; +use crate::dialects::constants::{DUCKDB, POSTGRES, SQLITE}; use crate::dialects::types::DialectProfile; use crate::lineage::_helpers::findings::statement::statement_findings; use crate::lineage::main::query_lineage::query_lineage; @@ -472,3 +472,43 @@ const CANONICAL_DUCKDB_NAMES: [(&[&str], &str); 19] = [ (&["TIMETZ", "TIME WITH TIME ZONE"], "TIMETZ"), (&["BLOB", "BYTEA", "VARBINARY"], "BLOB"), ]; + +/// A SQLite shop, with the declared types its tables are created with. +pub(crate) fn sqlite_shop_schema() -> TypedShopSchema { + TypedShopSchema { + closed: HashMap::from([ + ( + "orders".to_owned(), + vec![ + typed_column("id", "INTEGER"), + typed_column("quantity", "INTEGER"), + typed_column("price", "REAL"), + typed_column("status", "TEXT"), + ], + ), + ( + "customers".to_owned(), + vec![typed_column("id", "INTEGER"), typed_column("name", "TEXT")], + ), + ]), + ..TypedShopSchema::default() + } +} + +/// Each SQLite output's name, type and nullability. +pub(crate) fn sqlite_outputs( + schema: &TypedShopSchema, + sql: &str, +) -> Vec<(String, Option, Nullability)> { + let context = LineageContext { + profile: &SQLITE, + schema, + sql: Some(sql), + }; + query_lineage(sql, Dialect::SQLite, &ParseOptions::default(), context) + .unwrap_or_else(|error| panic!("{sql}: {error}")) + .outputs + .into_iter() + .map(|output| (output.name, output.data_type, output.nullability)) + .collect() +} diff --git a/crates/sqlquack-semantics/src/lineage/tests/test_checks.rs b/crates/sqlquack-semantics/src/lineage/tests/test_checks.rs index e1be90b71..a0a228e4b 100644 --- a/crates/sqlquack-semantics/src/lineage/tests/test_checks.rs +++ b/crates/sqlquack-semantics/src/lineage/tests/test_checks.rs @@ -61,6 +61,16 @@ fn given_a_duckdb_select_when_checking_its_scope_rules_then_broken_rules_are_fou sql: "SELECT status, COUNT(*) AS n FROM orders GROUP BY 1 HAVING COUNT(*) > 1 ORDER BY n", expected_findings: &[], }, + QueryFindingsTestCase { + description: "a quoted read of a grouped column in another case, which DuckDB matches", + sql: "SELECT \"STATUS\", COUNT(*) AS n FROM orders GROUP BY status", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "a qualified grouped column read in another case", + sql: "SELECT O.Status, COUNT(*) AS n FROM orders AS o GROUP BY o.status", + expected_findings: &[], + }, ]; let schema = typed_duckdb_shop_schema(); for test_case in test_cases { diff --git a/crates/sqlquack-semantics/src/lineage/tests/test_duckdb_types.rs b/crates/sqlquack-semantics/src/lineage/tests/test_duckdb_types.rs index 45db8c9ae..173046314 100644 --- a/crates/sqlquack-semantics/src/lineage/tests/test_duckdb_types.rs +++ b/crates/sqlquack-semantics/src/lineage/tests/test_duckdb_types.rs @@ -44,6 +44,29 @@ fn given_duckdb_functions_when_typing_outputs_then_each_has_the_engine_type() { sql: "SELECT LOWER(CAST(quantity AS VARCHAR)) AS v FROM t", expected_type: Some("VARCHAR"), }, + DuckdbTypeTestCase { + description: "a recursive CTE keeps its anchor's type when its recursive branch is text", + sql: "WITH RECURSIVE r(n) AS (SELECT 1 UNION ALL SELECT 'x' FROM r WHERE n < 3) \ + SELECT n AS v FROM r", + expected_type: Some("INTEGER"), + }, + DuckdbTypeTestCase { + description: "a recursive CTE keeps its anchor's text type over an integer branch", + sql: "WITH RECURSIVE r(n) AS (SELECT 'a' UNION ALL SELECT 1 FROM r WHERE length(n) < 3) \ + SELECT n AS v FROM r", + expected_type: Some("VARCHAR"), + }, + DuckdbTypeTestCase { + description: "a recursive CTE keeps an INTEGER anchor over a DECIMAL step", + sql: "WITH RECURSIVE r(n) AS (SELECT 1 UNION SELECT n + 1.5 FROM r WHERE n < 3) \ + SELECT n AS v FROM r", + expected_type: Some("INTEGER"), + }, + DuckdbTypeTestCase { + description: "a plain UNION ALL of an integer and a text literal unifies to text", + sql: "SELECT 1 AS v UNION ALL SELECT 'x'", + expected_type: Some("VARCHAR"), + }, DuckdbTypeTestCase { description: "ROW_NUMBER is BIGINT", sql: "SELECT ROW_NUMBER() OVER (ORDER BY quantity) AS v FROM t", diff --git a/crates/sqlquack-semantics/src/lineage/tests/test_sqlite.rs b/crates/sqlquack-semantics/src/lineage/tests/test_sqlite.rs new file mode 100644 index 000000000..e6570ba0a --- /dev/null +++ b/crates/sqlquack-semantics/src/lineage/tests/test_sqlite.rs @@ -0,0 +1,186 @@ +//! SQLite's scope rules, and the types and names it records, as sqlite3 3.53 prepares queries. + +use sqlquack_syntax::Dialect; + +use crate::dialects::constants::SQLITE; +use crate::lineage::models::Nullability; +use crate::lineage::tests::helpers::{ + found_findings, owned_findings, sqlite_outputs, sqlite_shop_schema, +}; +use crate::lineage::tests::test_types::{QueryFindingsTestCase, SqliteOutputsTestCase}; + +#[test] +fn given_a_sqlite_query_sqlite_rejects_when_checking_then_the_broken_rule_is_found() { + let test_cases = [ + QueryFindingsTestCase { + description: "two CTEs under one name", + sql: "WITH x AS (SELECT 1 AS a), x AS (SELECT 2 AS a) SELECT a FROM x", + expected_findings: &[("E233", "Duplicate CTE name 'x'", 55)], + }, + QueryFindingsTestCase { + description: "fewer CTE column aliases than its outputs", + sql: "WITH x(a) AS (SELECT 1, 2) SELECT a FROM x", + expected_findings: &[( + "E223", + "CTE column alias count does not match its output column count", + 0, + )], + }, + QueryFindingsTestCase { + description: "a ranking function without OVER", + sql: "SELECT ROW_NUMBER() AS r FROM orders", + expected_findings: &[("E232", "Window function requires an OVER clause", 0)], + }, + QueryFindingsTestCase { + description: "a column as the LIMIT", + sql: "SELECT id FROM orders LIMIT quantity", + expected_findings: &[( + "E234", + "LIMIT/OFFSET must be a non-negative constant numeric expression", + 28, + )], + }, + QueryFindingsTestCase { + description: "UNION branches of different widths", + sql: "SELECT id, status FROM orders UNION ALL SELECT id FROM orders", + expected_findings: &[( + "E216", + "UNION operands return different column counts: left 2, right 1", + 0, + )], + }, + QueryFindingsTestCase { + description: "an unknown column", + sql: "SELECT nope FROM orders", + expected_findings: &[("E201", "Unknown column 'nope' in table 'orders'", 7)], + }, + ]; + let schema = sqlite_shop_schema(); + for test_case in test_cases { + assert_eq!( + found_findings(test_case.sql, (Dialect::SQLite, &SQLITE, &schema)), + owned_findings(test_case.expected_findings), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_sqlite_query_sqlite_accepts_when_checking_then_nothing_is_found() { + let test_cases = [ + QueryFindingsTestCase { + description: "a bare column in an aggregate query, read from any row of its group", + sql: "SELECT id, status, SUM(price) AS total FROM orders GROUP BY id", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "HAVING reads a column neither grouped nor aggregated", + sql: "SELECT status FROM orders GROUP BY id HAVING price > 1", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "two FROM items under one alias, no column read through it", + sql: "SELECT 1 AS one FROM orders AS a, customers AS a", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "a negative LIMIT, which SQLite reads as no limit", + sql: "SELECT id FROM orders LIMIT -1", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "a text column as a predicate and text plus a number", + sql: "SELECT status + 1 AS bumped FROM orders WHERE status", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "a quoted name in another case, which SQLite matches", + sql: "SELECT \"STATUS\", COUNT(*) AS n FROM orders GROUP BY status", + expected_findings: &[], + }, + ]; + let schema = sqlite_shop_schema(); + for test_case in test_cases { + assert_eq!( + found_findings(test_case.sql, (Dialect::SQLite, &SQLITE, &schema)), + owned_findings(test_case.expected_findings), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_sqlite_outputs_when_typing_then_only_one_storage_class_values_are_typed() { + let test_cases = [ + SqliteOutputsTestCase { + description: "literals have the storage class typeof() reports", + sql: "SELECT 1 AS i, 1.5 AS r, 1e3 AS e, 'a' AS t, NULL AS n", + expected_outputs: &[ + ("i", Some("INTEGER"), Nullability::NonNull), + ("r", Some("REAL"), Nullability::NonNull), + ("e", Some("REAL"), Nullability::NonNull), + ("t", Some("TEXT"), Nullability::NonNull), + ("n", None, Nullability::Nullable), + ], + }, + SqliteOutputsTestCase { + description: "casts type by affinity; NUMERIC affinity is INTEGER or REAL by value", + sql: "SELECT CAST(price AS VARCHAR(10)) AS t, CAST(status AS BIGINT) AS i, \ + CAST(id AS DOUBLE) AS r, CAST(price AS DECIMAL(10,2)) AS d, \ + CAST(status AS DATE) AS dt FROM orders", + expected_outputs: &[ + ("t", Some("TEXT"), Nullability::Unknown), + ("i", Some("INTEGER"), Nullability::Unknown), + ("r", Some("REAL"), Nullability::Unknown), + ("d", None, Nullability::Unknown), + ("dt", None, Nullability::Unknown), + ], + }, + SqliteOutputsTestCase { + description: "computed values have no one type, and a division can be NULL", + sql: "SELECT SUM(quantity) AS s, quantity / 0 AS q, UPPER(status) AS u FROM orders", + expected_outputs: &[ + ("s", None, Nullability::Unknown), + ("q", None, Nullability::Unknown), + ("u", None, Nullability::Unknown), + ], + }, + SqliteOutputsTestCase { + description: "a column keeps its declared type; an unaliased expression is unnamed", + sql: "SELECT price, quantity * 2 FROM orders", + expected_outputs: &[ + ("price", Some("REAL"), Nullability::Unknown), + ("", None, Nullability::Unknown), + ], + }, + SqliteOutputsTestCase { + description: "VALUES columns are named column1, column2", + sql: "SELECT * FROM (VALUES (1, 'a'))", + expected_outputs: &[ + ("column1", Some("INTEGER"), Nullability::NonNull), + ("column2", Some("TEXT"), Nullability::NonNull), + ], + }, + ]; + let schema = sqlite_shop_schema(); + for test_case in test_cases { + assert_eq!( + sqlite_outputs(&schema, test_case.sql), + test_case + .expected_outputs + .iter() + .map(|(name, data_type, nullability)| { + ( + (*name).to_owned(), + data_type.map(str::to_owned), + *nullability, + ) + }) + .collect::, Nullability)>>(), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack-semantics/src/lineage/tests/test_types.rs b/crates/sqlquack-semantics/src/lineage/tests/test_types.rs index 85a068f27..e7a0003e6 100644 --- a/crates/sqlquack-semantics/src/lineage/tests/test_types.rs +++ b/crates/sqlquack-semantics/src/lineage/tests/test_types.rs @@ -149,3 +149,10 @@ pub(crate) struct QueryFindingsTestCase { pub(crate) sql: &'static str, pub(crate) expected_findings: &'static [(&'static str, &'static str, usize)], } + +/// SQLite outputs: the query and each output's name, type and nullability. +pub(crate) struct SqliteOutputsTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_outputs: &'static [(&'static str, Option<&'static str>, Nullability)], +} From 864ffd6873de6410b8cee4fea9706085b590bed8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 17:09:05 +0000 Subject: [PATCH 02/10] WIP (unreviewed): SQLite tokenizer, rule for rule from SQLite 3.53.4's tokenize.c New crate sqlquack-syntax-sqlite. The scanner follows sqlite3GetToken and the WINDOW, OVER and FILTER checks of sqlite3RunParser; against a C driver built on the 3.53.4 amalgamation it splits 203,085 inputs (the SQLite catalogue, the SQLBuild SQLite models and random fuzz) into the same tokens. The recursive-descent parser over parse.y is a first draft, not yet checked against the engine. --- Cargo.lock | 9 + Cargo.toml | 2 + crates/sqlquack-syntax-sqlite/Cargo.toml | 20 + .../src/keywords/constants.rs | 229 ++++++++ .../src/keywords/main/keyword.rs | 15 + .../src/keywords/main/mod.rs | 3 + .../src/keywords/mod.rs | 5 + .../src/keywords/models.rs | 10 + crates/sqlquack-syntax-sqlite/src/lib.rs | 16 + .../src/parse/_helpers/commands/from.rs | 283 ++++++++++ .../src/parse/_helpers/commands/mod.rs | 5 + .../src/parse/_helpers/commands/queries.rs | 436 +++++++++++++++ .../src/parse/_helpers/commands/statements.rs | 67 +++ .../src/parse/_helpers/mod.rs | 5 + .../src/parse/_helpers/tree/cursor.rs | 273 +++++++++ .../src/parse/_helpers/tree/mod.rs | 5 + .../src/parse/_helpers/tree/nodes.rs | 403 ++++++++++++++ .../src/parse/_helpers/tree/spans.rs | 110 ++++ .../src/parse/_helpers/values/expressions.rs | 335 +++++++++++ .../src/parse/_helpers/values/functions.rs | 243 ++++++++ .../src/parse/_helpers/values/mod.rs | 5 + .../src/parse/_helpers/values/primary.rs | 335 +++++++++++ .../src/parse/constants.rs | 88 +++ .../src/parse/errors.rs | 30 + .../src/parse/main/mod.rs | 4 + .../src/parse/main/parse.rs | 46 ++ .../src/parse/main/parse_strict.rs | 15 + .../sqlquack-syntax-sqlite/src/parse/mod.rs | 9 + .../sqlquack-syntax-sqlite/src/parse/tests.rs | 4 + .../src/parse/tests/test_parse.rs | 83 +++ .../src/parse/tests/test_types.rs | 10 + .../src/scan/_helpers/locations.rs | 39 ++ .../src/scan/_helpers/mod.rs | 2 + .../src/scan/_helpers/scanner.rs | 523 ++++++++++++++++++ .../src/scan/constants.rs | 13 + .../src/scan/main/mod.rs | 4 + .../src/scan/main/scan.rs | 12 + .../src/scan/main/uncovered.rs | 22 + crates/sqlquack-syntax-sqlite/src/scan/mod.rs | 9 + .../sqlquack-syntax-sqlite/src/scan/models.rs | 61 ++ .../sqlquack-syntax-sqlite/src/scan/tests.rs | 4 + .../src/scan/tests/test_scan.rs | 195 +++++++ .../src/scan/tests/test_types.rs | 17 + 43 files changed, 4004 insertions(+) create mode 100644 crates/sqlquack-syntax-sqlite/Cargo.toml create mode 100644 crates/sqlquack-syntax-sqlite/src/keywords/constants.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/keywords/main/keyword.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/keywords/main/mod.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/keywords/mod.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/keywords/models.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/lib.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/parse/_helpers/commands/from.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/parse/_helpers/commands/mod.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/parse/_helpers/commands/queries.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/parse/_helpers/commands/statements.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/parse/_helpers/mod.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/parse/_helpers/tree/cursor.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/parse/_helpers/tree/mod.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/parse/_helpers/tree/nodes.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/parse/_helpers/tree/spans.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/parse/_helpers/values/expressions.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/parse/_helpers/values/functions.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/parse/_helpers/values/mod.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/parse/_helpers/values/primary.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/parse/constants.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/parse/errors.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/parse/main/mod.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/parse/main/parse.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/parse/main/parse_strict.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/parse/mod.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/parse/tests.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/parse/tests/test_parse.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/parse/tests/test_types.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/scan/_helpers/locations.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/scan/_helpers/mod.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/scan/_helpers/scanner.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/scan/constants.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/scan/main/mod.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/scan/main/scan.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/scan/main/uncovered.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/scan/mod.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/scan/models.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/scan/tests.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/scan/tests/test_scan.rs create mode 100644 crates/sqlquack-syntax-sqlite/src/scan/tests/test_types.rs diff --git a/Cargo.lock b/Cargo.lock index 96d5f578d..b8a5c5864 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1521,6 +1521,15 @@ dependencies = [ "sqlquack-syntax", ] +[[package]] +name = "sqlquack-syntax-sqlite" +version = "0.2.0" +dependencies = [ + "serde_json", + "sqlquack-syntax", + "stacker", +] + [[package]] name = "stacker" version = "0.1.25" diff --git a/Cargo.toml b/Cargo.toml index fead36671..6d9e6380d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ members = [ "crates/sqlquack-polyglot-check", "crates/sqlquack-syntax-coverage", "crates/sqlquack-syntax-postgres", + "crates/sqlquack-syntax-sqlite", "crates/sqlquack-python", ] exclude = ["fuzz"] @@ -64,6 +65,7 @@ sqlquack-snowflake = { path = "crates/sqlquack-snowflake" } sqlquack-syntax = { path = "crates/sqlquack-syntax" } sqlquack-syntax-derive = { path = "crates/sqlquack-syntax-derive" } sqlquack-syntax-postgres = { path = "crates/sqlquack-syntax-postgres" } +sqlquack-syntax-sqlite = { path = "crates/sqlquack-syntax-sqlite" } stacker = "0.1" syn = { version = "2", features = ["full"] } tempfile = "3" diff --git a/crates/sqlquack-syntax-sqlite/Cargo.toml b/crates/sqlquack-syntax-sqlite/Cargo.toml new file mode 100644 index 000000000..b8cd94e58 --- /dev/null +++ b/crates/sqlquack-syntax-sqlite/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "sqlquack-syntax-sqlite" +version.workspace = true +description = "SQLQuack's SQLite 3.53 scanner and parser into the shared sqlquack-syntax tree." +edition.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +publish = false +rust-version.workspace = true + +[dependencies] +sqlquack-syntax.workspace = true +stacker.workspace = true + +[dev-dependencies] +serde_json.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlquack-syntax-sqlite/src/keywords/constants.rs b/crates/sqlquack-syntax-sqlite/src/keywords/constants.rs new file mode 100644 index 000000000..0032ba7f8 --- /dev/null +++ b/crates/sqlquack-syntax-sqlite/src/keywords/constants.rs @@ -0,0 +1,229 @@ +//! The keyword table and the `%fallback ID` list of SQLite 3.53.4. + +/// Every keyword `mkkeywordhash.c` defines in the release build (no `WITHIN`), and its token. +pub const KEYWORDS: [(&str, &str); 147] = [ + ("ABORT", "ABORT"), + ("ACTION", "ACTION"), + ("ADD", "ADD"), + ("AFTER", "AFTER"), + ("ALL", "ALL"), + ("ALTER", "ALTER"), + ("ALWAYS", "ALWAYS"), + ("ANALYZE", "ANALYZE"), + ("AND", "AND"), + ("AS", "AS"), + ("ASC", "ASC"), + ("ATTACH", "ATTACH"), + ("AUTOINCREMENT", "AUTOINCR"), + ("BEFORE", "BEFORE"), + ("BEGIN", "BEGIN"), + ("BETWEEN", "BETWEEN"), + ("BY", "BY"), + ("CASCADE", "CASCADE"), + ("CASE", "CASE"), + ("CAST", "CAST"), + ("CHECK", "CHECK"), + ("COLLATE", "COLLATE"), + ("COLUMN", "COLUMNKW"), + ("COMMIT", "COMMIT"), + ("CONFLICT", "CONFLICT"), + ("CONSTRAINT", "CONSTRAINT"), + ("CREATE", "CREATE"), + ("CROSS", "JOIN_KW"), + ("CURRENT", "CURRENT"), + ("CURRENT_DATE", "CTIME_KW"), + ("CURRENT_TIME", "CTIME_KW"), + ("CURRENT_TIMESTAMP", "CTIME_KW"), + ("DATABASE", "DATABASE"), + ("DEFAULT", "DEFAULT"), + ("DEFERRED", "DEFERRED"), + ("DEFERRABLE", "DEFERRABLE"), + ("DELETE", "DELETE"), + ("DESC", "DESC"), + ("DETACH", "DETACH"), + ("DISTINCT", "DISTINCT"), + ("DO", "DO"), + ("DROP", "DROP"), + ("END", "END"), + ("EACH", "EACH"), + ("ELSE", "ELSE"), + ("ESCAPE", "ESCAPE"), + ("EXCEPT", "EXCEPT"), + ("EXCLUSIVE", "EXCLUSIVE"), + ("EXCLUDE", "EXCLUDE"), + ("EXISTS", "EXISTS"), + ("EXPLAIN", "EXPLAIN"), + ("FAIL", "FAIL"), + ("FILTER", "FILTER"), + ("FIRST", "FIRST"), + ("FOLLOWING", "FOLLOWING"), + ("FOR", "FOR"), + ("FOREIGN", "FOREIGN"), + ("FROM", "FROM"), + ("FULL", "JOIN_KW"), + ("GENERATED", "GENERATED"), + ("GLOB", "LIKE_KW"), + ("GROUP", "GROUP"), + ("GROUPS", "GROUPS"), + ("HAVING", "HAVING"), + ("IF", "IF"), + ("IGNORE", "IGNORE"), + ("IMMEDIATE", "IMMEDIATE"), + ("IN", "IN"), + ("INDEX", "INDEX"), + ("INDEXED", "INDEXED"), + ("INITIALLY", "INITIALLY"), + ("INNER", "JOIN_KW"), + ("INSERT", "INSERT"), + ("INSTEAD", "INSTEAD"), + ("INTERSECT", "INTERSECT"), + ("INTO", "INTO"), + ("IS", "IS"), + ("ISNULL", "ISNULL"), + ("JOIN", "JOIN"), + ("KEY", "KEY"), + ("LAST", "LAST"), + ("LEFT", "JOIN_KW"), + ("LIKE", "LIKE_KW"), + ("LIMIT", "LIMIT"), + ("MATCH", "MATCH"), + ("MATERIALIZED", "MATERIALIZED"), + ("NATURAL", "JOIN_KW"), + ("NO", "NO"), + ("NOT", "NOT"), + ("NOTHING", "NOTHING"), + ("NOTNULL", "NOTNULL"), + ("NULL", "NULL"), + ("NULLS", "NULLS"), + ("OF", "OF"), + ("OFFSET", "OFFSET"), + ("ON", "ON"), + ("OR", "OR"), + ("ORDER", "ORDER"), + ("OTHERS", "OTHERS"), + ("OUTER", "JOIN_KW"), + ("OVER", "OVER"), + ("PARTITION", "PARTITION"), + ("PLAN", "PLAN"), + ("PRAGMA", "PRAGMA"), + ("PRECEDING", "PRECEDING"), + ("PRIMARY", "PRIMARY"), + ("QUERY", "QUERY"), + ("RAISE", "RAISE"), + ("RANGE", "RANGE"), + ("RECURSIVE", "RECURSIVE"), + ("REFERENCES", "REFERENCES"), + ("REGEXP", "LIKE_KW"), + ("REINDEX", "REINDEX"), + ("RELEASE", "RELEASE"), + ("RENAME", "RENAME"), + ("REPLACE", "REPLACE"), + ("RESTRICT", "RESTRICT"), + ("RETURNING", "RETURNING"), + ("RIGHT", "JOIN_KW"), + ("ROLLBACK", "ROLLBACK"), + ("ROW", "ROW"), + ("ROWS", "ROWS"), + ("SAVEPOINT", "SAVEPOINT"), + ("SELECT", "SELECT"), + ("SET", "SET"), + ("TABLE", "TABLE"), + ("TEMP", "TEMP"), + ("TEMPORARY", "TEMP"), + ("THEN", "THEN"), + ("TIES", "TIES"), + ("TO", "TO"), + ("TRANSACTION", "TRANSACTION"), + ("TRIGGER", "TRIGGER"), + ("UNBOUNDED", "UNBOUNDED"), + ("UNION", "UNION"), + ("UNIQUE", "UNIQUE"), + ("UPDATE", "UPDATE"), + ("USING", "USING"), + ("VACUUM", "VACUUM"), + ("VALUES", "VALUES"), + ("VIEW", "VIEW"), + ("VIRTUAL", "VIRTUAL"), + ("WHEN", "WHEN"), + ("WHERE", "WHERE"), + ("WINDOW", "WINDOW"), + ("WITH", "WITH"), + ("WITHOUT", "WITHOUT"), +]; + +/// The tokens parse.y's `%fallback ID` reads as `ID` where the keyword itself has no action. +pub const FALLBACK_TOKENS: [&str; 73] = [ + "ABORT", + "ACTION", + "AFTER", + "ANALYZE", + "ASC", + "ATTACH", + "BEFORE", + "BEGIN", + "BY", + "CASCADE", + "CAST", + "COLUMNKW", + "CONFLICT", + "DATABASE", + "DEFERRED", + "DESC", + "DETACH", + "DO", + "EACH", + "END", + "EXCLUSIVE", + "EXPLAIN", + "FAIL", + "FOR", + "IGNORE", + "IMMEDIATE", + "INITIALLY", + "INSTEAD", + "LIKE_KW", + "MATCH", + "NO", + "PLAN", + "QUERY", + "KEY", + "OF", + "OFFSET", + "PRAGMA", + "RAISE", + "RECURSIVE", + "RELEASE", + "REPLACE", + "RESTRICT", + "ROW", + "ROWS", + "ROLLBACK", + "SAVEPOINT", + "TEMP", + "TRIGGER", + "VACUUM", + "VIEW", + "VIRTUAL", + "WITH", + "WITHOUT", + "NULLS", + "FIRST", + "LAST", + "CURRENT", + "FOLLOWING", + "PARTITION", + "PRECEDING", + "RANGE", + "UNBOUNDED", + "EXCLUDE", + "GROUPS", + "OTHERS", + "TIES", + "GENERATED", + "ALWAYS", + "MATERIALIZED", + "REINDEX", + "RENAME", + "CTIME_KW", + "IF", +]; diff --git a/crates/sqlquack-syntax-sqlite/src/keywords/main/keyword.rs b/crates/sqlquack-syntax-sqlite/src/keywords/main/keyword.rs new file mode 100644 index 000000000..6b22da1b8 --- /dev/null +++ b/crates/sqlquack-syntax-sqlite/src/keywords/main/keyword.rs @@ -0,0 +1,15 @@ +//! `keyword`: the keyword a word is, as `keywordCode` finds it. + +use crate::keywords::constants::{FALLBACK_TOKENS, KEYWORDS}; +use crate::keywords::models::Keyword; + +/// The keyword `word` spells, compared without case, or none when it is an identifier. +pub fn keyword(word: &str) -> Option { + KEYWORDS + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(word)) + .map(|(_, token)| Keyword { + token, + fallback: FALLBACK_TOKENS.contains(token), + }) +} diff --git a/crates/sqlquack-syntax-sqlite/src/keywords/main/mod.rs b/crates/sqlquack-syntax-sqlite/src/keywords/main/mod.rs new file mode 100644 index 000000000..8dc448961 --- /dev/null +++ b/crates/sqlquack-syntax-sqlite/src/keywords/main/mod.rs @@ -0,0 +1,3 @@ +//! Entry points. + +pub mod keyword; diff --git a/crates/sqlquack-syntax-sqlite/src/keywords/mod.rs b/crates/sqlquack-syntax-sqlite/src/keywords/mod.rs new file mode 100644 index 000000000..e43b18e53 --- /dev/null +++ b/crates/sqlquack-syntax-sqlite/src/keywords/mod.rs @@ -0,0 +1,5 @@ +//! SQLite 3.53's keywords (`tool/mkkeywordhash.c`) and which fall back to identifiers (`parse.y`). + +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlquack-syntax-sqlite/src/keywords/models.rs b/crates/sqlquack-syntax-sqlite/src/keywords/models.rs new file mode 100644 index 000000000..398388229 --- /dev/null +++ b/crates/sqlquack-syntax-sqlite/src/keywords/models.rs @@ -0,0 +1,10 @@ +//! A keyword as the tokenizer hands it to the grammar. + +/// A keyword: the grammar token it is, and whether `%fallback ID` lets it stand for a name. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Keyword { + /// The grammar token without its `TK_` prefix, such as `SELECT`, `JOIN_KW` or `CTIME_KW`. + pub token: &'static str, + /// Whether the parser reads it as `ID` where the keyword itself has no action. + pub fallback: bool, +} diff --git a/crates/sqlquack-syntax-sqlite/src/lib.rs b/crates/sqlquack-syntax-sqlite/src/lib.rs new file mode 100644 index 000000000..783de4015 --- /dev/null +++ b/crates/sqlquack-syntax-sqlite/src/lib.rs @@ -0,0 +1,16 @@ +//! SQLQuack's SQLite 3.53 scanner and parser into the shared sqlquack-syntax tree. + +#![forbid(unsafe_code)] + +pub mod keywords; +pub mod parse; +pub mod scan; + +pub use crate::keywords::main::keyword::keyword; +pub use crate::keywords::models::Keyword; +pub use crate::parse::errors::{ParseError, StrictFailure}; +pub use crate::parse::main::parse::parse; +pub use crate::parse::main::parse_strict::parse_strict; +pub use crate::scan::main::scan::scan; +pub use crate::scan::main::uncovered::uncovered; +pub use sqlquack_syntax::{Gap, Parsed}; diff --git a/crates/sqlquack-syntax-sqlite/src/parse/_helpers/commands/from.rs b/crates/sqlquack-syntax-sqlite/src/parse/_helpers/commands/from.rs new file mode 100644 index 000000000..e9c8b0934 --- /dev/null +++ b/crates/sqlquack-syntax-sqlite/src/parse/_helpers/commands/from.rs @@ -0,0 +1,283 @@ +//! `seltablist`, `joinop` and `on_using` (parse.y): the `FROM` clause. + +use sqlquack_syntax::Expression; +use sqlquack_syntax::expressions::names::models::{Identifier, TableRef}; +use sqlquack_syntax::expressions::operators::models::Alias; +use sqlquack_syntax::expressions::queries::models::{From, Join, JoinKind, Select}; + +use crate::parse::_helpers::tree::cursor::{Parser, Parsing, is_ids, is_nm}; +use crate::parse::_helpers::tree::nodes::{ + JoinConstraint, TableAlias, function, join, joined_table, subquery, table, +}; +use crate::parse::constants::{ + JOIN_WORDS, JT_CROSS, JT_INNER, JT_LEFT, JT_NATURAL, JT_OUTER, JT_RIGHT, MAX_JOIN_WORDS, +}; +use crate::scan::models::Token; + +/// How an item joins the items before it: by a comma, or by a join of a kind. +#[derive(Clone, Copy)] +enum Joiner { + Comma, + Join { + kind: JoinKind, + inner: bool, + outer: bool, + }, +} + +/// One `seltablist` item: how it joins, the item, and its `ON` or `USING`. +struct Item { + joiner: Joiner, + this: Expression, + constraint: JoinConstraint, +} + +/// The shared tree's join kind for `sqlite3JoinType`'s bits. +fn join_kind(bits: u8) -> JoinKind { + let natural: bool = bits & JT_NATURAL != 0; + let (left, right) = (bits & JT_LEFT != 0, bits & JT_RIGHT != 0); + match (natural, left, right) { + (true, true, true) => JoinKind::NaturalFull, + (true, true, false) => JoinKind::NaturalLeft, + (true, false, true) => JoinKind::NaturalRight, + (true, false, false) => JoinKind::Natural, + (false, true, true) => JoinKind::Full, + (false, true, false) => JoinKind::Left, + (false, false, true) => JoinKind::Right, + (false, false, false) if bits & JT_CROSS != 0 => JoinKind::Cross, + (false, false, false) => JoinKind::Inner, + } +} + +/// `item` under `alias`, as the shared tree keeps an aliased item of each kind. +fn aliased(item: Expression, alias: Option) -> Expression { + let Some(alias) = alias else { + return item; + }; + match item { + Expression::Table(mut relation) => { + relation.alias = Some(alias.name); + relation.alias_explicit_as = alias.explicit; + Expression::Table(relation) + } + Expression::Subquery(mut node) => { + node.alias = Some(alias.name); + node.alias_explicit_as = alias.explicit; + node.alias_keyword = alias.explicit.then(|| "AS".to_owned()); + Expression::Subquery(node) + } + Expression::JoinedTable(mut node) => { + node.alias = Some(alias.name); + Expression::JoinedTable(node) + } + Expression::Alias(mut node) => { + node.alias = alias.name; + node.alias_explicit_as = alias.explicit; + node.alias_keyword = alias.explicit.then(|| "AS".to_owned()); + Expression::Alias(node) + } + other => { + let mut node: Alias = Alias::new(other, alias.name); + node.alias_explicit_as = alias.explicit; + node.alias_keyword = alias.explicit.then(|| "AS".to_owned()); + Expression::Alias(Box::new(node)) + } + } +} + +impl Parser<'_> { + /// `FROM seltablist`: comma-joined items before any join are sources, the rest are joins. + pub(crate) fn select_from(&mut self, mut select: Box { + Box::new(Select::new()) +} + +/// `body` with `with` as its common table expressions. +fn with_ctes(mut body: Expression, with: Option) -> Expression { + match &mut body { + Expression::Select(select) => select.with = with, + Expression::Union(node) => node.with = with, + Expression::Intersect(node) => node.with = with, + Expression::Except(node) => node.with = with, + _ => {} + } + body +} + +/// `sqlite3SelectOpName` of a compound operator, as the misplaced-clause message names it. +fn operator_name(operator: &str, all: bool) -> &'static str { + match (operator, all) { + ("UNION", true) => "UNION ALL", + ("UNION", false) => "UNION", + ("INTERSECT", _) => "INTERSECT", + _ => "EXCEPT", + } +} + +impl Parser<'_> { + /// `select`: `[WITH [RECURSIVE] wqlist] selectnowith`. + pub(crate) fn select(&mut self) -> Parsing { + self.deeper(|parser| { + let with: Option = match parser.kw("WITH") { + true => Some(parser.with_clause()?), + false => None, + }; + parser.select_with(with) + }) + } + + /// `selectnowith` after its `WITH` clause, if it has one. + pub(crate) fn select_with(&mut self, with: Option) -> Parsing { + let first: usize = self.at; + let start: usize = self.offset(); + let mut term: Term = self.one_select()?; + let mut terms: usize = 1; + let mut body: Expression = term.body; + let mut last_values: bool = term.values; + while let Some((operator, all)) = self.compound_operator() { + if term.order_by.is_some() || term.limit.is_some() { + let clause: &str = match term.order_by.is_some() { + true => "ORDER BY", + false => "LIMIT", + }; + let message: String = format!( + "{clause} clause should come after {} not before", + operator_name(operator, all) + ); + return Err(self.error_with(&message, start, true)); + } + term = self.one_select()?; + terms += 1; + last_values = term.values; + body = set_operation(operator, body, term.body, (all, false)); + } + if terms > MAX_COMPOUND_SELECT && !last_values { + return Err(self.error_with(COMPOUND_TERMS, start, false)); + } + let body: Expression = finished(body, term.order_by, term.limit); + Ok(self.respanned(first, with_ctes(body, with))) + } + + /// `multiselect_op`: `UNION [ALL]`, `INTERSECT` or `EXCEPT`, consumed. + fn compound_operator(&mut self) -> Option<(&'static str, bool)> { + let operator: &'static str = match self.kind_at(0)? { + TokenKind::Keyword("UNION") => "UNION", + TokenKind::Keyword("INTERSECT") => "INTERSECT", + TokenKind::Keyword("EXCEPT") => "EXCEPT", + _ => return None, + }; + self.at += 1; + let all: bool = matches!(operator, "UNION") && self.eat_kw("ALL"); + Some((operator, all)) + } + + /// `oneselect`: a `SELECT` with its clauses, or `VALUES` rows. + fn one_select(&mut self) -> Parsing { + let first: usize = self.at; + if self.eat_kw("VALUES") { + let mut rows: Vec> = vec![self.values_row()?]; + while self.eat_op("COMMA") { + rows.push(self.values_row()?); + } + return Ok(Term { + body: self.respanned(first, values(rows, None)), + order_by: None, + limit: None, + values: true, + }); + } + self.expect_kw("SELECT")?; + let mut select: Box