feat(rules): read custom-rule SQL syntax trees from SQLQuack parsers

crates/sqlbuild-analysis/src/assembly/analysis_session/main/prove_dynamic_contracts.rs

@@ -5,15 +5,11 @@ use sqlbuild_core::constants::PANIC_MESSAGE;
 use crate::assembly::analysis_session::_helpers::dynamic_pivot::pivot_outcomes;
 use crate::assembly::analysis_session::constants::PIVOT_WORKER_STACK_BYTES;
 use crate::assembly::analysis_session::models::{PivotBatchRequest, PivotOutcome};
+use crate::worker_stack::main::run_on_worker_stack::run_on_worker_stack;
 
 /// Each model's proof in order; an internal failure of any proof fails them all.
 pub fn prove_dynamic_contracts(request: &PivotBatchRequest) -> Result<Vec<PivotOutcome>, String> {
-    std::thread::scope(|scope| {
-        std::thread::Builder::new()
-            .stack_size(PIVOT_WORKER_STACK_BYTES)
-            .spawn_scoped(scope, || pivot_outcomes(&request.models, &request.tables))
-            .map_err(|error| error.to_string())?
-            .join()
-            .map_err(|_| PANIC_MESSAGE.to_owned())?
-    })
+    run_on_worker_stack(PIVOT_WORKER_STACK_BYTES, PANIC_MESSAGE, || {
+        pivot_outcomes(&request.models, &request.tables)
+    })?
 }

crates/sqlbuild-analysis/src/assembly/analysis_session/main/recover_cte_facts.rs

@@ -5,17 +5,15 @@ use crate::assembly::analysis_session::_helpers::cte_facts::{
 };
 use crate::assembly::analysis_session::constants::CTE_FACT_WORKER_STACK_BYTES;
 use crate::assembly::analysis_session::models::{CteFactRequest, CteFacts};
+use crate::worker_stack::main::run_on_worker_stack::run_on_worker_stack;
 
 /// The facts enrichment recovers for the query, or why they are left to Python.
 pub fn recover_cte_facts(request: &CteFactRequest) -> Result<CteFacts, String> {
-    std::thread::scope(|scope| {
-        std::thread::Builder::new()
-            .stack_size(CTE_FACT_WORKER_STACK_BYTES)
-            .spawn_scoped(scope, || recovered_facts(request))
-            .map_err(|error| error.to_string())?
-            .join()
-            .map_err(|_| "native CTE fact recovery panicked".to_owned())?
-    })
+    run_on_worker_stack(
+        CTE_FACT_WORKER_STACK_BYTES,
+        "native CTE fact recovery panicked",
+        || recovered_facts(request),
+    )?
 }
 
 fn recovered_facts(request: &CteFactRequest) -> Result<CteFacts, String> {

crates/sqlbuild-analysis/src/lib.rs

@@ -48,5 +48,7 @@ pub mod snowflake_cost;
 pub mod sql_macros;
 pub mod sql_test_attachment;
 pub mod sql_tokens;
+pub mod sql_tree;
 pub mod target_paths;
 pub mod type_system;
+pub mod worker_stack;

crates/sqlbuild-analysis/src/sql_tree/_helpers/clauses.rs

@@ -0,0 +1,67 @@
+//! Join and CTE clauses, which SQLQuack keeps as statement fields rather than expressions.
+
+use sqlquack_syntax::Expression;
+use sqlquack_syntax::expressions::queries::models::{Cte, Join, JoinKind, With};
+use sqlquack_syntax::tokens::models::Span;
+
+use crate::sql_tree::constants::{CTE_KIND, JOIN_KIND};
+
+/// One clause of a statement: its kind, its span, and the expressions it owns.
+pub(crate) struct Clause<'ast> {
+    pub(crate) kind: &'static str,
+    pub(crate) span: Option<Span>,
+    pub(crate) children: Vec<&'ast Expression>,
+}
+
+/// The CTEs and explicit joins `expression` holds directly, CTEs first.
+pub(crate) fn clauses(expression: &Expression) -> Vec<Clause<'_>> {
+    let (with, joins): (Option<&With>, Vec<&[Join]>) = match expression {
+        Expression::Select(select) => (select.with.as_ref(), vec![select.joins.as_slice()]),
+        Expression::Union(union) => (union.with.as_ref(), Vec::new()),
+        Expression::Intersect(intersect) => (intersect.with.as_ref(), Vec::new()),
+        Expression::Except(except) => (except.with.as_ref(), Vec::new()),
+        Expression::Insert(insert) => (insert.with.as_ref(), Vec::new()),
+        Expression::Update(update) => (
+            update.with.as_ref(),
+            vec![update.table_joins.as_slice(), update.from_joins.as_slice()],
+        ),
+        Expression::Delete(delete) => (delete.with.as_ref(), vec![delete.joins.as_slice()]),
+        Expression::JoinedTable(joined) => (None, vec![joined.joins.as_slice()]),
+        _ => (None, Vec::new()),
+    };
+    let ctes = with
+        .into_iter()
+        .flat_map(|with| with.ctes.iter())
+        .map(cte_clause);
+    let joins = joins
+        .into_iter()
+        .flatten()
+        .filter(|join| !is_comma_item(join))
+        .map(join_clause);
+    ctes.chain(joins).collect()
+}
+
+fn cte_clause(cte: &Cte) -> Clause<'_> {
+    Clause {
+        kind: CTE_KIND,
+        span: cte.span.0,
+        children: vec![&cte.this],
+    }
+}
+
+fn join_clause(join: &Join) -> Clause<'_> {
+    let mut children: Vec<&Expression> = vec![&join.this];
+    children.extend(join.on.as_ref());
+    children.extend(join.match_condition.as_ref());
+    children.extend(join.pivots.iter());
+    Clause {
+        kind: JOIN_KIND,
+        span: join.span.0,
+        children,
+    }
+}
+
+/// A comma-separated FROM item after an explicit join, which the shared parser keeps as a join.
+fn is_comma_item(join: &Join) -> bool {
+    join.kind == JoinKind::Implicit
+}

crates/sqlbuild-analysis/src/sql_tree/_helpers/mod.rs

@@ -0,0 +1,7 @@
+//! Parser selection, parsing, the tree walk, clause nodes and source positions.
+
+pub(crate) mod clauses;
+pub(crate) mod parse;
+pub(crate) mod positions;
+pub(crate) mod rules_tree_parser;
+pub(crate) mod walk;

crates/sqlbuild-analysis/src/sql_tree/_helpers/parse.rs

@@ -0,0 +1,44 @@
+//! Parse SQL into statements with node spans.
+
+use sqlquack_syntax::{CallMode, Expression, ParseOptions};
+
+use crate::sql_tree::constants::GUARD_CODE_PREFIX;
+use crate::sql_tree::errors::SqlTreeError;
+use crate::sql_tree::models::TreeParser;
+
+/// The statements of `sql` under `parser`; a non-syntax PostgreSQL failure or a skipped statement is unanalysable.
+pub(crate) fn parse(sql: &str, parser: TreeParser) -> Result<Vec<Expression>, SqlTreeError> {
+    match parser {
+        TreeParser::Shared(dialect) => sqlquack_syntax::parse_statements_with(
+            sql,
+            dialect,
+            &ParseOptions::default(),
+            CallMode::FaithfulLocated,
+        )
+        .map_err(|error| shared_error(&error.to_string())),
+        TreeParser::Postgres => {
+            let parsed = sqlquack_syntax_postgres::parse_strict(sql).map_err(|failure| {
+                match failure.parse_error() {
+                    Some(error) if error.syntax => SqlTreeError::Parse(error.message.clone()),
+                    Some(error) => SqlTreeError::Unanalysable(error.message.clone()),
+                    None => SqlTreeError::Parse("SQL has text outside any token".to_owned()),
+                }
+            })?;
+            if !parsed.gaps.is_empty() {
+                return Err(SqlTreeError::Unanalysable(
+                    "a statement uses syntax the PostgreSQL parser skips".to_owned(),
+                ));
+            }
+            Ok(parsed.statements)
+        }
+    }
+}
+
+/// A shared-parser failure: a guard limit, such as nesting depth, is unanalysable, not a syntax error.
+fn shared_error(message: &str) -> SqlTreeError {
+    if message.contains(GUARD_CODE_PREFIX) {
+        SqlTreeError::Unanalysable(message.to_owned())
+    } else {
+        SqlTreeError::Parse(message.to_owned())
+    }
+}

crates/sqlbuild-analysis/src/sql_tree/_helpers/positions.rs

@@ -0,0 +1,19 @@
+//! Character offsets as 1-based lines and columns.
+
+/// The character offset at which each line of `sql` starts.
+pub(crate) fn line_starts(sql: &str) -> Vec<usize> {
+    let mut starts: Vec<usize> = vec![0];
+    for (offset, character) in sql.chars().enumerate() {
+        if character == '\n' {
+            starts.push(offset + 1);
+        }
+    }
+    starts
+}
+
+/// The 1-based line and column of the character at `offset`.
+pub(crate) fn line_column(starts: &[usize], offset: usize) -> (usize, usize) {
+    let line: usize = starts.partition_point(|start| *start <= offset);
+    let start: usize = starts.get(line.saturating_sub(1)).copied().unwrap_or(0);
+    (line.max(1), offset - start + 1)
+}

crates/sqlbuild-analysis/src/sql_tree/_helpers/rules_tree_parser.rs

@@ -0,0 +1,23 @@
+//! `rules_tree_parser`: the parser that reads a dialect's SQL for rules.
+
+use sqlquack_syntax::Dialect;
+
+use crate::sql_tree::constants::{
+    DUCKDB_DIALECTS, POSTGRES_DIALECTS, SNOWFLAKE_DIALECT, SQLITE_DIALECT,
+};
+use crate::sql_tree::models::TreeParser;
+
+/// The parser for `dialect`, or `None`; add a dialect (one line) only once its parser passes the depth probe with an empty allowlist.
+pub(crate) fn rules_tree_parser(dialect: &str) -> Option<TreeParser> {
+    if POSTGRES_DIALECTS.contains(&dialect) {
+        return Some(TreeParser::Postgres);
+    }
+    if DUCKDB_DIALECTS.contains(&dialect) {
+        return Some(TreeParser::Shared(Dialect::DuckDB));
+    }
+    match dialect {
+        SNOWFLAKE_DIALECT => Some(TreeParser::Shared(Dialect::Snowflake)),
+        SQLITE_DIALECT => Some(TreeParser::Shared(Dialect::SQLite)),
+        _ => None,
+    }
+}

crates/sqlbuild-analysis/src/sql_tree/_helpers/walk.rs

@@ -0,0 +1,102 @@
+//! The iterative pre-order walk that turns parsed statements into rule tree nodes.
+
+use std::collections::HashMap;
+
+use sqlquack_syntax::tokens::models::Span;
+use sqlquack_syntax::{Expression, ExpressionWalk};
+
+use crate::sql_tree::_helpers::clauses::clauses;
+use crate::sql_tree::_helpers::positions::line_column;
+use crate::sql_tree::constants::OPAQUE_KINDS;
+use crate::sql_tree::errors::SqlTreeError;
+use crate::sql_tree::models::SqlTreeNode;
+
+/// Child expressions mapped to the index of the clause node that owns them.
+type Owners = HashMap<*const Expression, usize>;
+
+/// The nodes of `statements` in pre-order, walked with an explicit stack so depth cannot overflow.
+pub(crate) fn walk(
+    statements: &[Expression],
+    starts: &[usize],
+) -> Result<Vec<SqlTreeNode>, SqlTreeError> {
+    let mut nodes: Vec<SqlTreeNode> = Vec::new();
+    let mut stack: Vec<(&Expression, Option<usize>)> = statements
+        .iter()
+        .rev()
+        .map(|statement| (statement, None))
+        .collect();
+    while let Some((expression, parent)) = stack.pop() {
+        let index: usize = nodes.len();
+        let node: SqlTreeNode = expression_node(&nodes, expression, parent, starts)?;
+        let (clause_nodes, owners) = clause_nodes(expression, &node, index, starts);
+        nodes.push(node);
+        nodes.extend(clause_nodes);
+        for child in expression.children().into_iter().rev() {
+            let owner: usize = owners
+                .get(&std::ptr::from_ref(child))
+                .copied()
+                .unwrap_or(index);
+            stack.push((child, Some(owner)));
+        }
+    }
+    Ok(nodes)
+}
+
+/// `expression`'s node, at its own start or else at its parent's.
+fn expression_node(
+    nodes: &[SqlTreeNode],
+    expression: &Expression,
+    parent: Option<usize>,
+    starts: &[usize],
+) -> Result<SqlTreeNode, SqlTreeError> {
+    let kind: &'static str = expression.variant_name();
+    if OPAQUE_KINDS.contains(&kind) {
+        return Err(SqlTreeError::Unanalysable(
+            "a statement uses syntax the parser keeps as unparsed text".to_owned(),
+        ));
+    }
+    let inherited = parent.map_or((1, 1), |parent| (nodes[parent].line, nodes[parent].column));
+    let (line, column) =
+        own_span(expression).map_or(inherited, |span| line_column(starts, span.start));
+    Ok(SqlTreeNode {
+        kind,
+        line,
+        column,
+        parent,
+    })
+}
+
+/// The clause nodes `expression` holds, numbered after `index`, and the children each one owns.
+fn clause_nodes(
+    expression: &Expression,
+    node: &SqlTreeNode,
+    index: usize,
+    starts: &[usize],
+) -> (Vec<SqlTreeNode>, Owners) {
+    let mut built: Vec<SqlTreeNode> = Vec::new();
+    let mut owners: Owners = HashMap::new();
+    for clause in clauses(expression) {
+        let clause_index: usize = index + 1 + built.len();
+        let (line, column) = clause.span.map_or((node.line, node.column), |span| {
+            line_column(starts, span.start)
+        });
+        built.push(SqlTreeNode {
+            kind: clause.kind,
+            line,
+            column,
+            parent: Some(index),
+        });
+        for child in clause.children {
+            let _ = owners.insert(std::ptr::from_ref(child), clause_index);
+        }
+    }
+    (built, owners)
+}
+
+/// The node's span; a qualified star without one starts at its qualifier.
+fn own_span(expression: &Expression) -> Option<Span> {
+    expression.source_span().or_else(|| match expression {
+        Expression::Star(star) => star.table.as_ref().and_then(|table| table.span),
+        _ => None,
+    })
+}

crates/sqlbuild-analysis/src/sql_tree/constants.rs

@@ -0,0 +1,20 @@
+//! Node kinds the rule tree adds to SQLQuack's expression variants, and kinds it refuses.
+
+/// The kind of a join clause, which SQLQuack keeps as a field of its statement.
+pub(crate) const JOIN_KIND: &str = "join";
+/// The kind of a common table expression, which SQLQuack keeps as a field of its statement.
+pub(crate) const CTE_KIND: &str = "cte";
+/// Kinds that hold a statement the parser kept as unparsed text, so no rule can see inside it.
+pub(crate) const OPAQUE_KINDS: [&str; 2] = ["command", "raw"];
+/// Dialect names read with the shared DuckDB grammar; `generic` matches the fingerprint tokenizer.
+pub(crate) const DUCKDB_DIALECTS: [&str; 3] = ["duckdb", "motherduck", "generic"];
+/// Dialect names read with the native PostgreSQL parser.
+pub(crate) const POSTGRES_DIALECTS: [&str; 2] = ["postgres", "postgresql"];
+/// The Snowflake dialect name.
+pub(crate) const SNOWFLAKE_DIALECT: &str = "snowflake";
+/// The SQLite dialect name.
+pub(crate) const SQLITE_DIALECT: &str = "sqlite";
+/// Stack for the parse-and-walk worker, so deep SQL reaches the parser's depth limit, not a stack overflow.
+pub(crate) const TREE_WORKER_STACK_BYTES: usize = 16 * 1024 * 1024;
+/// The prefix of the shared parser's guard codes, such as `E_GUARD_NESTING_DEPTH_EXCEEDED`.
+pub(crate) const GUARD_CODE_PREFIX: &str = "E_GUARD_";

crates/sqlbuild-analysis/src/sql_tree/errors.rs

@@ -0,0 +1,34 @@
+//! Why a rule cannot have a SQL syntax tree.
+
+use std::fmt::{self, Display};
+
+/// A dialect SQLQuack does not parse, SQL it rejects, or SQL it read only as opaque text.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum SqlTreeError {
+    /// No SQLQuack parser reads this dialect yet.
+    UnavailableDialect(String),
+    /// The parser read the SQL but kept part of it as unparsed text or refused its depth.
+    Unanalysable(String),
+    /// The parser rejected the SQL.
+    Parse(String),
+    /// The parse worker could not start or panicked.
+    Worker(String),
+}
+
+impl Display for SqlTreeError {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::UnavailableDialect(dialect) => write!(
+                formatter,
+                "SQL syntax trees are unavailable for the {dialect} dialect: SQLQuack does not parse it yet"
+            ),
+            Self::Unanalysable(reason) => {
+                write!(formatter, "SQL syntax tree is unavailable: {reason}")
+            }
+            Self::Parse(message) => write!(formatter, "SQL could not be parsed: {message}"),
+            Self::Worker(message) => write!(formatter, "SQL syntax tree worker failed: {message}"),
+        }
+    }
+}
+
+impl std::error::Error for SqlTreeError {}

crates/sqlbuild-analysis/src/sql_tree/main/mod.rs

@@ -0,0 +1,3 @@
+//! Entry points.
+
+pub mod sql_tree_nodes;