Tokenizer and depth fixes, iterative tree drop, account and property quoting, interval qualifier precision, anonymous procedure gaps

crates/sqlquack-core/src/diagnostics/tests/test_render.rs

@@ -102,6 +102,18 @@ fn given_guard_failures_when_diagnosed_then_each_guard_has_its_own_code_and_form
             expected_code: ("SQ0001", None),
             expected_label: "Unterminated string",
         },
+        GuardTestCase {
+            description: "an unterminated block comment is refused with the parse code",
+            source: "SELECT 1 /* never closed".to_owned(),
+            expected_code: ("SQ0001", None),
+            expected_label: "Unterminated block comment",
+        },
+        GuardTestCase {
+            description: "50,000 added terms are one flat chain, not 50,000 levels of depth",
+            source: format!("SELECT {} FROM", vec!["1"; 50_000].join(" + ")),
+            expected_code: ("SQ0001", None),
+            expected_label: "Expected table name or subquery, got From",
+        },
     ];
     for test_case in test_cases {
         let diagnostics = diagnose(&[unparsed(&test_case.source)], &test_case.source);

crates/sqlquack-syntax-derive/src/ast_node/_helpers/mod.rs

@@ -3,4 +3,5 @@
 pub(crate) mod field_types;
 pub(crate) mod naming;
 pub(crate) mod spans;
+pub(crate) mod takers;
 pub(crate) mod visitors;

crates/sqlquack-syntax-derive/src/ast_node/_helpers/takers.rs

@@ -0,0 +1,124 @@
+//! Takers: move a node's direct expression children out so a tree can be freed without recursion.
+
+use quote::format_ident;
+use quote::quote;
+use syn::Data;
+use syn::DeriveInput;
+use syn::Field;
+use syn::Fields;
+use syn::Type;
+use syn::Variant;
+
+use crate::ast_node::_helpers::field_types::container_inner;
+use crate::ast_node::_helpers::field_types::is_expression;
+use crate::ast_node::_helpers::field_types::is_scalar;
+use crate::ast_node::_helpers::field_types::is_skipped;
+
+/// The body of `take_expressions`, which moves every direct expression child into `out`.
+pub(crate) fn node_taker(input: &DeriveInput) -> proc_macro2::TokenStream {
+    match &input.data {
+        Data::Struct(data) => struct_takes(&data.fields),
+        Data::Enum(data) => {
+            let mut arms: Vec<proc_macro2::TokenStream> = Vec::new();
+            for variant in &data.variants {
+                arms.push(variant_arm(&input.ident, variant));
+            }
+            quote!(match self { #(#arms),* })
+        }
+        Data::Union(_) => quote!(),
+    }
+}
+
+/// Takes from every traversed field of a struct.
+fn struct_takes(fields: &Fields) -> proc_macro2::TokenStream {
+    let mut takes: Vec<proc_macro2::TokenStream> = Vec::new();
+    for (index, field) in fields.iter().enumerate() {
+        if is_skipped(field) {
+            continue;
+        }
+        let access = match &field.ident {
+            Some(ident) => quote!(&mut self.#ident),
+            None => {
+                let index = syn::Index::from(index);
+                quote!(&mut self.#index)
+            }
+        };
+        takes.push(take(&field.ty, access));
+    }
+    quote!(#(#takes)*)
+}
+
+/// Whether a field can hold expressions and is traversed.
+fn is_kept(field: &Field) -> bool {
+    !is_skipped(field) && !is_scalar(&field.ty)
+}
+
+/// One match arm that binds an enum variant's kept fields and takes from them.
+fn variant_arm(enum_name: &syn::Ident, variant: &Variant) -> proc_macro2::TokenStream {
+    let variant_name = &variant.ident;
+    let mut bindings: Vec<proc_macro2::TokenStream> = Vec::new();
+    let mut takes: Vec<proc_macro2::TokenStream> = Vec::new();
+    for (index, field) in variant.fields.iter().enumerate() {
+        let binding = match &field.ident {
+            Some(ident) => ident.clone(),
+            None => format_ident!("field_{index}"),
+        };
+        let kept = is_kept(field);
+        bindings.push(match (&field.ident, kept) {
+            (Some(ident), true) => quote!(#ident),
+            (Some(ident), false) => quote!(#ident: _),
+            (None, true) => quote!(#binding),
+            (None, false) => quote!(_),
+        });
+        if kept {
+            takes.push(take(&field.ty, quote!(#binding)));
+        }
+    }
+    match &variant.fields {
+        Fields::Named(_) => quote!(#enum_name::#variant_name { #(#bindings),* } => { #(#takes)* }),
+        Fields::Unnamed(_) => quote!(#enum_name::#variant_name(#(#bindings),*) => { #(#takes)* }),
+        Fields::Unit => quote!(#enum_name::#variant_name => {}),
+    }
+}
+
+/// Moves the expressions in a field reached through the `&mut` place `access` into `out`.
+fn take(ty: &Type, access: proc_macro2::TokenStream) -> proc_macro2::TokenStream {
+    if is_expression(ty) {
+        return quote! {
+            {
+                let slot: &mut crate::expressions::tree::models::Expression = #access;
+                if !matches!(slot, crate::expressions::tree::models::Expression::Null(_)) {
+                    out.push(::core::mem::replace(
+                        slot,
+                        crate::expressions::tree::models::Expression::Null(
+                            crate::expressions::names::models::Null,
+                        ),
+                    ));
+                }
+            }
+        };
+    }
+    if let Some(inner) = container_inner(ty, "Option") {
+        let take = take(inner, quote!(value));
+        return quote!(if let Some(value) = (#access).as_mut() { #take });
+    }
+    if let Some(inner) = container_inner(ty, "Box") {
+        return take(inner, quote!(&mut **(#access)));
+    }
+    if let Some(inner) = container_inner(ty, "Vec") {
+        let take = take(inner, quote!(value));
+        return quote!(for value in (#access).iter_mut() { #take });
+    }
+    if let Type::Tuple(tuple) = ty {
+        let mut takes: Vec<proc_macro2::TokenStream> = Vec::new();
+        for (index, element) in tuple.elems.iter().enumerate() {
+            let index = syn::Index::from(index);
+            takes.push(take(element, quote!(&mut (#access).#index)));
+        }
+        return quote!(#(#takes)*);
+    }
+    if is_scalar(ty) {
+        return quote!();
+    }
+    quote!(crate::traversal::types::AstNode::take_expressions(#access, out);)
+}

crates/sqlquack-syntax-derive/src/ast_node/main/expand_ast_node.rs

@@ -6,6 +6,7 @@ use syn::DeriveInput;
 
 use crate::ast_node::_helpers::naming::serde_snake_case;
 use crate::ast_node::_helpers::spans::span_methods;
+use crate::ast_node::_helpers::takers::node_taker;
 use crate::ast_node::_helpers::visitors::node_visitor;
 use crate::constants::EXPRESSION_TYPE;
 
@@ -14,6 +15,7 @@ pub(crate) fn expand_ast_node(input: &DeriveInput) -> proc_macro2::TokenStream {
     let immutable = node_visitor(input, true);
     let untracked = node_visitor(input, false);
     let spans = span_methods(input);
+    let taker = node_taker(input);
     let serialized_variant_names = if name == EXPRESSION_TYPE {
         if let Data::Enum(data) = &input.data {
             let names = data.variants.iter().map(|variant| {
@@ -58,6 +60,13 @@ pub(crate) fn expand_ast_node(input: &DeriveInput) -> proc_macro2::TokenStream {
                 #untracked
             }
 
+            fn take_expressions(
+                &mut self,
+                out: &mut Vec<crate::expressions::tree::models::Expression>,
+            ) {
+                #taker
+            }
+
             #spans
         }
 

crates/sqlquack-syntax/src/expressions/operators/_helpers/boolean_chains.rs

@@ -44,13 +44,9 @@ pub(crate) fn flatten_boolean_terms_owned(expr: Expression, is_and: bool) -> Vec
 
     while let Some(node) = stack.pop() {
         match (is_and, node) {
-            (true, Expression::And(op)) => {
-                stack.push(op.right);
-                stack.push(op.left);
-            }
-            (false, Expression::Or(op)) => {
-                stack.push(op.right);
-                stack.push(op.left);
+            (true, Expression::And(mut op)) | (false, Expression::Or(mut op)) => {
+                stack.push(std::mem::replace(&mut op.right, Expression::Null(Null)));
+                stack.push(std::mem::replace(&mut op.left, Expression::Null(Null)));
             }
             (_, other) => terms.push(other),
         }

crates/sqlquack-syntax/src/expressions/tree/_helpers/drops.rs

@@ -0,0 +1,54 @@
+//! Iterative drops for nodes that the parser chains in a loop, such as `x::INT::INT` or `v[1][1]`.
+
+use crate::expressions::operators::models::Alias;
+use crate::expressions::operators::models::Between;
+use crate::expressions::operators::models::BinaryOp;
+use crate::expressions::operators::models::Cast;
+use crate::expressions::operators::models::CollationExpr;
+use crate::expressions::operators::models::DotAccess;
+use crate::expressions::operators::models::In;
+use crate::expressions::operators::models::IsNull;
+use crate::expressions::operators::models::IsTrueFalse;
+use crate::expressions::operators::models::LikeOp;
+use crate::expressions::operators::models::Subscript;
+use crate::expressions::operators::models::UnaryOp;
+use crate::expressions::tree::models::Expression;
+use crate::expressions::typed_functions::models::JsonExtractFunc;
+use crate::traversal::types::AstNode;
+
+/// Frees a node's expression descendants with a worklist, so dropping a long chain cannot overflow the stack.
+fn drop_descendants(node: &mut impl AstNode) {
+    let mut pending: Vec<Expression> = Vec::new();
+    node.take_expressions(&mut pending);
+    while let Some(mut expression) = pending.pop() {
+        expression.take_expressions(&mut pending);
+    }
+}
+
+macro_rules! drop_iteratively {
+    ($($node:ty),* $(,)?) => {
+        $(
+            impl Drop for $node {
+                fn drop(&mut self) {
+                    drop_descendants(self);
+                }
+            }
+        )*
+    };
+}
+
+drop_iteratively!(
+    Alias,
+    Between,
+    BinaryOp,
+    Cast,
+    CollationExpr,
+    DotAccess,
+    In,
+    IsNull,
+    IsTrueFalse,
+    JsonExtractFunc,
+    LikeOp,
+    Subscript,
+    UnaryOp,
+);

crates/sqlquack-syntax/src/expressions/tree/_helpers/mod.rs

@@ -2,3 +2,4 @@
 
 pub(crate) mod accessors;
 pub(crate) mod api;
+pub(crate) mod drops;

crates/sqlquack-syntax/src/expressions/types/models.rs

@@ -464,6 +464,21 @@ pub enum IntervalUnitSpec {
     ExprSpan(IntervalSpanExpr),
     /// Expression as unit (e.g., CURRENT_DATE, CAST(GETDATE() AS DATE))
     Expr(Box<Expression>),
+    /// Snowflake qualifier with a field precision, such as `YEAR(9) TO MONTH` or `SECOND(9), (3)`.
+    Qualifier(IntervalQualifier),
+}
+
+/// Snowflake interval qualifier: fields with their leading and fractional seconds precisions.
+#[derive(sqlquack_syntax_derive::AstNode, Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct IntervalQualifier {
+    /// Leading field (e.g., DAY)
+    pub this: IntervalUnit,
+    /// Leading field precision, as 9 in `DAY(9)`
+    pub leading_precision: Option<u32>,
+    /// Trailing field after TO (e.g., SECOND)
+    pub expression: Option<IntervalUnit>,
+    /// Fractional seconds precision, as 3 in `DAY TO SECOND(3)` or `SECOND(9), (3)`
+    pub fractional_precision: Option<u32>,
 }
 
 /// Interval span for ranges like HOUR TO SECOND

crates/sqlquack-syntax/src/guard/_helpers/ast.rs

@@ -72,17 +72,47 @@ pub(crate) fn push_ast_children<'a>(
             stack = push_connector_child(stack, &op.right, depth, ConnectorKind::Or);
             stack = push_connector_child(stack, &op.left, depth, ConnectorKind::Or);
         }
-        _ => {
-            let child_start = stack.len();
-            crate::traversal::main::for_each_child::for_each_child(expr, |_, child| {
-                stack.push((child, depth + 1));
-            });
-            stack[child_start..].reverse();
-        }
+        _ => match chain_operator(expr).filter(|op| is_commentless_binary_op(op)) {
+            Some(op) => {
+                stack.push((&op.right, depth + 1));
+                let left_depth = match chain_operator(&op.left) {
+                    Some(left) if is_commentless_binary_op(left) => depth,
+                    _ => depth + 1,
+                };
+                stack.push((&op.left, left_depth));
+            }
+            None => {
+                let child_start = stack.len();
+                crate::traversal::main::for_each_child::for_each_child(expr, |_, child| {
+                    stack.push((child, depth + 1));
+                });
+                stack[child_start..].reverse();
+            }
+        },
     }
     stack
 }
 
+/// A left-associative operator whose flat chain, such as `1 + 1 + 1`, is one level, not nesting.
+pub(crate) fn chain_operator(
+    expr: &Expression,
+) -> Option<&crate::expressions::operators::models::BinaryOp> {
+    match expr {
+        Expression::Add(op)
+        | Expression::Sub(op)
+        | Expression::Mul(op)
+        | Expression::Div(op)
+        | Expression::Mod(op)
+        | Expression::Concat(op)
+        | Expression::BitwiseAnd(op)
+        | Expression::BitwiseOr(op)
+        | Expression::BitwiseXor(op)
+        | Expression::BitwiseLeftShift(op)
+        | Expression::BitwiseRightShift(op) => Some(op),
+        _ => None,
+    }
+}
+
 pub(crate) fn push_connector_child<'a>(
     mut stack: Vec<(&'a Expression, usize)>,
     child: &'a Expression,

crates/sqlquack-syntax/src/guard/tests.rs

@@ -1,5 +1,7 @@
 #[path = "tests/helpers.rs"]
 mod helpers;
+#[path = "tests/test_ast_guard.rs"]
+mod test_ast_guard;
 #[path = "tests/test_token_guard.rs"]
 mod test_token_guard;
 #[path = "tests/test_types.rs"]

crates/sqlquack-syntax/src/guard/tests/helpers.rs

@@ -61,3 +61,15 @@ pub(crate) fn limit_json_problems(field: &str) -> Vec<String> {
 pub(crate) fn guard_json_accepted(json: &str) -> bool {
     serde_json::from_str::<ComplexityGuardOptions>(json).is_ok()
 }
+
+/// The default-guarded Snowflake parse of `sql`, or its error message.
+pub(crate) fn guarded_parse(sql: &str) -> Result<(), String> {
+    crate::parse_statements_with(
+        sql,
+        crate::Dialect::Snowflake,
+        &crate::ParseOptions::default(),
+        crate::CallMode::Faithful,
+    )
+    .map(|_| ())
+    .map_err(|error| error.to_string())
+}