feat: parse rule facts, types and syntax trees with SQLQuack

Cargo.toml

@@ -176,9 +176,10 @@ sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" }
 sqlbuild-state = { path = "crates/sqlbuild-state" }
 # SQL lint Rules, quality Rules and the formatter.
 sqlclean = { git = "https://github.com/sqlstack-io/sqlclean.git", rev = "cf79656d247277a30bc81ef39b6dee420e721221" }
-# The lossless SQL tokenizer behind query fingerprints.
-sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "4d0c6bc56da0e44eb164601a2af673ae7998257d" }
-sqlquack-syntax-postgres = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "4d0c6bc56da0e44eb164601a2af673ae7998257d" }
+# SQLQuack's lossless tokenizer behind query fingerprints and its parsers for warehouse types.
+sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "edfa30b26a0bab86661f96490b86a4f24b92db0a" }
+sqlquack-syntax-bigquery = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "edfa30b26a0bab86661f96490b86a4f24b92db0a" }
+sqlquack-syntax-postgres = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "edfa30b26a0bab86661f96490b86a4f24b92db0a" }
 stacker = "0.1"
 syn = { version = "2", features = ["full", "visit", "extra-traits"] }
 subtle = "2"

crates/sqlbuild-analysis/Cargo.toml

@@ -27,7 +27,9 @@ sqlbuild-scopes.workspace = true
 sqlbuild-sqltext.workspace = true
 sqlclean.workspace = true
 sqlquack-syntax.workspace = true
+sqlquack-syntax-bigquery.workspace = true
 sqlquack-syntax-postgres.workspace = true
+stacker.workspace = true
 
 [lints]
 workspace = true

crates/sqlbuild-analysis/src/contracts/tests/test_evaluate_model_contracts.rs

@@ -282,11 +282,11 @@ fn given_models_sharing_types_when_evaluating_together_then_each_matches_its_own
 }
 
 #[test]
-fn given_a_dialect_polyglot_does_not_know_when_comparing_types_then_python_error_is_returned() {
+fn given_a_dialect_no_adapter_ships_when_comparing_types_then_python_error_is_returned() {
     let test_cases = [UnknownDialectTestCase {
         description: "an enforced contract compared under an unknown dialect",
-        dialect: "motherduck",
-        expected_error: "Unknown dialect: motherduck",
+        dialect: "mysql",
+        expected_error: "Unknown dialect: mysql",
     }];
     for test_case in test_cases {
         let request = ContractRequest {

crates/sqlbuild-analysis/src/lib.rs

@@ -37,6 +37,7 @@ pub mod python_plan;
 pub mod python_selection;
 pub mod python_values;
 pub mod query_analysis;
+pub mod relation_markers;
 pub mod relation_names;
 pub mod required_columns;
 pub mod resource_diagnostics;
@@ -51,4 +52,5 @@ pub mod sql_tokens;
 pub mod sql_tree;
 pub mod target_paths;
 pub mod type_system;
+pub mod warehouse_types;
 pub mod worker_stack;

crates/sqlbuild-analysis/src/planner_strategy/main/classify_cursor_sql_type.rs

@@ -7,9 +7,9 @@ use crate::planner_strategy::_helpers::type_names::{argument, named_type};
 use crate::planner_strategy::constants::{GENERIC_DIALECT, POLYGLOT_CUSTOM_TYPE_NAME};
 use crate::planner_strategy::errors::TypeParseFailure;
 use crate::planner_strategy::models::CursorType;
-use crate::type_system::main::polyglot_type_args::polyglot_type_args;
+use crate::type_system::main::type_args::type_args;
 
-/// Timestamp or integer for supported cross-adapter types; `Err` when Polyglot cannot parse an unknown type.
+/// Timestamp or integer for supported cross-adapter types; `Err` when an unknown type does not parse.
 pub fn classify_cursor_sql_type(sql_type: &str) -> Result<Option<CursorType>, TypeParseFailure> {
     let normalized: String = sql_type
         .to_uppercase()
@@ -29,7 +29,7 @@ pub fn classify_cursor_sql_type(sql_type: &str) -> Result<Option<CursorType>, Ty
         return Ok(Some(cursor_type));
     }
     let args: Map<String, Value> =
-        polyglot_type_args(&normalized, GENERIC_DIALECT).map_err(|message| TypeParseFailure {
+        type_args(&normalized, GENERIC_DIALECT).map_err(|message| TypeParseFailure {
             normalized_type: normalized.clone(),
             message,
         })?;

crates/sqlbuild-analysis/src/relation_markers/_helpers/mod.rs

@@ -0,0 +1 @@
+pub(crate) mod pattern;

crates/sqlbuild-analysis/src/relation_markers/_helpers/pattern.rs

@@ -0,0 +1,15 @@
+//! The compiled relation marker call pattern.
+
+use std::sync::LazyLock;
+
+use regex::Regex;
+
+use crate::relation_markers::constants::RELATION_MARKER_PATTERN;
+
+static RELATION_MARKER: LazyLock<Result<Regex, String>> =
+    LazyLock::new(|| Regex::new(RELATION_MARKER_PATTERN).map_err(|error| error.to_string()));
+
+/// The compiled relation marker call pattern, or its compile error.
+pub(crate) fn relation_marker_pattern() -> Result<&'static Regex, String> {
+    RELATION_MARKER.as_ref().map_err(Clone::clone)
+}

crates/sqlbuild-analysis/src/relation_markers/constants.rs

@@ -0,0 +1,11 @@
+//! The relation marker call pattern and the names marker calls stand as.
+
+/// The prefix of the name a marker call stands as while parsed; the call's index follows it.
+pub const MARKER_NAME_PREFIX: &str = "__sqb_";
+/// Any relation marker call: `__name("x")`, or `__name("a", "b")` as `__dbt_ref` writes it.
+pub(crate) const RELATION_MARKER_PATTERN: &str =
+    r#"(?i)\b(__[a-z_]+)\(\s*"([^"]+)"\s*(?:,\s*"([^"]+)"\s*)?\)"#;
+/// The marker whose two names read as one, joined by `__`.
+pub(crate) const DBT_REF_FUNCTION: &str = "__dbt_ref";
+/// The marker of a table function call, which Snowflake reads only inside `TABLE( )`.
+pub(crate) const TABLE_FUNCTION_MARKER: &str = "__table_fn";

crates/sqlbuild-analysis/src/relation_markers/main/markers_as_names.rs

@@ -0,0 +1,60 @@
+//! `markers_as_names`: relation marker calls in code as plain names that keep every position.
+
+use regex::Regex;
+use sqlbuild_sqltext::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges;
+use sqlbuild_sqltext::sql_scan::models::LexicalSyntax;
+
+use crate::relation_markers::_helpers::pattern::relation_marker_pattern;
+use crate::relation_markers::constants::{DBT_REF_FUNCTION, MARKER_NAME_PREFIX};
+use crate::relation_markers::types::MarkerCall;
+
+/// `sql` with each marker call in code as `__sqb_<index>`, padded so later text keeps its line and column, and the calls by index.
+pub fn markers_as_names(
+    sql: &str,
+    syntax: &LexicalSyntax,
+) -> Result<(String, Vec<MarkerCall>), String> {
+    let pattern: &Regex = relation_marker_pattern()?;
+    let mut protected: Option<Vec<(usize, usize)>> = None;
+    let mut named: String = String::with_capacity(sql.len());
+    let mut calls: Vec<MarkerCall> = Vec::new();
+    let mut cursor: usize = 0;
+    for captures in pattern.captures_iter(sql) {
+        let (Some(full), Some(function), Some(first)) =
+            (captures.get(0), captures.get(1), captures.get(2))
+        else {
+            continue;
+        };
+        let ranges = protected.get_or_insert_with(|| dialect_non_code_ranges(sql, syntax));
+        let following: usize = ranges.partition_point(|(start, _)| *start <= full.start());
+        if following > 0 && full.start() < ranges[following - 1].1 {
+            continue;
+        }
+        let function_name: String = function.as_str().to_ascii_lowercase();
+        let referenced_name: String = match (captures.get(3), function_name == DBT_REF_FUNCTION) {
+            (None, _) => first.as_str().to_owned(),
+            (Some(second), true) => format!("{}__{}", first.as_str(), second.as_str()),
+            (Some(_), false) => continue,
+        };
+        named.push_str(&sql[cursor..full.start()]);
+        named.push_str(&padded_name(
+            &format!("{MARKER_NAME_PREFIX}{}", calls.len()),
+            full.as_str(),
+        ));
+        cursor = full.end();
+        calls.push((function_name, referenced_name));
+    }
+    named.push_str(&sql[cursor..]);
+    Ok((named, calls))
+}
+
+/// `name`, then the call's line breaks, then spaces as wide as the call's last line.
+fn padded_name(name: &str, call: &str) -> String {
+    let breaks: usize = call.matches('\n').count();
+    let last_line: &str = call.rsplit('\n').next().unwrap_or(call);
+    let width: usize = if breaks == 0 {
+        call.chars().count().saturating_sub(name.chars().count())
+    } else {
+        last_line.chars().count()
+    };
+    format!("{name}{}{}", "\n".repeat(breaks), " ".repeat(width))
+}

crates/sqlbuild-analysis/src/relation_markers/main/mod.rs

@@ -0,0 +1 @@
+pub mod markers_as_names;

crates/sqlbuild-analysis/src/relation_markers/mod.rs

@@ -0,0 +1,9 @@
+//! Relation marker calls, such as `__ref("orders")`, read as plain relation names by strict parsers.
+
+mod _helpers;
+pub mod constants;
+pub mod main;
+pub mod types;
+
+#[cfg(test)]
+mod tests;

crates/sqlbuild-analysis/src/relation_markers/tests.rs

@@ -0,0 +1,4 @@
+#[path = "tests/test_markers_as_names.rs"]
+mod test_markers_as_names;
+#[path = "tests/test_types.rs"]
+mod test_types;

crates/sqlbuild-analysis/src/relation_markers/tests/test_markers_as_names.rs

@@ -0,0 +1,72 @@
+use sqlbuild_sqltext::sql_scan::main::dialect_lexical_syntax::dialect_lexical_syntax;
+
+use crate::relation_markers::main::markers_as_names::markers_as_names;
+use crate::relation_markers::tests::test_types::MarkersAsNamesTestCase;
+
+#[test]
+fn given_marker_calls_when_reading_them_as_names_then_code_calls_become_names_and_positions_hold() {
+    let test_cases = [
+        MarkersAsNamesTestCase {
+            description: "each call becomes its indexed name, padded to the call's width",
+            dialect: "snowflake",
+            sql: r#"SELECT * FROM __ref("orders") o JOIN __source("raw") r ON TRUE"#,
+            expected_sql: format!(
+                "SELECT * FROM __sqb_0{} o JOIN __sqb_1{} r ON TRUE",
+                " ".repeat(8),
+                " ".repeat(8)
+            ),
+            expected_calls: &[("__ref", "orders"), ("__source", "raw")],
+        },
+        MarkersAsNamesTestCase {
+            description: "calls in quoted text and comments stay; an upper-case call folds",
+            dialect: "snowflake",
+            sql: "SELECT '__ref(\"a\")' AS q -- __ref(\"b\")\nFROM __REF(\"Orders\") // __ref(\"c\")",
+            expected_sql: format!(
+                "SELECT '__ref(\"a\")' AS q -- __ref(\"b\")\nFROM __sqb_0{} // __ref(\"c\")",
+                " ".repeat(8)
+            ),
+            expected_calls: &[("__ref", "Orders")],
+        },
+        MarkersAsNamesTestCase {
+            description: "a call over three lines keeps its line breaks and its last line's width",
+            dialect: "duckdb",
+            sql: "FROM __ref(\n  \"orders\"\n) AS o",
+            expected_sql: "FROM __sqb_0\n\n  AS o".to_owned(),
+            expected_calls: &[("__ref", "orders")],
+        },
+        MarkersAsNamesTestCase {
+            description: "two dbt names join; a non-ASCII name pads by characters, not bytes",
+            dialect: "postgres",
+            sql: r#"FROM __dbt_ref("shop", "customers") d JOIN __ref("café") c ON TRUE"#,
+            expected_sql: format!(
+                "FROM __sqb_0{} d JOIN __sqb_1{} c ON TRUE",
+                " ".repeat(23),
+                " ".repeat(6)
+            ),
+            expected_calls: &[("__dbt_ref", "shop__customers"), ("__ref", "café")],
+        },
+        MarkersAsNamesTestCase {
+            description: "calls with a second name, or a non-literal argument, stay calls",
+            dialect: "duckdb",
+            sql: r#"FROM __ref("a", "b") JOIN __table_fn("orders", 2) ON TRUE"#,
+            expected_sql: r#"FROM __ref("a", "b") JOIN __table_fn("orders", 2) ON TRUE"#.to_owned(),
+            expected_calls: &[],
+        },
+    ];
+    for test_case in test_cases {
+        let (named, calls) =
+            markers_as_names(test_case.sql, &dialect_lexical_syntax(test_case.dialect))
+                .expect(test_case.description);
+        let expected_calls: Vec<(String, String)> = test_case
+            .expected_calls
+            .iter()
+            .map(|(function, name)| ((*function).to_owned(), (*name).to_owned()))
+            .collect();
+        assert_eq!(
+            (named, calls),
+            (test_case.expected_sql, expected_calls),
+            "{}",
+            test_case.description
+        );
+    }
+}

crates/sqlbuild-analysis/src/relation_markers/tests/test_types.rs

@@ -0,0 +1,10 @@
+//! Test case types for relation markers read as names.
+
+/// SQL in one dialect, the SQL its markers read as, and the calls they stand for.
+pub(crate) struct MarkersAsNamesTestCase {
+    pub(crate) description: &'static str,
+    pub(crate) dialect: &'static str,
+    pub(crate) sql: &'static str,
+    pub(crate) expected_sql: String,
+    pub(crate) expected_calls: &'static [(&'static str, &'static str)],
+}

crates/sqlbuild-analysis/src/relation_markers/types.rs

@@ -0,0 +1,4 @@
+//! Shapes of relation marker calls.
+
+/// A marker call's lower-case function, such as `__ref`, and the name it references.
+pub type MarkerCall = (String, String);

crates/sqlbuild-analysis/src/semantic_checks/tests/test_metadata.rs

@@ -17,8 +17,8 @@ fn given_project_metadata_when_checking_then_reports_python_errors() {
             expected_outcome: Ok(expected_checked_outcome()),
         },
         MetadataTestCase {
-            description: "a dialect the all-dialects parser build added",
-            dialect: "mysql",
+            description: "a shipped dialect without a native type parser",
+            dialect: "tsql",
             mart_shape: MART_SHAPE,
             expected_outcome: Ok(expected_checked_outcome()),
         },

crates/sqlbuild-analysis/src/sql_tree/_helpers/mod.rs

@@ -1,7 +1,8 @@
-//! Parser selection, parsing, the tree walk, clause nodes and source positions.
+//! Parser selection, parsing, the tree walk, clause nodes, source positions and table wrapping.
 
 pub(crate) mod clauses;
 pub(crate) mod parse;
 pub(crate) mod positions;
 pub(crate) mod rules_tree_parser;
+pub(crate) mod table_wrapped;
 pub(crate) mod walk;

crates/sqlbuild-analysis/src/sql_tree/_helpers/rules_tree_parser.rs

@@ -1,23 +1,13 @@
 //! `rules_tree_parser`: the parser that reads a dialect's SQL for rules.
 
-use sqlquack_syntax::Dialect;
-
-use crate::sql_tree::constants::{
-    DUCKDB_DIALECTS, POSTGRES_DIALECTS, SNOWFLAKE_DIALECT, SQLITE_DIALECT,
-};
+use crate::sql_tree::constants::POSTGRES_DIALECTS;
+use crate::sql_tree::main::shared_dialect::shared_dialect;
 use crate::sql_tree::models::TreeParser;
 
 /// The parser for `dialect`, or `None`; add a dialect (one line) only once its parser passes the depth probe with an empty allowlist.
 pub(crate) fn rules_tree_parser(dialect: &str) -> Option<TreeParser> {
     if POSTGRES_DIALECTS.contains(&dialect) {
         return Some(TreeParser::Postgres);
     }
-    if DUCKDB_DIALECTS.contains(&dialect) {
-        return Some(TreeParser::Shared(Dialect::DuckDB));
-    }
-    match dialect {
-        SNOWFLAKE_DIALECT => Some(TreeParser::Shared(Dialect::Snowflake)),
-        SQLITE_DIALECT => Some(TreeParser::Shared(Dialect::SQLite)),
-        _ => None,
-    }
+    shared_dialect(dialect).map(TreeParser::Shared)
 }