feat!: compute query fingerprints from SQLQuack tokens

Cargo.toml

@@ -174,8 +174,11 @@ sqlbuild-recall = { path = "crates/sqlbuild-recall" }
 sqlbuild-scopes = { path = "crates/sqlbuild-scopes" }
 sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" }
 sqlbuild-state = { path = "crates/sqlbuild-state" }
-# SQL lint Rules, quality Rules, the formatter and the canonical token stream.
+# SQL lint Rules, quality Rules and the formatter.
 sqlclean = { git = "https://github.com/sqlstack-io/sqlclean.git", rev = "cf79656d247277a30bc81ef39b6dee420e721221" }
+# The lossless SQL tokenizer behind query fingerprints.
+sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "4d0c6bc56da0e44eb164601a2af673ae7998257d" }
+sqlquack-syntax-postgres = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "4d0c6bc56da0e44eb164601a2af673ae7998257d" }
 stacker = "0.1"
 syn = { version = "2", features = ["full", "visit", "extra-traits"] }
 subtle = "2"

crates/sqlbuild-analysis/Cargo.toml

@@ -26,6 +26,8 @@ sqlbuild-render.workspace = true
 sqlbuild-scopes.workspace = true
 sqlbuild-sqltext.workspace = true
 sqlclean.workspace = true
+sqlquack-syntax.workspace = true
+sqlquack-syntax-postgres.workspace = true
 
 [lints]
 workspace = true

crates/sqlbuild-analysis/src/sql_tokens/_helpers/canonical_tokens.rs

@@ -0,0 +1,99 @@
+//! The canonical token stream of SQL under SQLQuack's tokenizers.
+
+use sqlclean::sql_tokens::main::is_unquoted_word::is_unquoted_word;
+use sqlclean::sql_tokens::main::name_positions::name_positions;
+use sqlquack_syntax::lossless::models::TokenKind;
+use sqlquack_syntax_postgres::scan::main::scan::scan;
+use sqlquack_syntax_postgres::scan::models::TokenKind as PostgresTokenKind;
+
+use crate::sql_tokens::_helpers::reserved_keywords::is_reserved_keyword;
+use crate::sql_tokens::constants::{
+    CALL_CLOSER, CALL_OPENER, COLUMN_LIST_RELATION_KEYWORDS, CTE_BODY_OPENERS, CTE_BODY_PREFIX,
+    PLACEHOLDER_PREFIX,
+};
+use crate::sql_tokens::models::{FingerprintTokenizer, SqlquackFingerprintDialect};
+
+/// Significant token texts with reserved keywords and called names, except `__` placeholders, upper-cased.
+pub(crate) fn canonical_tokens(
+    sql: &str,
+    dialect: SqlquackFingerprintDialect,
+) -> Result<Vec<String>, String> {
+    let raws: Vec<&str> = significant_texts(sql, dialect.tokenizer)?;
+    let uppers: Vec<String> = raws.iter().map(|raw| raw.to_ascii_uppercase()).collect();
+    let names: Vec<bool> = name_positions(&uppers);
+    let closers: Vec<Option<usize>> = matching_closers(&uppers);
+    let mut texts: Vec<String> = Vec::with_capacity(raws.len());
+    for (index, raw) in raws.iter().enumerate() {
+        let folds = !names[index]
+            && is_unquoted_word(raw)
+            && (is_reserved_keyword(dialect.keywords, &uppers[index])
+                || calls(raw, &uppers, &closers, index));
+        texts.push(if folds {
+            uppers[index].clone()
+        } else {
+            (*raw).to_owned()
+        });
+    }
+    Ok(texts)
+}
+
+/// The text of each significant token of `sql`, in order.
+fn significant_texts(sql: &str, tokenizer: FingerprintTokenizer) -> Result<Vec<&str>, String> {
+    match tokenizer {
+        FingerprintTokenizer::Lossless(dialect) => Ok(sqlquack_syntax::tokenize(sql, dialect)
+            .map_err(|error| format!("SQL tokenization failed: {error}"))?
+            .iter()
+            .filter(|token| matches!(token.kind, TokenKind::Significant(_)))
+            .map(|token| token.text(sql))
+            .collect()),
+        FingerprintTokenizer::PostgresScanner => scan(sql)
+            .into_iter()
+            .map(|token| match token.kind {
+                PostgresTokenKind::Fault(_) => {
+                    Err(format!("SQL tokenization failed: {}", token.value))
+                }
+                _ => sql
+                    .get(token.start..token.end)
+                    .ok_or_else(|| format!("SQL tokenization failed at byte {}", token.start)),
+            })
+            .collect(),
+    }
+}
+
+/// The index of the `)` closing each `(`, by the index of the `(`.
+fn matching_closers(uppers: &[String]) -> Vec<Option<usize>> {
+    let mut closers: Vec<Option<usize>> = vec![None; uppers.len()];
+    let mut open: Vec<usize> = Vec::new();
+    for (index, token) in uppers.iter().enumerate() {
+        if token == CALL_OPENER {
+            open.push(index);
+        } else if token == CALL_CLOSER
+            && let Some(opener) = open.pop()
+        {
+            closers[opener] = Some(index);
+        }
+    }
+    closers
+}
+
+/// Whether the word at `index` is called: followed by `(` and neither a placeholder nor a relation.
+fn calls(raw: &str, uppers: &[String], closers: &[Option<usize>], index: usize) -> bool {
+    if uppers.get(index + 1).is_none_or(|next| next != CALL_OPENER)
+        || raw.starts_with(PLACEHOLDER_PREFIX)
+    {
+        return false;
+    }
+    let previous: Option<&str> = index.checked_sub(1).map(|before| uppers[before].as_str());
+    if previous.is_some_and(|word| COLUMN_LIST_RELATION_KEYWORDS.contains(&word)) {
+        return false;
+    }
+    let Some(closer) = closers[index + 1] else {
+        return true;
+    };
+    !(uppers
+        .get(closer + 1)
+        .is_some_and(|next| next == CTE_BODY_PREFIX)
+        && uppers
+            .get(closer + 2)
+            .is_some_and(|next| CTE_BODY_OPENERS.contains(&next.as_str())))
+}

crates/sqlbuild-analysis/src/sql_tokens/_helpers/mod.rs

@@ -0,0 +1,2 @@
+pub(crate) mod canonical_tokens;
+pub(crate) mod reserved_keywords;

crates/sqlbuild-analysis/src/sql_tokens/_helpers/reserved_keywords.rs

@@ -0,0 +1,34 @@
+//! Per-dialect reserved keywords, which can never be unquoted identifiers in that dialect.
+
+use std::collections::HashSet;
+use std::sync::LazyLock;
+
+use crate::sql_tokens::models::KeywordList;
+
+static DUCKDB: LazyLock<HashSet<String>> =
+    LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/duckdb.txt")));
+static POSTGRES: LazyLock<HashSet<String>> =
+    LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/postgres.txt")));
+static SNOWFLAKE: LazyLock<HashSet<String>> =
+    LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/snowflake.txt")));
+static GENERIC: LazyLock<HashSet<String>> =
+    LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/generic.txt")));
+
+/// Whether the upper-cased word is reserved in `list`.
+pub(crate) fn is_reserved_keyword(list: KeywordList, upper_word: &str) -> bool {
+    let keywords: &HashSet<String> = match list {
+        KeywordList::DuckDB => &DUCKDB,
+        KeywordList::PostgreSQL => &POSTGRES,
+        KeywordList::Snowflake => &SNOWFLAKE,
+        KeywordList::Generic => &GENERIC,
+    };
+    keywords.contains(upper_word)
+}
+
+fn keyword_set(data: &str) -> HashSet<String> {
+    data.lines()
+        .map(str::trim)
+        .filter(|line| !line.is_empty() && !line.starts_with('#'))
+        .map(str::to_ascii_uppercase)
+        .collect()
+}

crates/sqlbuild-analysis/src/sql_tokens/constants.rs

@@ -1,5 +1,9 @@
 //! Fingerprint dialects and the function language fingerprinted like a query.
 
+use sqlquack_syntax::tokens::models::Dialect;
+
+use crate::sql_tokens::models::{FingerprintTokenizer, KeywordList, SqlquackFingerprintDialect};
+
 /// The fingerprint dialect when a project names none.
 pub(crate) const GENERIC_FINGERPRINT_DIALECT: &str = "generic";
 /// Python's `NATIVE_DIALECT_ALIASES`.
@@ -10,3 +14,48 @@ pub(crate) const FINGERPRINT_DIALECT_ALIASES: [(&str, &str); 3] = [
 ];
 /// The function language whose definitions are fingerprinted like queries.
 pub(crate) const SQL_FUNCTION_LANGUAGE: &str = "sql";
+/// Fingerprint dialects SQLQuack tokenizes; generic uses the DuckDB tokenizer, PostgreSQL its native scanner.
+pub(crate) const SQLQUACK_FINGERPRINT_DIALECTS: [SqlquackFingerprintDialect; 6] = [
+    SqlquackFingerprintDialect {
+        name: "generic",
+        tokenizer: FingerprintTokenizer::Lossless(Dialect::DuckDB),
+        keywords: KeywordList::Generic,
+    },
+    SqlquackFingerprintDialect {
+        name: "duckdb",
+        tokenizer: FingerprintTokenizer::Lossless(Dialect::DuckDB),
+        keywords: KeywordList::DuckDB,
+    },
+    SqlquackFingerprintDialect {
+        name: "postgres",
+        tokenizer: FingerprintTokenizer::PostgresScanner,
+        keywords: KeywordList::PostgreSQL,
+    },
+    SqlquackFingerprintDialect {
+        name: "postgresql",
+        tokenizer: FingerprintTokenizer::PostgresScanner,
+        keywords: KeywordList::PostgreSQL,
+    },
+    SqlquackFingerprintDialect {
+        name: "snowflake",
+        tokenizer: FingerprintTokenizer::Lossless(Dialect::Snowflake),
+        keywords: KeywordList::Snowflake,
+    },
+    SqlquackFingerprintDialect {
+        name: "sqlite",
+        tokenizer: FingerprintTokenizer::Lossless(Dialect::SQLite),
+        keywords: KeywordList::Generic,
+    },
+];
+/// The token that follows a called function's name.
+pub(crate) const CALL_OPENER: &str = "(";
+/// The token that closes a call's arguments or a column list.
+pub(crate) const CALL_CLOSER: &str = ")";
+/// Keywords after which a word followed by `(` names a relation and its column list.
+pub(crate) const COLUMN_LIST_RELATION_KEYWORDS: [&str; 2] = ["INTO", "TABLE"];
+/// The keyword between a CTE's column list and its body.
+pub(crate) const CTE_BODY_PREFIX: &str = "AS";
+/// The tokens that can follow `AS` after a CTE's column list.
+pub(crate) const CTE_BODY_OPENERS: [&str; 3] = ["(", "MATERIALIZED", "NOT"];
+/// The prefix of SQLBuild placeholder calls such as `__ref` and `__source`.
+pub(crate) const PLACEHOLDER_PREFIX: &str = "__";

crates/sqlbuild-analysis/src/sql_tokens/main/query_fingerprint.rs

@@ -7,19 +7,37 @@ use sha2::{Digest, Sha256};
 
 use sqlclean::sql_tokens::main::canonical_tokens::canonical_tokens;
 
+use crate::sql_tokens::_helpers::canonical_tokens::canonical_tokens as sqlquack_canonical_tokens;
+use crate::sql_tokens::constants::SQLQUACK_FINGERPRINT_DIALECTS;
+
 /// Return the SHA-256 hex digest of the canonical token stream of `sql` under `dialect_name`.
 pub fn query_fingerprint(sql: &str, dialect_name: &str) -> Result<String, String> {
+    let texts: Vec<String> = match SQLQUACK_FINGERPRINT_DIALECTS
+        .iter()
+        .find(|dialect| dialect.name == dialect_name)
+    {
+        Some(dialect) => sqlquack_canonical_tokens(sql, *dialect)?,
+        None => polyglot_canonical_tokens(sql, dialect_name)?,
+    };
+    let mut digest = Sha256::new();
+    for text in texts {
+        digest.update(text.len().to_string().as_bytes());
+        digest.update(b":");
+        digest.update(text.as_bytes());
+    }
+    Ok(format!("{:x}", digest.finalize()))
+}
+
+/// Dialects SQLQuack cannot tokenize yet keep SQLClean's Polyglot token stream.
+fn polyglot_canonical_tokens(sql: &str, dialect_name: &str) -> Result<Vec<String>, String> {
     let dialect_type = DialectType::from_str(dialect_name)
         .map_err(|_| format!("unknown SQL fingerprint dialect '{dialect_name}'"))?;
     let dialect = Dialect::get(dialect_type);
-    let mut digest = Sha256::new();
     let tokens = dialect
         .tokenize(sql)
         .map_err(|error| format!("SQL tokenization failed: {error}"))?;
-    for token in canonical_tokens(sql, &tokens, &dialect)? {
-        digest.update(token.text.len().to_string().as_bytes());
-        digest.update(b":");
-        digest.update(token.text.as_bytes());
-    }
-    Ok(format!("{:x}", digest.finalize()))
+    Ok(canonical_tokens(sql, &tokens, &dialect)?
+        .into_iter()
+        .map(|token| token.text)
+        .collect())
 }

crates/sqlbuild-analysis/src/sql_tokens/mod.rs

@@ -1,4 +1,6 @@
-//! SQLBuild's query and definition fingerprints over SQLClean's canonical SQL token stream.
+//! SQLBuild's query and definition fingerprints over a canonical SQL token stream.
 
+mod _helpers;
 pub mod constants;
 pub mod main;
+pub(crate) mod models;

crates/sqlbuild-analysis/src/sql_tokens/models.rs

@@ -0,0 +1,29 @@
+//! How a fingerprint dialect is tokenized and which of its words fold.
+
+use sqlquack_syntax::tokens::models::Dialect;
+
+/// The reserved-keyword list a fingerprint dialect folds.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) enum KeywordList {
+    DuckDB,
+    PostgreSQL,
+    Snowflake,
+    Generic,
+}
+
+/// The SQLQuack tokenizer a fingerprint dialect reads its tokens with.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) enum FingerprintTokenizer {
+    /// `sqlquack_syntax`'s lossless tokenizer for one dialect.
+    Lossless(Dialect),
+    /// The native PostgreSQL scanner of `sqlquack_syntax_postgres`.
+    PostgresScanner,
+}
+
+/// A fingerprint dialect SQLQuack tokenizes: its tokenizer and its reserved keywords.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) struct SqlquackFingerprintDialect {
+    pub(crate) name: &'static str,
+    pub(crate) tokenizer: FingerprintTokenizer,
+    pub(crate) keywords: KeywordList,
+}