diff --git a/Cargo.lock b/Cargo.lock index 9a409db2f1..5255394799 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4142,6 +4142,8 @@ dependencies = [ "sqlbuild-scopes", "sqlbuild-sqltext", "sqlclean", + "sqlquack-syntax", + "sqlquack-syntax-postgres", ] [[package]] @@ -4494,6 +4496,37 @@ dependencies = [ "sha2 0.10.9", ] +[[package]] +name = "sqlquack-syntax" +version = "0.2.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=4d0c6bc56da0e44eb164601a2af673ae7998257d#4d0c6bc56da0e44eb164601a2af673ae7998257d" +dependencies = [ + "serde", + "serde_json", + "sqlquack-syntax-derive", + "stacker", + "thiserror 1.0.69", + "unicode-segmentation", +] + +[[package]] +name = "sqlquack-syntax-derive" +version = "0.2.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=4d0c6bc56da0e44eb164601a2af673ae7998257d#4d0c6bc56da0e44eb164601a2af673ae7998257d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "sqlquack-syntax-postgres" +version = "0.2.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=4d0c6bc56da0e44eb164601a2af673ae7998257d#4d0c6bc56da0e44eb164601a2af673ae7998257d" +dependencies = [ + "sqlquack-syntax", +] + [[package]] name = "sqlx-core" version = "0.8.6" diff --git a/Cargo.toml b/Cargo.toml index 9ef850601d..f03cafe024 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -174,8 +174,11 @@ sqlbuild-recall = { path = "crates/sqlbuild-recall" } sqlbuild-scopes = { path = "crates/sqlbuild-scopes" } sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" } sqlbuild-state = { path = "crates/sqlbuild-state" } -# SQL lint Rules, quality Rules, the formatter and the canonical token stream. +# SQL lint Rules, quality Rules and the formatter. sqlclean = { git = "https://github.com/sqlstack-io/sqlclean.git", rev = "cf79656d247277a30bc81ef39b6dee420e721221" } +# The lossless SQL tokenizer behind query fingerprints. +sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "4d0c6bc56da0e44eb164601a2af673ae7998257d" } +sqlquack-syntax-postgres = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "4d0c6bc56da0e44eb164601a2af673ae7998257d" } stacker = "0.1" syn = { version = "2", features = ["full", "visit", "extra-traits"] } subtle = "2" diff --git a/crates/sqlbuild-analysis/Cargo.toml b/crates/sqlbuild-analysis/Cargo.toml index 864c22d7e7..80a5ae2313 100644 --- a/crates/sqlbuild-analysis/Cargo.toml +++ b/crates/sqlbuild-analysis/Cargo.toml @@ -26,6 +26,8 @@ sqlbuild-render.workspace = true sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true sqlclean.workspace = true +sqlquack-syntax.workspace = true +sqlquack-syntax-postgres.workspace = true [lints] workspace = true diff --git a/crates/sqlbuild-analysis/src/sql_tokens/_helpers/canonical_tokens.rs b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/canonical_tokens.rs new file mode 100644 index 0000000000..c5d1988159 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/canonical_tokens.rs @@ -0,0 +1,99 @@ +//! The canonical token stream of SQL under SQLQuack's tokenizers. + +use sqlclean::sql_tokens::main::is_unquoted_word::is_unquoted_word; +use sqlclean::sql_tokens::main::name_positions::name_positions; +use sqlquack_syntax::lossless::models::TokenKind; +use sqlquack_syntax_postgres::scan::main::scan::scan; +use sqlquack_syntax_postgres::scan::models::TokenKind as PostgresTokenKind; + +use crate::sql_tokens::_helpers::reserved_keywords::is_reserved_keyword; +use crate::sql_tokens::constants::{ + CALL_CLOSER, CALL_OPENER, COLUMN_LIST_RELATION_KEYWORDS, CTE_BODY_OPENERS, CTE_BODY_PREFIX, + PLACEHOLDER_PREFIX, +}; +use crate::sql_tokens::models::{FingerprintTokenizer, SqlquackFingerprintDialect}; + +/// Significant token texts with reserved keywords and called names, except `__` placeholders, upper-cased. +pub(crate) fn canonical_tokens( + sql: &str, + dialect: SqlquackFingerprintDialect, +) -> Result, String> { + let raws: Vec<&str> = significant_texts(sql, dialect.tokenizer)?; + let uppers: Vec = raws.iter().map(|raw| raw.to_ascii_uppercase()).collect(); + let names: Vec = name_positions(&uppers); + let closers: Vec> = matching_closers(&uppers); + let mut texts: Vec = Vec::with_capacity(raws.len()); + for (index, raw) in raws.iter().enumerate() { + let folds = !names[index] + && is_unquoted_word(raw) + && (is_reserved_keyword(dialect.keywords, &uppers[index]) + || calls(raw, &uppers, &closers, index)); + texts.push(if folds { + uppers[index].clone() + } else { + (*raw).to_owned() + }); + } + Ok(texts) +} + +/// The text of each significant token of `sql`, in order. +fn significant_texts(sql: &str, tokenizer: FingerprintTokenizer) -> Result, String> { + match tokenizer { + FingerprintTokenizer::Lossless(dialect) => Ok(sqlquack_syntax::tokenize(sql, dialect) + .map_err(|error| format!("SQL tokenization failed: {error}"))? + .iter() + .filter(|token| matches!(token.kind, TokenKind::Significant(_))) + .map(|token| token.text(sql)) + .collect()), + FingerprintTokenizer::PostgresScanner => scan(sql) + .into_iter() + .map(|token| match token.kind { + PostgresTokenKind::Fault(_) => { + Err(format!("SQL tokenization failed: {}", token.value)) + } + _ => sql + .get(token.start..token.end) + .ok_or_else(|| format!("SQL tokenization failed at byte {}", token.start)), + }) + .collect(), + } +} + +/// The index of the `)` closing each `(`, by the index of the `(`. +fn matching_closers(uppers: &[String]) -> Vec> { + let mut closers: Vec> = vec![None; uppers.len()]; + let mut open: Vec = Vec::new(); + for (index, token) in uppers.iter().enumerate() { + if token == CALL_OPENER { + open.push(index); + } else if token == CALL_CLOSER + && let Some(opener) = open.pop() + { + closers[opener] = Some(index); + } + } + closers +} + +/// Whether the word at `index` is called: followed by `(` and neither a placeholder nor a relation. +fn calls(raw: &str, uppers: &[String], closers: &[Option], index: usize) -> bool { + if uppers.get(index + 1).is_none_or(|next| next != CALL_OPENER) + || raw.starts_with(PLACEHOLDER_PREFIX) + { + return false; + } + let previous: Option<&str> = index.checked_sub(1).map(|before| uppers[before].as_str()); + if previous.is_some_and(|word| COLUMN_LIST_RELATION_KEYWORDS.contains(&word)) { + return false; + } + let Some(closer) = closers[index + 1] else { + return true; + }; + !(uppers + .get(closer + 1) + .is_some_and(|next| next == CTE_BODY_PREFIX) + && uppers + .get(closer + 2) + .is_some_and(|next| CTE_BODY_OPENERS.contains(&next.as_str()))) +} diff --git a/crates/sqlbuild-analysis/src/sql_tokens/_helpers/mod.rs b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/mod.rs new file mode 100644 index 0000000000..5e7000dd1e --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod canonical_tokens; +pub(crate) mod reserved_keywords; diff --git a/crates/sqlbuild-analysis/src/sql_tokens/_helpers/reserved_keywords.rs b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/reserved_keywords.rs new file mode 100644 index 0000000000..dae99b7712 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/reserved_keywords.rs @@ -0,0 +1,34 @@ +//! Per-dialect reserved keywords, which can never be unquoted identifiers in that dialect. + +use std::collections::HashSet; +use std::sync::LazyLock; + +use crate::sql_tokens::models::KeywordList; + +static DUCKDB: LazyLock> = + LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/duckdb.txt"))); +static POSTGRES: LazyLock> = + LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/postgres.txt"))); +static SNOWFLAKE: LazyLock> = + LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/snowflake.txt"))); +static GENERIC: LazyLock> = + LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/generic.txt"))); + +/// Whether the upper-cased word is reserved in `list`. +pub(crate) fn is_reserved_keyword(list: KeywordList, upper_word: &str) -> bool { + let keywords: &HashSet = match list { + KeywordList::DuckDB => &DUCKDB, + KeywordList::PostgreSQL => &POSTGRES, + KeywordList::Snowflake => &SNOWFLAKE, + KeywordList::Generic => &GENERIC, + }; + keywords.contains(upper_word) +} + +fn keyword_set(data: &str) -> HashSet { + data.lines() + .map(str::trim) + .filter(|line| !line.is_empty() && !line.starts_with('#')) + .map(str::to_ascii_uppercase) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/sql_tokens/constants.rs b/crates/sqlbuild-analysis/src/sql_tokens/constants.rs index 44e22ce088..7a59c6115f 100644 --- a/crates/sqlbuild-analysis/src/sql_tokens/constants.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/constants.rs @@ -1,5 +1,9 @@ //! Fingerprint dialects and the function language fingerprinted like a query. +use sqlquack_syntax::tokens::models::Dialect; + +use crate::sql_tokens::models::{FingerprintTokenizer, KeywordList, SqlquackFingerprintDialect}; + /// The fingerprint dialect when a project names none. pub(crate) const GENERIC_FINGERPRINT_DIALECT: &str = "generic"; /// Python's `NATIVE_DIALECT_ALIASES`. @@ -10,3 +14,48 @@ pub(crate) const FINGERPRINT_DIALECT_ALIASES: [(&str, &str); 3] = [ ]; /// The function language whose definitions are fingerprinted like queries. pub(crate) const SQL_FUNCTION_LANGUAGE: &str = "sql"; +/// Fingerprint dialects SQLQuack tokenizes; generic uses the DuckDB tokenizer, PostgreSQL its native scanner. +pub(crate) const SQLQUACK_FINGERPRINT_DIALECTS: [SqlquackFingerprintDialect; 6] = [ + SqlquackFingerprintDialect { + name: "generic", + tokenizer: FingerprintTokenizer::Lossless(Dialect::DuckDB), + keywords: KeywordList::Generic, + }, + SqlquackFingerprintDialect { + name: "duckdb", + tokenizer: FingerprintTokenizer::Lossless(Dialect::DuckDB), + keywords: KeywordList::DuckDB, + }, + SqlquackFingerprintDialect { + name: "postgres", + tokenizer: FingerprintTokenizer::PostgresScanner, + keywords: KeywordList::PostgreSQL, + }, + SqlquackFingerprintDialect { + name: "postgresql", + tokenizer: FingerprintTokenizer::PostgresScanner, + keywords: KeywordList::PostgreSQL, + }, + SqlquackFingerprintDialect { + name: "snowflake", + tokenizer: FingerprintTokenizer::Lossless(Dialect::Snowflake), + keywords: KeywordList::Snowflake, + }, + SqlquackFingerprintDialect { + name: "sqlite", + tokenizer: FingerprintTokenizer::Lossless(Dialect::SQLite), + keywords: KeywordList::Generic, + }, +]; +/// The token that follows a called function's name. +pub(crate) const CALL_OPENER: &str = "("; +/// The token that closes a call's arguments or a column list. +pub(crate) const CALL_CLOSER: &str = ")"; +/// Keywords after which a word followed by `(` names a relation and its column list. +pub(crate) const COLUMN_LIST_RELATION_KEYWORDS: [&str; 2] = ["INTO", "TABLE"]; +/// The keyword between a CTE's column list and its body. +pub(crate) const CTE_BODY_PREFIX: &str = "AS"; +/// The tokens that can follow `AS` after a CTE's column list. +pub(crate) const CTE_BODY_OPENERS: [&str; 3] = ["(", "MATERIALIZED", "NOT"]; +/// The prefix of SQLBuild placeholder calls such as `__ref` and `__source`. +pub(crate) const PLACEHOLDER_PREFIX: &str = "__"; diff --git a/crates/sqlbuild-analysis/src/sql_tokens/main/query_fingerprint.rs b/crates/sqlbuild-analysis/src/sql_tokens/main/query_fingerprint.rs index 9cde608f7d..6162534d1a 100644 --- a/crates/sqlbuild-analysis/src/sql_tokens/main/query_fingerprint.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/main/query_fingerprint.rs @@ -7,19 +7,37 @@ use sha2::{Digest, Sha256}; use sqlclean::sql_tokens::main::canonical_tokens::canonical_tokens; +use crate::sql_tokens::_helpers::canonical_tokens::canonical_tokens as sqlquack_canonical_tokens; +use crate::sql_tokens::constants::SQLQUACK_FINGERPRINT_DIALECTS; + /// Return the SHA-256 hex digest of the canonical token stream of `sql` under `dialect_name`. pub fn query_fingerprint(sql: &str, dialect_name: &str) -> Result { + let texts: Vec = match SQLQUACK_FINGERPRINT_DIALECTS + .iter() + .find(|dialect| dialect.name == dialect_name) + { + Some(dialect) => sqlquack_canonical_tokens(sql, *dialect)?, + None => polyglot_canonical_tokens(sql, dialect_name)?, + }; + let mut digest = Sha256::new(); + for text in texts { + digest.update(text.len().to_string().as_bytes()); + digest.update(b":"); + digest.update(text.as_bytes()); + } + Ok(format!("{:x}", digest.finalize())) +} + +/// Dialects SQLQuack cannot tokenize yet keep SQLClean's Polyglot token stream. +fn polyglot_canonical_tokens(sql: &str, dialect_name: &str) -> Result, String> { let dialect_type = DialectType::from_str(dialect_name) .map_err(|_| format!("unknown SQL fingerprint dialect '{dialect_name}'"))?; let dialect = Dialect::get(dialect_type); - let mut digest = Sha256::new(); let tokens = dialect .tokenize(sql) .map_err(|error| format!("SQL tokenization failed: {error}"))?; - for token in canonical_tokens(sql, &tokens, &dialect)? { - digest.update(token.text.len().to_string().as_bytes()); - digest.update(b":"); - digest.update(token.text.as_bytes()); - } - Ok(format!("{:x}", digest.finalize())) + Ok(canonical_tokens(sql, &tokens, &dialect)? + .into_iter() + .map(|token| token.text) + .collect()) } diff --git a/crates/sqlbuild-analysis/src/sql_tokens/mod.rs b/crates/sqlbuild-analysis/src/sql_tokens/mod.rs index 4d3a25f7d4..a574d034fe 100644 --- a/crates/sqlbuild-analysis/src/sql_tokens/mod.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/mod.rs @@ -1,4 +1,6 @@ -//! SQLBuild's query and definition fingerprints over SQLClean's canonical SQL token stream. +//! SQLBuild's query and definition fingerprints over a canonical SQL token stream. +mod _helpers; pub mod constants; pub mod main; +pub(crate) mod models; diff --git a/crates/sqlbuild-analysis/src/sql_tokens/models.rs b/crates/sqlbuild-analysis/src/sql_tokens/models.rs new file mode 100644 index 0000000000..1221a29855 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_tokens/models.rs @@ -0,0 +1,29 @@ +//! How a fingerprint dialect is tokenized and which of its words fold. + +use sqlquack_syntax::tokens::models::Dialect; + +/// The reserved-keyword list a fingerprint dialect folds. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum KeywordList { + DuckDB, + PostgreSQL, + Snowflake, + Generic, +} + +/// The SQLQuack tokenizer a fingerprint dialect reads its tokens with. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum FingerprintTokenizer { + /// `sqlquack_syntax`'s lossless tokenizer for one dialect. + Lossless(Dialect), + /// The native PostgreSQL scanner of `sqlquack_syntax_postgres`. + PostgresScanner, +} + +/// A fingerprint dialect SQLQuack tokenizes: its tokenizer and its reserved keywords. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct SqlquackFingerprintDialect { + pub(crate) name: &'static str, + pub(crate) tokenizer: FingerprintTokenizer, + pub(crate) keywords: KeywordList, +} diff --git a/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/duckdb.txt b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/duckdb.txt new file mode 100644 index 0000000000..4374c4e538 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/duckdb.txt @@ -0,0 +1,77 @@ +# DuckDB reserved keywords: SELECT keyword_name FROM duckdb_keywords() +# WHERE keyword_category = 'reserved' (DuckDB 1.5.2). +all +analyse +analyze +and +any +array +as +asc +asymmetric +both +case +cast +check +collate +column +constraint +create +default +deferrable +desc +describe +distinct +do +else +end +except +false +fetch +for +foreign +from +group +having +in +initially +intersect +into +lambda +lateral +leading +limit +not +null +offset +on +only +or +order +pivot +pivot_longer +pivot_wider +placing +primary +qualify +references +returning +select +show +some +summarize +symmetric +table +then +to +trailing +true +union +unique +unpivot +using +variadic +when +where +window +with diff --git a/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/generic.txt b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/generic.txt new file mode 100644 index 0000000000..05f8ee42cd --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/generic.txt @@ -0,0 +1,28 @@ +# Generic dialect: keywords reserved in every dialect file above except Databricks +# (DuckDB, PostgreSQL, BigQuery, Snowflake and T-SQL). +all +and +any +as +create +distinct +else +for +from +group +having +in +intersect +into +not +null +on +or +order +select +some +then +to +union +where +with diff --git a/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/postgres.txt b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/postgres.txt new file mode 100644 index 0000000000..7a57911cb5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/postgres.txt @@ -0,0 +1,80 @@ +# PostgreSQL reserved keywords (non-function): RESERVED_KEYWORD entries of +# src/include/parser/kwlist.h (REL_17_STABLE); Appendix C 'reserved'. +all +analyse +analyze +and +any +array +as +asc +asymmetric +both +case +cast +check +collate +column +constraint +create +current_catalog +current_date +current_role +current_time +current_timestamp +current_user +default +deferrable +desc +distinct +do +else +end +except +false +fetch +for +foreign +from +grant +group +having +in +initially +intersect +into +lateral +leading +limit +localtime +localtimestamp +not +null +offset +on +only +or +order +placing +primary +references +returning +select +session_user +some +symmetric +system_user +table +then +to +trailing +true +union +unique +user +using +variadic +when +where +window +with diff --git a/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/snowflake.txt b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/snowflake.txt new file mode 100644 index 0000000000..511b64668b --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/snowflake.txt @@ -0,0 +1,65 @@ +# Snowflake reserved keywords: entries marked 'Reserved by ANSI', 'Reserved by Snowflake' +# or 'Reserved by Snowflake and others' in https://docs.snowflake.com/en/sql-reference/reserved-keywords. +# Limited keywords (usable as identifiers in some positions) are excluded. +all +alter +and +any +as +between +by +check +column +connect +create +current +delete +distinct +drop +else +exists +following +for +from +grant +group +having +ilike +in +increment +insert +intersect +into +is +like +minus +not +null +of +on +or +order +qualify +regexp +revoke +rlike +row +rows +sample +select +set +some +start +table +tablesample +then +to +trigger +union +unique +update +values +whenever +where +window +with diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_fingerprint_changes.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_fingerprint_changes.py index 609fc99929..fd2b83ab95 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_fingerprint_changes.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_fingerprint_changes.py @@ -54,6 +54,13 @@ expected_format_summary="3 files checked, 2 changed", expected_reasons={"stg_orders": "no_change", "large_orders": "no_change"}, ), + QueryFingerprintEditE2ETestCase( + description="an unedited project plans nothing after a build", + edits=(), + format_flags=("--check",), + expected_format_summary="3 files checked, 2 changed", + expected_reasons={"stg_orders": "no_change", "large_orders": "no_change"}, + ), QueryFingerprintEditE2ETestCase( description="comment and keyword case edits are unchanged", edits=( @@ -71,6 +78,13 @@ expected_format_summary="3 files checked, 2 changed", expected_reasons={"stg_orders": "query_changed"}, ), + QueryFingerprintEditE2ETestCase( + description="identifier case is a query change", + edits=(("models/large_orders.sql", "AS is_large", "AS Is_Large"),), + format_flags=("--check",), + expected_format_summary="3 files checked, 2 changed", + expected_reasons={"stg_orders": "no_change", "large_orders": "query_changed"}, + ), ], ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/_test_types.py index 03bb59ce74..c2e1831c4f 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/_test_types.py @@ -104,3 +104,11 @@ class QueryFingerprintFailureTestCase: dialect: str query_sql: str expected_message: str + + +@dataclass(frozen=True) +class QueryFingerprintTimeLimitTestCase: + description: str + dialect: str + query_sql: str + expected_max_seconds: float diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py index efc8b96a50..5d8ebbfc3d 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py @@ -1,5 +1,7 @@ from __future__ import annotations +import time + import pytest from sqlbuild._native import QueryFingerprintError, compute_query_hash, normalize_query_sql @@ -8,8 +10,11 @@ ComputeQueryHashTestCase, NormalizeQuerySqlTestCase, QueryFingerprintFailureTestCase, + QueryFingerprintTimeLimitTestCase, ) +_NESTING: int = 32_000 + @pytest.mark.parametrize( "test_case", @@ -79,6 +84,12 @@ def test_given_raw_query_when_normalizing_then_returns_expected_text( "QUALIFY ROW_NUMBER() OVER (PARTITION BY o.id ORDER BY o.id) = 1", "c75f7b5931a18e1a074ffa7ab113c256fc838285af75a3f9feb62d61cc76d94d", ), + ComputeQueryHashTestCase( + "DuckDB lower-case keywords and SQLBuild placeholder", + "duckdb", + 'select o.id from __ref("orders") as o where o.id > 0', + "fda88b9b0f26486bdd25b8e3bab5a982fbb387901ab229091b17772d044aebd0", + ), ComputeQueryHashTestCase( "BigQuery raw string and backtick path", "bigquery", @@ -161,10 +172,80 @@ def test_given_dialect_query_when_computing_hash_then_golden_fingerprint_is_stab False, ), ComputeQueryHashStabilityTestCase( - "user-defined function case is significant", + "unquoted call name case is ignored", "duckdb", "SELECT MyFunc(id) FROM orders", "SELECT myfunc(id) FROM orders", + True, + ), + ComputeQueryHashStabilityTestCase( + "quoted call name case is significant", + "snowflake", + 'SELECT "MyFunc"(id) FROM orders', + 'SELECT "myfunc"(id) FROM orders', + False, + ), + ComputeQueryHashStabilityTestCase( + "PostgreSQL multi-character operator is one token", + "postgres", + "SELECT a |>> b FROM orders", + "SELECT a |> > b FROM orders", + False, + ), + ComputeQueryHashStabilityTestCase( + "CTE column-list name case is significant", + "duckdb", + "WITH Tree(id) AS (SELECT 1) SELECT id FROM orders", + "WITH tree(id) AS (SELECT 1) SELECT id FROM orders", + False, + ), + ComputeQueryHashStabilityTestCase( + "SQLBuild placeholder case is significant", + "duckdb", + 'SELECT id FROM __ref("orders")', + 'SELECT id FROM __REF("orders")', + False, + ), + ComputeQueryHashStabilityTestCase( + "PostgreSQL comments, layout and keyword case are ignored", + "postgres", + "SELECT o.id -- key\nFROM orders AS o WHERE o.id > 0", + "select o.id from orders as o /* all */ where o.id > 0", + True, + ), + ComputeQueryHashStabilityTestCase( + "PostgreSQL dollar-quoted body is significant", + "postgres", + "SELECT $tag$it's$tag$ AS note", + "SELECT $tag$it s$tag$ AS note", + False, + ), + ComputeQueryHashStabilityTestCase( + "SQLite comments and keyword case are ignored", + "sqlite", + "SELECT id FROM orders -- trailing", + "select id\nfrom orders", + True, + ), + ComputeQueryHashStabilityTestCase( + "SQLite identifier case is significant", + "sqlite", + "SELECT Id FROM orders", + "SELECT id FROM orders", + False, + ), + ComputeQueryHashStabilityTestCase( + "Snowflake double-slash comments are ignored", + "snowflake", + "SELECT id // order key\nFROM orders", + "SELECT id FROM orders", + True, + ), + ComputeQueryHashStabilityTestCase( + "generic keyword case is ignored and identifier case is not", + None, + "select Id from orders", + "SELECT id FROM orders", False, ), ComputeQueryHashStabilityTestCase( @@ -231,3 +312,36 @@ def test_given_untokenizable_sql_when_computing_hash_then_raises_fingerprint_err ) -> None: with pytest.raises(QueryFingerprintError, match=test_case.expected_message): _ = compute_query_hash(query_sql=test_case.query_sql, dialect=test_case.dialect) + + +@pytest.mark.parametrize( + "test_case", + [ + QueryFingerprintTimeLimitTestCase( + "deeply nested parentheses", + "duckdb", + "SELECT 1 + " + "(" * _NESTING + "1" + ")" * _NESTING, + 0.25, + ), + QueryFingerprintTimeLimitTestCase( + "deeply nested calls", + "postgres", + "SELECT " + "f(" * _NESTING + "1" + ")" * _NESTING, + 0.25, + ), + QueryFingerprintTimeLimitTestCase( + "unclosed calls", + "snowflake", + "SELECT " + "f(" * _NESTING, + 0.25, + ), + ], + ids=lambda case: case.description, +) +def test_given_pathological_nesting_when_computing_hash_then_time_stays_linear( + test_case: QueryFingerprintTimeLimitTestCase, +) -> None: + started: float = time.perf_counter() + _ = compute_query_hash(query_sql=test_case.query_sql, dialect=test_case.dialect) + + assert time.perf_counter() - started < test_case.expected_max_seconds