From ba15039ddccbceb32aa515af23bfd17253995357 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 15:52:05 +0000 Subject: [PATCH 1/4] feat!: compute query fingerprints from SQLQuack tokens Query fingerprints for generic, DuckDB, PostgreSQL, Snowflake and SQLite SQL now come from SQLQuack's lossless tokenizer instead of Polyglot's. Reserved keywords and called names fold case outside name positions; SQLBuild placeholders such as __ref keep their case. BigQuery, Databricks, SQL Server and ClickHouse keep the previous token stream until SQLQuack tokenizes them. BREAKING CHANGE: the first run after upgrading can plan some models as query_changed even though their SQL did not change. On the switched dialects this affects models that call user-defined functions, whose name case now folds like built-in function names. Plan that run: affected incremental and microbatch models whose replay_on_change is bounded or full replay or rebuild, and sqb build --changed reruns them. Models on the default forward policy only warn. --- Cargo.lock | 24 ++++++ Cargo.toml | 4 +- crates/sqlbuild-analysis/Cargo.toml | 1 + .../sql_tokens/_helpers/canonical_tokens.rs | 41 ++++++++++ .../src/sql_tokens/_helpers/mod.rs | 2 + .../sql_tokens/_helpers/reserved_keywords.rs | 34 ++++++++ .../src/sql_tokens/constants.rs | 41 ++++++++++ .../src/sql_tokens/main/query_fingerprint.rs | 32 ++++++-- .../sqlbuild-analysis/src/sql_tokens/mod.rs | 4 +- .../src/sql_tokens/models.rs | 20 +++++ .../sql_tokens/reserved_keywords/duckdb.txt | 77 ++++++++++++++++++ .../sql_tokens/reserved_keywords/generic.txt | 28 +++++++ .../sql_tokens/reserved_keywords/postgres.txt | 80 +++++++++++++++++++ .../reserved_keywords/snowflake.txt | 65 +++++++++++++++ .../build/test_query_fingerprint_changes.py | 14 ++++ .../main/test_query_fingerprint.py | 57 ++++++++++++- 16 files changed, 514 insertions(+), 10 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/sql_tokens/_helpers/canonical_tokens.rs create mode 100644 crates/sqlbuild-analysis/src/sql_tokens/_helpers/mod.rs create mode 100644 crates/sqlbuild-analysis/src/sql_tokens/_helpers/reserved_keywords.rs create mode 100644 crates/sqlbuild-analysis/src/sql_tokens/models.rs create mode 100644 crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/duckdb.txt create mode 100644 crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/generic.txt create mode 100644 crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/postgres.txt create mode 100644 crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/snowflake.txt diff --git a/Cargo.lock b/Cargo.lock index 7306a408d1..4afd0e11c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4142,6 +4142,7 @@ dependencies = [ "sqlbuild-scopes", "sqlbuild-sqltext", "sqlclean", + "sqlquack-syntax", ] [[package]] @@ -4494,6 +4495,29 @@ dependencies = [ "sha2 0.10.9", ] +[[package]] +name = "sqlquack-syntax" +version = "0.2.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=4d0c6bc56da0e44eb164601a2af673ae7998257d#4d0c6bc56da0e44eb164601a2af673ae7998257d" +dependencies = [ + "serde", + "serde_json", + "sqlquack-syntax-derive", + "stacker", + "thiserror 1.0.69", + "unicode-segmentation", +] + +[[package]] +name = "sqlquack-syntax-derive" +version = "0.2.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=4d0c6bc56da0e44eb164601a2af673ae7998257d#4d0c6bc56da0e44eb164601a2af673ae7998257d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "sqlx-core" version = "0.8.6" diff --git a/Cargo.toml b/Cargo.toml index 9ef850601d..03369eec88 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -174,8 +174,10 @@ sqlbuild-recall = { path = "crates/sqlbuild-recall" } sqlbuild-scopes = { path = "crates/sqlbuild-scopes" } sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" } sqlbuild-state = { path = "crates/sqlbuild-state" } -# SQL lint Rules, quality Rules, the formatter and the canonical token stream. +# SQL lint Rules, quality Rules and the formatter. sqlclean = { git = "https://github.com/sqlstack-io/sqlclean.git", rev = "cf79656d247277a30bc81ef39b6dee420e721221" } +# The lossless SQL tokenizer behind query fingerprints. +sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "4d0c6bc56da0e44eb164601a2af673ae7998257d" } stacker = "0.1" syn = { version = "2", features = ["full", "visit", "extra-traits"] } subtle = "2" diff --git a/crates/sqlbuild-analysis/Cargo.toml b/crates/sqlbuild-analysis/Cargo.toml index 864c22d7e7..e943fae308 100644 --- a/crates/sqlbuild-analysis/Cargo.toml +++ b/crates/sqlbuild-analysis/Cargo.toml @@ -26,6 +26,7 @@ sqlbuild-render.workspace = true sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true sqlclean.workspace = true +sqlquack-syntax.workspace = true [lints] workspace = true diff --git a/crates/sqlbuild-analysis/src/sql_tokens/_helpers/canonical_tokens.rs b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/canonical_tokens.rs new file mode 100644 index 0000000000..cdadca4c78 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/canonical_tokens.rs @@ -0,0 +1,41 @@ +//! The canonical token stream of SQL under SQLQuack's lossless tokenizer. + +use sqlclean::sql_tokens::main::is_unquoted_word::is_unquoted_word; +use sqlclean::sql_tokens::main::name_positions::name_positions; +use sqlquack_syntax::lossless::models::TokenKind; + +use crate::sql_tokens::_helpers::reserved_keywords::is_reserved_keyword; +use crate::sql_tokens::constants::{CALL_OPENER, PLACEHOLDER_PREFIX}; +use crate::sql_tokens::models::SqlquackFingerprintDialect; + +/// Significant token texts with reserved keywords and called names, except `__` placeholders, upper-cased. +pub(crate) fn canonical_tokens( + sql: &str, + dialect: SqlquackFingerprintDialect, +) -> Result, String> { + let tokens = sqlquack_syntax::tokenize(sql, dialect.tokenizer) + .map_err(|error| format!("SQL tokenization failed: {error}"))?; + let raws: Vec<&str> = tokens + .iter() + .filter(|token| matches!(token.kind, TokenKind::Significant(_))) + .map(|token| token.text(sql)) + .collect(); + let uppers: Vec = raws.iter().map(|raw| raw.to_ascii_uppercase()).collect(); + let names: Vec = name_positions(&uppers); + let mut texts: Vec = Vec::with_capacity(raws.len()); + for (index, raw) in raws.iter().enumerate() { + let calls = uppers + .get(index + 1) + .is_some_and(|next| next == CALL_OPENER) + && !raw.starts_with(PLACEHOLDER_PREFIX); + let folds = !names[index] + && is_unquoted_word(raw) + && (calls || is_reserved_keyword(dialect.keywords, &uppers[index])); + texts.push(if folds { + uppers[index].clone() + } else { + (*raw).to_owned() + }); + } + Ok(texts) +} diff --git a/crates/sqlbuild-analysis/src/sql_tokens/_helpers/mod.rs b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/mod.rs new file mode 100644 index 0000000000..5e7000dd1e --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod canonical_tokens; +pub(crate) mod reserved_keywords; diff --git a/crates/sqlbuild-analysis/src/sql_tokens/_helpers/reserved_keywords.rs b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/reserved_keywords.rs new file mode 100644 index 0000000000..dae99b7712 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/reserved_keywords.rs @@ -0,0 +1,34 @@ +//! Per-dialect reserved keywords, which can never be unquoted identifiers in that dialect. + +use std::collections::HashSet; +use std::sync::LazyLock; + +use crate::sql_tokens::models::KeywordList; + +static DUCKDB: LazyLock> = + LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/duckdb.txt"))); +static POSTGRES: LazyLock> = + LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/postgres.txt"))); +static SNOWFLAKE: LazyLock> = + LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/snowflake.txt"))); +static GENERIC: LazyLock> = + LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/generic.txt"))); + +/// Whether the upper-cased word is reserved in `list`. +pub(crate) fn is_reserved_keyword(list: KeywordList, upper_word: &str) -> bool { + let keywords: &HashSet = match list { + KeywordList::DuckDB => &DUCKDB, + KeywordList::PostgreSQL => &POSTGRES, + KeywordList::Snowflake => &SNOWFLAKE, + KeywordList::Generic => &GENERIC, + }; + keywords.contains(upper_word) +} + +fn keyword_set(data: &str) -> HashSet { + data.lines() + .map(str::trim) + .filter(|line| !line.is_empty() && !line.starts_with('#')) + .map(str::to_ascii_uppercase) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/sql_tokens/constants.rs b/crates/sqlbuild-analysis/src/sql_tokens/constants.rs index 44e22ce088..9c452c67db 100644 --- a/crates/sqlbuild-analysis/src/sql_tokens/constants.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/constants.rs @@ -1,5 +1,9 @@ //! Fingerprint dialects and the function language fingerprinted like a query. +use sqlquack_syntax::tokens::models::Dialect; + +use crate::sql_tokens::models::{KeywordList, SqlquackFingerprintDialect}; + /// The fingerprint dialect when a project names none. pub(crate) const GENERIC_FINGERPRINT_DIALECT: &str = "generic"; /// Python's `NATIVE_DIALECT_ALIASES`. @@ -10,3 +14,40 @@ pub(crate) const FINGERPRINT_DIALECT_ALIASES: [(&str, &str); 3] = [ ]; /// The function language whose definitions are fingerprinted like queries. pub(crate) const SQL_FUNCTION_LANGUAGE: &str = "sql"; +/// Fingerprint dialects SQLQuack tokenizes; the generic stream uses its DuckDB tokenizer. +pub(crate) const SQLQUACK_FINGERPRINT_DIALECTS: [SqlquackFingerprintDialect; 6] = [ + SqlquackFingerprintDialect { + name: "generic", + tokenizer: Dialect::DuckDB, + keywords: KeywordList::Generic, + }, + SqlquackFingerprintDialect { + name: "duckdb", + tokenizer: Dialect::DuckDB, + keywords: KeywordList::DuckDB, + }, + SqlquackFingerprintDialect { + name: "postgres", + tokenizer: Dialect::PostgreSQL, + keywords: KeywordList::PostgreSQL, + }, + SqlquackFingerprintDialect { + name: "postgresql", + tokenizer: Dialect::PostgreSQL, + keywords: KeywordList::PostgreSQL, + }, + SqlquackFingerprintDialect { + name: "snowflake", + tokenizer: Dialect::Snowflake, + keywords: KeywordList::Snowflake, + }, + SqlquackFingerprintDialect { + name: "sqlite", + tokenizer: Dialect::SQLite, + keywords: KeywordList::Generic, + }, +]; +/// The token that follows a called function's name. +pub(crate) const CALL_OPENER: &str = "("; +/// The prefix of SQLBuild placeholder calls such as `__ref` and `__source`. +pub(crate) const PLACEHOLDER_PREFIX: &str = "__"; diff --git a/crates/sqlbuild-analysis/src/sql_tokens/main/query_fingerprint.rs b/crates/sqlbuild-analysis/src/sql_tokens/main/query_fingerprint.rs index 9cde608f7d..6162534d1a 100644 --- a/crates/sqlbuild-analysis/src/sql_tokens/main/query_fingerprint.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/main/query_fingerprint.rs @@ -7,19 +7,37 @@ use sha2::{Digest, Sha256}; use sqlclean::sql_tokens::main::canonical_tokens::canonical_tokens; +use crate::sql_tokens::_helpers::canonical_tokens::canonical_tokens as sqlquack_canonical_tokens; +use crate::sql_tokens::constants::SQLQUACK_FINGERPRINT_DIALECTS; + /// Return the SHA-256 hex digest of the canonical token stream of `sql` under `dialect_name`. pub fn query_fingerprint(sql: &str, dialect_name: &str) -> Result { + let texts: Vec = match SQLQUACK_FINGERPRINT_DIALECTS + .iter() + .find(|dialect| dialect.name == dialect_name) + { + Some(dialect) => sqlquack_canonical_tokens(sql, *dialect)?, + None => polyglot_canonical_tokens(sql, dialect_name)?, + }; + let mut digest = Sha256::new(); + for text in texts { + digest.update(text.len().to_string().as_bytes()); + digest.update(b":"); + digest.update(text.as_bytes()); + } + Ok(format!("{:x}", digest.finalize())) +} + +/// Dialects SQLQuack cannot tokenize yet keep SQLClean's Polyglot token stream. +fn polyglot_canonical_tokens(sql: &str, dialect_name: &str) -> Result, String> { let dialect_type = DialectType::from_str(dialect_name) .map_err(|_| format!("unknown SQL fingerprint dialect '{dialect_name}'"))?; let dialect = Dialect::get(dialect_type); - let mut digest = Sha256::new(); let tokens = dialect .tokenize(sql) .map_err(|error| format!("SQL tokenization failed: {error}"))?; - for token in canonical_tokens(sql, &tokens, &dialect)? { - digest.update(token.text.len().to_string().as_bytes()); - digest.update(b":"); - digest.update(token.text.as_bytes()); - } - Ok(format!("{:x}", digest.finalize())) + Ok(canonical_tokens(sql, &tokens, &dialect)? + .into_iter() + .map(|token| token.text) + .collect()) } diff --git a/crates/sqlbuild-analysis/src/sql_tokens/mod.rs b/crates/sqlbuild-analysis/src/sql_tokens/mod.rs index 4d3a25f7d4..a574d034fe 100644 --- a/crates/sqlbuild-analysis/src/sql_tokens/mod.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/mod.rs @@ -1,4 +1,6 @@ -//! SQLBuild's query and definition fingerprints over SQLClean's canonical SQL token stream. +//! SQLBuild's query and definition fingerprints over a canonical SQL token stream. +mod _helpers; pub mod constants; pub mod main; +pub(crate) mod models; diff --git a/crates/sqlbuild-analysis/src/sql_tokens/models.rs b/crates/sqlbuild-analysis/src/sql_tokens/models.rs new file mode 100644 index 0000000000..eabc82a198 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_tokens/models.rs @@ -0,0 +1,20 @@ +//! How a fingerprint dialect is tokenized and which of its words fold. + +use sqlquack_syntax::tokens::models::Dialect; + +/// The reserved-keyword list a fingerprint dialect folds. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum KeywordList { + DuckDB, + PostgreSQL, + Snowflake, + Generic, +} + +/// A fingerprint dialect SQLQuack tokenizes: its tokenizer and its reserved keywords. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct SqlquackFingerprintDialect { + pub(crate) name: &'static str, + pub(crate) tokenizer: Dialect, + pub(crate) keywords: KeywordList, +} diff --git a/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/duckdb.txt b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/duckdb.txt new file mode 100644 index 0000000000..4374c4e538 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/duckdb.txt @@ -0,0 +1,77 @@ +# DuckDB reserved keywords: SELECT keyword_name FROM duckdb_keywords() +# WHERE keyword_category = 'reserved' (DuckDB 1.5.2). +all +analyse +analyze +and +any +array +as +asc +asymmetric +both +case +cast +check +collate +column +constraint +create +default +deferrable +desc +describe +distinct +do +else +end +except +false +fetch +for +foreign +from +group +having +in +initially +intersect +into +lambda +lateral +leading +limit +not +null +offset +on +only +or +order +pivot +pivot_longer +pivot_wider +placing +primary +qualify +references +returning +select +show +some +summarize +symmetric +table +then +to +trailing +true +union +unique +unpivot +using +variadic +when +where +window +with diff --git a/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/generic.txt b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/generic.txt new file mode 100644 index 0000000000..05f8ee42cd --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/generic.txt @@ -0,0 +1,28 @@ +# Generic dialect: keywords reserved in every dialect file above except Databricks +# (DuckDB, PostgreSQL, BigQuery, Snowflake and T-SQL). +all +and +any +as +create +distinct +else +for +from +group +having +in +intersect +into +not +null +on +or +order +select +some +then +to +union +where +with diff --git a/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/postgres.txt b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/postgres.txt new file mode 100644 index 0000000000..7a57911cb5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/postgres.txt @@ -0,0 +1,80 @@ +# PostgreSQL reserved keywords (non-function): RESERVED_KEYWORD entries of +# src/include/parser/kwlist.h (REL_17_STABLE); Appendix C 'reserved'. +all +analyse +analyze +and +any +array +as +asc +asymmetric +both +case +cast +check +collate +column +constraint +create +current_catalog +current_date +current_role +current_time +current_timestamp +current_user +default +deferrable +desc +distinct +do +else +end +except +false +fetch +for +foreign +from +grant +group +having +in +initially +intersect +into +lateral +leading +limit +localtime +localtimestamp +not +null +offset +on +only +or +order +placing +primary +references +returning +select +session_user +some +symmetric +system_user +table +then +to +trailing +true +union +unique +user +using +variadic +when +where +window +with diff --git a/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/snowflake.txt b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/snowflake.txt new file mode 100644 index 0000000000..511b64668b --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/snowflake.txt @@ -0,0 +1,65 @@ +# Snowflake reserved keywords: entries marked 'Reserved by ANSI', 'Reserved by Snowflake' +# or 'Reserved by Snowflake and others' in https://docs.snowflake.com/en/sql-reference/reserved-keywords. +# Limited keywords (usable as identifiers in some positions) are excluded. +all +alter +and +any +as +between +by +check +column +connect +create +current +delete +distinct +drop +else +exists +following +for +from +grant +group +having +ilike +in +increment +insert +intersect +into +is +like +minus +not +null +of +on +or +order +qualify +regexp +revoke +rlike +row +rows +sample +select +set +some +start +table +tablesample +then +to +trigger +union +unique +update +values +whenever +where +window +with diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_fingerprint_changes.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_fingerprint_changes.py index 609fc99929..fd2b83ab95 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_fingerprint_changes.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_fingerprint_changes.py @@ -54,6 +54,13 @@ expected_format_summary="3 files checked, 2 changed", expected_reasons={"stg_orders": "no_change", "large_orders": "no_change"}, ), + QueryFingerprintEditE2ETestCase( + description="an unedited project plans nothing after a build", + edits=(), + format_flags=("--check",), + expected_format_summary="3 files checked, 2 changed", + expected_reasons={"stg_orders": "no_change", "large_orders": "no_change"}, + ), QueryFingerprintEditE2ETestCase( description="comment and keyword case edits are unchanged", edits=( @@ -71,6 +78,13 @@ expected_format_summary="3 files checked, 2 changed", expected_reasons={"stg_orders": "query_changed"}, ), + QueryFingerprintEditE2ETestCase( + description="identifier case is a query change", + edits=(("models/large_orders.sql", "AS is_large", "AS Is_Large"),), + format_flags=("--check",), + expected_format_summary="3 files checked, 2 changed", + expected_reasons={"stg_orders": "no_change", "large_orders": "query_changed"}, + ), ], ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py index efc8b96a50..e4ef165a4f 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py @@ -79,6 +79,12 @@ def test_given_raw_query_when_normalizing_then_returns_expected_text( "QUALIFY ROW_NUMBER() OVER (PARTITION BY o.id ORDER BY o.id) = 1", "c75f7b5931a18e1a074ffa7ab113c256fc838285af75a3f9feb62d61cc76d94d", ), + ComputeQueryHashTestCase( + "DuckDB lower-case keywords and SQLBuild placeholder", + "duckdb", + 'select o.id from __ref("orders") as o where o.id > 0', + "fda88b9b0f26486bdd25b8e3bab5a982fbb387901ab229091b17772d044aebd0", + ), ComputeQueryHashTestCase( "BigQuery raw string and backtick path", "bigquery", @@ -161,10 +167,59 @@ def test_given_dialect_query_when_computing_hash_then_golden_fingerprint_is_stab False, ), ComputeQueryHashStabilityTestCase( - "user-defined function case is significant", + "unquoted call name case is ignored", "duckdb", "SELECT MyFunc(id) FROM orders", "SELECT myfunc(id) FROM orders", + True, + ), + ComputeQueryHashStabilityTestCase( + "SQLBuild placeholder case is significant", + "duckdb", + 'SELECT id FROM __ref("orders")', + 'SELECT id FROM __REF("orders")', + False, + ), + ComputeQueryHashStabilityTestCase( + "PostgreSQL comments, layout and keyword case are ignored", + "postgres", + "SELECT o.id -- key\nFROM orders AS o WHERE o.id > 0", + "select o.id from orders as o /* all */ where o.id > 0", + True, + ), + ComputeQueryHashStabilityTestCase( + "PostgreSQL dollar-quoted body is significant", + "postgres", + "SELECT $tag$it's$tag$ AS note", + "SELECT $tag$it s$tag$ AS note", + False, + ), + ComputeQueryHashStabilityTestCase( + "SQLite comments and keyword case are ignored", + "sqlite", + "SELECT id FROM orders -- trailing", + "select id\nfrom orders", + True, + ), + ComputeQueryHashStabilityTestCase( + "SQLite identifier case is significant", + "sqlite", + "SELECT Id FROM orders", + "SELECT id FROM orders", + False, + ), + ComputeQueryHashStabilityTestCase( + "Snowflake double-slash comments are ignored", + "snowflake", + "SELECT id // order key\nFROM orders", + "SELECT id FROM orders", + True, + ), + ComputeQueryHashStabilityTestCase( + "generic keyword case is ignored and identifier case is not", + None, + "select Id from orders", + "SELECT id FROM orders", False, ), ComputeQueryHashStabilityTestCase( From 18a1bd0e8dc5d0aaa6fde32af5638eb68331806f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 15:56:32 +0000 Subject: [PATCH 2/4] fix: keep CTE and relation column-list name case significant in fingerprints --- .../sql_tokens/_helpers/canonical_tokens.rs | 36 +++++++++++++++++-- .../src/sql_tokens/constants.rs | 8 +++++ .../main/test_query_fingerprint.py | 7 ++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/crates/sqlbuild-analysis/src/sql_tokens/_helpers/canonical_tokens.rs b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/canonical_tokens.rs index cdadca4c78..2bc4a9f67f 100644 --- a/crates/sqlbuild-analysis/src/sql_tokens/_helpers/canonical_tokens.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/canonical_tokens.rs @@ -5,7 +5,10 @@ use sqlclean::sql_tokens::main::name_positions::name_positions; use sqlquack_syntax::lossless::models::TokenKind; use crate::sql_tokens::_helpers::reserved_keywords::is_reserved_keyword; -use crate::sql_tokens::constants::{CALL_OPENER, PLACEHOLDER_PREFIX}; +use crate::sql_tokens::constants::{ + CALL_CLOSER, CALL_OPENER, COLUMN_LIST_RELATION_KEYWORDS, CTE_BODY_OPENERS, CTE_BODY_PREFIX, + PLACEHOLDER_PREFIX, +}; use crate::sql_tokens::models::SqlquackFingerprintDialect; /// Significant token texts with reserved keywords and called names, except `__` placeholders, upper-cased. @@ -27,7 +30,8 @@ pub(crate) fn canonical_tokens( let calls = uppers .get(index + 1) .is_some_and(|next| next == CALL_OPENER) - && !raw.starts_with(PLACEHOLDER_PREFIX); + && !raw.starts_with(PLACEHOLDER_PREFIX) + && !names_column_list(&uppers, index, dialect); let folds = !names[index] && is_unquoted_word(raw) && (calls || is_reserved_keyword(dialect.keywords, &uppers[index])); @@ -39,3 +43,31 @@ pub(crate) fn canonical_tokens( } Ok(texts) } + +/// Whether the word at `index` names the relation a column list follows, as in `cte(a, b) AS (`. +fn names_column_list(uppers: &[String], index: usize, dialect: SqlquackFingerprintDialect) -> bool { + let previous: Option<&str> = index.checked_sub(1).map(|before| uppers[before].as_str()); + if previous.is_some_and(|word| { + COLUMN_LIST_RELATION_KEYWORDS.contains(&word) + || (is_unquoted_word(word) && !is_reserved_keyword(dialect.keywords, word)) + }) { + return true; + } + let mut depth: usize = 0; + for (offset, token) in uppers.iter().enumerate().skip(index + 1) { + if token == CALL_OPENER { + depth += 1; + } else if token == CALL_CLOSER { + depth -= 1; + if depth == 0 { + return uppers + .get(offset + 1) + .is_some_and(|next| next == CTE_BODY_PREFIX) + && uppers + .get(offset + 2) + .is_some_and(|next| CTE_BODY_OPENERS.contains(&next.as_str())); + } + } + } + false +} diff --git a/crates/sqlbuild-analysis/src/sql_tokens/constants.rs b/crates/sqlbuild-analysis/src/sql_tokens/constants.rs index 9c452c67db..aa44590b35 100644 --- a/crates/sqlbuild-analysis/src/sql_tokens/constants.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/constants.rs @@ -49,5 +49,13 @@ pub(crate) const SQLQUACK_FINGERPRINT_DIALECTS: [SqlquackFingerprintDialect; 6] ]; /// The token that follows a called function's name. pub(crate) const CALL_OPENER: &str = "("; +/// The token that closes a call's arguments or a column list. +pub(crate) const CALL_CLOSER: &str = ")"; +/// Keywords after which a word followed by `(` names a relation and its column list. +pub(crate) const COLUMN_LIST_RELATION_KEYWORDS: [&str; 2] = ["INTO", "TABLE"]; +/// The keyword between a CTE's column list and its body. +pub(crate) const CTE_BODY_PREFIX: &str = "AS"; +/// The tokens that can follow `AS` after a CTE's column list. +pub(crate) const CTE_BODY_OPENERS: [&str; 3] = ["(", "MATERIALIZED", "NOT"]; /// The prefix of SQLBuild placeholder calls such as `__ref` and `__source`. pub(crate) const PLACEHOLDER_PREFIX: &str = "__"; diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py index e4ef165a4f..262d9da933 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py @@ -173,6 +173,13 @@ def test_given_dialect_query_when_computing_hash_then_golden_fingerprint_is_stab "SELECT myfunc(id) FROM orders", True, ), + ComputeQueryHashStabilityTestCase( + "CTE column-list name case is significant", + "duckdb", + "WITH Tree(id) AS (SELECT 1) SELECT id FROM orders", + "WITH tree(id) AS (SELECT 1) SELECT id FROM orders", + False, + ), ComputeQueryHashStabilityTestCase( "SQLBuild placeholder case is significant", "duckdb", From 49b8c611882b5d934c27f7da415f2323da974a06 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 16:12:06 +0000 Subject: [PATCH 3/4] feat: fingerprint PostgreSQL with SQLQuack's native scanner Fold call names after JOIN again; only INTO, TABLE and CTE column lists keep a called name's case. --- Cargo.lock | 9 ++++ Cargo.toml | 1 + crates/sqlbuild-analysis/Cargo.toml | 1 + .../sql_tokens/_helpers/canonical_tokens.rs | 46 +++++++++++++------ .../src/sql_tokens/constants.rs | 16 +++---- .../src/sql_tokens/models.rs | 11 ++++- .../main/test_query_fingerprint.py | 14 ++++++ 7 files changed, 74 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4afd0e11c9..2fece4274a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4143,6 +4143,7 @@ dependencies = [ "sqlbuild-sqltext", "sqlclean", "sqlquack-syntax", + "sqlquack-syntax-postgres", ] [[package]] @@ -4518,6 +4519,14 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "sqlquack-syntax-postgres" +version = "0.2.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=4d0c6bc56da0e44eb164601a2af673ae7998257d#4d0c6bc56da0e44eb164601a2af673ae7998257d" +dependencies = [ + "sqlquack-syntax", +] + [[package]] name = "sqlx-core" version = "0.8.6" diff --git a/Cargo.toml b/Cargo.toml index 03369eec88..f03cafe024 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -178,6 +178,7 @@ sqlbuild-state = { path = "crates/sqlbuild-state" } sqlclean = { git = "https://github.com/sqlstack-io/sqlclean.git", rev = "cf79656d247277a30bc81ef39b6dee420e721221" } # The lossless SQL tokenizer behind query fingerprints. sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "4d0c6bc56da0e44eb164601a2af673ae7998257d" } +sqlquack-syntax-postgres = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "4d0c6bc56da0e44eb164601a2af673ae7998257d" } stacker = "0.1" syn = { version = "2", features = ["full", "visit", "extra-traits"] } subtle = "2" diff --git a/crates/sqlbuild-analysis/Cargo.toml b/crates/sqlbuild-analysis/Cargo.toml index e943fae308..80a5ae2313 100644 --- a/crates/sqlbuild-analysis/Cargo.toml +++ b/crates/sqlbuild-analysis/Cargo.toml @@ -27,6 +27,7 @@ sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true sqlclean.workspace = true sqlquack-syntax.workspace = true +sqlquack-syntax-postgres.workspace = true [lints] workspace = true diff --git a/crates/sqlbuild-analysis/src/sql_tokens/_helpers/canonical_tokens.rs b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/canonical_tokens.rs index 2bc4a9f67f..2c37563cdd 100644 --- a/crates/sqlbuild-analysis/src/sql_tokens/_helpers/canonical_tokens.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/canonical_tokens.rs @@ -1,28 +1,24 @@ -//! The canonical token stream of SQL under SQLQuack's lossless tokenizer. +//! The canonical token stream of SQL under SQLQuack's tokenizers. use sqlclean::sql_tokens::main::is_unquoted_word::is_unquoted_word; use sqlclean::sql_tokens::main::name_positions::name_positions; use sqlquack_syntax::lossless::models::TokenKind; +use sqlquack_syntax_postgres::scan::main::scan::scan; +use sqlquack_syntax_postgres::scan::models::TokenKind as PostgresTokenKind; use crate::sql_tokens::_helpers::reserved_keywords::is_reserved_keyword; use crate::sql_tokens::constants::{ CALL_CLOSER, CALL_OPENER, COLUMN_LIST_RELATION_KEYWORDS, CTE_BODY_OPENERS, CTE_BODY_PREFIX, PLACEHOLDER_PREFIX, }; -use crate::sql_tokens::models::SqlquackFingerprintDialect; +use crate::sql_tokens::models::{FingerprintTokenizer, SqlquackFingerprintDialect}; /// Significant token texts with reserved keywords and called names, except `__` placeholders, upper-cased. pub(crate) fn canonical_tokens( sql: &str, dialect: SqlquackFingerprintDialect, ) -> Result, String> { - let tokens = sqlquack_syntax::tokenize(sql, dialect.tokenizer) - .map_err(|error| format!("SQL tokenization failed: {error}"))?; - let raws: Vec<&str> = tokens - .iter() - .filter(|token| matches!(token.kind, TokenKind::Significant(_))) - .map(|token| token.text(sql)) - .collect(); + let raws: Vec<&str> = significant_texts(sql, dialect.tokenizer)?; let uppers: Vec = raws.iter().map(|raw| raw.to_ascii_uppercase()).collect(); let names: Vec = name_positions(&uppers); let mut texts: Vec = Vec::with_capacity(raws.len()); @@ -31,7 +27,7 @@ pub(crate) fn canonical_tokens( .get(index + 1) .is_some_and(|next| next == CALL_OPENER) && !raw.starts_with(PLACEHOLDER_PREFIX) - && !names_column_list(&uppers, index, dialect); + && !names_column_list(&uppers, index); let folds = !names[index] && is_unquoted_word(raw) && (calls || is_reserved_keyword(dialect.keywords, &uppers[index])); @@ -44,13 +40,33 @@ pub(crate) fn canonical_tokens( Ok(texts) } +/// The text of each significant token of `sql`, in order. +fn significant_texts(sql: &str, tokenizer: FingerprintTokenizer) -> Result, String> { + match tokenizer { + FingerprintTokenizer::Lossless(dialect) => Ok(sqlquack_syntax::tokenize(sql, dialect) + .map_err(|error| format!("SQL tokenization failed: {error}"))? + .iter() + .filter(|token| matches!(token.kind, TokenKind::Significant(_))) + .map(|token| token.text(sql)) + .collect()), + FingerprintTokenizer::PostgresScanner => scan(sql) + .into_iter() + .map(|token| match token.kind { + PostgresTokenKind::Fault(_) => { + Err(format!("SQL tokenization failed: {}", token.value)) + } + _ => sql + .get(token.start..token.end) + .ok_or_else(|| format!("SQL tokenization failed at byte {}", token.start)), + }) + .collect(), + } +} + /// Whether the word at `index` names the relation a column list follows, as in `cte(a, b) AS (`. -fn names_column_list(uppers: &[String], index: usize, dialect: SqlquackFingerprintDialect) -> bool { +fn names_column_list(uppers: &[String], index: usize) -> bool { let previous: Option<&str> = index.checked_sub(1).map(|before| uppers[before].as_str()); - if previous.is_some_and(|word| { - COLUMN_LIST_RELATION_KEYWORDS.contains(&word) - || (is_unquoted_word(word) && !is_reserved_keyword(dialect.keywords, word)) - }) { + if previous.is_some_and(|word| COLUMN_LIST_RELATION_KEYWORDS.contains(&word)) { return true; } let mut depth: usize = 0; diff --git a/crates/sqlbuild-analysis/src/sql_tokens/constants.rs b/crates/sqlbuild-analysis/src/sql_tokens/constants.rs index aa44590b35..7a59c6115f 100644 --- a/crates/sqlbuild-analysis/src/sql_tokens/constants.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/constants.rs @@ -2,7 +2,7 @@ use sqlquack_syntax::tokens::models::Dialect; -use crate::sql_tokens::models::{KeywordList, SqlquackFingerprintDialect}; +use crate::sql_tokens::models::{FingerprintTokenizer, KeywordList, SqlquackFingerprintDialect}; /// The fingerprint dialect when a project names none. pub(crate) const GENERIC_FINGERPRINT_DIALECT: &str = "generic"; @@ -14,36 +14,36 @@ pub(crate) const FINGERPRINT_DIALECT_ALIASES: [(&str, &str); 3] = [ ]; /// The function language whose definitions are fingerprinted like queries. pub(crate) const SQL_FUNCTION_LANGUAGE: &str = "sql"; -/// Fingerprint dialects SQLQuack tokenizes; the generic stream uses its DuckDB tokenizer. +/// Fingerprint dialects SQLQuack tokenizes; generic uses the DuckDB tokenizer, PostgreSQL its native scanner. pub(crate) const SQLQUACK_FINGERPRINT_DIALECTS: [SqlquackFingerprintDialect; 6] = [ SqlquackFingerprintDialect { name: "generic", - tokenizer: Dialect::DuckDB, + tokenizer: FingerprintTokenizer::Lossless(Dialect::DuckDB), keywords: KeywordList::Generic, }, SqlquackFingerprintDialect { name: "duckdb", - tokenizer: Dialect::DuckDB, + tokenizer: FingerprintTokenizer::Lossless(Dialect::DuckDB), keywords: KeywordList::DuckDB, }, SqlquackFingerprintDialect { name: "postgres", - tokenizer: Dialect::PostgreSQL, + tokenizer: FingerprintTokenizer::PostgresScanner, keywords: KeywordList::PostgreSQL, }, SqlquackFingerprintDialect { name: "postgresql", - tokenizer: Dialect::PostgreSQL, + tokenizer: FingerprintTokenizer::PostgresScanner, keywords: KeywordList::PostgreSQL, }, SqlquackFingerprintDialect { name: "snowflake", - tokenizer: Dialect::Snowflake, + tokenizer: FingerprintTokenizer::Lossless(Dialect::Snowflake), keywords: KeywordList::Snowflake, }, SqlquackFingerprintDialect { name: "sqlite", - tokenizer: Dialect::SQLite, + tokenizer: FingerprintTokenizer::Lossless(Dialect::SQLite), keywords: KeywordList::Generic, }, ]; diff --git a/crates/sqlbuild-analysis/src/sql_tokens/models.rs b/crates/sqlbuild-analysis/src/sql_tokens/models.rs index eabc82a198..1221a29855 100644 --- a/crates/sqlbuild-analysis/src/sql_tokens/models.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/models.rs @@ -11,10 +11,19 @@ pub(crate) enum KeywordList { Generic, } +/// The SQLQuack tokenizer a fingerprint dialect reads its tokens with. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum FingerprintTokenizer { + /// `sqlquack_syntax`'s lossless tokenizer for one dialect. + Lossless(Dialect), + /// The native PostgreSQL scanner of `sqlquack_syntax_postgres`. + PostgresScanner, +} + /// A fingerprint dialect SQLQuack tokenizes: its tokenizer and its reserved keywords. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) struct SqlquackFingerprintDialect { pub(crate) name: &'static str, - pub(crate) tokenizer: Dialect, + pub(crate) tokenizer: FingerprintTokenizer, pub(crate) keywords: KeywordList, } diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py index 262d9da933..702eb9e639 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py @@ -173,6 +173,20 @@ def test_given_dialect_query_when_computing_hash_then_golden_fingerprint_is_stab "SELECT myfunc(id) FROM orders", True, ), + ComputeQueryHashStabilityTestCase( + "quoted call name case is significant", + "snowflake", + 'SELECT "MyFunc"(id) FROM orders', + 'SELECT "myfunc"(id) FROM orders', + False, + ), + ComputeQueryHashStabilityTestCase( + "PostgreSQL multi-character operator is one token", + "postgres", + "SELECT a |>> b FROM orders", + "SELECT a |> > b FROM orders", + False, + ), ComputeQueryHashStabilityTestCase( "CTE column-list name case is significant", "duckdb", From 227b63326d9c9722bc128f9874709e4b6d253dc8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 16:13:32 +0000 Subject: [PATCH 4/4] fix: match fingerprint parentheses in one pass Check the word before looking for a column list and find each call's closing parenthesis with one stack pass, so deep or unbalanced nesting stays linear. --- .../sql_tokens/_helpers/canonical_tokens.rs | 62 +++++++++++-------- .../compiler/fingerprints/main/_test_types.py | 8 +++ .../main/test_query_fingerprint.py | 38 ++++++++++++ 3 files changed, 82 insertions(+), 26 deletions(-) diff --git a/crates/sqlbuild-analysis/src/sql_tokens/_helpers/canonical_tokens.rs b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/canonical_tokens.rs index 2c37563cdd..c5d1988159 100644 --- a/crates/sqlbuild-analysis/src/sql_tokens/_helpers/canonical_tokens.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/canonical_tokens.rs @@ -21,16 +21,13 @@ pub(crate) fn canonical_tokens( let raws: Vec<&str> = significant_texts(sql, dialect.tokenizer)?; let uppers: Vec = raws.iter().map(|raw| raw.to_ascii_uppercase()).collect(); let names: Vec = name_positions(&uppers); + let closers: Vec> = matching_closers(&uppers); let mut texts: Vec = Vec::with_capacity(raws.len()); for (index, raw) in raws.iter().enumerate() { - let calls = uppers - .get(index + 1) - .is_some_and(|next| next == CALL_OPENER) - && !raw.starts_with(PLACEHOLDER_PREFIX) - && !names_column_list(&uppers, index); let folds = !names[index] && is_unquoted_word(raw) - && (calls || is_reserved_keyword(dialect.keywords, &uppers[index])); + && (is_reserved_keyword(dialect.keywords, &uppers[index]) + || calls(raw, &uppers, &closers, index)); texts.push(if folds { uppers[index].clone() } else { @@ -63,27 +60,40 @@ fn significant_texts(sql: &str, tokenizer: FingerprintTokenizer) -> Result bool { - let previous: Option<&str> = index.checked_sub(1).map(|before| uppers[before].as_str()); - if previous.is_some_and(|word| COLUMN_LIST_RELATION_KEYWORDS.contains(&word)) { - return true; - } - let mut depth: usize = 0; - for (offset, token) in uppers.iter().enumerate().skip(index + 1) { +/// The index of the `)` closing each `(`, by the index of the `(`. +fn matching_closers(uppers: &[String]) -> Vec> { + let mut closers: Vec> = vec![None; uppers.len()]; + let mut open: Vec = Vec::new(); + for (index, token) in uppers.iter().enumerate() { if token == CALL_OPENER { - depth += 1; - } else if token == CALL_CLOSER { - depth -= 1; - if depth == 0 { - return uppers - .get(offset + 1) - .is_some_and(|next| next == CTE_BODY_PREFIX) - && uppers - .get(offset + 2) - .is_some_and(|next| CTE_BODY_OPENERS.contains(&next.as_str())); - } + open.push(index); + } else if token == CALL_CLOSER + && let Some(opener) = open.pop() + { + closers[opener] = Some(index); } } - false + closers +} + +/// Whether the word at `index` is called: followed by `(` and neither a placeholder nor a relation. +fn calls(raw: &str, uppers: &[String], closers: &[Option], index: usize) -> bool { + if uppers.get(index + 1).is_none_or(|next| next != CALL_OPENER) + || raw.starts_with(PLACEHOLDER_PREFIX) + { + return false; + } + let previous: Option<&str> = index.checked_sub(1).map(|before| uppers[before].as_str()); + if previous.is_some_and(|word| COLUMN_LIST_RELATION_KEYWORDS.contains(&word)) { + return false; + } + let Some(closer) = closers[index + 1] else { + return true; + }; + !(uppers + .get(closer + 1) + .is_some_and(|next| next == CTE_BODY_PREFIX) + && uppers + .get(closer + 2) + .is_some_and(|next| CTE_BODY_OPENERS.contains(&next.as_str()))) } diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/_test_types.py index 03bb59ce74..c2e1831c4f 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/_test_types.py @@ -104,3 +104,11 @@ class QueryFingerprintFailureTestCase: dialect: str query_sql: str expected_message: str + + +@dataclass(frozen=True) +class QueryFingerprintTimeLimitTestCase: + description: str + dialect: str + query_sql: str + expected_max_seconds: float diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py index 702eb9e639..5d8ebbfc3d 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py @@ -1,5 +1,7 @@ from __future__ import annotations +import time + import pytest from sqlbuild._native import QueryFingerprintError, compute_query_hash, normalize_query_sql @@ -8,8 +10,11 @@ ComputeQueryHashTestCase, NormalizeQuerySqlTestCase, QueryFingerprintFailureTestCase, + QueryFingerprintTimeLimitTestCase, ) +_NESTING: int = 32_000 + @pytest.mark.parametrize( "test_case", @@ -307,3 +312,36 @@ def test_given_untokenizable_sql_when_computing_hash_then_raises_fingerprint_err ) -> None: with pytest.raises(QueryFingerprintError, match=test_case.expected_message): _ = compute_query_hash(query_sql=test_case.query_sql, dialect=test_case.dialect) + + +@pytest.mark.parametrize( + "test_case", + [ + QueryFingerprintTimeLimitTestCase( + "deeply nested parentheses", + "duckdb", + "SELECT 1 + " + "(" * _NESTING + "1" + ")" * _NESTING, + 0.25, + ), + QueryFingerprintTimeLimitTestCase( + "deeply nested calls", + "postgres", + "SELECT " + "f(" * _NESTING + "1" + ")" * _NESTING, + 0.25, + ), + QueryFingerprintTimeLimitTestCase( + "unclosed calls", + "snowflake", + "SELECT " + "f(" * _NESTING, + 0.25, + ), + ], + ids=lambda case: case.description, +) +def test_given_pathological_nesting_when_computing_hash_then_time_stays_linear( + test_case: QueryFingerprintTimeLimitTestCase, +) -> None: + started: float = time.perf_counter() + _ = compute_query_hash(query_sql=test_case.query_sql, dialect=test_case.dialect) + + assert time.perf_counter() - started < test_case.expected_max_seconds