From 795a7acbaca5f2a8ad5203401bc5e8373106a959 Mon Sep 17 00:00:00 2001 From: s5dsn-eqee Date: Mon, 9 Nov 2026 15:56:01 +0100 Subject: [PATCH 1/3] ClickHouse: Accept an unterminated dollar-quote tag as an identifier --- ...lickhouse_fuzz_unterminated_dollar_tag.sql | 1 + src/dialect/clickhouse.rs | 6 ++++ src/dialect/mod.rs | 6 ++++ src/tokenizer.rs | 11 +++++++ tests/sqlparser_clickhouse.rs | 31 +++++++++++++++++++ 5 files changed, 55 insertions(+) create mode 100644 fuzz/fuzz_seeds/clickhouse_fuzz_unterminated_dollar_tag.sql diff --git a/fuzz/fuzz_seeds/clickhouse_fuzz_unterminated_dollar_tag.sql b/fuzz/fuzz_seeds/clickhouse_fuzz_unterminated_dollar_tag.sql new file mode 100644 index 0000000000..a0df919ce1 --- /dev/null +++ b/fuzz/fuzz_seeds/clickhouse_fuzz_unterminated_dollar_tag.sql @@ -0,0 +1 @@ +SELECT $a$ x FROM t \ No newline at end of file diff --git a/src/dialect/clickhouse.rs b/src/dialect/clickhouse.rs index e670db211c..81981f9367 100644 --- a/src/dialect/clickhouse.rs +++ b/src/dialect/clickhouse.rs @@ -92,6 +92,12 @@ impl Dialect for ClickHouseDialect { true } + /// Undocumented; the lexer reads a heredoc only up to a matching closing tag, and + /// otherwise reads the tag as a bare word, see "start of heredoc" in `Lexer.cpp`. + fn supports_unterminated_dollar_tag_as_word(&self) -> bool { + true + } + fn supports_partition_by_after_order_by(&self) -> bool { true } diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index 22e3fa367a..c1c2a367f3 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -1336,6 +1336,12 @@ pub trait Dialect: Debug + Any { false } + /// Returns true if a dollar-quote opening tag with no matching closing tag, such as + /// `$a$` in `SELECT $a$ x`, is a word rather than an unterminated dollar-quoted string. + fn supports_unterminated_dollar_tag_as_word(&self) -> bool { + false + } + /// Returns true if this dialect supports `$` as a prefix for money literals /// e.g. `SELECT $123.45` (SQL Server) fn supports_dollar_as_money_prefix(&self) -> bool { diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 02a54f08ff..71249089d6 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -2240,6 +2240,17 @@ impl<'a> Tokenizer<'a> { let mut temp = String::new(); let end_delimiter = format!("${value}$"); + if self.dialect.supports_unterminated_dollar_tag_as_word() + && !chars + .peekable + .clone() + .collect::() + .contains(&end_delimiter) + { + let word = self.tokenize_word(end_delimiter, chars); + return Ok(Token::make_word(&word, None)); + } + loop { match chars.next() { Some(ch) => { diff --git a/tests/sqlparser_clickhouse.rs b/tests/sqlparser_clickhouse.rs index 3e6fe93270..1bdc3c5174 100644 --- a/tests/sqlparser_clickhouse.rs +++ b/tests/sqlparser_clickhouse.rs @@ -2367,3 +2367,34 @@ fn parse_create_table_engine_name_quoting() { clickhouse_and_generic().verified_stmt("CREATE TABLE t (a INT) ENGINE = ``"); clickhouse().verified_stmt("CREATE TABLE t (a INT) ENGINE = MergeTree"); } + +#[test] +fn parse_unterminated_dollar_tag_as_identifier() { + // Not in the syntax docs; clickhouse-local 26.9 runs `SELECT $a$ x FROM t`. + let select = clickhouse().verified_only_select("SELECT $a$ AS x, $b$c$ FROM t"); + assert_eq!( + select.projection, + vec![ + SelectItem::ExprWithAlias { + expr: Identifier(Ident::new("$a$")), + alias: Ident::new("x"), + }, + UnnamedExpr(Identifier(Ident::new("$b$c$"))), + ] + ); + clickhouse().one_statement_parses_to("SELECT $a$ x", "SELECT $a$ AS x"); + + // A matching closing tag still makes a dollar-quoted string. + clickhouse().verified_stmt("SELECT $a$ x $a$"); + clickhouse() + .parse_sql_statements("SELECT $$ x") + .unwrap_err(); + + all_dialects_where(|d| { + !d.supports_unterminated_dollar_tag_as_word() + && !d.supports_dollar_placeholder() + && !d.is_identifier_start('$') + }) + .parse_sql_statements("SELECT $a$ x") + .unwrap_err(); +} From 1a70d5e49d0ac0930672ff792ecfed1912944323 Mon Sep 17 00:00:00 2001 From: s5dsn-eqee Date: Mon, 9 Nov 2026 16:15:16 +0100 Subject: [PATCH 2/3] Add edge cases and exact errors to unterminated dollar-tag test --- tests/sqlparser_clickhouse.rs | 69 +++++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 11 deletions(-) diff --git a/tests/sqlparser_clickhouse.rs b/tests/sqlparser_clickhouse.rs index 1bdc3c5174..77707df32b 100644 --- a/tests/sqlparser_clickhouse.rs +++ b/tests/sqlparser_clickhouse.rs @@ -2383,18 +2383,65 @@ fn parse_unterminated_dollar_tag_as_identifier() { ] ); clickhouse().one_statement_parses_to("SELECT $a$ x", "SELECT $a$ AS x"); + for sql in [ + "SELECT $a$", + "SELECT 1 AS $a$", + "SELECT $a$$ FROM t", + "SELECT $1$ FROM t", + "SELECT $a$b$c FROM t", + "SELECT $a_1$ FROM t", + "SELECT $a$.x FROM t", + "SELECT f($a$)", + ] { + clickhouse().verified_stmt(sql); + } + // ClickHouse rejects non-ASCII tags; accepted, as with a closing tag. + clickhouse().one_statement_parses_to("SELECT $é$ x", "SELECT $é$ AS x"); - // A matching closing tag still makes a dollar-quoted string. + // Any later matching tag closes a dollar-quoted string. + let select = clickhouse().verified_only_select("SELECT $a$ x, $a$ AS y"); + assert_eq!( + select.projection, + vec![SelectItem::ExprWithAlias { + expr: Expr::Value( + Value::DollarQuotedString(DollarQuotedString { + value: " x, ".to_string(), + tag: Some("a".to_string()), + }) + .with_empty_span() + ), + alias: Ident::new("y"), + }] + ); clickhouse().verified_stmt("SELECT $a$ x $a$"); - clickhouse() - .parse_sql_statements("SELECT $$ x") - .unwrap_err(); + clickhouse().verified_stmt("SELECT $a$$a$"); - all_dialects_where(|d| { - !d.supports_unterminated_dollar_tag_as_word() - && !d.supports_dollar_placeholder() - && !d.is_identifier_start('$') - }) - .parse_sql_statements("SELECT $a$ x") - .unwrap_err(); + assert_eq!( + clickhouse() + .parse_sql_statements("SELECT $$ x") + .unwrap_err(), + sqlparser::parser::ParserError::TokenizerError( + "Unterminated dollar-quoted string at Line: 1, Column: 12".to_string() + ) + ); + // Two different tags are two words, which ClickHouse rejects too. + assert_eq!( + clickhouse() + .parse_sql_statements("SELECT $a$ x $b$") + .unwrap_err(), + ParserError("Expected: end of statement, found: $b$".to_string()) + ); + + assert_eq!( + all_dialects_where(|d| { + !d.supports_unterminated_dollar_tag_as_word() + && !d.supports_dollar_placeholder() + && !d.is_identifier_start('$') + }) + .parse_sql_statements("SELECT $a$ x") + .unwrap_err(), + sqlparser::parser::ParserError::TokenizerError( + "Unterminated dollar-quoted, expected $ at Line: 1, Column: 13".to_string() + ) + ); } From f4af460c0ddf052e4867983a6600bf1090cbbc16 Mon Sep 17 00:00:00 2001 From: s5dsn-eqee Date: Mon, 9 Nov 2026 18:29:38 +0100 Subject: [PATCH 3/3] Detect an unterminated dollar tag with a lookahead, link the ClickHouse docs, add tag matching and span tests --- src/dialect/clickhouse.rs | 4 ++-- src/tokenizer.rs | 28 ++++++++++++------------ tests/sqlparser_clickhouse.rs | 41 +++++++++++++++++++++++++++++++++-- 3 files changed, 55 insertions(+), 18 deletions(-) diff --git a/src/dialect/clickhouse.rs b/src/dialect/clickhouse.rs index 81981f9367..a1107983c2 100644 --- a/src/dialect/clickhouse.rs +++ b/src/dialect/clickhouse.rs @@ -92,8 +92,8 @@ impl Dialect for ClickHouseDialect { true } - /// Undocumented; the lexer reads a heredoc only up to a matching closing tag, and - /// otherwise reads the tag as a bare word, see "start of heredoc" in `Lexer.cpp`. + /// `$tag$` starts a heredoc only if the same tag appears again later in the query, see + /// . fn supports_unterminated_dollar_tag_as_word(&self) -> bool { true } diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 71249089d6..fce3158412 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -2239,24 +2239,18 @@ impl<'a> Tokenizer<'a> { let mut temp = String::new(); let end_delimiter = format!("${value}$"); - - if self.dialect.supports_unterminated_dollar_tag_as_word() - && !chars - .peekable - .clone() - .collect::() - .contains(&end_delimiter) - { - let word = self.tokenize_word(end_delimiter, chars); - return Ok(Token::make_word(&word, None)); - } + let mut lookahead = State { + peekable: chars.peekable.clone(), + line: chars.line, + col: chars.col, + }; loop { - match chars.next() { + match lookahead.next() { Some(ch) => { temp.push(ch); - if temp.ends_with(&end_delimiter) { + if ch == '$' && temp.ends_with(&end_delimiter) { if let Some(temp) = temp.strip_suffix(&end_delimiter) { s.push_str(temp); } @@ -2271,13 +2265,19 @@ impl<'a> Tokenizer<'a> { break; } + if self.dialect.supports_unterminated_dollar_tag_as_word() { + let word = self.tokenize_word(end_delimiter, chars); + return Ok(Token::make_word(&word, None)); + } + return self.tokenizer_error( - chars.location(), + lookahead.location(), "Unterminated dollar-quoted, expected $", ); } } } + *chars = lookahead; } else { return Ok(Token::Placeholder(format!("${value}"))); } diff --git a/tests/sqlparser_clickhouse.rs b/tests/sqlparser_clickhouse.rs index 77707df32b..89525d118c 100644 --- a/tests/sqlparser_clickhouse.rs +++ b/tests/sqlparser_clickhouse.rs @@ -22,7 +22,7 @@ mod test_utils; use helpers::attached_token::AttachedToken; -use sqlparser::tokenizer::Span; +use sqlparser::tokenizer::{Location, Span}; use test_utils::*; use sqlparser::ast::Expr::{BinaryOp, Identifier}; @@ -2370,7 +2370,7 @@ fn parse_create_table_engine_name_quoting() { #[test] fn parse_unterminated_dollar_tag_as_identifier() { - // Not in the syntax docs; clickhouse-local 26.9 runs `SELECT $a$ x FROM t`. + // clickhouse-local 26.9 runs `SELECT $a$ x FROM t`. let select = clickhouse().verified_only_select("SELECT $a$ AS x, $b$c$ FROM t"); assert_eq!( select.projection, @@ -2445,3 +2445,40 @@ fn parse_unterminated_dollar_tag_as_identifier() { ) ); } + +#[test] +fn parse_unterminated_dollar_tag_matching() { + // The closing tag must match in full and case, and cannot overlap the opening tag. + clickhouse().verified_stmt("SELECT $a$a$ FROM t"); + clickhouse().one_statement_parses_to("SELECT $ab$ x, $a$", "SELECT $ab$ AS x, $a$"); + assert_eq!( + clickhouse() + .parse_sql_statements("SELECT $A$ x $a$") + .unwrap_err(), + ParserError("Expected: end of statement, found: $a$".to_string()) + ); +} + +#[test] +fn parse_unterminated_dollar_tag_spans() { + let sql = r#"SELECT $a$, + $b$ x $b$, y"#; + let Statement::Query(query) = sqlparser::parser::Parser::parse_sql(&ClickHouseDialect {}, sql) + .unwrap() + .remove(0) + else { + unreachable!() + }; + let SetExpr::Select(select) = *query.body else { + unreachable!() + }; + let spans: Vec = select.projection.iter().map(|item| item.span()).collect(); + assert_eq!( + spans, + vec![ + Span::new(Location::new(1, 8), Location::new(1, 11)), + Span::new(Location::new(2, 3), Location::new(2, 12)), + Span::new(Location::new(2, 14), Location::new(2, 15)), + ] + ); +}