ClickHouse: Accept an unterminated dollar-quote tag as an identifier

fuzz/fuzz_seeds/clickhouse_fuzz_unterminated_dollar_tag.sql

@@ -0,0 +1 @@
+SELECT $a$ x FROM t

src/dialect/clickhouse.rs

@@ -92,6 +92,12 @@ impl Dialect for ClickHouseDialect {
         true
     }
 
+    /// `$tag$` starts a heredoc only if the same tag appears again later in the query, see
+    /// <https://clickhouse.com/docs/sql-reference/syntax#identifiers>.
+    fn supports_unterminated_dollar_tag_as_word(&self) -> bool {
+        true
+    }
+
     fn supports_partition_by_after_order_by(&self) -> bool {
         true
     }

src/dialect/mod.rs

@@ -1336,6 +1336,12 @@ pub trait Dialect: Debug + Any {
         false
     }
 
+    /// Returns true if a dollar-quote opening tag with no matching closing tag, such as
+    /// `$a$` in `SELECT $a$ x`, is a word rather than an unterminated dollar-quoted string.
+    fn supports_unterminated_dollar_tag_as_word(&self) -> bool {
+        false
+    }
+
     /// Returns true if this dialect supports `$` as a prefix for money literals
     /// e.g. `SELECT $123.45` (SQL Server)
     fn supports_dollar_as_money_prefix(&self) -> bool {

src/tokenizer.rs

@@ -2239,13 +2239,18 @@ impl<'a> Tokenizer<'a> {
 
                 let mut temp = String::new();
                 let end_delimiter = format!("${value}$");
+                let mut lookahead = State {
+                    peekable: chars.peekable.clone(),
+                    line: chars.line,
+                    col: chars.col,
+                };
 
                 loop {
-                    match chars.next() {
+                    match lookahead.next() {
                         Some(ch) => {
                             temp.push(ch);
 
-                            if temp.ends_with(&end_delimiter) {
+                            if ch == '$' && temp.ends_with(&end_delimiter) {
                                 if let Some(temp) = temp.strip_suffix(&end_delimiter) {
                                     s.push_str(temp);
                                 }
@@ -2260,13 +2265,19 @@ impl<'a> Tokenizer<'a> {
                                 break;
                             }
 
+                            if self.dialect.supports_unterminated_dollar_tag_as_word() {
+                                let word = self.tokenize_word(end_delimiter, chars);
+                                return Ok(Token::make_word(&word, None));
+                            }
+
                             return self.tokenizer_error(
-                                chars.location(),
+                                lookahead.location(),
                                 "Unterminated dollar-quoted, expected $",
                             );
                         }
                     }
                 }
+                *chars = lookahead;
             } else {
                 return Ok(Token::Placeholder(format!("${value}")));
             }

tests/sqlparser_clickhouse.rs

@@ -22,7 +22,7 @@
 mod test_utils;
 
 use helpers::attached_token::AttachedToken;
-use sqlparser::tokenizer::Span;
+use sqlparser::tokenizer::{Location, Span};
 use test_utils::*;
 
 use sqlparser::ast::Expr::{BinaryOp, Identifier};
@@ -2367,3 +2367,118 @@ fn parse_create_table_engine_name_quoting() {
     clickhouse_and_generic().verified_stmt("CREATE TABLE t (a INT) ENGINE = ``");
     clickhouse().verified_stmt("CREATE TABLE t (a INT) ENGINE = MergeTree");
 }
+
+#[test]
+fn parse_unterminated_dollar_tag_as_identifier() {
+    // clickhouse-local 26.9 runs `SELECT $a$ x FROM t`.
+    let select = clickhouse().verified_only_select("SELECT $a$ AS x, $b$c$ FROM t");
+    assert_eq!(
+        select.projection,
+        vec![
+            SelectItem::ExprWithAlias {
+                expr: Identifier(Ident::new("$a$")),
+                alias: Ident::new("x"),
+            },
+            UnnamedExpr(Identifier(Ident::new("$b$c$"))),
+        ]
+    );
+    clickhouse().one_statement_parses_to("SELECT $a$ x", "SELECT $a$ AS x");
+    for sql in [
+        "SELECT $a$",
+        "SELECT 1 AS $a$",
+        "SELECT $a$$ FROM t",
+        "SELECT $1$ FROM t",
+        "SELECT $a$b$c FROM t",
+        "SELECT $a_1$ FROM t",
+        "SELECT $a$.x FROM t",
+        "SELECT f($a$)",
+    ] {
+        clickhouse().verified_stmt(sql);
+    }
+    // ClickHouse rejects non-ASCII tags; accepted, as with a closing tag.
+    clickhouse().one_statement_parses_to("SELECT $é$ x", "SELECT $é$ AS x");
+
+    // Any later matching tag closes a dollar-quoted string.
+    let select = clickhouse().verified_only_select("SELECT $a$ x, $a$ AS y");
+    assert_eq!(
+        select.projection,
+        vec![SelectItem::ExprWithAlias {
+            expr: Expr::Value(
+                Value::DollarQuotedString(DollarQuotedString {
+                    value: " x, ".to_string(),
+                    tag: Some("a".to_string()),
+                })
+                .with_empty_span()
+            ),
+            alias: Ident::new("y"),
+        }]
+    );
+    clickhouse().verified_stmt("SELECT $a$ x $a$");
+    clickhouse().verified_stmt("SELECT $a$$a$");
+
+    assert_eq!(
+        clickhouse()
+            .parse_sql_statements("SELECT $$ x")
+            .unwrap_err(),
+        sqlparser::parser::ParserError::TokenizerError(
+            "Unterminated dollar-quoted string at Line: 1, Column: 12".to_string()
+        )
+    );
+    // Two different tags are two words, which ClickHouse rejects too.
+    assert_eq!(
+        clickhouse()
+            .parse_sql_statements("SELECT $a$ x $b$")
+            .unwrap_err(),
+        ParserError("Expected: end of statement, found: $b$".to_string())
+    );
+
+    assert_eq!(
+        all_dialects_where(|d| {
+            !d.supports_unterminated_dollar_tag_as_word()
+                && !d.supports_dollar_placeholder()
+                && !d.is_identifier_start('$')
+        })
+        .parse_sql_statements("SELECT $a$ x")
+        .unwrap_err(),
+        sqlparser::parser::ParserError::TokenizerError(
+            "Unterminated dollar-quoted, expected $ at Line: 1, Column: 13".to_string()
+        )
+    );
+}
+
+#[test]
+fn parse_unterminated_dollar_tag_matching() {
+    // The closing tag must match in full and case, and cannot overlap the opening tag.
+    clickhouse().verified_stmt("SELECT $a$a$ FROM t");
+    clickhouse().one_statement_parses_to("SELECT $ab$ x, $a$", "SELECT $ab$ AS x, $a$");
+    assert_eq!(
+        clickhouse()
+            .parse_sql_statements("SELECT $A$ x $a$")
+            .unwrap_err(),
+        ParserError("Expected: end of statement, found: $a$".to_string())
+    );
+}
+
+#[test]
+fn parse_unterminated_dollar_tag_spans() {
+    let sql = r#"SELECT $a$,
+  $b$ x $b$, y"#;
+    let Statement::Query(query) = sqlparser::parser::Parser::parse_sql(&ClickHouseDialect {}, sql)
+        .unwrap()
+        .remove(0)
+    else {
+        unreachable!()
+    };
+    let SetExpr::Select(select) = *query.body else {
+        unreachable!()
+    };
+    let spans: Vec<Span> = select.projection.iter().map(|item| item.span()).collect();
+    assert_eq!(
+        spans,
+        vec![
+            Span::new(Location::new(1, 8), Location::new(1, 11)),
+            Span::new(Location::new(2, 3), Location::new(2, 12)),
+            Span::new(Location::new(2, 14), Location::new(2, 15)),
+        ]
+    );
+}