diff --git a/fuzz/fuzz_seeds/clickhouse_fuzz_unterminated_dollar_tag.sql b/fuzz/fuzz_seeds/clickhouse_fuzz_unterminated_dollar_tag.sql new file mode 100644 index 0000000000..a0df919ce1 --- /dev/null +++ b/fuzz/fuzz_seeds/clickhouse_fuzz_unterminated_dollar_tag.sql @@ -0,0 +1 @@ +SELECT $a$ x FROM t \ No newline at end of file diff --git a/src/dialect/clickhouse.rs b/src/dialect/clickhouse.rs index e670db211c..a1107983c2 100644 --- a/src/dialect/clickhouse.rs +++ b/src/dialect/clickhouse.rs @@ -92,6 +92,12 @@ impl Dialect for ClickHouseDialect { true } + /// `$tag$` starts a heredoc only if the same tag appears again later in the query, see + /// . + fn supports_unterminated_dollar_tag_as_word(&self) -> bool { + true + } + fn supports_partition_by_after_order_by(&self) -> bool { true } diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index 22e3fa367a..c1c2a367f3 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -1336,6 +1336,12 @@ pub trait Dialect: Debug + Any { false } + /// Returns true if a dollar-quote opening tag with no matching closing tag, such as + /// `$a$` in `SELECT $a$ x`, is a word rather than an unterminated dollar-quoted string. + fn supports_unterminated_dollar_tag_as_word(&self) -> bool { + false + } + /// Returns true if this dialect supports `$` as a prefix for money literals /// e.g. `SELECT $123.45` (SQL Server) fn supports_dollar_as_money_prefix(&self) -> bool { diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 02a54f08ff..fce3158412 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -2239,13 +2239,18 @@ impl<'a> Tokenizer<'a> { let mut temp = String::new(); let end_delimiter = format!("${value}$"); + let mut lookahead = State { + peekable: chars.peekable.clone(), + line: chars.line, + col: chars.col, + }; loop { - match chars.next() { + match lookahead.next() { Some(ch) => { temp.push(ch); - if temp.ends_with(&end_delimiter) { + if ch == '$' && temp.ends_with(&end_delimiter) { if let Some(temp) = temp.strip_suffix(&end_delimiter) { s.push_str(temp); } @@ -2260,13 +2265,19 @@ impl<'a> Tokenizer<'a> { break; } + if self.dialect.supports_unterminated_dollar_tag_as_word() { + let word = self.tokenize_word(end_delimiter, chars); + return Ok(Token::make_word(&word, None)); + } + return self.tokenizer_error( - chars.location(), + lookahead.location(), "Unterminated dollar-quoted, expected $", ); } } } + *chars = lookahead; } else { return Ok(Token::Placeholder(format!("${value}"))); } diff --git a/tests/sqlparser_clickhouse.rs b/tests/sqlparser_clickhouse.rs index 3e6fe93270..89525d118c 100644 --- a/tests/sqlparser_clickhouse.rs +++ b/tests/sqlparser_clickhouse.rs @@ -22,7 +22,7 @@ mod test_utils; use helpers::attached_token::AttachedToken; -use sqlparser::tokenizer::Span; +use sqlparser::tokenizer::{Location, Span}; use test_utils::*; use sqlparser::ast::Expr::{BinaryOp, Identifier}; @@ -2367,3 +2367,118 @@ fn parse_create_table_engine_name_quoting() { clickhouse_and_generic().verified_stmt("CREATE TABLE t (a INT) ENGINE = ``"); clickhouse().verified_stmt("CREATE TABLE t (a INT) ENGINE = MergeTree"); } + +#[test] +fn parse_unterminated_dollar_tag_as_identifier() { + // clickhouse-local 26.9 runs `SELECT $a$ x FROM t`. + let select = clickhouse().verified_only_select("SELECT $a$ AS x, $b$c$ FROM t"); + assert_eq!( + select.projection, + vec![ + SelectItem::ExprWithAlias { + expr: Identifier(Ident::new("$a$")), + alias: Ident::new("x"), + }, + UnnamedExpr(Identifier(Ident::new("$b$c$"))), + ] + ); + clickhouse().one_statement_parses_to("SELECT $a$ x", "SELECT $a$ AS x"); + for sql in [ + "SELECT $a$", + "SELECT 1 AS $a$", + "SELECT $a$$ FROM t", + "SELECT $1$ FROM t", + "SELECT $a$b$c FROM t", + "SELECT $a_1$ FROM t", + "SELECT $a$.x FROM t", + "SELECT f($a$)", + ] { + clickhouse().verified_stmt(sql); + } + // ClickHouse rejects non-ASCII tags; accepted, as with a closing tag. + clickhouse().one_statement_parses_to("SELECT $é$ x", "SELECT $é$ AS x"); + + // Any later matching tag closes a dollar-quoted string. + let select = clickhouse().verified_only_select("SELECT $a$ x, $a$ AS y"); + assert_eq!( + select.projection, + vec![SelectItem::ExprWithAlias { + expr: Expr::Value( + Value::DollarQuotedString(DollarQuotedString { + value: " x, ".to_string(), + tag: Some("a".to_string()), + }) + .with_empty_span() + ), + alias: Ident::new("y"), + }] + ); + clickhouse().verified_stmt("SELECT $a$ x $a$"); + clickhouse().verified_stmt("SELECT $a$$a$"); + + assert_eq!( + clickhouse() + .parse_sql_statements("SELECT $$ x") + .unwrap_err(), + sqlparser::parser::ParserError::TokenizerError( + "Unterminated dollar-quoted string at Line: 1, Column: 12".to_string() + ) + ); + // Two different tags are two words, which ClickHouse rejects too. + assert_eq!( + clickhouse() + .parse_sql_statements("SELECT $a$ x $b$") + .unwrap_err(), + ParserError("Expected: end of statement, found: $b$".to_string()) + ); + + assert_eq!( + all_dialects_where(|d| { + !d.supports_unterminated_dollar_tag_as_word() + && !d.supports_dollar_placeholder() + && !d.is_identifier_start('$') + }) + .parse_sql_statements("SELECT $a$ x") + .unwrap_err(), + sqlparser::parser::ParserError::TokenizerError( + "Unterminated dollar-quoted, expected $ at Line: 1, Column: 13".to_string() + ) + ); +} + +#[test] +fn parse_unterminated_dollar_tag_matching() { + // The closing tag must match in full and case, and cannot overlap the opening tag. + clickhouse().verified_stmt("SELECT $a$a$ FROM t"); + clickhouse().one_statement_parses_to("SELECT $ab$ x, $a$", "SELECT $ab$ AS x, $a$"); + assert_eq!( + clickhouse() + .parse_sql_statements("SELECT $A$ x $a$") + .unwrap_err(), + ParserError("Expected: end of statement, found: $a$".to_string()) + ); +} + +#[test] +fn parse_unterminated_dollar_tag_spans() { + let sql = r#"SELECT $a$, + $b$ x $b$, y"#; + let Statement::Query(query) = sqlparser::parser::Parser::parse_sql(&ClickHouseDialect {}, sql) + .unwrap() + .remove(0) + else { + unreachable!() + }; + let SetExpr::Select(select) = *query.body else { + unreachable!() + }; + let spans: Vec = select.projection.iter().map(|item| item.span()).collect(); + assert_eq!( + spans, + vec![ + Span::new(Location::new(1, 8), Location::new(1, 11)), + Span::new(Location::new(2, 3), Location::new(2, 12)), + Span::new(Location::new(2, 14), Location::new(2, 15)), + ] + ); +}