diff --git a/fuzz/fuzz_seeds/spark_2912_numeric_suffix.sql b/fuzz/fuzz_seeds/spark_2912_numeric_suffix.sql new file mode 100644 index 0000000000..3fb556a89e --- /dev/null +++ b/fuzz/fuzz_seeds/spark_2912_numeric_suffix.sql @@ -0,0 +1 @@ +SELECT 1Y, 1.5bd, 1e5D, 1Dx diff --git a/src/ast/value.rs b/src/ast/value.rs index 5b05af694c..9a2ae3c16a 100644 --- a/src/ast/value.rs +++ b/src/ast/value.rs @@ -137,7 +137,10 @@ impl DerefMut for ValueWithSpan { #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub enum Value { - /// Numeric literal + /// Numeric literal, with any type suffix such as the `Y` of `10Y`. + /// See [`Dialect::is_numeric_literal_suffix`]. + /// + /// [`Dialect::is_numeric_literal_suffix`]: crate::dialect::Dialect::is_numeric_literal_suffix #[cfg(not(feature = "bigdecimal"))] Number(String, bool), #[cfg(feature = "bigdecimal")] diff --git a/src/dialect/databricks.rs b/src/dialect/databricks.rs index d660cec9eb..08752ee607 100644 --- a/src/dialect/databricks.rs +++ b/src/dialect/databricks.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use crate::dialect::Dialect; +use crate::dialect::{Dialect, SparkSqlDialect}; use crate::keywords::Keyword; use crate::parser::Parser; @@ -72,6 +72,13 @@ impl Dialect for DatabricksDialect { true } + /// The same suffixes as in Spark SQL, as in `+1Y`. + /// + /// See + fn is_numeric_literal_suffix(&self, number: &str, suffix: &str) -> bool { + SparkSqlDialect {}.is_numeric_literal_suffix(number, suffix) + } + fn supports_filter_during_aggregation(&self) -> bool { true } diff --git a/src/dialect/hive.rs b/src/dialect/hive.rs index e80bc75d2b..f11e0cca3a 100644 --- a/src/dialect/hive.rs +++ b/src/dialect/hive.rs @@ -78,6 +78,18 @@ impl Dialect for HiveDialect { true } + /// `L`, `S` and `Y` after an integer, and `D` and `BD` after any number, in + /// any case. + /// + /// See `IntegralLiteral` and `NumberLiteral` in + fn is_numeric_literal_suffix(&self, number: &str, suffix: &str) -> bool { + match suffix.as_bytes() { + [b'L' | b'l' | b'S' | b's' | b'Y' | b'y'] => !number.contains(['.', 'e', 'E']), + [b'D' | b'd'] | [b'B' | b'b', b'D' | b'd'] => true, + _ => false, + } + } + fn require_interval_qualifier(&self) -> bool { true } diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index 22e3fa367a..32e59ab45a 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -615,6 +615,15 @@ pub trait Dialect: Debug + Any { false } + /// Returns true if `suffix`, the identifier characters directly after the + /// numeric literal `number`, is a type suffix of the number, such as the `Y` + /// of `10Y` in Spark SQL. The suffix stays in the number's text, so `10Y` + /// fails to parse with the `bigdecimal` feature. An `L` suffix, in either + /// case, makes the number long instead. + fn is_numeric_literal_suffix(&self, _number: &str, _suffix: &str) -> bool { + false + } + /// Returns true if the dialect supports integer literals in hexadecimal, /// octal and binary, e.g. `0x1F`, `0o17` and `0b101`, which then parse as /// numbers instead of `0x` hexadecimal strings. diff --git a/src/dialect/oracle.rs b/src/dialect/oracle.rs index 40c91cea35..dbf305cba5 100644 --- a/src/dialect/oracle.rs +++ b/src/dialect/oracle.rs @@ -52,6 +52,14 @@ impl Dialect for OracleDialect { ch.is_alphanumeric() || ch == '_' || ch == '$' || ch == '#' || ch == '@' } + /// `F` for `BINARY_FLOAT` and `D` for `BINARY_DOUBLE`, after any number, in + /// any case. + /// + /// See + fn is_numeric_literal_suffix(&self, _number: &str, suffix: &str) -> bool { + matches!(suffix.as_bytes(), [b'F' | b'f' | b'D' | b'd']) + } + /// LIMIT is not a reserved word in Oracle SQL. /// See fn is_table_factor(&self, kw: &Keyword, _parser: &mut Parser) -> bool { diff --git a/src/dialect/spark.rs b/src/dialect/spark.rs index c3f369cfbf..390e36d812 100644 --- a/src/dialect/spark.rs +++ b/src/dialect/spark.rs @@ -107,6 +107,18 @@ impl Dialect for SparkSqlDialect { true } + /// `L`, `S` and `Y` after an integer, and `F`, `D` and `BD` after any number, + /// in any case. + /// + /// See + fn is_numeric_literal_suffix(&self, number: &str, suffix: &str) -> bool { + match suffix.as_bytes() { + [b'L' | b'l' | b'S' | b's' | b'Y' | b'y'] => !number.contains(['.', 'e', 'E']), + [b'F' | b'f' | b'D' | b'd'] | [b'B' | b'b', b'D' | b'd'] => true, + _ => false, + } + } + /// See fn supports_values_as_table_factor(&self) -> bool { true diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 02a54f08ff..39bac9e2cb 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -60,7 +60,8 @@ pub enum Token { EOF, /// A keyword (like SELECT) or an optionally quoted SQL identifier Word(Word), - /// An unsigned numeric literal + /// An unsigned numeric literal, with any type suffix that + /// [`Dialect::is_numeric_literal_suffix`] accepts, and whether it is long, as in `1L` Number(String, bool), /// A character that could not be tokenized Char(char), @@ -1686,6 +1687,26 @@ impl<'a> Tokenizer<'a> { } } + // After a period, as in `t.1D`, the number may start a column name. + if prev_token != Some(&Token::Period) { + let suffix: String = chars + .peekable + .clone() + .take_while(|&ch| self.dialect.is_identifier_part(ch)) + .collect(); + if !suffix.is_empty() && self.dialect.is_numeric_literal_suffix(&s, &suffix) + { + for _ in 0..suffix.chars().count() { + chars.next(); + } + if suffix.eq_ignore_ascii_case("L") { + return Ok(Some(Token::Number(s, true))); + } + s += &suffix; + return Ok(Some(Token::Number(s, false))); + } + } + // If the dialect supports identifiers that start with a numeric prefix, // we need to check if the value is in fact an identifier and must thus // be tokenized as a word. @@ -2935,7 +2956,7 @@ mod tests { use super::*; use crate::dialect::{ BigQueryDialect, ClickHouseDialect, HiveDialect, MsSqlDialect, MySqlDialect, - PostgreSqlDialect, SQLiteDialect, + PostgreSqlDialect, SQLiteDialect, SparkSqlDialect, }; use crate::test_utils::{all_dialects, all_dialects_except, all_dialects_where}; use core::fmt::Debug; @@ -5167,4 +5188,38 @@ mod tests { ); } } + + #[test] + fn tokenize_numeric_literal_suffix() { + let number = |n: &str, long| Token::Number(n.to_string(), long); + let space = || Token::Whitespace(Whitespace::Space); + compare( + vec![ + number("10Y", false), + space(), + number("1.5bd", false), + space(), + number("10", true), + space(), + number("1", false), + Token::make_word("Dx", None), + ], + Tokenizer::new(&SparkSqlDialect {}, "10Y 1.5bd 10l 1Dx") + .tokenize() + .unwrap(), + ); + // With numeric prefixes, a column name can follow a period. + compare( + vec![ + Token::make_word("t", None), + Token::Period, + Token::make_word("1D", None), + space(), + number("1D", false), + ], + Tokenizer::new(&HiveDialect {}, "t.1D 1D") + .tokenize() + .unwrap(), + ); + } } diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index aeb3e67a47..f6790a06af 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -23481,3 +23481,9 @@ fn parse_trailing_end_keyword_is_not_silently_dropped() { ParserError::ParserError("Expected: end of statement, found: END".to_string()) ); } + +#[test] +fn parse_number_without_suffix() { + all_dialects_where(|d| !d.supports_numeric_prefix() && !d.is_numeric_literal_suffix("1", "D")) + .one_statement_parses_to("SELECT 1D", "SELECT 1 AS D"); +} diff --git a/tests/sqlparser_databricks.rs b/tests/sqlparser_databricks.rs index ce26acc621..a0e17120e1 100644 --- a/tests/sqlparser_databricks.rs +++ b/tests/sqlparser_databricks.rs @@ -797,3 +797,23 @@ fn parse_databricks_collated_data_types() { )) ); } + +#[test] +fn parse_numeric_literal_suffixes() { + #[cfg(not(feature = "bigdecimal"))] + databricks().verified_stmt("SELECT +1Y, 10S, 1.5F, 1.5D, 1.5BD"); + + assert_eq!( + databricks().verified_expr("1L"), + Expr::value(Value::Number("1".parse().unwrap(), true)) + ); + assert_eq!( + databricks() + .verified_only_select("SELECT 1La, 1Dx") + .projection, + vec![ + SelectItem::UnnamedExpr(Expr::Identifier(Ident::new("1La"))), + SelectItem::UnnamedExpr(Expr::Identifier(Ident::new("1Dx"))), + ] + ); +} diff --git a/tests/sqlparser_hive.rs b/tests/sqlparser_hive.rs index 355fd5c8d5..5d8b56e8e0 100644 --- a/tests/sqlparser_hive.rs +++ b/tests/sqlparser_hive.rs @@ -22,7 +22,7 @@ use sqlparser::ast::{ ClusteredBy, CommentDef, CreateFunction, CreateFunctionBody, CreateFunctionUsing, CreateTable, - Expr, Ident, ObjectName, OrderByExpr, OrderByOptions, OrderBySort, Set, Statement, + Expr, Ident, ObjectName, OrderByExpr, OrderByOptions, OrderBySort, SelectItem, Set, Statement, UnaryOperator, Use, Value, }; use sqlparser::dialect::{AnsiDialect, GenericDialect, HiveDialect}; @@ -518,3 +518,17 @@ fn hive() -> TestedDialects { fn hive_and_generic() -> TestedDialects { TestedDialects::new(vec![Box::new(HiveDialect {}), Box::new(GenericDialect {})]) } + +#[test] +fn parse_numeric_literal_suffixes() { + #[cfg(not(feature = "bigdecimal"))] + hive().verified_stmt("SELECT 10Y, 10S, 1.5D, 1.5BD, 1e5d"); + + // Hive has no `F` suffix, so `1.5F` is a column name. + assert_eq!( + hive().verified_only_select("SELECT 1.5F").projection, + vec![SelectItem::UnnamedExpr(Expr::Identifier(Ident::new( + "1.5F" + )))] + ); +} diff --git a/tests/sqlparser_oracle.rs b/tests/sqlparser_oracle.rs index 0ed573be68..2f4b08b6dd 100644 --- a/tests/sqlparser_oracle.rs +++ b/tests/sqlparser_oracle.rs @@ -545,3 +545,14 @@ fn parse_non_reserved_keyword_table_names() { oracle().verified_stmt("SELECT * FROM limit"); oracle().verified_stmt("SELECT * FROM t JOIN limit ON TRUE"); } + +#[test] +fn parse_numeric_literal_suffixes() { + #[cfg(not(feature = "bigdecimal"))] + oracle().verified_stmt("SELECT 25f, +6.34F, 0.5d, -1D, 1e3d FROM dual"); + + oracle().one_statement_parses_to( + "SELECT 10Y, 1.5BD FROM dual", + "SELECT 10 AS Y, 1.5 AS BD FROM dual", + ); +} diff --git a/tests/sqlparser_spark.rs b/tests/sqlparser_spark.rs index 21a6477559..51dbc3de5f 100644 --- a/tests/sqlparser_spark.rs +++ b/tests/sqlparser_spark.rs @@ -381,3 +381,28 @@ fn test_substring() { fn test_pipe_operator() { spark().verified_stmt("SELECT * FROM t |> WHERE x > 1 |> SELECT x AS y |> ORDER BY y"); } + +#[test] +fn parse_numeric_literal_suffixes() { + #[cfg(not(feature = "bigdecimal"))] + { + spark().verified_stmt("SELECT 10Y, 10s, 1.5F, 1.5d, 1.5BD, .5bd, 5.F, 1e5D, -32Y"); + assert_eq!(spark().verified_expr("1.5BD"), Expr::value(number("1.5BD"))); + } + // A `BigDecimal` can't hold the suffix. + #[cfg(feature = "bigdecimal")] + assert_eq!( + spark().parse_sql_statements("SELECT 10Y").unwrap_err(), + sqlparser::parser::ParserError::ParserError( + "Could not parse '10Y' as bigdecimal::BigDecimal: invalid digit found in string" + .to_string() + ) + ); + + spark().one_statement_parses_to("SELECT 10l", "SELECT 10L"); + // `L`, `S` and `Y` only follow an integer, and a suffix ends the word. + spark().one_statement_parses_to( + "SELECT 1.5S, 1.5Y, 1Dx, 1B, 1BDx", + "SELECT 1.5 AS S, 1.5 AS Y, 1 AS Dx, 1 AS B, 1 AS BDx", + ); +}