From 2a4f7c4a9137c893d523781d6aa57b303d26f06f Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 11 Nov 2026 11:30:36 -0700 Subject: [PATCH 1/3] Spark: Parse typed numeric literal suffixes Spark SQL types a numeric literal by a case-insensitive suffix: `Y`, `S` and `L` after an integer, and `F`, `D` and `BD` after any number. The tokenizer read `1D` as the number `1` followed by the alias `D`. --- fuzz/fuzz_seeds/spark_TBD_numeric_suffix.sql | 1 + src/ast/mod.rs | 2 +- src/ast/value.rs | 40 +++++++++++++++ src/dialect/mod.rs | 9 ++++ src/dialect/spark.rs | 5 ++ src/parser/mod.rs | 2 + src/tokenizer.rs | 42 +++++++++++++++- tests/sqlparser_spark.rs | 52 +++++++++++++++++++- 8 files changed, 150 insertions(+), 3 deletions(-) create mode 100644 fuzz/fuzz_seeds/spark_TBD_numeric_suffix.sql diff --git a/fuzz/fuzz_seeds/spark_TBD_numeric_suffix.sql b/fuzz/fuzz_seeds/spark_TBD_numeric_suffix.sql new file mode 100644 index 0000000000..3fb556a89e --- /dev/null +++ b/fuzz/fuzz_seeds/spark_TBD_numeric_suffix.sql @@ -0,0 +1 @@ +SELECT 1Y, 1.5bd, 1e5D, 1Dx diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 473b11a37b..9c39ee14ba 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -130,7 +130,7 @@ pub use self::trigger::{ pub use self::value::{ escape_double_quote_string, escape_quoted_string, DateTimeField, DollarQuotedString, - NormalizationForm, QuoteDelimitedString, TrimWhereField, Value, ValueWithSpan, + NormalizationForm, NumberSuffix, QuoteDelimitedString, TrimWhereField, Value, ValueWithSpan, }; use crate::ast::helpers::key_value_options::KeyValueOptions; diff --git a/src/ast/value.rs b/src/ast/value.rs index 5b05af694c..3331587a54 100644 --- a/src/ast/value.rs +++ b/src/ast/value.rs @@ -146,6 +146,13 @@ pub enum Value { /// but fail on CI with the `--all-features` flag enabled /// Numeric literal (uses `BigDecimal` when the `bigdecimal` feature is enabled). Number(BigDecimal, bool), + /// Numeric literal with a type suffix, e.g. `1Y` or `1.5BD`. The string is the + /// number as written without the suffix, also with the `bigdecimal` feature. + /// + /// See [`Dialect::supports_numeric_literal_suffixes`]. + /// + /// [`Dialect::supports_numeric_literal_suffixes`]: crate::dialect::Dialect::supports_numeric_literal_suffixes + SuffixedNumber(String, NumberSuffix), /// 'string value' SingleQuotedString(String), /// Dollar-quoted string literal, e.g. `$$...$$` or `$tag$...$tag$` (Postgres syntax). @@ -262,6 +269,7 @@ impl fmt::Display for Value { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { Value::Number(v, l) => write!(f, "{}{long}", v, long = if *l { "L" } else { "" }), + Value::SuffixedNumber(v, suffix) => write!(f, "{v}{suffix}"), Value::DoubleQuotedString(v) => write!(f, "\"{}\"", escape_double_quote_string(v)), Value::SingleQuotedString(v) => write!(f, "'{}'", escape_single_quote_string(v)), Value::TripleSingleQuotedString(v) => { @@ -538,6 +546,38 @@ impl fmt::Display for NormalizationForm { } } +/// The type suffix of a [`Value::SuffixedNumber`]. An `L` suffix makes a +/// [`Value::Number`] long instead. +/// +/// See [Spark SQL](https://spark.apache.org/docs/latest/sql-ref-literals.html#numeric-literal). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub enum NumberSuffix { + /// `Y`, a `TINYINT` + TinyInt, + /// `S`, a `SMALLINT` + SmallInt, + /// `F`, a `FLOAT` + Float, + /// `D`, a `DOUBLE` + Double, + /// `BD`, a `DECIMAL` + Decimal, +} + +impl fmt::Display for NumberSuffix { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str(match self { + NumberSuffix::TinyInt => "Y", + NumberSuffix::SmallInt => "S", + NumberSuffix::Float => "F", + NumberSuffix::Double => "D", + NumberSuffix::Decimal => "BD", + }) + } +} + pub struct EscapeQuotedString<'a> { string: &'a str, quote: char, diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index 22e3fa367a..c443240d20 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -615,6 +615,15 @@ pub trait Dialect: Debug + Any { false } + /// Returns true if a case-insensitive suffix gives a numeric literal its type: + /// `L`, `S` or `Y` after an integer, as in `1Y`, and `F`, `D` or `BD` after + /// any number, as in `1.5BD`. See [`NumberSuffix`]. + /// + /// [`NumberSuffix`]: crate::ast::NumberSuffix + fn supports_numeric_literal_suffixes(&self) -> bool { + false + } + /// Returns true if the dialect supports integer literals in hexadecimal, /// octal and binary, e.g. `0x1F`, `0o17` and `0b101`, which then parse as /// numbers instead of `0x` hexadecimal strings. diff --git a/src/dialect/spark.rs b/src/dialect/spark.rs index c3f369cfbf..d299dcdb3b 100644 --- a/src/dialect/spark.rs +++ b/src/dialect/spark.rs @@ -112,6 +112,11 @@ impl Dialect for SparkSqlDialect { true } + /// See + fn supports_numeric_literal_suffixes(&self) -> bool { + true + } + fn require_interval_qualifier(&self) -> bool { true } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 1d6cc53417..777e1fc4bb 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -2083,6 +2083,7 @@ impl<'a> Parser<'a> { Ok(Expr::Value(self.parse_value()?)) } Token::Number(_, _) + | Token::SuffixedNumber(_, _) | Token::SingleQuotedString(_) | Token::DoubleQuotedString(_) | Token::TripleSingleQuotedString(_) @@ -13327,6 +13328,7 @@ impl<'a> Parser<'a> { Token::Number(n, l) => { ok_value(Value::Number(Self::parse_number_literal(n, span.start)?, l)) } + Token::SuffixedNumber(n, suffix) => ok_value(Value::SuffixedNumber(n, suffix)), Token::SingleQuotedString(ref s) => ok_value(Value::SingleQuotedString( self.maybe_concat_string_literal(s.to_string()), )), diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 02a54f08ff..578bbaeb1e 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -40,7 +40,7 @@ use serde::{Deserialize, Serialize}; #[cfg(feature = "visitor")] use sqlparser_derive::{Visit, VisitMut}; -use crate::ast::{DollarQuotedString, QuoteDelimitedString}; +use crate::ast::{DollarQuotedString, NumberSuffix, QuoteDelimitedString}; use crate::dialect::Dialect; use crate::dialect::{ BigQueryDialect, DuckDbDialect, GenericDialect, MySqlDialect, PostgreSqlDialect, @@ -62,6 +62,8 @@ pub enum Token { Word(Word), /// An unsigned numeric literal Number(String, bool), + /// An unsigned numeric literal with a type suffix, e.g. `1Y` + SuffixedNumber(String, NumberSuffix), /// A character that could not be tokenized Char(char), /// Single quoted string: i.e: 'string' @@ -303,6 +305,7 @@ impl fmt::Display for Token { Token::EOF => f.write_str("EOF"), Token::Word(ref w) => write!(f, "{w}"), Token::Number(ref n, l) => write!(f, "{}{long}", n, long = if *l { "L" } else { "" }), + Token::SuffixedNumber(ref n, suffix) => write!(f, "{n}{suffix}"), Token::Char(ref c) => write!(f, "{c}"), Token::SingleQuotedString(ref s) => write!(f, "'{s}'"), Token::TripleSingleQuotedString(ref s) => write!(f, "'''{s}'''"), @@ -1707,6 +1710,10 @@ impl<'a> Tokenizer<'a> { } } + if self.dialect.supports_numeric_literal_suffixes() { + return Ok(Some(self.tokenize_number_suffix(chars, s))); + } + let long = if chars.peek() == Some(&'L') { chars.next(); true @@ -2342,6 +2349,39 @@ impl<'a> Tokenizer<'a> { Ok(Token::Number(s, false)) } + /// Tokenizes the number `s` with its type suffix if one follows, as in `1Y` or + /// `1.5BD`. A suffix directly followed by an identifier character is not one. + fn tokenize_number_suffix(&self, chars: &mut State, s: String) -> Token { + let is_integer = !s.contains(['.', 'e', 'E']); + let mut ahead = chars.peekable.clone(); + // `None` is the `L` of a long number. + let (suffix, len) = match ahead.next().map(|ch| ch.to_ascii_uppercase()) { + Some('L') if is_integer => (None, 1), + Some('S') if is_integer => (Some(NumberSuffix::SmallInt), 1), + Some('Y') if is_integer => (Some(NumberSuffix::TinyInt), 1), + Some('F') => (Some(NumberSuffix::Float), 1), + Some('D') => (Some(NumberSuffix::Double), 1), + Some('B') if ahead.peek().is_some_and(|ch| ch.eq_ignore_ascii_case(&'D')) => { + ahead.next(); + (Some(NumberSuffix::Decimal), 2) + } + _ => return Token::Number(s, false), + }; + if ahead + .next() + .is_some_and(|ch| self.dialect.is_identifier_part(ch)) + { + return Token::Number(s, false); + } + for _ in 0..len { + chars.next(); + } + match suffix { + Some(suffix) => Token::SuffixedNumber(s, suffix), + None => Token::Number(s, true), + } + } + // Consume characters until newline fn tokenize_single_line_comment(&self, chars: &mut State) -> String { peeking_take_while(chars, |ch| match ch { diff --git a/tests/sqlparser_spark.rs b/tests/sqlparser_spark.rs index 21a6477559..fe716c381f 100644 --- a/tests/sqlparser_spark.rs +++ b/tests/sqlparser_spark.rs @@ -19,7 +19,7 @@ //! Test SQL syntax specific to Apache Spark SQL. use sqlparser::ast::*; -use sqlparser::dialect::SparkSqlDialect; +use sqlparser::dialect::{GenericDialect, SparkSqlDialect}; use test_utils::*; #[macro_use] @@ -381,3 +381,53 @@ fn test_substring() { fn test_pipe_operator() { spark().verified_stmt("SELECT * FROM t |> WHERE x > 1 |> SELECT x AS y |> ORDER BY y"); } + +#[test] +fn test_numeric_literal_suffixes() { + let suffixed = |n: &str, suffix| Expr::value(Value::SuffixedNumber(n.into(), suffix)); + assert_eq!( + spark().verified_expr("10Y"), + suffixed("10", NumberSuffix::TinyInt) + ); + assert_eq!( + spark().verified_expr("10S"), + suffixed("10", NumberSuffix::SmallInt) + ); + assert_eq!( + spark().verified_expr("1e5F"), + suffixed("1e5", NumberSuffix::Float) + ); + assert_eq!( + spark().verified_expr("1.5D"), + suffixed("1.5", NumberSuffix::Double) + ); + assert_eq!( + spark().verified_expr(".5BD"), + suffixed(".5", NumberSuffix::Decimal) + ); + assert_eq!( + spark().verified_expr("10L"), + Expr::value(Value::Number("10".parse().unwrap(), true)) + ); + + spark().verified_stmt("SELECT 1Y, 1S, 1L, 1F, 1D, 1BD"); + spark().verified_stmt("SELECT 1.5F, 1.5D, 1.5BD, 5.F, 1.5E-3BD, -32Y"); + spark().verified_stmt("SELECT 2 * 1.5D - 1Y FROM t WHERE x > 1BD"); + spark().one_statement_parses_to( + "SELECT 1y, 1s, 1l, 1f, 1d, 1bd, 1.5bD, 9223372036854775807l", + "SELECT 1Y, 1S, 1L, 1F, 1D, 1BD, 1.5BD, 9223372036854775807L", + ); + + // `L`, `S` and `Y` only follow an integer, and no suffix runs into an identifier. + spark().one_statement_parses_to( + "SELECT 1.5S, 1.5Y, 1.5L, 1Dx, 1D5, 1B, 1BDx, 1La", + "SELECT 1.5 AS S, 1.5 AS Y, 1.5 AS L, 1 AS Dx, 1 AS D5, 1 AS B, 1 AS BDx, 1 AS La", + ); + assert_eq!( + spark().parse_sql_statements("SELECT 1e5L").unwrap(), + spark().parse_sql_statements("SELECT 1e5 AS L").unwrap() + ); + + TestedDialects::new(vec![Box::new(GenericDialect {})]) + .one_statement_parses_to("SELECT 1D, 1Y, 1l, 1L", "SELECT 1 AS D, 1 AS Y, 1 AS l, 1L"); +} From 57fa38c830b8749e8d30422c79661bdeaa0895b3 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 11 Nov 2026 12:09:37 -0700 Subject: [PATCH 2/3] Name the fuzz seed after the PR --- ...spark_TBD_numeric_suffix.sql => spark_2912_numeric_suffix.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename fuzz/fuzz_seeds/{spark_TBD_numeric_suffix.sql => spark_2912_numeric_suffix.sql} (100%) diff --git a/fuzz/fuzz_seeds/spark_TBD_numeric_suffix.sql b/fuzz/fuzz_seeds/spark_2912_numeric_suffix.sql similarity index 100% rename from fuzz/fuzz_seeds/spark_TBD_numeric_suffix.sql rename to fuzz/fuzz_seeds/spark_2912_numeric_suffix.sql From 40445a4c87c0eb39d9d11c1ab5f7d684524a4110 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 11 Nov 2026 12:37:35 -0700 Subject: [PATCH 3/3] Let each dialect accept its numeric literal suffixes, kept in the number's text Instead of a NumberSuffix enum and new Token and Value variants, the tokenizer asks Dialect::is_numeric_literal_suffix about the identifier characters after a number and keeps an accepted suffix in the Token::Number text. Spark, Databricks, Hive and Oracle each list their own suffixes. --- src/ast/mod.rs | 2 +- src/ast/value.rs | 45 ++------------- src/dialect/databricks.rs | 9 ++- src/dialect/hive.rs | 12 ++++ src/dialect/mod.rs | 12 ++-- src/dialect/oracle.rs | 8 +++ src/dialect/spark.rs | 17 ++++-- src/parser/mod.rs | 2 - src/tokenizer.rs | 101 +++++++++++++++++++--------------- tests/sqlparser_common.rs | 6 ++ tests/sqlparser_databricks.rs | 20 +++++++ tests/sqlparser_hive.rs | 16 +++++- tests/sqlparser_oracle.rs | 11 ++++ tests/sqlparser_spark.rs | 61 ++++++-------------- 14 files changed, 179 insertions(+), 143 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 9c39ee14ba..473b11a37b 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -130,7 +130,7 @@ pub use self::trigger::{ pub use self::value::{ escape_double_quote_string, escape_quoted_string, DateTimeField, DollarQuotedString, - NormalizationForm, NumberSuffix, QuoteDelimitedString, TrimWhereField, Value, ValueWithSpan, + NormalizationForm, QuoteDelimitedString, TrimWhereField, Value, ValueWithSpan, }; use crate::ast::helpers::key_value_options::KeyValueOptions; diff --git a/src/ast/value.rs b/src/ast/value.rs index 3331587a54..9a2ae3c16a 100644 --- a/src/ast/value.rs +++ b/src/ast/value.rs @@ -137,7 +137,10 @@ impl DerefMut for ValueWithSpan { #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub enum Value { - /// Numeric literal + /// Numeric literal, with any type suffix such as the `Y` of `10Y`. + /// See [`Dialect::is_numeric_literal_suffix`]. + /// + /// [`Dialect::is_numeric_literal_suffix`]: crate::dialect::Dialect::is_numeric_literal_suffix #[cfg(not(feature = "bigdecimal"))] Number(String, bool), #[cfg(feature = "bigdecimal")] @@ -146,13 +149,6 @@ pub enum Value { /// but fail on CI with the `--all-features` flag enabled /// Numeric literal (uses `BigDecimal` when the `bigdecimal` feature is enabled). Number(BigDecimal, bool), - /// Numeric literal with a type suffix, e.g. `1Y` or `1.5BD`. The string is the - /// number as written without the suffix, also with the `bigdecimal` feature. - /// - /// See [`Dialect::supports_numeric_literal_suffixes`]. - /// - /// [`Dialect::supports_numeric_literal_suffixes`]: crate::dialect::Dialect::supports_numeric_literal_suffixes - SuffixedNumber(String, NumberSuffix), /// 'string value' SingleQuotedString(String), /// Dollar-quoted string literal, e.g. `$$...$$` or `$tag$...$tag$` (Postgres syntax). @@ -269,7 +265,6 @@ impl fmt::Display for Value { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { Value::Number(v, l) => write!(f, "{}{long}", v, long = if *l { "L" } else { "" }), - Value::SuffixedNumber(v, suffix) => write!(f, "{v}{suffix}"), Value::DoubleQuotedString(v) => write!(f, "\"{}\"", escape_double_quote_string(v)), Value::SingleQuotedString(v) => write!(f, "'{}'", escape_single_quote_string(v)), Value::TripleSingleQuotedString(v) => { @@ -546,38 +541,6 @@ impl fmt::Display for NormalizationForm { } } -/// The type suffix of a [`Value::SuffixedNumber`]. An `L` suffix makes a -/// [`Value::Number`] long instead. -/// -/// See [Spark SQL](https://spark.apache.org/docs/latest/sql-ref-literals.html#numeric-literal). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Hash)] -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] -pub enum NumberSuffix { - /// `Y`, a `TINYINT` - TinyInt, - /// `S`, a `SMALLINT` - SmallInt, - /// `F`, a `FLOAT` - Float, - /// `D`, a `DOUBLE` - Double, - /// `BD`, a `DECIMAL` - Decimal, -} - -impl fmt::Display for NumberSuffix { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.write_str(match self { - NumberSuffix::TinyInt => "Y", - NumberSuffix::SmallInt => "S", - NumberSuffix::Float => "F", - NumberSuffix::Double => "D", - NumberSuffix::Decimal => "BD", - }) - } -} - pub struct EscapeQuotedString<'a> { string: &'a str, quote: char, diff --git a/src/dialect/databricks.rs b/src/dialect/databricks.rs index d660cec9eb..08752ee607 100644 --- a/src/dialect/databricks.rs +++ b/src/dialect/databricks.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use crate::dialect::Dialect; +use crate::dialect::{Dialect, SparkSqlDialect}; use crate::keywords::Keyword; use crate::parser::Parser; @@ -72,6 +72,13 @@ impl Dialect for DatabricksDialect { true } + /// The same suffixes as in Spark SQL, as in `+1Y`. + /// + /// See + fn is_numeric_literal_suffix(&self, number: &str, suffix: &str) -> bool { + SparkSqlDialect {}.is_numeric_literal_suffix(number, suffix) + } + fn supports_filter_during_aggregation(&self) -> bool { true } diff --git a/src/dialect/hive.rs b/src/dialect/hive.rs index e80bc75d2b..f11e0cca3a 100644 --- a/src/dialect/hive.rs +++ b/src/dialect/hive.rs @@ -78,6 +78,18 @@ impl Dialect for HiveDialect { true } + /// `L`, `S` and `Y` after an integer, and `D` and `BD` after any number, in + /// any case. + /// + /// See `IntegralLiteral` and `NumberLiteral` in + fn is_numeric_literal_suffix(&self, number: &str, suffix: &str) -> bool { + match suffix.as_bytes() { + [b'L' | b'l' | b'S' | b's' | b'Y' | b'y'] => !number.contains(['.', 'e', 'E']), + [b'D' | b'd'] | [b'B' | b'b', b'D' | b'd'] => true, + _ => false, + } + } + fn require_interval_qualifier(&self) -> bool { true } diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index c443240d20..32e59ab45a 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -615,12 +615,12 @@ pub trait Dialect: Debug + Any { false } - /// Returns true if a case-insensitive suffix gives a numeric literal its type: - /// `L`, `S` or `Y` after an integer, as in `1Y`, and `F`, `D` or `BD` after - /// any number, as in `1.5BD`. See [`NumberSuffix`]. - /// - /// [`NumberSuffix`]: crate::ast::NumberSuffix - fn supports_numeric_literal_suffixes(&self) -> bool { + /// Returns true if `suffix`, the identifier characters directly after the + /// numeric literal `number`, is a type suffix of the number, such as the `Y` + /// of `10Y` in Spark SQL. The suffix stays in the number's text, so `10Y` + /// fails to parse with the `bigdecimal` feature. An `L` suffix, in either + /// case, makes the number long instead. + fn is_numeric_literal_suffix(&self, _number: &str, _suffix: &str) -> bool { false } diff --git a/src/dialect/oracle.rs b/src/dialect/oracle.rs index 40c91cea35..dbf305cba5 100644 --- a/src/dialect/oracle.rs +++ b/src/dialect/oracle.rs @@ -52,6 +52,14 @@ impl Dialect for OracleDialect { ch.is_alphanumeric() || ch == '_' || ch == '$' || ch == '#' || ch == '@' } + /// `F` for `BINARY_FLOAT` and `D` for `BINARY_DOUBLE`, after any number, in + /// any case. + /// + /// See + fn is_numeric_literal_suffix(&self, _number: &str, suffix: &str) -> bool { + matches!(suffix.as_bytes(), [b'F' | b'f' | b'D' | b'd']) + } + /// LIMIT is not a reserved word in Oracle SQL. /// See fn is_table_factor(&self, kw: &Keyword, _parser: &mut Parser) -> bool { diff --git a/src/dialect/spark.rs b/src/dialect/spark.rs index d299dcdb3b..390e36d812 100644 --- a/src/dialect/spark.rs +++ b/src/dialect/spark.rs @@ -107,13 +107,20 @@ impl Dialect for SparkSqlDialect { true } - /// See - fn supports_values_as_table_factor(&self) -> bool { - true + /// `L`, `S` and `Y` after an integer, and `F`, `D` and `BD` after any number, + /// in any case. + /// + /// See + fn is_numeric_literal_suffix(&self, number: &str, suffix: &str) -> bool { + match suffix.as_bytes() { + [b'L' | b'l' | b'S' | b's' | b'Y' | b'y'] => !number.contains(['.', 'e', 'E']), + [b'F' | b'f' | b'D' | b'd'] | [b'B' | b'b', b'D' | b'd'] => true, + _ => false, + } } - /// See - fn supports_numeric_literal_suffixes(&self) -> bool { + /// See + fn supports_values_as_table_factor(&self) -> bool { true } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 777e1fc4bb..1d6cc53417 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -2083,7 +2083,6 @@ impl<'a> Parser<'a> { Ok(Expr::Value(self.parse_value()?)) } Token::Number(_, _) - | Token::SuffixedNumber(_, _) | Token::SingleQuotedString(_) | Token::DoubleQuotedString(_) | Token::TripleSingleQuotedString(_) @@ -13328,7 +13327,6 @@ impl<'a> Parser<'a> { Token::Number(n, l) => { ok_value(Value::Number(Self::parse_number_literal(n, span.start)?, l)) } - Token::SuffixedNumber(n, suffix) => ok_value(Value::SuffixedNumber(n, suffix)), Token::SingleQuotedString(ref s) => ok_value(Value::SingleQuotedString( self.maybe_concat_string_literal(s.to_string()), )), diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 578bbaeb1e..39bac9e2cb 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -40,7 +40,7 @@ use serde::{Deserialize, Serialize}; #[cfg(feature = "visitor")] use sqlparser_derive::{Visit, VisitMut}; -use crate::ast::{DollarQuotedString, NumberSuffix, QuoteDelimitedString}; +use crate::ast::{DollarQuotedString, QuoteDelimitedString}; use crate::dialect::Dialect; use crate::dialect::{ BigQueryDialect, DuckDbDialect, GenericDialect, MySqlDialect, PostgreSqlDialect, @@ -60,10 +60,9 @@ pub enum Token { EOF, /// A keyword (like SELECT) or an optionally quoted SQL identifier Word(Word), - /// An unsigned numeric literal + /// An unsigned numeric literal, with any type suffix that + /// [`Dialect::is_numeric_literal_suffix`] accepts, and whether it is long, as in `1L` Number(String, bool), - /// An unsigned numeric literal with a type suffix, e.g. `1Y` - SuffixedNumber(String, NumberSuffix), /// A character that could not be tokenized Char(char), /// Single quoted string: i.e: 'string' @@ -305,7 +304,6 @@ impl fmt::Display for Token { Token::EOF => f.write_str("EOF"), Token::Word(ref w) => write!(f, "{w}"), Token::Number(ref n, l) => write!(f, "{}{long}", n, long = if *l { "L" } else { "" }), - Token::SuffixedNumber(ref n, suffix) => write!(f, "{n}{suffix}"), Token::Char(ref c) => write!(f, "{c}"), Token::SingleQuotedString(ref s) => write!(f, "'{s}'"), Token::TripleSingleQuotedString(ref s) => write!(f, "'''{s}'''"), @@ -1689,6 +1687,26 @@ impl<'a> Tokenizer<'a> { } } + // After a period, as in `t.1D`, the number may start a column name. + if prev_token != Some(&Token::Period) { + let suffix: String = chars + .peekable + .clone() + .take_while(|&ch| self.dialect.is_identifier_part(ch)) + .collect(); + if !suffix.is_empty() && self.dialect.is_numeric_literal_suffix(&s, &suffix) + { + for _ in 0..suffix.chars().count() { + chars.next(); + } + if suffix.eq_ignore_ascii_case("L") { + return Ok(Some(Token::Number(s, true))); + } + s += &suffix; + return Ok(Some(Token::Number(s, false))); + } + } + // If the dialect supports identifiers that start with a numeric prefix, // we need to check if the value is in fact an identifier and must thus // be tokenized as a word. @@ -1710,10 +1728,6 @@ impl<'a> Tokenizer<'a> { } } - if self.dialect.supports_numeric_literal_suffixes() { - return Ok(Some(self.tokenize_number_suffix(chars, s))); - } - let long = if chars.peek() == Some(&'L') { chars.next(); true @@ -2349,39 +2363,6 @@ impl<'a> Tokenizer<'a> { Ok(Token::Number(s, false)) } - /// Tokenizes the number `s` with its type suffix if one follows, as in `1Y` or - /// `1.5BD`. A suffix directly followed by an identifier character is not one. - fn tokenize_number_suffix(&self, chars: &mut State, s: String) -> Token { - let is_integer = !s.contains(['.', 'e', 'E']); - let mut ahead = chars.peekable.clone(); - // `None` is the `L` of a long number. - let (suffix, len) = match ahead.next().map(|ch| ch.to_ascii_uppercase()) { - Some('L') if is_integer => (None, 1), - Some('S') if is_integer => (Some(NumberSuffix::SmallInt), 1), - Some('Y') if is_integer => (Some(NumberSuffix::TinyInt), 1), - Some('F') => (Some(NumberSuffix::Float), 1), - Some('D') => (Some(NumberSuffix::Double), 1), - Some('B') if ahead.peek().is_some_and(|ch| ch.eq_ignore_ascii_case(&'D')) => { - ahead.next(); - (Some(NumberSuffix::Decimal), 2) - } - _ => return Token::Number(s, false), - }; - if ahead - .next() - .is_some_and(|ch| self.dialect.is_identifier_part(ch)) - { - return Token::Number(s, false); - } - for _ in 0..len { - chars.next(); - } - match suffix { - Some(suffix) => Token::SuffixedNumber(s, suffix), - None => Token::Number(s, true), - } - } - // Consume characters until newline fn tokenize_single_line_comment(&self, chars: &mut State) -> String { peeking_take_while(chars, |ch| match ch { @@ -2975,7 +2956,7 @@ mod tests { use super::*; use crate::dialect::{ BigQueryDialect, ClickHouseDialect, HiveDialect, MsSqlDialect, MySqlDialect, - PostgreSqlDialect, SQLiteDialect, + PostgreSqlDialect, SQLiteDialect, SparkSqlDialect, }; use crate::test_utils::{all_dialects, all_dialects_except, all_dialects_where}; use core::fmt::Debug; @@ -5207,4 +5188,38 @@ mod tests { ); } } + + #[test] + fn tokenize_numeric_literal_suffix() { + let number = |n: &str, long| Token::Number(n.to_string(), long); + let space = || Token::Whitespace(Whitespace::Space); + compare( + vec![ + number("10Y", false), + space(), + number("1.5bd", false), + space(), + number("10", true), + space(), + number("1", false), + Token::make_word("Dx", None), + ], + Tokenizer::new(&SparkSqlDialect {}, "10Y 1.5bd 10l 1Dx") + .tokenize() + .unwrap(), + ); + // With numeric prefixes, a column name can follow a period. + compare( + vec![ + Token::make_word("t", None), + Token::Period, + Token::make_word("1D", None), + space(), + number("1D", false), + ], + Tokenizer::new(&HiveDialect {}, "t.1D 1D") + .tokenize() + .unwrap(), + ); + } } diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index aeb3e67a47..f6790a06af 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -23481,3 +23481,9 @@ fn parse_trailing_end_keyword_is_not_silently_dropped() { ParserError::ParserError("Expected: end of statement, found: END".to_string()) ); } + +#[test] +fn parse_number_without_suffix() { + all_dialects_where(|d| !d.supports_numeric_prefix() && !d.is_numeric_literal_suffix("1", "D")) + .one_statement_parses_to("SELECT 1D", "SELECT 1 AS D"); +} diff --git a/tests/sqlparser_databricks.rs b/tests/sqlparser_databricks.rs index ce26acc621..a0e17120e1 100644 --- a/tests/sqlparser_databricks.rs +++ b/tests/sqlparser_databricks.rs @@ -797,3 +797,23 @@ fn parse_databricks_collated_data_types() { )) ); } + +#[test] +fn parse_numeric_literal_suffixes() { + #[cfg(not(feature = "bigdecimal"))] + databricks().verified_stmt("SELECT +1Y, 10S, 1.5F, 1.5D, 1.5BD"); + + assert_eq!( + databricks().verified_expr("1L"), + Expr::value(Value::Number("1".parse().unwrap(), true)) + ); + assert_eq!( + databricks() + .verified_only_select("SELECT 1La, 1Dx") + .projection, + vec![ + SelectItem::UnnamedExpr(Expr::Identifier(Ident::new("1La"))), + SelectItem::UnnamedExpr(Expr::Identifier(Ident::new("1Dx"))), + ] + ); +} diff --git a/tests/sqlparser_hive.rs b/tests/sqlparser_hive.rs index 355fd5c8d5..5d8b56e8e0 100644 --- a/tests/sqlparser_hive.rs +++ b/tests/sqlparser_hive.rs @@ -22,7 +22,7 @@ use sqlparser::ast::{ ClusteredBy, CommentDef, CreateFunction, CreateFunctionBody, CreateFunctionUsing, CreateTable, - Expr, Ident, ObjectName, OrderByExpr, OrderByOptions, OrderBySort, Set, Statement, + Expr, Ident, ObjectName, OrderByExpr, OrderByOptions, OrderBySort, SelectItem, Set, Statement, UnaryOperator, Use, Value, }; use sqlparser::dialect::{AnsiDialect, GenericDialect, HiveDialect}; @@ -518,3 +518,17 @@ fn hive() -> TestedDialects { fn hive_and_generic() -> TestedDialects { TestedDialects::new(vec![Box::new(HiveDialect {}), Box::new(GenericDialect {})]) } + +#[test] +fn parse_numeric_literal_suffixes() { + #[cfg(not(feature = "bigdecimal"))] + hive().verified_stmt("SELECT 10Y, 10S, 1.5D, 1.5BD, 1e5d"); + + // Hive has no `F` suffix, so `1.5F` is a column name. + assert_eq!( + hive().verified_only_select("SELECT 1.5F").projection, + vec![SelectItem::UnnamedExpr(Expr::Identifier(Ident::new( + "1.5F" + )))] + ); +} diff --git a/tests/sqlparser_oracle.rs b/tests/sqlparser_oracle.rs index 0ed573be68..2f4b08b6dd 100644 --- a/tests/sqlparser_oracle.rs +++ b/tests/sqlparser_oracle.rs @@ -545,3 +545,14 @@ fn parse_non_reserved_keyword_table_names() { oracle().verified_stmt("SELECT * FROM limit"); oracle().verified_stmt("SELECT * FROM t JOIN limit ON TRUE"); } + +#[test] +fn parse_numeric_literal_suffixes() { + #[cfg(not(feature = "bigdecimal"))] + oracle().verified_stmt("SELECT 25f, +6.34F, 0.5d, -1D, 1e3d FROM dual"); + + oracle().one_statement_parses_to( + "SELECT 10Y, 1.5BD FROM dual", + "SELECT 10 AS Y, 1.5 AS BD FROM dual", + ); +} diff --git a/tests/sqlparser_spark.rs b/tests/sqlparser_spark.rs index fe716c381f..51dbc3de5f 100644 --- a/tests/sqlparser_spark.rs +++ b/tests/sqlparser_spark.rs @@ -19,7 +19,7 @@ //! Test SQL syntax specific to Apache Spark SQL. use sqlparser::ast::*; -use sqlparser::dialect::{GenericDialect, SparkSqlDialect}; +use sqlparser::dialect::SparkSqlDialect; use test_utils::*; #[macro_use] @@ -383,51 +383,26 @@ fn test_pipe_operator() { } #[test] -fn test_numeric_literal_suffixes() { - let suffixed = |n: &str, suffix| Expr::value(Value::SuffixedNumber(n.into(), suffix)); - assert_eq!( - spark().verified_expr("10Y"), - suffixed("10", NumberSuffix::TinyInt) - ); - assert_eq!( - spark().verified_expr("10S"), - suffixed("10", NumberSuffix::SmallInt) - ); - assert_eq!( - spark().verified_expr("1e5F"), - suffixed("1e5", NumberSuffix::Float) - ); - assert_eq!( - spark().verified_expr("1.5D"), - suffixed("1.5", NumberSuffix::Double) - ); - assert_eq!( - spark().verified_expr(".5BD"), - suffixed(".5", NumberSuffix::Decimal) - ); +fn parse_numeric_literal_suffixes() { + #[cfg(not(feature = "bigdecimal"))] + { + spark().verified_stmt("SELECT 10Y, 10s, 1.5F, 1.5d, 1.5BD, .5bd, 5.F, 1e5D, -32Y"); + assert_eq!(spark().verified_expr("1.5BD"), Expr::value(number("1.5BD"))); + } + // A `BigDecimal` can't hold the suffix. + #[cfg(feature = "bigdecimal")] assert_eq!( - spark().verified_expr("10L"), - Expr::value(Value::Number("10".parse().unwrap(), true)) - ); - - spark().verified_stmt("SELECT 1Y, 1S, 1L, 1F, 1D, 1BD"); - spark().verified_stmt("SELECT 1.5F, 1.5D, 1.5BD, 5.F, 1.5E-3BD, -32Y"); - spark().verified_stmt("SELECT 2 * 1.5D - 1Y FROM t WHERE x > 1BD"); - spark().one_statement_parses_to( - "SELECT 1y, 1s, 1l, 1f, 1d, 1bd, 1.5bD, 9223372036854775807l", - "SELECT 1Y, 1S, 1L, 1F, 1D, 1BD, 1.5BD, 9223372036854775807L", + spark().parse_sql_statements("SELECT 10Y").unwrap_err(), + sqlparser::parser::ParserError::ParserError( + "Could not parse '10Y' as bigdecimal::BigDecimal: invalid digit found in string" + .to_string() + ) ); - // `L`, `S` and `Y` only follow an integer, and no suffix runs into an identifier. + spark().one_statement_parses_to("SELECT 10l", "SELECT 10L"); + // `L`, `S` and `Y` only follow an integer, and a suffix ends the word. spark().one_statement_parses_to( - "SELECT 1.5S, 1.5Y, 1.5L, 1Dx, 1D5, 1B, 1BDx, 1La", - "SELECT 1.5 AS S, 1.5 AS Y, 1.5 AS L, 1 AS Dx, 1 AS D5, 1 AS B, 1 AS BDx, 1 AS La", - ); - assert_eq!( - spark().parse_sql_statements("SELECT 1e5L").unwrap(), - spark().parse_sql_statements("SELECT 1e5 AS L").unwrap() + "SELECT 1.5S, 1.5Y, 1Dx, 1B, 1BDx", + "SELECT 1.5 AS S, 1.5 AS Y, 1 AS Dx, 1 AS B, 1 AS BDx", ); - - TestedDialects::new(vec![Box::new(GenericDialect {})]) - .one_statement_parses_to("SELECT 1D, 1Y, 1l, 1L", "SELECT 1 AS D, 1 AS Y, 1 AS l, 1L"); }