From 0b45379d43e682ffe1d99e226c0cc4e6e638be0a Mon Sep 17 00:00:00 2001 From: Aleksandr Efimov Date: Thu, 20 Aug 2026 17:11:16 +0300 Subject: [PATCH] fix(spark): report interval_year overflow instead of wrapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit years and months are both int32 on the wire and Spark's physical type is a months Int, so nothing widens the arithmetic the way the day-time literal is widened. Above 178,956,970 years the product wrapped, and a large positive interval came out negative — Int.MaxValue years became -12 months. Far outside the spec's 10,000-year bound, so it takes a producer that is already well past it; the point is that the two adjacent literal conversions now take the same stance on the same arithmetic. --- .../spark/expression/ToSparkExpression.scala | 6 ++++-- .../substrait/spark/TypesAndLiteralsSuite.scala | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/spark/src/main/scala/io/substrait/spark/expression/ToSparkExpression.scala b/spark/src/main/scala/io/substrait/spark/expression/ToSparkExpression.scala index 7aaca95bd..8a66d5ae3 100644 --- a/spark/src/main/scala/io/substrait/spark/expression/ToSparkExpression.scala +++ b/spark/src/main/scala/io/substrait/spark/expression/ToSparkExpression.scala @@ -148,8 +148,10 @@ class ToSparkExpression( override def visit( expr: SExpression.IntervalYearLiteral, context: EmptyVisitationContext): Literal = { - // Spark uses a single months Int as the "physical" type for YearMonthInterval - val months = expr.years() * 12 + expr.months() + // Spark uses a single months Int as the "physical" type for YearMonthInterval. Both operands + // are int32 on the wire, so unlike the day-time case nothing widens the arithmetic and a large + // year count wraps a positive interval into a negative one; report it instead. + val months = Math.addExact(Math.multiplyExact(expr.years(), 12), expr.months()) Literal(months, ToSparkType.convert(expr.getType)) } diff --git a/spark/src/test/scala/io/substrait/spark/TypesAndLiteralsSuite.scala b/spark/src/test/scala/io/substrait/spark/TypesAndLiteralsSuite.scala index 8d1b1c491..ec86cdbd1 100644 --- a/spark/src/test/scala/io/substrait/spark/TypesAndLiteralsSuite.scala +++ b/spark/src/test/scala/io/substrait/spark/TypesAndLiteralsSuite.scala @@ -307,6 +307,22 @@ class TypesAndLiteralsSuite extends SparkFunSuite { sparkLiteral(ExpressionCreator.intervalDay(false, Int.MaxValue, 0, 0L, 6))) } + test("a year-month interval reports overflow instead of wrapping") { + // years and months are int32 on the wire and Spark's physical type is a months Int, so there + // is no Long operand to widen this the way the day-time case is widened. Far outside the + // spec's 10,000-year bound, but it used to turn a large positive interval into a negative one: + // 178,956,971 years came out as -2,147,483,644 months, and Int.MaxValue years as -12. + intercept[ArithmeticException]( + sparkLiteral(ExpressionCreator.intervalYear(false, 178956971, 0))) + intercept[ArithmeticException]( + sparkLiteral(ExpressionCreator.intervalYear(false, Int.MaxValue, 0))) + + // The spec's maximum still converts, and the months carry through. + val atTheBound = sparkLiteral(ExpressionCreator.intervalYear(false, 10000, 0)) + assert(atTheBound.value === 120000) + assert(atTheBound.dataType === YearMonthIntervalType.DEFAULT) + } + test("a coarser precision on a type is rejected, since a type has no value to rescale") { // A Spark type carries no precision of its own, so mapping precision_timestamp<3> onto // TimestampNTZType would reinterpret millisecond counts as microsecond ones. Only the literal