diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rs index e188b4543f284..afb1e272aaef1 100644 --- a/datafusion/datasource-parquet/src/opener/mod.rs +++ b/datafusion/datasource-parquet/src/opener/mod.rs @@ -1973,6 +1973,20 @@ fn stats_alone_unsatisfiable( } /// Extract constant column values from statistics, keyed by column name in the logical file schema. +/// Returns true when `data_type` stores IEEE-754 floating point values, +/// including dictionary encoded floats. +/// +/// `min == max` does not prove that such a column is constant: parquet writers +/// exclude `NaN` from `min`/`max` statistics as the format requires, and +/// `-0.0 == 0.0` compares equal even though the rows differ. Substituting the +/// statistic for the data would silently rewrite those rows. +fn is_float_like(data_type: &DataType) -> bool { + match data_type { + DataType::Dictionary(_, value_type) => value_type.is_floating(), + _ => data_type.is_floating(), + } +} + fn constant_columns_from_stats( statistics: Option<&Statistics>, file_schema: &SchemaRef, @@ -2014,6 +2028,8 @@ fn constant_value_from_stats( && min == max && !min.is_null() && matches!(column_stats.null_count, Precision::Exact(0)) + && !is_float_like(&min.data_type()) + && !is_float_like(data_type) { // Cast to the expected data type if needed (e.g., Utf8 -> Dictionary) if min.data_type() != *data_type { diff --git a/datafusion/sqllogictest/test_files/parquet_constant_columns.slt b/datafusion/sqllogictest/test_files/parquet_constant_columns.slt new file mode 100644 index 0000000000000..5b5fa59bad6a0 --- /dev/null +++ b/datafusion/sqllogictest/test_files/parquet_constant_columns.slt @@ -0,0 +1,23 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +# Parquet writers exclude NaN from floating point min/max statistics, as the +# format requires, so a column holding 1.0, NaN and 1.0 has min == max == 1.0 +# and null_count = 0. Statistics must not be used to replace such a projected +# column with that value: doing so silently turns the NaN row into a 1.0 row. +statement ok +COPY (SELECT v::double AS f FROM (VALUES (1.0::double), ('NaN'::double), (1.0::double)) AS t(v)) +TO 'test_files/scratch/parquet_constant_columns/nan.parquet' +STORED AS PARQUET; + +statement ok +CREATE EXTERNAL TABLE nan_statistics STORED AS PARQUET +LOCATION 'test_files/scratch/parquet_constant_columns/nan.parquet'; + +query R +SELECT f FROM nan_statistics ORDER BY f; +---- +1 +1 +NaN