Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions datafusion/datasource-parquet/src/opener/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1973,6 +1973,20 @@ fn stats_alone_unsatisfiable(
}

/// Extract constant column values from statistics, keyed by column name in the logical file schema.
/// Returns true when `data_type` stores IEEE-754 floating point values,
/// including dictionary encoded floats.
///
/// `min == max` does not prove that such a column is constant: parquet writers
/// exclude `NaN` from `min`/`max` statistics as the format requires, and
/// `-0.0 == 0.0` compares equal even though the rows differ. Substituting the
/// statistic for the data would silently rewrite those rows.
fn is_float_like(data_type: &DataType) -> bool {
match data_type {
DataType::Dictionary(_, value_type) => value_type.is_floating(),
_ => data_type.is_floating(),
}
}

fn constant_columns_from_stats(
statistics: Option<&Statistics>,
file_schema: &SchemaRef,
Expand Down Expand Up @@ -2014,6 +2028,8 @@ fn constant_value_from_stats(
&& min == max
&& !min.is_null()
&& matches!(column_stats.null_count, Precision::Exact(0))
&& !is_float_like(&min.data_type())
&& !is_float_like(data_type)
{
// Cast to the expected data type if needed (e.g., Utf8 -> Dictionary)
if min.data_type() != *data_type {
Expand Down
23 changes: 23 additions & 0 deletions datafusion/sqllogictest/test_files/parquet_constant_columns.slt
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

# Parquet writers exclude NaN from floating point min/max statistics, as the
# format requires, so a column holding 1.0, NaN and 1.0 has min == max == 1.0
# and null_count = 0. Statistics must not be used to replace such a projected
# column with that value: doing so silently turns the NaN row into a 1.0 row.
statement ok
COPY (SELECT v::double AS f FROM (VALUES (1.0::double), ('NaN'::double), (1.0::double)) AS t(v))
TO 'test_files/scratch/parquet_constant_columns/nan.parquet'
STORED AS PARQUET;

statement ok
CREATE EXTERNAL TABLE nan_statistics STORED AS PARQUET
LOCATION 'test_files/scratch/parquet_constant_columns/nan.parquet';

query R
SELECT f FROM nan_statistics ORDER BY f;
----
1
1
NaN