From 75d41261c44f69ed8483e4b33daa1420a7b35124 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sun, 8 Nov 2026 08:55:50 -0500 Subject: [PATCH 1/5] docs: clarify EquivalenceClass, ConstExpr and OrderingEquivalenceClass docs Co-Authored-By: Claude Fable 5.1 --- .../physical-expr/src/equivalence/class.rs | 58 ++++++++++--------- .../physical-expr/src/equivalence/mod.rs | 9 +++ .../physical-expr/src/equivalence/ordering.rs | 43 +++++++------- 3 files changed, 63 insertions(+), 47 deletions(-) diff --git a/datafusion/physical-expr/src/equivalence/class.rs b/datafusion/physical-expr/src/equivalence/class.rs index a94a38f47f75d..8b5d6803c24a3 100644 --- a/datafusion/physical-expr/src/equivalence/class.rs +++ b/datafusion/physical-expr/src/equivalence/class.rs @@ -24,16 +24,19 @@ use datafusion_physical_expr_common::physical_expr::{ use indexmap::{IndexMap, IndexSet}; -/// Represents whether a constant expression's value is uniform or varies across -/// partitions. Has two variants: -/// - `Heterogeneous`: The constant expression may have different values for -/// different partitions. -/// - `Uniform(Option)`: The constant expression has the same value -/// across all partitions, or is `None` if the value is unknown. +/// How a constant expression's value varies across partitions. +/// +/// See [Constants] for why the difference matters. +/// +/// [Constants]: crate::equivalence::EquivalenceProperties#constants #[derive(Clone, Debug, Default, Eq, PartialEq)] pub enum AcrossPartitions { + /// The constant may have different values for different partitions. #[default] Heterogeneous, + /// The constant has the same value across all partitions. + /// + /// `Some` if the value is exactly known, `None` otherwise. Uniform(Option), } @@ -52,20 +55,11 @@ impl Display for AcrossPartitions { } } -/// A structure representing a expression known to be constant in a physical -/// execution plan. -/// -/// The `ConstExpr` struct encapsulates an expression that is constant during -/// the execution of a query. For example if a filter like `A = 5` appears -/// earlier in the plan, `A` would become a constant in subsequent operations. -/// -/// # Fields +/// An expression known to be constant during the execution of a query. /// -/// - `expr`: Constant expression for a node in the physical plan. -/// - `across_partitions`: A boolean flag indicating whether the constant -/// expression is the same across partitions. If set to `true`, the constant -/// expression has same value for all partitions. If set to `false`, the -/// constant expression may have different values for different partitions. +/// For example, after a filter like `A = 5` in the plan, the expression `A` +/// is known to be constant in subsequent operations. See [Constants] for more +/// details. /// /// # Example /// @@ -76,11 +70,13 @@ impl Display for AcrossPartitions { /// // Create a constant expression from a physical expression: /// let const_expr = ConstExpr::from(col); /// ``` +/// +/// [Constants]: crate::equivalence::EquivalenceProperties#constants #[derive(Clone, Debug)] pub struct ConstExpr { /// The expression that is known to be constant (e.g. a `Column`). pub expr: Arc, - /// Indicates whether the constant have the same value across all partitions. + /// Does the constant have the same value across all partitions? pub across_partitions: AcrossPartitions, } // TODO: The `ConstExpr` definition above can be in an inconsistent state where @@ -157,13 +153,17 @@ impl From> for ConstExpr { } } -/// An `EquivalenceClass` is a set of [`Arc`]s that are known -/// to have the same value for all tuples in a relation. These are generated by -/// equality predicates (e.g. `a = b`), typically equi-join conditions and -/// equality conditions in filters. +/// A set of [`Arc`]s that are known +/// to have the same value for all tuples in a relation. +/// +/// `EquivalenceClass`es are generated by equality predicates such as `a = b`, +/// typically equi-join conditions and conditions in filters. See +/// [Equivalent expressions] for more detail. /// -/// Two `EquivalenceClass`es are equal if they contains the same expressions in +/// Two `EquivalenceClass`es are equal if they contain the same expressions in /// without any ordering. +/// +/// [Equivalent expressions]: crate::equivalence::EquivalenceProperties#equivalent-expressions #[derive(Clone, Debug, Default, Eq, PartialEq)] pub struct EquivalenceClass { /// The expressions in this equivalence class. The order doesn't matter for @@ -300,8 +300,12 @@ type AugmentedMapping<'a> = IndexMap< (&'a ProjectionTargets, Option<&'a EquivalenceClass>), >; -/// A collection of distinct `EquivalenceClass`es. This object supports fast -/// lookups of expressions and their equivalence classes. +/// A set of distinct [`EquivalenceClass`]es. +/// +/// `EquivalenceGroup` supports fast lookups of expressions and their +/// equivalence classes. See [`EquivalenceProperties`] for more details. +/// +/// [`EquivalenceProperties`]: crate::equivalence::EquivalenceProperties #[derive(Clone, Debug, Default)] pub struct EquivalenceGroup { /// A mapping from expressions to their equivalence class key. diff --git a/datafusion/physical-expr/src/equivalence/mod.rs b/datafusion/physical-expr/src/equivalence/mod.rs index ff57146efcce9..3724469083451 100644 --- a/datafusion/physical-expr/src/equivalence/mod.rs +++ b/datafusion/physical-expr/src/equivalence/mod.rs @@ -2,6 +2,15 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +//! Equivalence properties: the orderings, equality, constants and +//! groupings known to hold for the output of a plan node, which the planner +//! uses to avoid sorts and repartitions. +//! +//! # See Also +//! +//! * [`EquivalenceProperties`], which explains the concepts and how the +//! pieces below fit together. + use std::borrow::Borrow; use std::sync::Arc; diff --git a/datafusion/physical-expr/src/equivalence/ordering.rs b/datafusion/physical-expr/src/equivalence/ordering.rs index 7de47767f62fd..da10586483773 100644 --- a/datafusion/physical-expr/src/equivalence/ordering.rs +++ b/datafusion/physical-expr/src/equivalence/ordering.rs @@ -15,8 +15,9 @@ use arrow::datatypes::SchemaRef; use datafusion_common::Result; use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr; -/// An `OrderingEquivalenceClass` keeps track of distinct alternative orderings -/// than can describe a table. For example, consider the following table: +/// Tracks distinct alternative orderings that describe a table. +/// +/// For example, consider the following table: /// /// ```text /// ┌───┬───┬───┬───┐ @@ -30,16 +31,25 @@ use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr; /// ``` /// /// Here, both `[a ASC, b ASC]` and `[c DESC, d ASC]` describe the table -/// ordering. In this case, we say that these orderings are equivalent. +/// ordering, so these orderings are equivalent. See [Equivalent sort +/// expressions] for more details. +/// +/// An `OrderingEquivalenceClass` is a set of such equivalent orderings. +/// +/// [`GroupingEquivalenceClass`] is used when groups of equal values are +/// contiguous, but not in order. /// -/// An `OrderingEquivalenceClass` is a set of such equivalent orderings, which -/// is represented by a vector of `LexOrdering`s. The set does not store any -/// redundant information by enforcing the invariant that no suffix of an -/// ordering in the equivalence class is a prefix of another ordering in the -/// equivalence class. The set can be empty, which means that there are no -/// orderings that describe the table. +/// [Equivalent sort expressions]: crate::equivalence::EquivalenceProperties#equivalent-sort-expressions +/// [`GroupingEquivalenceClass`]: crate::equivalence::GroupingEquivalenceClass #[derive(Clone, Debug, Default, Eq, PartialEq)] pub struct OrderingEquivalenceClass { + /// Set of equivalent orderings + /// + /// Invariant: no suffix of an ordering in the equivalence class is a prefix + /// of another ordering in the equivalence class. This avoids redundant + /// information. + /// + /// If empty, means the table is unordered orderings: Vec, } @@ -216,18 +226,11 @@ impl OrderingEquivalenceClass { /// `ASC NULLS LAST` and `DESC NULLS FIRST`. Rows sorted both ways all have /// the same value (or are all null). /// - /// This is how a value that is constant only within groups of rows is - /// recorded, since a `ConstExpr` is constant across all rows. For example, - /// `SUM(x) OVER (PARTITION BY k)` has one value per `k`, so the window adds - /// `[k, w]` once for each of the four directions of `w`. - /// - /// ### Future Improvements + /// This is how a whole-partition window aggregate records that its value + /// is constant within each window partition. See + /// [Per-partition constants from window functions]. /// - /// This function may become unnecessary if any of the following improvements - /// are implemented: - /// 1. `SortOptions` supports encoding constantness information. - /// 2. `EquivalenceProperties` gains `FunctionalDependency` awareness, eliminating - /// the need for `Constant` and `Constraints`. + /// [Per-partition constants from window functions]: crate::equivalence::EquivalenceProperties#per-partition-constants-from-window-functions pub fn is_expr_partial_const(&self, expr: &Arc) -> bool { is_partial_const(self.iter().map(|ordering| ordering.first()), expr) } From 7dae3816b7b2cd699b6f577145dc80f225e6e23c Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 9 Nov 2026 14:33:21 -0500 Subject: [PATCH 2/5] Update datafusion/physical-expr/src/equivalence/class.rs Co-authored-by: xavlee <52435309+xavlee@users.noreply.github.com> --- datafusion/physical-expr/src/equivalence/class.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datafusion/physical-expr/src/equivalence/class.rs b/datafusion/physical-expr/src/equivalence/class.rs index 8b5d6803c24a3..09dd79f12de13 100644 --- a/datafusion/physical-expr/src/equivalence/class.rs +++ b/datafusion/physical-expr/src/equivalence/class.rs @@ -160,8 +160,8 @@ impl From> for ConstExpr { /// typically equi-join conditions and conditions in filters. See /// [Equivalent expressions] for more detail. /// -/// Two `EquivalenceClass`es are equal if they contain the same expressions in -/// without any ordering. +/// Two `EquivalenceClass`es are equal if they contain the same expressions, regardless +/// of their order. /// /// [Equivalent expressions]: crate::equivalence::EquivalenceProperties#equivalent-expressions #[derive(Clone, Debug, Default, Eq, PartialEq)] From 1024c618986d40a9e8ef710bb2aabfa283260824 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 11 Nov 2026 09:22:44 -0500 Subject: [PATCH 3/5] Update datafusion/physical-expr/src/equivalence/mod.rs Co-authored-by: Jay Zhan --- datafusion/physical-expr/src/equivalence/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/physical-expr/src/equivalence/mod.rs b/datafusion/physical-expr/src/equivalence/mod.rs index 3724469083451..093afac6a6914 100644 --- a/datafusion/physical-expr/src/equivalence/mod.rs +++ b/datafusion/physical-expr/src/equivalence/mod.rs @@ -9,7 +9,7 @@ //! # See Also //! //! * [`EquivalenceProperties`], which explains the concepts and how the -//! pieces below fit together. +//! pieces below fit together. use std::borrow::Borrow; use std::sync::Arc; From 0da0be2883998c10a625291efb639a6da5a5887d Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 11 Nov 2026 09:23:02 -0500 Subject: [PATCH 4/5] Apply suggestion from @jayzhan211 Co-authored-by: Jay Zhan --- datafusion/physical-expr/src/equivalence/ordering.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/physical-expr/src/equivalence/ordering.rs b/datafusion/physical-expr/src/equivalence/ordering.rs index da10586483773..a908cce4fe4af 100644 --- a/datafusion/physical-expr/src/equivalence/ordering.rs +++ b/datafusion/physical-expr/src/equivalence/ordering.rs @@ -230,7 +230,7 @@ impl OrderingEquivalenceClass { /// is constant within each window partition. See /// [Per-partition constants from window functions]. /// - /// [Per-partition constants from window functions]: crate::equivalence::EquivalenceProperties#per-partition-constants-from-window-functions + /// [Per-partition constants from window functions]: crate::equivalence::EquivalenceProperties#example-window-functions-with-partition-by pub fn is_expr_partial_const(&self, expr: &Arc) -> bool { is_partial_const(self.iter().map(|ordering| ordering.first()), expr) } From ddb7abc610d4805643b3ed7a3d6bd62049f04445 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 11 Nov 2026 09:47:08 -0500 Subject: [PATCH 5/5] docs: planner --> physical optimizezr --- datafusion/physical-expr/src/equivalence/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datafusion/physical-expr/src/equivalence/mod.rs b/datafusion/physical-expr/src/equivalence/mod.rs index 093afac6a6914..1f869cf06c862 100644 --- a/datafusion/physical-expr/src/equivalence/mod.rs +++ b/datafusion/physical-expr/src/equivalence/mod.rs @@ -3,8 +3,8 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements //! Equivalence properties: the orderings, equality, constants and -//! groupings known to hold for the output of a plan node, which the planner -//! uses to avoid sorts and repartitions. +//! groupings known to hold for the output of a plan node, which the physical +//! optimizer uses to avoid sorts and repartitions. //! //! # See Also //!