diff --git a/datafusion/physical-expr/src/equivalence/class.rs b/datafusion/physical-expr/src/equivalence/class.rs index a94a38f47f75d..09dd79f12de13 100644 --- a/datafusion/physical-expr/src/equivalence/class.rs +++ b/datafusion/physical-expr/src/equivalence/class.rs @@ -24,16 +24,19 @@ use datafusion_physical_expr_common::physical_expr::{ use indexmap::{IndexMap, IndexSet}; -/// Represents whether a constant expression's value is uniform or varies across -/// partitions. Has two variants: -/// - `Heterogeneous`: The constant expression may have different values for -/// different partitions. -/// - `Uniform(Option)`: The constant expression has the same value -/// across all partitions, or is `None` if the value is unknown. +/// How a constant expression's value varies across partitions. +/// +/// See [Constants] for why the difference matters. +/// +/// [Constants]: crate::equivalence::EquivalenceProperties#constants #[derive(Clone, Debug, Default, Eq, PartialEq)] pub enum AcrossPartitions { + /// The constant may have different values for different partitions. #[default] Heterogeneous, + /// The constant has the same value across all partitions. + /// + /// `Some` if the value is exactly known, `None` otherwise. Uniform(Option), } @@ -52,20 +55,11 @@ impl Display for AcrossPartitions { } } -/// A structure representing a expression known to be constant in a physical -/// execution plan. -/// -/// The `ConstExpr` struct encapsulates an expression that is constant during -/// the execution of a query. For example if a filter like `A = 5` appears -/// earlier in the plan, `A` would become a constant in subsequent operations. -/// -/// # Fields +/// An expression known to be constant during the execution of a query. /// -/// - `expr`: Constant expression for a node in the physical plan. -/// - `across_partitions`: A boolean flag indicating whether the constant -/// expression is the same across partitions. If set to `true`, the constant -/// expression has same value for all partitions. If set to `false`, the -/// constant expression may have different values for different partitions. +/// For example, after a filter like `A = 5` in the plan, the expression `A` +/// is known to be constant in subsequent operations. See [Constants] for more +/// details. /// /// # Example /// @@ -76,11 +70,13 @@ impl Display for AcrossPartitions { /// // Create a constant expression from a physical expression: /// let const_expr = ConstExpr::from(col); /// ``` +/// +/// [Constants]: crate::equivalence::EquivalenceProperties#constants #[derive(Clone, Debug)] pub struct ConstExpr { /// The expression that is known to be constant (e.g. a `Column`). pub expr: Arc, - /// Indicates whether the constant have the same value across all partitions. + /// Does the constant have the same value across all partitions? pub across_partitions: AcrossPartitions, } // TODO: The `ConstExpr` definition above can be in an inconsistent state where @@ -157,13 +153,17 @@ impl From> for ConstExpr { } } -/// An `EquivalenceClass` is a set of [`Arc`]s that are known -/// to have the same value for all tuples in a relation. These are generated by -/// equality predicates (e.g. `a = b`), typically equi-join conditions and -/// equality conditions in filters. +/// A set of [`Arc`]s that are known +/// to have the same value for all tuples in a relation. +/// +/// `EquivalenceClass`es are generated by equality predicates such as `a = b`, +/// typically equi-join conditions and conditions in filters. See +/// [Equivalent expressions] for more detail. +/// +/// Two `EquivalenceClass`es are equal if they contain the same expressions, regardless +/// of their order. /// -/// Two `EquivalenceClass`es are equal if they contains the same expressions in -/// without any ordering. +/// [Equivalent expressions]: crate::equivalence::EquivalenceProperties#equivalent-expressions #[derive(Clone, Debug, Default, Eq, PartialEq)] pub struct EquivalenceClass { /// The expressions in this equivalence class. The order doesn't matter for @@ -300,8 +300,12 @@ type AugmentedMapping<'a> = IndexMap< (&'a ProjectionTargets, Option<&'a EquivalenceClass>), >; -/// A collection of distinct `EquivalenceClass`es. This object supports fast -/// lookups of expressions and their equivalence classes. +/// A set of distinct [`EquivalenceClass`]es. +/// +/// `EquivalenceGroup` supports fast lookups of expressions and their +/// equivalence classes. See [`EquivalenceProperties`] for more details. +/// +/// [`EquivalenceProperties`]: crate::equivalence::EquivalenceProperties #[derive(Clone, Debug, Default)] pub struct EquivalenceGroup { /// A mapping from expressions to their equivalence class key. diff --git a/datafusion/physical-expr/src/equivalence/mod.rs b/datafusion/physical-expr/src/equivalence/mod.rs index ff57146efcce9..1f869cf06c862 100644 --- a/datafusion/physical-expr/src/equivalence/mod.rs +++ b/datafusion/physical-expr/src/equivalence/mod.rs @@ -2,6 +2,15 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +//! Equivalence properties: the orderings, equality, constants and +//! groupings known to hold for the output of a plan node, which the physical +//! optimizer uses to avoid sorts and repartitions. +//! +//! # See Also +//! +//! * [`EquivalenceProperties`], which explains the concepts and how the +//! pieces below fit together. + use std::borrow::Borrow; use std::sync::Arc; diff --git a/datafusion/physical-expr/src/equivalence/ordering.rs b/datafusion/physical-expr/src/equivalence/ordering.rs index 7de47767f62fd..a908cce4fe4af 100644 --- a/datafusion/physical-expr/src/equivalence/ordering.rs +++ b/datafusion/physical-expr/src/equivalence/ordering.rs @@ -15,8 +15,9 @@ use arrow::datatypes::SchemaRef; use datafusion_common::Result; use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr; -/// An `OrderingEquivalenceClass` keeps track of distinct alternative orderings -/// than can describe a table. For example, consider the following table: +/// Tracks distinct alternative orderings that describe a table. +/// +/// For example, consider the following table: /// /// ```text /// ┌───┬───┬───┬───┐ @@ -30,16 +31,25 @@ use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr; /// ``` /// /// Here, both `[a ASC, b ASC]` and `[c DESC, d ASC]` describe the table -/// ordering. In this case, we say that these orderings are equivalent. +/// ordering, so these orderings are equivalent. See [Equivalent sort +/// expressions] for more details. +/// +/// An `OrderingEquivalenceClass` is a set of such equivalent orderings. +/// +/// [`GroupingEquivalenceClass`] is used when groups of equal values are +/// contiguous, but not in order. /// -/// An `OrderingEquivalenceClass` is a set of such equivalent orderings, which -/// is represented by a vector of `LexOrdering`s. The set does not store any -/// redundant information by enforcing the invariant that no suffix of an -/// ordering in the equivalence class is a prefix of another ordering in the -/// equivalence class. The set can be empty, which means that there are no -/// orderings that describe the table. +/// [Equivalent sort expressions]: crate::equivalence::EquivalenceProperties#equivalent-sort-expressions +/// [`GroupingEquivalenceClass`]: crate::equivalence::GroupingEquivalenceClass #[derive(Clone, Debug, Default, Eq, PartialEq)] pub struct OrderingEquivalenceClass { + /// Set of equivalent orderings + /// + /// Invariant: no suffix of an ordering in the equivalence class is a prefix + /// of another ordering in the equivalence class. This avoids redundant + /// information. + /// + /// If empty, means the table is unordered orderings: Vec, } @@ -216,18 +226,11 @@ impl OrderingEquivalenceClass { /// `ASC NULLS LAST` and `DESC NULLS FIRST`. Rows sorted both ways all have /// the same value (or are all null). /// - /// This is how a value that is constant only within groups of rows is - /// recorded, since a `ConstExpr` is constant across all rows. For example, - /// `SUM(x) OVER (PARTITION BY k)` has one value per `k`, so the window adds - /// `[k, w]` once for each of the four directions of `w`. - /// - /// ### Future Improvements + /// This is how a whole-partition window aggregate records that its value + /// is constant within each window partition. See + /// [Per-partition constants from window functions]. /// - /// This function may become unnecessary if any of the following improvements - /// are implemented: - /// 1. `SortOptions` supports encoding constantness information. - /// 2. `EquivalenceProperties` gains `FunctionalDependency` awareness, eliminating - /// the need for `Constant` and `Constraints`. + /// [Per-partition constants from window functions]: crate::equivalence::EquivalenceProperties#example-window-functions-with-partition-by pub fn is_expr_partial_const(&self, expr: &Arc) -> bool { is_partial_const(self.iter().map(|ordering| ordering.first()), expr) }