docs: Improve documentation for SortKeyWalk

datafusion/physical-expr-common/src/sort_expr.rs

@@ -344,6 +344,11 @@ pub fn options_compatible(
 /// * `ORDER BY A DESC NULLS FIRST` matches
 /// * `ORDER BY A ASC  NULLS FIRST` matches (`ASC` and `NULL` options ignored)
 /// * `ORDER BY B DESC NULLS FIRST` does not match  (different expr)
+///
+/// See [Required orderings] for where requirements come from and how they
+/// are checked against the orderings an input provides.
+///
+/// [Required orderings]: https://docs.rs/datafusion-physical-expr/latest/datafusion_physical_expr/equivalence/struct.EquivalenceProperties.html#required-orderings
 #[derive(Clone, Debug)]
 pub struct PhysicalSortRequirement {
     /// Physical expression representing the column to sort
@@ -697,6 +702,11 @@ impl From<LexOrdering> for Vec<PhysicalSortExpr> {
 /// expressions after construction like a `LexOrdering` does, because it provides
 /// no mutation methods. If such methods become necessary, we will need to
 /// enforce uniqueness like the latter object.
+///
+/// See [Required orderings] for where requirements come from and how they
+/// are checked against the orderings an input provides.
+///
+/// [Required orderings]: https://docs.rs/datafusion-physical-expr/latest/datafusion_physical_expr/equivalence/struct.EquivalenceProperties.html#required-orderings
 #[derive(Debug, Clone, PartialEq)]
 pub struct LexRequirement {
     reqs: Vec<PhysicalSortRequirement>,
@@ -807,6 +817,11 @@ impl From<LexRequirement> for LexOrdering {
 /// 1. It is non-degenerate, meaning it contains at least one ordering. The
 ///    absence of an input ordering requirement is represented by a `None` value
 ///    in `ExecutionPlan` APIs, which return an `Option<OrderingRequirements>`.
+///
+/// See [Required orderings] for where requirements come from and how they
+/// are checked against the orderings an input provides.
+///
+/// [Required orderings]: https://docs.rs/datafusion-physical-expr/latest/datafusion_physical_expr/equivalence/struct.EquivalenceProperties.html#required-orderings
 #[derive(Debug, Clone, PartialEq)]
 pub enum OrderingRequirements {
     /// The operator is not able to work without one of these requirements.

datafusion/physical-expr/src/equivalence/class.rs

@@ -24,16 +24,19 @@ use datafusion_physical_expr_common::physical_expr::{
 
 use indexmap::{IndexMap, IndexSet};
 
-/// Represents whether a constant expression's value is uniform or varies across
-/// partitions. Has two variants:
-/// - `Heterogeneous`: The constant expression may have different values for
-///   different partitions.
-/// - `Uniform(Option<ScalarValue>)`: The constant expression has the same value
-///   across all partitions, or is `None` if the value is unknown.
+/// How a constant expression value varies  across partitions.
+///
+/// See [Constants] for why the difference matters.
+///
+/// [Constants]: crate::equivalence::EquivalenceProperties#constants
 #[derive(Clone, Debug, Default, Eq, PartialEq)]
 pub enum AcrossPartitions {
+    /// The constant may have different values for different partitions.
     #[default]
     Heterogeneous,
+    /// The constant has the same value across all partitions.
+    ///
+    /// `Some` if the value is exactly known, `None` otherwise.
     Uniform(Option<ScalarValue>),
 }
 
@@ -52,20 +55,10 @@ impl Display for AcrossPartitions {
     }
 }
 
-/// A structure representing a expression known to be constant in a physical
-/// execution plan.
-///
-/// The `ConstExpr` struct encapsulates an expression that is constant during
-/// the execution of a query. For example if a filter like `A = 5` appears
-/// earlier in the plan, `A` would become a constant in subsequent operations.
-///
-/// # Fields
+/// An expression known to be constant during the execution of a query.
 ///
-/// - `expr`: Constant expression for a node in the physical plan.
-/// - `across_partitions`: A boolean flag indicating whether the constant
-///   expression is the same across partitions. If set to `true`, the constant
-///   expression has same value for all partitions. If set to `false`, the
-///   constant expression may have different values for different partitions.
+/// For example after a filter like `A = 5` in a the plan, the expression `A`
+/// is know to be constant in subsequent operations. See [Constants] for more details.
 ///
 /// # Example
 ///
@@ -76,11 +69,13 @@ impl Display for AcrossPartitions {
 /// // Create a constant expression from a physical expression:
 /// let const_expr = ConstExpr::from(col);
 /// ```
+///
+/// [Constants]: crate::equivalence::EquivalenceProperties#constants
 #[derive(Clone, Debug)]
 pub struct ConstExpr {
     /// The expression that is known to be constant (e.g. a `Column`).
     pub expr: Arc<dyn PhysicalExpr>,
-    /// Indicates whether the constant have the same value across all partitions.
+    /// Does the constant have the same value across all partitions?
     pub across_partitions: AcrossPartitions,
 }
 // TODO: The `ConstExpr` definition above can be in an inconsistent state where
@@ -157,13 +152,17 @@ impl From<Arc<dyn PhysicalExpr>> for ConstExpr {
     }
 }
 
-/// An `EquivalenceClass` is a set of [`Arc<dyn PhysicalExpr>`]s that are known
-/// to have the same value for all tuples in a relation. These are generated by
-/// equality predicates (e.g. `a = b`), typically equi-join conditions and
-/// equality conditions in filters.
+/// A set of [`Arc<dyn PhysicalExpr>`]s that are known
+/// to have the same value for all tuples in a relation.
+///
+/// `EquivaluenceClass`es are generated by equality predicates such as `a = b`,
+/// typically equi-join conditions and conditions in filters. See
+/// [Equivalent expressions] for more detail.
 ///
-/// Two `EquivalenceClass`es are equal if they contains the same expressions in
+/// Two `EquivalenceClass`es are equal if they contain the same expressions in
 /// without any ordering.
+///
+/// [Equivalent expressions]: crate::equivalence::EquivalenceProperties#equivalent-expressions
 #[derive(Clone, Debug, Default, Eq, PartialEq)]
 pub struct EquivalenceClass {
     /// The expressions in this equivalence class. The order doesn't matter for
@@ -300,8 +299,12 @@ type AugmentedMapping<'a> = IndexMap<
     (&'a ProjectionTargets, Option<&'a EquivalenceClass>),
 >;
 
-/// A collection of distinct `EquivalenceClass`es. This object supports fast
-/// lookups of expressions and their equivalence classes.
+/// A set of distinct [`EquivalenceClass`]es.
+///
+/// `EquivalenceGroup` supports fast lookups of expressions and their
+/// equivalence classes. See [`EquivalenceProperties`] for more details.
+///
+/// [`EquivalenceProperties`]: crate::equivalence::EquivalenceProperties
 #[derive(Clone, Debug, Default)]
 pub struct EquivalenceGroup {
     /// A mapping from expressions to their equivalence class key.

datafusion/physical-expr/src/equivalence/grouping.rs

@@ -22,10 +22,38 @@ use datafusion_physical_expr_common::physical_expr::format_physical_expr_list;
 /// order. Contiguity applies to the complete partition stream, not separately
 /// to each record batch.
 ///
+/// For example, consider the following table:
+///
+/// ```text
+/// ┌───┬───┬───┐
+/// │ a │ b │ c │
+/// ├───┼───┼───┤
+/// │ 2 │ 1 │ 5 │
+/// │ 2 │ 1 │ 3 │
+/// │ 1 │ 3 │ 4 │
+/// │ 2 │ 2 │ 1 │
+/// │ 2 │ 2 │ 2 │
+/// └───┴───┴───┘
+/// ```
+///
+/// * Is grouped by `[a, b]`, as rows with the same `(a, b)` are contiguous,
+///   even though the table is not sorted by `a`, `b` nor `(a, b)`.
+/// * Is grouped by `[b]`, since each value of `b` is in one run.
+/// * Is NOT grouped by `[a]`, because `a = 2` appears in two separate runs.
+///
 /// Expression order within an entry is not significant: `[a, b]` and `[b, a]`
 /// describe the same grouping. Entries do not imply properties for subsets;
-/// `[a, b]` alone says nothing about whether all rows with the same `a` are
+/// as the table shows, `[a, b]` does not imply `[a]` as the same `a` may not be
 /// contiguous.
+///
+/// # See also
+/// * [`OrderingEquivalenceClass`] for the stronger property that rows are
+///   sorted.
+/// * [`EquivalenceProperties`] for how groupings relate to the other
+///   properties of a plan node.
+///
+/// [`EquivalenceProperties`]: crate::equivalence::EquivalenceProperties
+/// [`OrderingEquivalenceClass`]: crate::equivalence::OrderingEquivalenceClass
 #[derive(Clone, Debug, Default)]
 pub struct GroupingEquivalenceClass {
     groupings: Vec<Vec<Arc<dyn PhysicalExpr>>>,

datafusion/physical-expr/src/equivalence/mod.rs

@@ -2,6 +2,15 @@
 // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
 // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
 
+//! Equivalence properties: the orderings, equality, constants and
+//! groupings known to hold for the output of a plan node, which the planner
+//! uses to avoid sorts and repartitions.
+//!
+//! # See Also
+//!
+//! * [`EquivalenceProperties`], which explains the concepts and how the
+//! pieces below fit together.
+
 use std::borrow::Borrow;
 use std::sync::Arc;
 

datafusion/physical-expr/src/equivalence/ordering.rs

@@ -15,8 +15,9 @@ use arrow::datatypes::SchemaRef;
 use datafusion_common::Result;
 use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
 
-/// An `OrderingEquivalenceClass` keeps track of distinct alternative orderings
-/// than can describe a table. For example, consider the following table:
+/// Tracks distinct alternative orderings than describe a table.
+///
+/// For example, consider the following table:
 ///
 /// ```text
 /// ┌───┬───┬───┬───┐
@@ -30,16 +31,25 @@ use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
 /// ```
 ///
 /// Here, both `[a ASC, b ASC]` and `[c DESC, d ASC]` describe the table
-/// ordering. In this case, we say that these orderings are equivalent.
+/// ordering, so these orderings are equivalent. See [Equivalent sort
+/// expressions] for more details.
+///
+/// An `OrderingEquivalenceClass` is a set of such equivalent orderings.
+///
+/// [`GroupingEquivalenceClass`] is used when groups of equal values are
+/// contiguous, but not in order.
 ///
-/// An `OrderingEquivalenceClass` is a set of such equivalent orderings, which
-/// is represented by a vector of `LexOrdering`s. The set does not store any
-/// redundant information by enforcing the invariant that no suffix of an
-/// ordering in the equivalence class is a prefix of another ordering in the
-/// equivalence class. The set can be empty, which means that there are no
-/// orderings that describe the table.
+/// [Equivalent sort expressions]: crate::equivalence::EquivalenceProperties#equivalent-sort-expressions
+/// [`GroupingEquivalenceClass`]: crate::equivalence::GroupingEquivalenceClass
 #[derive(Clone, Debug, Default, Eq, PartialEq)]
 pub struct OrderingEquivalenceClass {
+    /// Set of equivalent orderings
+    ///
+    /// Invariant: no suffix of an ordering in the equivalence class is a prefix
+    /// of another ordering in the equivalence class. This avoids redundant
+    /// information.
+    ///
+    /// If empty, means the table is unordered
     orderings: Vec<LexOrdering>,
 }
 
@@ -216,18 +226,11 @@ impl OrderingEquivalenceClass {
     /// `ASC NULLS LAST` and `DESC NULLS FIRST`. Rows sorted both ways all have
     /// the same value (or are all null).
     ///
-    /// This is how a value that is constant only within groups of rows is
-    /// recorded, since a `ConstExpr` is constant across all rows. For example,
-    /// `SUM(x) OVER (PARTITION BY k)` has one value per `k`, so the window adds
-    /// `[k, w]` once for each of the four directions of `w`.
-    ///
-    /// ### Future Improvements
+    /// This is how a whole-partition window aggregate records that its value
+    /// is constant within each window partition. See
+    /// [Per-partition constants from window functions].
     ///
-    /// This function may become unnecessary if any of the following improvements
-    /// are implemented:
-    /// 1. `SortOptions` supports encoding constantness information.
-    /// 2. `EquivalenceProperties` gains `FunctionalDependency` awareness, eliminating
-    ///    the need for `Constant` and `Constraints`.
+    /// [Per-partition constants from window functions]: crate::equivalence::EquivalenceProperties#per-partition-constants-from-window-functions
     pub fn is_expr_partial_const(&self, expr: &Arc<dyn PhysicalExpr>) -> bool {
         is_partial_const(self.iter().map(|ordering| ordering.first()), expr)
     }