From 03f34aca5becbb7f831e0118de54b87e20240f7c Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sun, 8 Nov 2026 07:26:54 -0500 Subject: [PATCH 01/16] docs: expand SortKeyWalk rustdoc Co-Authored-By: Claude Fable 5.1 --- .../src/equivalence/properties/walk.rs | 71 ++++++++++++++++--- 1 file changed, 60 insertions(+), 11 deletions(-) diff --git a/datafusion/physical-expr/src/equivalence/properties/walk.rs b/datafusion/physical-expr/src/equivalence/properties/walk.rs index e0487cdd23dfa..dc4418a5ebcca 100644 --- a/datafusion/physical-expr/src/equivalence/properties/walk.rs +++ b/datafusion/physical-expr/src/equivalence/properties/walk.rs @@ -22,18 +22,67 @@ use itertools::Itertools; /// The state of one walk along a sequence of sort keys, such as the keys of a /// required ordering. /// -/// A sort key only matters among the rows that tie on the keys before it. So -/// once a key is satisfied, the walk *pins* it: for the rest of the walk, the -/// key has one value, like a constant. An expression is *fixed* if it has one -/// value among those rows: it is a literal, a constant, pinned, a leaf that -/// reports itself as a singleton (such as an uncorrelated scalar subquery), or -/// not volatile and computed only from fixed expressions. The *head* of an -/// ordering is its first expression that is not fixed; an ordering whose -/// expressions are all fixed has no head. With orderings `[a, b]` and -/// `[a, c]`, the requirement `[a, c]` is satisfied, because pinning `a` makes -/// `c` the head of the second ordering. +/// [`EquivalenceProperties`] uses a `SortKeyWalk` to decide whether the +/// orderings it already knows satisfy a requirement, one key at a time. The +/// walk reads the properties but never changes them: everything it learns +/// along the way lives in the walk itself and is dropped with it. /// -/// Pins live in the walk; the equivalence properties it reads do not change. +/// # Why a walk +/// +/// A sort key only matters among the rows that tie on the keys before it. For +/// the requirement `[a, b]`, `a` must be sorted over all rows, but `b` only +/// has to be sorted within each group of rows with the same `a`. So once a +/// key is satisfied, the walk *pins* it: for the rest of the walk, the key has +/// one value, like a constant, and every known ordering can skip it. +/// +/// # Terms +/// +/// - An expression is *pinned* if the walk has pinned it. Pinning a member of +/// an equivalence class pins all of its members. +/// - An expression is *fixed* if it has one value among the rows that tie on +/// the pinned expressions: it is a literal, a constant, pinned, a leaf that +/// reports itself as a singleton (such as an uncorrelated scalar subquery), +/// or not volatile and computed only from fixed expressions. See +/// [`Self::is_fixed`]. +/// - The *head* of an ordering is its first expression that is not fixed. An +/// ordering whose expressions are all fixed has no head. See +/// [`Self::heads`]. +/// - An ordering is *implied* if it follows from the heads through a known +/// equality, as [`EquivalenceProperties::implied_ordering`] finds. The walk +/// collects implied orderings as it pins, and treats them like the +/// orderings of the properties from then on. +/// +/// A key is satisfied if it is fixed, if it is the head of some ordering in +/// the required direction, or if its order follows from the heads (for +/// example, `CAST(a AS BIGINT)` when `a` is a head). See +/// [`Self::sorted_prefix_len`]. +/// +/// # Example +/// +/// With orderings `[a, b]` and `[a, c]`, the requirement `[a, c]` is +/// satisfied: +/// +/// 1. Nothing is pinned, so `a` heads both orderings. `a` is satisfied and +/// pinned. +/// 2. `a` is now fixed, so the heads move to `b` and `c`. `c` heads the second +/// ordering, so it is satisfied. +/// +/// With orderings `[a, b]` and `[b, c]`, the requirement `[a, b, c]` is +/// satisfied too: pinning `a` moves the head of the first ordering to `b`, +/// which now heads both orderings. Pinning `b` uses up the first ordering and +/// moves the head of the second to `c`. +/// +/// # Where it is used +/// +/// - [`EquivalenceProperties::ordering_satisfy_requirement`] and +/// [`EquivalenceProperties::common_sort_prefix_length`] walk the keys of a +/// requirement with [`Self::sorted_prefix_len`]. +/// - [`EquivalenceProperties::find_longest_permutation`] pins each expression +/// it finds with [`Self::pin`], so that the next expression is checked among +/// the rows that tie on the ones found so far. +/// - [`EquivalenceProperties::get_expr_properties`] reads the properties of +/// an expression through a walk with nothing pinned, via +/// [`Self::expr_properties`]. pub(super) struct SortKeyWalk<'a> { props: &'a EquivalenceProperties, /// Pinned expressions. Pinning a member of an equivalence class pins all From 40ac3c9b3bb7ceb6f6738f072edebd9307b6a6fc Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sun, 8 Nov 2026 07:39:05 -0500 Subject: [PATCH 02/16] docs: consolidate equivalence concepts on EquivalenceProperties Co-Authored-By: Claude Fable 5.1 --- .../physical-expr/src/equivalence/class.rs | 18 ++- .../physical-expr/src/equivalence/grouping.rs | 3 + .../physical-expr/src/equivalence/mod.rs | 7 ++ .../physical-expr/src/equivalence/ordering.rs | 18 +-- .../src/equivalence/properties/mod.rs | 110 +++++++++++++++++- .../src/equivalence/properties/walk.rs | 10 +- datafusion/physical-plan/src/windows/mod.rs | 10 ++ 7 files changed, 151 insertions(+), 25 deletions(-) diff --git a/datafusion/physical-expr/src/equivalence/class.rs b/datafusion/physical-expr/src/equivalence/class.rs index a94a38f47f75d..9f7385c8f49f9 100644 --- a/datafusion/physical-expr/src/equivalence/class.rs +++ b/datafusion/physical-expr/src/equivalence/class.rs @@ -30,6 +30,9 @@ use indexmap::{IndexMap, IndexSet}; /// different partitions. /// - `Uniform(Option)`: The constant expression has the same value /// across all partitions, or is `None` if the value is unknown. +/// +/// See [Constants](crate::equivalence::EquivalenceProperties#constants) for +/// why the difference matters. #[derive(Clone, Debug, Default, Eq, PartialEq)] pub enum AcrossPartitions { #[default] @@ -58,14 +61,14 @@ impl Display for AcrossPartitions { /// The `ConstExpr` struct encapsulates an expression that is constant during /// the execution of a query. For example if a filter like `A = 5` appears /// earlier in the plan, `A` would become a constant in subsequent operations. +/// See [Constants](crate::equivalence::EquivalenceProperties#constants). /// /// # Fields /// /// - `expr`: Constant expression for a node in the physical plan. -/// - `across_partitions`: A boolean flag indicating whether the constant -/// expression is the same across partitions. If set to `true`, the constant -/// expression has same value for all partitions. If set to `false`, the -/// constant expression may have different values for different partitions. +/// - `across_partitions`: An [`AcrossPartitions`] saying whether the constant +/// expression has the same value in every partition (`Uniform`) or may have +/// different values in different partitions (`Heterogeneous`). /// /// # Example /// @@ -160,7 +163,8 @@ impl From> for ConstExpr { /// An `EquivalenceClass` is a set of [`Arc`]s that are known /// to have the same value for all tuples in a relation. These are generated by /// equality predicates (e.g. `a = b`), typically equi-join conditions and -/// equality conditions in filters. +/// equality conditions in filters. See +/// [Equivalent expressions](crate::equivalence::EquivalenceProperties#equivalent-expressions). /// /// Two `EquivalenceClass`es are equal if they contains the same expressions in /// without any ordering. @@ -301,7 +305,9 @@ type AugmentedMapping<'a> = IndexMap< >; /// A collection of distinct `EquivalenceClass`es. This object supports fast -/// lookups of expressions and their equivalence classes. +/// lookups of expressions and their equivalence classes. See +/// [`EquivalenceProperties`](crate::equivalence::EquivalenceProperties) for +/// how it is used. #[derive(Clone, Debug, Default)] pub struct EquivalenceGroup { /// A mapping from expressions to their equivalence class key. diff --git a/datafusion/physical-expr/src/equivalence/grouping.rs b/datafusion/physical-expr/src/equivalence/grouping.rs index 2fce6dad8292d..718fd6e7df28a 100644 --- a/datafusion/physical-expr/src/equivalence/grouping.rs +++ b/datafusion/physical-expr/src/equivalence/grouping.rs @@ -26,6 +26,9 @@ use datafusion_physical_expr_common::physical_expr::format_physical_expr_list; /// describe the same grouping. Entries do not imply properties for subsets; /// `[a, b]` alone says nothing about whether all rows with the same `a` are /// contiguous. +/// +/// See [`EquivalenceProperties`](crate::equivalence::EquivalenceProperties) +/// for how groupings relate to the other properties of a plan node. #[derive(Clone, Debug, Default)] pub struct GroupingEquivalenceClass { groupings: Vec>>, diff --git a/datafusion/physical-expr/src/equivalence/mod.rs b/datafusion/physical-expr/src/equivalence/mod.rs index ff57146efcce9..e8a377f654848 100644 --- a/datafusion/physical-expr/src/equivalence/mod.rs +++ b/datafusion/physical-expr/src/equivalence/mod.rs @@ -2,6 +2,13 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +//! Equivalence properties: the orderings, equal expressions, constants and +//! groupings known to hold for the output of a plan node, which the planner +//! uses to avoid sorts and repartitions. +//! +//! Start with the documentation of [`EquivalenceProperties`], which explains +//! the concepts and how the pieces below fit together. + use std::borrow::Borrow; use std::sync::Arc; diff --git a/datafusion/physical-expr/src/equivalence/ordering.rs b/datafusion/physical-expr/src/equivalence/ordering.rs index 7de47767f62fd..180a9bb4b8743 100644 --- a/datafusion/physical-expr/src/equivalence/ordering.rs +++ b/datafusion/physical-expr/src/equivalence/ordering.rs @@ -30,7 +30,8 @@ use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr; /// ``` /// /// Here, both `[a ASC, b ASC]` and `[c DESC, d ASC]` describe the table -/// ordering. In this case, we say that these orderings are equivalent. +/// ordering. In this case, we say that these orderings are equivalent. See +/// [Equivalent sort expressions](crate::equivalence::EquivalenceProperties#equivalent-sort-expressions). /// /// An `OrderingEquivalenceClass` is a set of such equivalent orderings, which /// is represented by a vector of `LexOrdering`s. The set does not store any @@ -216,18 +217,9 @@ impl OrderingEquivalenceClass { /// `ASC NULLS LAST` and `DESC NULLS FIRST`. Rows sorted both ways all have /// the same value (or are all null). /// - /// This is how a value that is constant only within groups of rows is - /// recorded, since a `ConstExpr` is constant across all rows. For example, - /// `SUM(x) OVER (PARTITION BY k)` has one value per `k`, so the window adds - /// `[k, w]` once for each of the four directions of `w`. - /// - /// ### Future Improvements - /// - /// This function may become unnecessary if any of the following improvements - /// are implemented: - /// 1. `SortOptions` supports encoding constantness information. - /// 2. `EquivalenceProperties` gains `FunctionalDependency` awareness, eliminating - /// the need for `Constant` and `Constraints`. + /// This is how a whole-partition window aggregate records that its value + /// is constant within each window partition. See + /// [Per-partition constants from window functions](crate::equivalence::EquivalenceProperties#per-partition-constants-from-window-functions). pub fn is_expr_partial_const(&self, expr: &Arc) -> bool { is_partial_const(self.iter().map(|ordering| ordering.first()), expr) } diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rs index 1206315bbb274..77adc22a66c7c 100644 --- a/datafusion/physical-expr/src/equivalence/properties/mod.rs +++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs @@ -50,7 +50,22 @@ use itertools::{Either, Itertools}; /// /// [Using Ordering for Better Plans]: https://datafusion.apache.org/blog/2025/03/11/ordering-analysis/ /// -/// # Example equivalent sort expressions +/// This page explains the concepts and how the pieces fit together. Each +/// struct documents what it represents. +/// +/// # Overview +/// +/// | Concept | Example | Stored in | +/// |---|---|---| +/// | Equivalent expressions | `a` and `b` after the filter `a = b` | An [`EquivalenceGroup`] of [`EquivalenceClass`]es | +/// | Constants | `a` after the filter `a = 5` | The `constant` of an [`EquivalenceClass`], added as a [`ConstExpr`] | +/// | Orderings | `[a ASC, b DESC]` after a sort | An [`OrderingEquivalenceClass`] | +/// | Groupings | `[a, b]` after a hash aggregate on `a, b` | A [`GroupingEquivalenceClass`] | +/// | Constraints | A primary key or unique constraint | [`Constraints`] | +/// +/// [`EquivalenceClass`]: crate::equivalence::EquivalenceClass +/// +/// # Equivalent sort expressions /// /// Consider table below: /// @@ -71,7 +86,7 @@ use itertools::{Either, Itertools}; /// specifies the output sorted by EITHER `a ASC` or `b DESC`, the sort can be /// avoided. /// -/// # Example equivalent expressions +/// # Equivalent expressions /// /// Similarly, consider the table below: /// @@ -92,6 +107,97 @@ use itertools::{Either, Itertools}; /// `Hash(b)`, then DataFusion avoids repartitioning the data as the existing /// partitioning satisfies the requirement. /// +/// # Constants +/// +/// A constant is an expression with a single value in the output of a plan +/// node. For example, after the filter `a = 5`, `a` is constant. A constant +/// never changes the order of rows, so a sort on `[a, b]` is the same as a +/// sort on `[b]`, and a requirement that mentions `a` can drop it (see +/// [Normal form](#normal-form)). +/// +/// A constant is *uniform* if it has the same value in every output +/// partition of the node, and *heterogeneous* if it has one value within each +/// partition but may differ between partitions (see [`AcrossPartitions`]). A +/// literal and the `a` of the filter `a = 5` are uniform. In the output of a +/// `UNION ALL` whose inputs each have their own constant value of `c`, `c` is +/// heterogeneous: each partition comes from one input, so it has one value, +/// but the inputs differ. A `ConstExpr` created from a non-literal expression +/// is heterogeneous unless stated otherwise, which is the safe assumption. +/// +/// The difference matters once partitions are merged. Within a partition, a +/// heterogeneous constant never changes the order of rows, but an operator +/// that merges partitions, such as a sort preserving merge, sees several +/// values, so the constant still discriminates the order and must be kept in +/// an ordering. Only uniform constants are removed from orderings and +/// requirements. +/// +/// # Per-partition constants from window functions +/// +/// Here *partition* means a window partition: the rows with equal +/// `PARTITION BY` keys, not an execution partition. A whole-partition window +/// aggregate such as `SUM(x) OVER (PARTITION BY k) AS w` has one value per +/// `k`. This is not a constant, since `w` differs across the output, and it +/// is not an ordering on its own. But within each group of rows with the same +/// `k`, `w` has one value, so if the input is sorted by `[k]`, the output is +/// sorted by `[k, w]` in *every* direction of `w`: ascending or descending, +/// nulls first or last. +/// +/// The window operator records this by adding `[k, w]` once for each of the +/// four directions of `w`. The encoding is read back in two places: +/// +/// - [`OrderingEquivalenceClass::is_expr_partial_const`] reports that `w` +/// has a single value when one ordering leads with `w` in some direction +/// and another leads with it in the exact reverse, since rows sorted both +/// ways all have the same value (or are all null). +/// - When a requirement is checked (see [below](#checking-whether-a-requirement-is-satisfied)), +/// pinning `k` makes `w` the leading expression of orderings in opposite +/// directions, so `w` is a singleton among the rows that tie on `k`. +/// +/// This encoding may become unnecessary if `SortOptions` can express that an +/// expression is constant, or if `EquivalenceProperties` gains functional +/// dependency awareness, which would also remove the need for constants and +/// constraints. +/// +/// # Normal form +/// +/// Orderings and requirements are compared in *normal form*: every +/// expression is replaced by the first member of its equivalence class, +/// uniform constants are removed, and so are the expressions that repeat an +/// earlier one. If `a` and `b` are equal and `d` is a uniform constant, the +/// requirement `[b ASC, d DESC, c ASC, a ASC]` normalizes to +/// `[a ASC, c ASC]`. Orderings are kept in normal form in a cache along with +/// a map from each leading expression to the orderings it leads, so that a +/// requirement is checked against orderings in the same form. +/// +/// # Checking whether a requirement is satisfied +/// +/// The requirement `[a, b]` holds if the rows are sorted by `a`, and the rows +/// that tie on `a` are sorted by `b`. So each key only has to be ordered +/// among the rows that tie on the keys before it. [`Self::ordering_satisfy_requirement`] +/// checks the keys in order. Once a key is satisfied, it is *pinned*: it is +/// treated like a constant for the rest of the keys, so every known ordering +/// can skip it. With orderings `[a, b]` and `[a, c]`, the requirement +/// `[a, c]` holds: pinning `a` leaves `c` as the first unpinned expression of +/// the second ordering. Pinning is kept in a `SortKeyWalk` (see +/// `properties/walk.rs`), which defines its terms; the properties being read +/// never change. +/// +/// A key need not appear in an ordering itself. Its order can follow from an +/// expression that does, as [`PhysicalExpr::get_properties`] reports: +/// `CAST(a AS BIGINT)` is sorted when `a` is, and +/// [`Self::get_expr_properties`] exposes this. +/// +/// # Discovering orderings +/// +/// Adding an ordering, an equality or a constant can imply orderings that +/// were not stated. If the rows are sorted by `[c, a, b]` and `c = f(a, b)` +/// for a function `f` that preserves the lexicographical order of its +/// arguments, such as `c = CAST(a AS BIGINT)`, the rows are also sorted by +/// `[a, b]`: rows with different `(a, b)` are in the order of `f(a, b)`, +/// which is the order of `c`, and rows with equal `c` are sorted by `[a, b]` +/// anyway. [`Self::add_ordering`], [`Self::add_equal_conditions`] and +/// [`Self::add_constants`] look for such orderings. +/// /// # Code Example /// ``` /// # use std::sync::Arc; diff --git a/datafusion/physical-expr/src/equivalence/properties/walk.rs b/datafusion/physical-expr/src/equivalence/properties/walk.rs index dc4418a5ebcca..fdf421bf2f38e 100644 --- a/datafusion/physical-expr/src/equivalence/properties/walk.rs +++ b/datafusion/physical-expr/src/equivalence/properties/walk.rs @@ -25,7 +25,8 @@ use itertools::Itertools; /// [`EquivalenceProperties`] uses a `SortKeyWalk` to decide whether the /// orderings it already knows satisfy a requirement, one key at a time. The /// walk reads the properties but never changes them: everything it learns -/// along the way lives in the walk itself and is dropped with it. +/// along the way lives in the walk itself and is dropped with it. For the +/// concepts, see [Checking whether a requirement is satisfied](EquivalenceProperties#checking-whether-a-requirement-is-satisfied). /// /// # Why a walk /// @@ -301,9 +302,10 @@ impl<'a> SortKeyWalk<'a> { } // Now, check what we know about orderings. A whole-partition window // such as `SUM(x) OVER (PARTITION BY k)` adds `[k, w]` in all four - // directions of `w` (see `OrderingEquivalenceClass::is_expr_partial_const`). - // Once `k` is pinned, `w` heads orderings in opposite directions, so it - // has one value among the rows that tie on `k`: + // directions of `w` (see "Per-partition constants from window + // functions" on `EquivalenceProperties`). Once `k` is pinned, `w` + // heads orderings in opposite directions, so it has one value among + // the rows that tie on `k`: let normal_expr = self.props.eq_group.normalize_expr(Arc::clone(&node.expr)); if self.is_fixed(&normal_expr) || is_partial_const(self.heads(), &normal_expr) { node.data.sort_properties = SortProperties::Singleton; diff --git a/datafusion/physical-plan/src/windows/mod.rs b/datafusion/physical-plan/src/windows/mod.rs index 1e5eea0b01285..6f57eb34ce288 100644 --- a/datafusion/physical-plan/src/windows/mod.rs +++ b/datafusion/physical-plan/src/windows/mod.rs @@ -364,6 +364,16 @@ pub(crate) fn get_partition_by_sort_exprs( } } +/// Computes the [`EquivalenceProperties`] of a window operator's output: the +/// input's properties, plus what each window expression adds. +/// +/// A whole-partition aggregate such as `SUM(x) OVER (PARTITION BY k)` is +/// constant across the table when there is no `PARTITION BY`, and otherwise +/// has one value per window partition, which is recorded by adding `[k, w]` +/// in all four sort directions of `w`. See +/// [Per-partition constants from window functions](EquivalenceProperties#per-partition-constants-from-window-functions). +/// A set-monotonic aggregate over an unbounded frame adds an ordering on its +/// output instead. pub(crate) fn window_equivalence_properties( schema: &SchemaRef, input: &Arc, From 4bc3e8fa3f95748cd8d5b297ecfa2feeec171b90 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sun, 8 Nov 2026 08:00:45 -0500 Subject: [PATCH 03/16] docs: use reference-style links in equivalence docs Co-Authored-By: Claude Fable 5.1 --- .../physical-expr/src/equivalence/class.rs | 33 +++++++++++-------- .../physical-expr/src/equivalence/grouping.rs | 6 ++-- .../physical-expr/src/equivalence/ordering.rs | 8 +++-- .../src/equivalence/properties/mod.rs | 6 ++-- .../src/equivalence/properties/walk.rs | 4 ++- datafusion/physical-plan/src/windows/mod.rs | 7 ++-- 6 files changed, 40 insertions(+), 24 deletions(-) diff --git a/datafusion/physical-expr/src/equivalence/class.rs b/datafusion/physical-expr/src/equivalence/class.rs index 9f7385c8f49f9..a34115b9a79a6 100644 --- a/datafusion/physical-expr/src/equivalence/class.rs +++ b/datafusion/physical-expr/src/equivalence/class.rs @@ -24,19 +24,20 @@ use datafusion_physical_expr_common::physical_expr::{ use indexmap::{IndexMap, IndexSet}; -/// Represents whether a constant expression's value is uniform or varies across -/// partitions. Has two variants: -/// - `Heterogeneous`: The constant expression may have different values for -/// different partitions. -/// - `Uniform(Option)`: The constant expression has the same value -/// across all partitions, or is `None` if the value is unknown. +/// Represents whether a constant expression's value is the same or different across +/// partitions. /// -/// See [Constants](crate::equivalence::EquivalenceProperties#constants) for -/// why the difference matters. +/// See [Constants] for why the difference matters. +/// +/// [Constants]: crate::equivalence::EquivalenceProperties#constants #[derive(Clone, Debug, Default, Eq, PartialEq)] pub enum AcrossPartitions { - #[default] + /// The constant expression may have different values for different + /// partitions. [default] Heterogeneous, + /// The constant expression has the same value across all partitions. + /// + /// `Some` if the value is exactly known, `None` otherwise. Uniform(Option), } @@ -61,7 +62,7 @@ impl Display for AcrossPartitions { /// The `ConstExpr` struct encapsulates an expression that is constant during /// the execution of a query. For example if a filter like `A = 5` appears /// earlier in the plan, `A` would become a constant in subsequent operations. -/// See [Constants](crate::equivalence::EquivalenceProperties#constants). +/// See [Constants]. /// /// # Fields /// @@ -79,6 +80,8 @@ impl Display for AcrossPartitions { /// // Create a constant expression from a physical expression: /// let const_expr = ConstExpr::from(col); /// ``` +/// +/// [Constants]: crate::equivalence::EquivalenceProperties#constants #[derive(Clone, Debug)] pub struct ConstExpr { /// The expression that is known to be constant (e.g. a `Column`). @@ -163,11 +166,12 @@ impl From> for ConstExpr { /// An `EquivalenceClass` is a set of [`Arc`]s that are known /// to have the same value for all tuples in a relation. These are generated by /// equality predicates (e.g. `a = b`), typically equi-join conditions and -/// equality conditions in filters. See -/// [Equivalent expressions](crate::equivalence::EquivalenceProperties#equivalent-expressions). +/// equality conditions in filters. See [Equivalent expressions]. /// /// Two `EquivalenceClass`es are equal if they contains the same expressions in /// without any ordering. +/// +/// [Equivalent expressions]: crate::equivalence::EquivalenceProperties#equivalent-expressions #[derive(Clone, Debug, Default, Eq, PartialEq)] pub struct EquivalenceClass { /// The expressions in this equivalence class. The order doesn't matter for @@ -306,8 +310,9 @@ type AugmentedMapping<'a> = IndexMap< /// A collection of distinct `EquivalenceClass`es. This object supports fast /// lookups of expressions and their equivalence classes. See -/// [`EquivalenceProperties`](crate::equivalence::EquivalenceProperties) for -/// how it is used. +/// [`EquivalenceProperties`] for how it is used. +/// +/// [`EquivalenceProperties`]: crate::equivalence::EquivalenceProperties #[derive(Clone, Debug, Default)] pub struct EquivalenceGroup { /// A mapping from expressions to their equivalence class key. diff --git a/datafusion/physical-expr/src/equivalence/grouping.rs b/datafusion/physical-expr/src/equivalence/grouping.rs index 718fd6e7df28a..156ae21d91b0b 100644 --- a/datafusion/physical-expr/src/equivalence/grouping.rs +++ b/datafusion/physical-expr/src/equivalence/grouping.rs @@ -27,8 +27,10 @@ use datafusion_physical_expr_common::physical_expr::format_physical_expr_list; /// `[a, b]` alone says nothing about whether all rows with the same `a` are /// contiguous. /// -/// See [`EquivalenceProperties`](crate::equivalence::EquivalenceProperties) -/// for how groupings relate to the other properties of a plan node. +/// See [`EquivalenceProperties`] for how groupings relate to the other +/// properties of a plan node. +/// +/// [`EquivalenceProperties`]: crate::equivalence::EquivalenceProperties #[derive(Clone, Debug, Default)] pub struct GroupingEquivalenceClass { groupings: Vec>>, diff --git a/datafusion/physical-expr/src/equivalence/ordering.rs b/datafusion/physical-expr/src/equivalence/ordering.rs index 180a9bb4b8743..d06d0afd325d2 100644 --- a/datafusion/physical-expr/src/equivalence/ordering.rs +++ b/datafusion/physical-expr/src/equivalence/ordering.rs @@ -31,7 +31,7 @@ use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr; /// /// Here, both `[a ASC, b ASC]` and `[c DESC, d ASC]` describe the table /// ordering. In this case, we say that these orderings are equivalent. See -/// [Equivalent sort expressions](crate::equivalence::EquivalenceProperties#equivalent-sort-expressions). +/// [Equivalent sort expressions]. /// /// An `OrderingEquivalenceClass` is a set of such equivalent orderings, which /// is represented by a vector of `LexOrdering`s. The set does not store any @@ -39,6 +39,8 @@ use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr; /// ordering in the equivalence class is a prefix of another ordering in the /// equivalence class. The set can be empty, which means that there are no /// orderings that describe the table. +/// +/// [Equivalent sort expressions]: crate::equivalence::EquivalenceProperties#equivalent-sort-expressions #[derive(Clone, Debug, Default, Eq, PartialEq)] pub struct OrderingEquivalenceClass { orderings: Vec, @@ -219,7 +221,9 @@ impl OrderingEquivalenceClass { /// /// This is how a whole-partition window aggregate records that its value /// is constant within each window partition. See - /// [Per-partition constants from window functions](crate::equivalence::EquivalenceProperties#per-partition-constants-from-window-functions). + /// [Per-partition constants from window functions]. + /// + /// [Per-partition constants from window functions]: crate::equivalence::EquivalenceProperties#per-partition-constants-from-window-functions pub fn is_expr_partial_const(&self, expr: &Arc) -> bool { is_partial_const(self.iter().map(|ordering| ordering.first()), expr) } diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rs index 77adc22a66c7c..fc08ba7e0065c 100644 --- a/datafusion/physical-expr/src/equivalence/properties/mod.rs +++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs @@ -64,6 +64,8 @@ use itertools::{Either, Itertools}; /// | Constraints | A primary key or unique constraint | [`Constraints`] | /// /// [`EquivalenceClass`]: crate::equivalence::EquivalenceClass +/// [Normal form]: #normal-form +/// [Checking whether a requirement is satisfied]: #checking-whether-a-requirement-is-satisfied /// /// # Equivalent sort expressions /// @@ -113,7 +115,7 @@ use itertools::{Either, Itertools}; /// node. For example, after the filter `a = 5`, `a` is constant. A constant /// never changes the order of rows, so a sort on `[a, b]` is the same as a /// sort on `[b]`, and a requirement that mentions `a` can drop it (see -/// [Normal form](#normal-form)). +/// [Normal form]). /// /// A constant is *uniform* if it has the same value in every output /// partition of the node, and *heterogeneous* if it has one value within each @@ -149,7 +151,7 @@ use itertools::{Either, Itertools}; /// has a single value when one ordering leads with `w` in some direction /// and another leads with it in the exact reverse, since rows sorted both /// ways all have the same value (or are all null). -/// - When a requirement is checked (see [below](#checking-whether-a-requirement-is-satisfied)), +/// - When a requirement is checked (see [Checking whether a requirement is satisfied]), /// pinning `k` makes `w` the leading expression of orderings in opposite /// directions, so `w` is a singleton among the rows that tie on `k`. /// diff --git a/datafusion/physical-expr/src/equivalence/properties/walk.rs b/datafusion/physical-expr/src/equivalence/properties/walk.rs index fdf421bf2f38e..6ac7a69615538 100644 --- a/datafusion/physical-expr/src/equivalence/properties/walk.rs +++ b/datafusion/physical-expr/src/equivalence/properties/walk.rs @@ -26,7 +26,7 @@ use itertools::Itertools; /// orderings it already knows satisfy a requirement, one key at a time. The /// walk reads the properties but never changes them: everything it learns /// along the way lives in the walk itself and is dropped with it. For the -/// concepts, see [Checking whether a requirement is satisfied](EquivalenceProperties#checking-whether-a-requirement-is-satisfied). +/// concepts, see [Checking whether a requirement is satisfied]. /// /// # Why a walk /// @@ -84,6 +84,8 @@ use itertools::Itertools; /// - [`EquivalenceProperties::get_expr_properties`] reads the properties of /// an expression through a walk with nothing pinned, via /// [`Self::expr_properties`]. +/// +/// [Checking whether a requirement is satisfied]: EquivalenceProperties#checking-whether-a-requirement-is-satisfied pub(super) struct SortKeyWalk<'a> { props: &'a EquivalenceProperties, /// Pinned expressions. Pinning a member of an equivalence class pins all diff --git a/datafusion/physical-plan/src/windows/mod.rs b/datafusion/physical-plan/src/windows/mod.rs index 6f57eb34ce288..06718dc976ca7 100644 --- a/datafusion/physical-plan/src/windows/mod.rs +++ b/datafusion/physical-plan/src/windows/mod.rs @@ -371,9 +371,10 @@ pub(crate) fn get_partition_by_sort_exprs( /// constant across the table when there is no `PARTITION BY`, and otherwise /// has one value per window partition, which is recorded by adding `[k, w]` /// in all four sort directions of `w`. See -/// [Per-partition constants from window functions](EquivalenceProperties#per-partition-constants-from-window-functions). -/// A set-monotonic aggregate over an unbounded frame adds an ordering on its -/// output instead. +/// [Per-partition constants from window functions]. A set-monotonic aggregate +/// over an unbounded frame adds an ordering on its output instead. +/// +/// [Per-partition constants from window functions]: EquivalenceProperties#per-partition-constants-from-window-functions pub(crate) fn window_equivalence_properties( schema: &SchemaRef, input: &Arc, From 5df65789fec33c03f898cd0c2b85ac32c3432000 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sun, 8 Nov 2026 08:06:57 -0500 Subject: [PATCH 04/16] docs: clean up docs --- .../physical-expr/src/equivalence/class.rs | 46 ++++++++----------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/datafusion/physical-expr/src/equivalence/class.rs b/datafusion/physical-expr/src/equivalence/class.rs index a34115b9a79a6..9723feb99a8ae 100644 --- a/datafusion/physical-expr/src/equivalence/class.rs +++ b/datafusion/physical-expr/src/equivalence/class.rs @@ -24,18 +24,17 @@ use datafusion_physical_expr_common::physical_expr::{ use indexmap::{IndexMap, IndexSet}; -/// Represents whether a constant expression's value is the same or different across -/// partitions. +/// How a constant expression value varies across partitions. /// /// See [Constants] for why the difference matters. /// /// [Constants]: crate::equivalence::EquivalenceProperties#constants #[derive(Clone, Debug, Default, Eq, PartialEq)] pub enum AcrossPartitions { - /// The constant expression may have different values for different - /// partitions. [default] + /// The constant may have different values for different partitions. + /// [default] Heterogeneous, - /// The constant expression has the same value across all partitions. + /// The constant has the same value across all partitions. /// /// `Some` if the value is exactly known, `None` otherwise. Uniform(Option), @@ -56,20 +55,10 @@ impl Display for AcrossPartitions { } } -/// A structure representing a expression known to be constant in a physical -/// execution plan. +/// An expression known to be constant during the execution of a query. /// -/// The `ConstExpr` struct encapsulates an expression that is constant during -/// the execution of a query. For example if a filter like `A = 5` appears -/// earlier in the plan, `A` would become a constant in subsequent operations. -/// See [Constants]. -/// -/// # Fields -/// -/// - `expr`: Constant expression for a node in the physical plan. -/// - `across_partitions`: An [`AcrossPartitions`] saying whether the constant -/// expression has the same value in every partition (`Uniform`) or may have -/// different values in different partitions (`Heterogeneous`). +/// For example after a filter like `A = 5` in a the plan, the expression `A` +/// is know to be constant in subsequent operations. See [Constants] for more details. /// /// # Example /// @@ -86,7 +75,7 @@ impl Display for AcrossPartitions { pub struct ConstExpr { /// The expression that is known to be constant (e.g. a `Column`). pub expr: Arc, - /// Indicates whether the constant have the same value across all partitions. + /// Does the constant have the same value across all partitions? pub across_partitions: AcrossPartitions, } // TODO: The `ConstExpr` definition above can be in an inconsistent state where @@ -163,12 +152,14 @@ impl From> for ConstExpr { } } -/// An `EquivalenceClass` is a set of [`Arc`]s that are known -/// to have the same value for all tuples in a relation. These are generated by -/// equality predicates (e.g. `a = b`), typically equi-join conditions and -/// equality conditions in filters. See [Equivalent expressions]. +/// A set of [`Arc`]s that are known +/// to have the same value for all tuples in a relation. /// -/// Two `EquivalenceClass`es are equal if they contains the same expressions in +/// `EquivaluenceClass`es are generated by equality predicates such as `a = b`, +/// typically equi-join conditions and conditions in filters. See +/// [Equivalent expressions] for more detail. +/// +/// Two `EquivalenceClass`es are equal if they contain the same expressions in /// without any ordering. /// /// [Equivalent expressions]: crate::equivalence::EquivalenceProperties#equivalent-expressions @@ -308,9 +299,10 @@ type AugmentedMapping<'a> = IndexMap< (&'a ProjectionTargets, Option<&'a EquivalenceClass>), >; -/// A collection of distinct `EquivalenceClass`es. This object supports fast -/// lookups of expressions and their equivalence classes. See -/// [`EquivalenceProperties`] for how it is used. +/// A set of distinct [`EquivalenceClass`]es. +/// +/// `EquivalenceGroup` supports fast lookups of expressions and their +/// equivalence classes. See [`EquivalenceProperties`] for more details. /// /// [`EquivalenceProperties`]: crate::equivalence::EquivalenceProperties #[derive(Clone, Debug, Default)] From f8ca64ebcf4b89d059b93bacff544eb9c1f0d955 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sun, 8 Nov 2026 08:17:10 -0500 Subject: [PATCH 05/16] docs: improve --- .../physical-expr/src/equivalence/class.rs | 2 +- .../physical-expr/src/equivalence/grouping.rs | 31 ++++++++++++++++--- .../physical-expr/src/equivalence/mod.rs | 8 +++-- .../physical-expr/src/equivalence/ordering.rs | 25 ++++++++++----- 4 files changed, 50 insertions(+), 16 deletions(-) diff --git a/datafusion/physical-expr/src/equivalence/class.rs b/datafusion/physical-expr/src/equivalence/class.rs index 9723feb99a8ae..7a5c9e1f2f05a 100644 --- a/datafusion/physical-expr/src/equivalence/class.rs +++ b/datafusion/physical-expr/src/equivalence/class.rs @@ -32,7 +32,7 @@ use indexmap::{IndexMap, IndexSet}; #[derive(Clone, Debug, Default, Eq, PartialEq)] pub enum AcrossPartitions { /// The constant may have different values for different partitions. - /// [default] + #[default] Heterogeneous, /// The constant has the same value across all partitions. /// diff --git a/datafusion/physical-expr/src/equivalence/grouping.rs b/datafusion/physical-expr/src/equivalence/grouping.rs index 156ae21d91b0b..f5ac6769ccca7 100644 --- a/datafusion/physical-expr/src/equivalence/grouping.rs +++ b/datafusion/physical-expr/src/equivalence/grouping.rs @@ -22,15 +22,38 @@ use datafusion_physical_expr_common::physical_expr::format_physical_expr_list; /// order. Contiguity applies to the complete partition stream, not separately /// to each record batch. /// +/// For example, consider the following table: +/// +/// ```text +/// ┌───┬───┬───┐ +/// │ a │ b │ c │ +/// ├───┼───┼───┤ +/// │ 2 │ 1 │ 5 │ +/// │ 2 │ 1 │ 3 │ +/// │ 1 │ 3 │ 4 │ +/// │ 2 │ 2 │ 1 │ +/// │ 2 │ 2 │ 2 │ +/// └───┴───┴───┘ +/// ``` +/// +/// Rows with the same `(a, b)` are contiguous, so the grouping `[a, b]` +/// describes the table, even though the table is not sorted by `a` or by +/// `b`: the runs appear in the order `(2, 1)`, `(1, 3)`, `(2, 2)`. The +/// grouping `[b]` describes it too, since each value of `b` is in one run. +/// The grouping `[a]` does not, because `a = 2` appears in two separate runs. +/// /// Expression order within an entry is not significant: `[a, b]` and `[b, a]` /// describe the same grouping. Entries do not imply properties for subsets; -/// `[a, b]` alone says nothing about whether all rows with the same `a` are -/// contiguous. +/// as the table shows, `[a, b]` alone says nothing about whether all rows +/// with the same `a` are contiguous. /// -/// See [`EquivalenceProperties`] for how groupings relate to the other -/// properties of a plan node. +/// # See also +/// * [`OrderingEquivalenceClass`] for the stronger property that rows are sorted. +/// * [`EquivalenceProperties`] for how groupings relate to the +/// other properties of a plan node. /// /// [`EquivalenceProperties`]: crate::equivalence::EquivalenceProperties +/// [`OrderingEquivalenceClass`]: crate::equivalence::OrderingEquivalenceClass #[derive(Clone, Debug, Default)] pub struct GroupingEquivalenceClass { groupings: Vec>>, diff --git a/datafusion/physical-expr/src/equivalence/mod.rs b/datafusion/physical-expr/src/equivalence/mod.rs index e8a377f654848..3724469083451 100644 --- a/datafusion/physical-expr/src/equivalence/mod.rs +++ b/datafusion/physical-expr/src/equivalence/mod.rs @@ -2,12 +2,14 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -//! Equivalence properties: the orderings, equal expressions, constants and +//! Equivalence properties: the orderings, equality, constants and //! groupings known to hold for the output of a plan node, which the planner //! uses to avoid sorts and repartitions. //! -//! Start with the documentation of [`EquivalenceProperties`], which explains -//! the concepts and how the pieces below fit together. +//! # See Also +//! +//! * [`EquivalenceProperties`], which explains the concepts and how the +//! pieces below fit together. use std::borrow::Borrow; use std::sync::Arc; diff --git a/datafusion/physical-expr/src/equivalence/ordering.rs b/datafusion/physical-expr/src/equivalence/ordering.rs index d06d0afd325d2..747d90014f1f1 100644 --- a/datafusion/physical-expr/src/equivalence/ordering.rs +++ b/datafusion/physical-expr/src/equivalence/ordering.rs @@ -16,7 +16,9 @@ use datafusion_common::Result; use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr; /// An `OrderingEquivalenceClass` keeps track of distinct alternative orderings -/// than can describe a table. For example, consider the following table: +/// than can describe a table. +/// +/// For example, consider the following table: /// /// ```text /// ┌───┬───┬───┬───┐ @@ -30,19 +32,26 @@ use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr; /// ``` /// /// Here, both `[a ASC, b ASC]` and `[c DESC, d ASC]` describe the table -/// ordering. In this case, we say that these orderings are equivalent. See -/// [Equivalent sort expressions]. +/// ordering. We say that these orderings are equivalent. See [Equivalent sort +/// expressions]. /// /// An `OrderingEquivalenceClass` is a set of such equivalent orderings, which -/// is represented by a vector of `LexOrdering`s. The set does not store any -/// redundant information by enforcing the invariant that no suffix of an -/// ordering in the equivalence class is a prefix of another ordering in the -/// equivalence class. The set can be empty, which means that there are no -/// orderings that describe the table. +/// is represented by a vector of `LexOrdering`s. +/// +/// [`GroupingEquivalenceClass`] is used when groups of equal values are +/// contiguous, but not in order. /// /// [Equivalent sort expressions]: crate::equivalence::EquivalenceProperties#equivalent-sort-expressions +/// [`GroupingEquivalenceClass`]: crate::equivalence::GroupingEquivalenceClass #[derive(Clone, Debug, Default, Eq, PartialEq)] pub struct OrderingEquivalenceClass { + /// Set of equivalent orderings + /// + /// invariant: no suffix of an ordering in the equivalence class is a prefix + /// of another ordering in the equivalence class. This avoids redundant + /// information. + /// + /// If empty, means the table is unordered orderings: Vec, } From 7646c08c8e579e015e1121503787aa7a4268494a Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sun, 8 Nov 2026 08:23:42 -0500 Subject: [PATCH 06/16] tweaks --- .../physical-expr/src/equivalence/grouping.rs | 20 +++++++++---------- .../physical-expr/src/equivalence/ordering.rs | 12 +++++------ 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/datafusion/physical-expr/src/equivalence/grouping.rs b/datafusion/physical-expr/src/equivalence/grouping.rs index f5ac6769ccca7..ab84fb1f49836 100644 --- a/datafusion/physical-expr/src/equivalence/grouping.rs +++ b/datafusion/physical-expr/src/equivalence/grouping.rs @@ -36,21 +36,21 @@ use datafusion_physical_expr_common::physical_expr::format_physical_expr_list; /// └───┴───┴───┘ /// ``` /// -/// Rows with the same `(a, b)` are contiguous, so the grouping `[a, b]` -/// describes the table, even though the table is not sorted by `a` or by -/// `b`: the runs appear in the order `(2, 1)`, `(1, 3)`, `(2, 2)`. The -/// grouping `[b]` describes it too, since each value of `b` is in one run. -/// The grouping `[a]` does not, because `a = 2` appears in two separate runs. +/// * Is grouped by `[a, b]`, as rows with the same `(a, b)` are contiguous, +/// even though the table is not sorted by `a`, `b` nor `(a, b)`. +/// * Is grouped by `[b]`, since each value of `b` is in one run. +/// * Is NOT grouped by `[a]`, because `a = 2` appears in two separate runs. /// /// Expression order within an entry is not significant: `[a, b]` and `[b, a]` /// describe the same grouping. Entries do not imply properties for subsets; -/// as the table shows, `[a, b]` alone says nothing about whether all rows -/// with the same `a` are contiguous. +/// as the table shows, `[a, b]` does not imply `[a]` as the same `a` may not be +/// contiguous. /// /// # See also -/// * [`OrderingEquivalenceClass`] for the stronger property that rows are sorted. -/// * [`EquivalenceProperties`] for how groupings relate to the -/// other properties of a plan node. +/// * [`OrderingEquivalenceClass`] for the stronger property that rows are +/// sorted. +/// * [`EquivalenceProperties`] for how groupings relate to the other +/// properties of a plan node. /// /// [`EquivalenceProperties`]: crate::equivalence::EquivalenceProperties /// [`OrderingEquivalenceClass`]: crate::equivalence::OrderingEquivalenceClass diff --git a/datafusion/physical-expr/src/equivalence/ordering.rs b/datafusion/physical-expr/src/equivalence/ordering.rs index 747d90014f1f1..05fe1d2c4610e 100644 --- a/datafusion/physical-expr/src/equivalence/ordering.rs +++ b/datafusion/physical-expr/src/equivalence/ordering.rs @@ -15,8 +15,7 @@ use arrow::datatypes::SchemaRef; use datafusion_common::Result; use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr; -/// An `OrderingEquivalenceClass` keeps track of distinct alternative orderings -/// than can describe a table. +/// Tracks distinct alternative orderings than describe a table. /// /// For example, consider the following table: /// @@ -32,11 +31,10 @@ use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr; /// ``` /// /// Here, both `[a ASC, b ASC]` and `[c DESC, d ASC]` describe the table -/// ordering. We say that these orderings are equivalent. See [Equivalent sort -/// expressions]. +/// ordering, so these orderings are equivalent. See [Equivalent sort +/// expressions] for more details. /// -/// An `OrderingEquivalenceClass` is a set of such equivalent orderings, which -/// is represented by a vector of `LexOrdering`s. +/// An `OrderingEquivalenceClass` is a set of such equivalent orderings. /// /// [`GroupingEquivalenceClass`] is used when groups of equal values are /// contiguous, but not in order. @@ -47,7 +45,7 @@ use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr; pub struct OrderingEquivalenceClass { /// Set of equivalent orderings /// - /// invariant: no suffix of an ordering in the equivalence class is a prefix + /// Invariant: no suffix of an ordering in the equivalence class is a prefix /// of another ordering in the equivalence class. This avoids redundant /// information. /// From 964de733f4c7296c0b576fb28bba0a663bb1a0eb Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sun, 8 Nov 2026 08:37:35 -0500 Subject: [PATCH 07/16] move --- .../src/equivalence/properties/mod.rs | 86 +++++++++++++++++++ datafusion/physical-plan/src/windows/mod.rs | 11 +-- 2 files changed, 89 insertions(+), 8 deletions(-) diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rs index fc08ba7e0065c..714d77a597351 100644 --- a/datafusion/physical-expr/src/equivalence/properties/mod.rs +++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs @@ -160,6 +160,92 @@ use itertools::{Either, Itertools}; /// dependency awareness, which would also remove the need for constants and /// constraints. /// +/// ## Example: Window Functions with `PARTITION BY` +/// +/// Consider an input sorted by `k ASC` and the window expression +/// `SUM(x) OVER (PARTITION BY k) AS w`, which produces the following table: +/// +/// ```text +/// ┌───┬───┬────┐ +/// │ k │ x │ w │ +/// ├───┼───┼────┤ +/// │ 1 │ 3 │ 5 │ +/// │ 1 │ 2 │ 5 │ +/// │ 2 │ 4 │ 10 │ +/// │ 2 │ 6 │ 10 │ +/// │ 3 │ 1 │ 1 │ +/// └───┴───┴────┘ +/// ``` +/// +/// Observations: +/// - `w` is not a constant because it differs between partitions. +/// - The output is sorted by `k` (the partitioning), because DataFusion's +/// window implementation keeps the input order. +/// - Within each run of equal `k`, `w` has one value. +/// +/// Given those observations, the output sorting is described by all of: +/// +/// - `[k ASC, w ASC NULLS FIRST]` +/// - `[k ASC, w ASC NULLS LAST]` +/// - `[k ASC, w DESC NULLS FIRST]` +/// - `[k ASC, w DESC NULLS LAST]` +/// +/// ## Example: Window Functions without `PARTITION BY` +/// +/// Consider the same input and the window expression `SUM(x) OVER () AS w`, +/// which produces the following table: +/// +/// ```text +/// ┌───┬───┬────┐ +/// │ k │ x │ w │ +/// ├───┼───┼────┤ +/// │ 1 │ 3 │ 16 │ +/// │ 1 │ 2 │ 16 │ +/// │ 2 │ 4 │ 16 │ +/// │ 2 │ 6 │ 16 │ +/// │ 3 │ 1 │ 16 │ +/// └───┴───┴────┘ +/// ``` +/// +/// Observations: +/// - The frame of every row is the whole table, so `w` has one value. +/// - The output is still sorted by `k`, as the input was. +/// +/// Given those observations, `w` is added as a constant (a [`ConstExpr`]) +/// rather than as orderings. +/// +/// ## Example: Set-Monotonic Window Functions +/// +/// Consider an input sorted by `ts ASC` and the window expression +/// `MAX(x) OVER (ORDER BY ts) AS m`, which produces the following table: +/// +/// ```text +/// ┌────┬───┬───┐ +/// │ ts │ x │ m │ +/// ├────┼───┼───┤ +/// │ 1 │ 3 │ 3 │ +/// │ 2 │ 1 │ 3 │ +/// │ 3 │ 5 │ 5 │ +/// │ 4 │ 2 │ 5 │ +/// └────┴───┴───┘ +/// ``` +/// +/// Observations: +/// - The default frame with an `ORDER BY` is `UNBOUNDED PRECEDING` to +/// `CURRENT ROW`, so the frame of each row contains the frames of all +/// earlier rows. +/// - `MAX` is set-monotonically increasing: adding rows to its input can only +/// raise its value or leave it unchanged (see [`SetMonotonicity`]). +/// +/// Given those observations, `m` never decreases as `ts` increases, so the +/// output sorting is described by `[m ASC NULLS LAST]`, and with a +/// `PARTITION BY k` it would be `[k ASC, m ASC NULLS LAST]`. A decreasing +/// aggregate such as `MIN` yields `DESC NULLS LAST`, and a frame that shrinks +/// instead, such as `CURRENT ROW` to `UNBOUNDED FOLLOWING`, reverses the +/// direction. +/// +/// [`SetMonotonicity`]: datafusion_expr::SetMonotonicity +/// /// # Normal form /// /// Orderings and requirements are compared in *normal form*: every diff --git a/datafusion/physical-plan/src/windows/mod.rs b/datafusion/physical-plan/src/windows/mod.rs index 06718dc976ca7..b6ff93b2682d7 100644 --- a/datafusion/physical-plan/src/windows/mod.rs +++ b/datafusion/physical-plan/src/windows/mod.rs @@ -364,15 +364,10 @@ pub(crate) fn get_partition_by_sort_exprs( } } -/// Computes the [`EquivalenceProperties`] of a window operator's output: the -/// input's properties, plus what each window expression adds. +/// Computes the [`EquivalenceProperties`] of a window operator's output. /// -/// A whole-partition aggregate such as `SUM(x) OVER (PARTITION BY k)` is -/// constant across the table when there is no `PARTITION BY`, and otherwise -/// has one value per window partition, which is recorded by adding `[k, w]` -/// in all four sort directions of `w`. See -/// [Per-partition constants from window functions]. A set-monotonic aggregate -/// over an unbounded frame adds an ordering on its output instead. +/// See [Per-partition constants from window functions] for what this computes +/// along with examples. /// /// [Per-partition constants from window functions]: EquivalenceProperties#per-partition-constants-from-window-functions pub(crate) fn window_equivalence_properties( From ee71843de9d26ce10e6aa27d574fa4bfddc9ea8b Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sun, 8 Nov 2026 08:44:09 -0500 Subject: [PATCH 08/16] more examples --- .../src/equivalence/properties/mod.rs | 96 +++++++++++++++---- 1 file changed, 75 insertions(+), 21 deletions(-) diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rs index 714d77a597351..08a7836a6e1f2 100644 --- a/datafusion/physical-expr/src/equivalence/properties/mod.rs +++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs @@ -38,10 +38,12 @@ use datafusion_expr::sort_properties::{ExprProperties, SortProperties}; use indexmap::IndexSet; use itertools::{Either, Itertools}; -/// `EquivalenceProperties` stores information about the output of a plan node -/// that can be used to optimize the plan. Currently, it keeps track of: +/// `EquivalenceProperties` stores information about how values are distributed +/// across the output of a plan node that can be used to optimize the plan. +/// +/// Currently, it tracks: /// - Sort expressions (orderings), -/// - Complete expression tuples whose values are contiguous, +/// - When all rows of equal values are contiguous for certain tuples (groupings) /// - Equivalent expressions; i.e. expressions known to have the same value. /// - Constants expressions; i.e. expressions known to contain a single constant /// value. @@ -50,18 +52,17 @@ use itertools::{Either, Itertools}; /// /// [Using Ordering for Better Plans]: https://datafusion.apache.org/blog/2025/03/11/ordering-analysis/ /// -/// This page explains the concepts and how the pieces fit together. Each -/// struct documents what it represents. +/// This document explains the concepts and how the pieces fit together. /// /// # Overview /// -/// | Concept | Example | Stored in | -/// |---|---|---| -/// | Equivalent expressions | `a` and `b` after the filter `a = b` | An [`EquivalenceGroup`] of [`EquivalenceClass`]es | -/// | Constants | `a` after the filter `a = 5` | The `constant` of an [`EquivalenceClass`], added as a [`ConstExpr`] | -/// | Orderings | `[a ASC, b DESC]` after a sort | An [`OrderingEquivalenceClass`] | -/// | Groupings | `[a, b]` after a hash aggregate on `a, b` | A [`GroupingEquivalenceClass`] | -/// | Constraints | A primary key or unique constraint | [`Constraints`] | +/// | Concept | Example | Stored in | +/// |------------------------|-------------------------------------------|---------------------------------------------------------------------| +/// | Equivalent expressions | `a` and `b` after the filter `a = b` | An [`EquivalenceGroup`] of [`EquivalenceClass`]es | +/// | Constants | `a` after the filter `a = 5` | A [`ConstExpr`] of [`EquivalenceClass`]es | +/// | Orderings | `[a ASC, b DESC]` after a sort | An [`OrderingEquivalenceClass`] | +/// | Groupings | `[a, b]` after a hash aggregate on `a, b` | A [`GroupingEquivalenceClass`] | +/// | Constraints | A primary key or unique constraint | [`Constraints`] | /// /// [`EquivalenceClass`]: crate::equivalence::EquivalenceClass /// [Normal form]: #normal-form @@ -83,10 +84,9 @@ use itertools::{Either, Itertools}; /// ``` /// /// In this case, both `a ASC` and `b DESC` can describe the table ordering. -/// `EquivalenceProperties` tracks these different valid sort expressions and -/// treat `a ASC` and `b DESC` on an equal footing. For example, if the query -/// specifies the output sorted by EITHER `a ASC` or `b DESC`, the sort can be -/// avoided. +/// `EquivalenceProperties` tracks these different, valid sort expressions. For +/// example, if the query specifies the output should be sorted by EITHER `a +/// ASC` or `b DESC`, the sort can be avoided. /// /// # Equivalent expressions /// @@ -103,11 +103,11 @@ use itertools::{Either, Itertools}; /// └---┴---┘ /// ``` /// -/// In this case, columns `a` and `b` always have the same value. With this -/// information, Datafusion can optimize various operations. For example, if -/// the partition requirement is `Hash(a)` and output partitioning is -/// `Hash(b)`, then DataFusion avoids repartitioning the data as the existing -/// partitioning satisfies the requirement. +/// In this case, columns `a` and `b` always have the same value. DataFusion +/// uses this information to optimize various operations. For example, if the +/// partition requirement is `Hash(a)` and output partitioning is `Hash(b)`, +/// then DataFusion avoids repartitioning the data as the existing partitioning +/// satisfies the requirement. /// /// # Constants /// @@ -133,6 +133,60 @@ use itertools::{Either, Itertools}; /// an ordering. Only uniform constants are removed from orderings and /// requirements. /// +/// ## Example: Uniform Constant from a Filter +/// +/// Consider an input sorted by `b ASC` and the filter `a = 5`, which produces +/// the following table: +/// +/// ```text +/// ┌───┬───┐ +/// │ a │ b │ +/// ├───┼───┤ +/// │ 5 │ 1 │ +/// │ 5 │ 2 │ +/// │ 5 │ 4 │ +/// └───┴───┘ +/// ``` +/// +/// Observations: +/// - Every row that passes the filter has `a = 5`, in every partition, so `a` +/// is a uniform constant. +/// - The rows are in the same order whether sorted by `[b ASC]`, +/// `[a ASC, b ASC]` or `[b ASC, a DESC]`. +/// +/// Given those observations, the filter adds `a` as a uniform [`ConstExpr`]. +/// A requirement such as `[a ASC, b ASC]` normalizes to `[b ASC]`, which the +/// input ordering satisfies, so no sort is needed. +/// +/// ## Example: Heterogeneous Constant from `UNION ALL` +/// +/// Consider `SELECT 1 AS c, x FROM t1 UNION ALL SELECT 2 AS c, x FROM t2`, +/// where each input is sorted by `x ASC` and has one partition. The union has +/// two output partitions, one from each input: +/// +/// ```text +/// Partition 0 Partition 1 +/// ┌───┬───┐ ┌───┬───┐ +/// │ c │ x │ │ c │ x │ +/// ├───┼───┤ ├───┼───┤ +/// │ 1 │ 1 │ │ 2 │ 2 │ +/// │ 1 │ 3 │ │ 2 │ 5 │ +/// └───┴───┘ └───┴───┘ +/// ``` +/// +/// Observations: +/// - Within each partition `c` has one value, but the values differ between +/// partitions, so `c` is a heterogeneous constant. +/// - Each partition is sorted by `[x ASC]`, and therefore also by +/// `[c ASC, x ASC]`. +/// - Merging the partitions by `[x ASC]` alone gives `c` values `1, 2, 1, 2`, +/// which is not sorted by `c`. +/// +/// Given those observations, `c` stays in the ordering `[c ASC, x ASC]` +/// rather than being removed as a constant would be. A sort preserving merge +/// that needs `[c ASC, x ASC]` then compares `c` first and produces +/// `1, 1, 2, 2`, which is correct, whereas merging on `[x ASC]` would not be. +/// /// # Per-partition constants from window functions /// /// Here *partition* means a window partition: the rows with equal From ebb4c4026d057d335556a6a49b42fefe2cdb795a Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sun, 8 Nov 2026 10:52:32 -0500 Subject: [PATCH 09/16] more examples --- .../src/equivalence/properties/mod.rs | 145 +++++++----------- .../src/equivalence/properties/walk.rs | 16 +- datafusion/physical-plan/src/windows/mod.rs | 6 +- 3 files changed, 67 insertions(+), 100 deletions(-) diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rs index 08a7836a6e1f2..ec7e6c60b9ea5 100644 --- a/datafusion/physical-expr/src/equivalence/properties/mod.rs +++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs @@ -84,9 +84,10 @@ use itertools::{Either, Itertools}; /// ``` /// /// In this case, both `a ASC` and `b DESC` can describe the table ordering. -/// `EquivalenceProperties` tracks these different, valid sort expressions. For -/// example, if the query specifies the output should be sorted by EITHER `a -/// ASC` or `b DESC`, the sort can be avoided. +/// `EquivalenceProperties` tracks these different valid sort expressions and +/// treats `a ASC` and `b DESC` on an equal footing. For example, if the query +/// specifies the output sorted by EITHER `a ASC` or `b DESC`, the sort can be +/// avoided. /// /// # Equivalent expressions /// @@ -112,26 +113,10 @@ use itertools::{Either, Itertools}; /// # Constants /// /// A constant is an expression with a single value in the output of a plan -/// node. For example, after the filter `a = 5`, `a` is constant. A constant -/// never changes the order of rows, so a sort on `[a, b]` is the same as a -/// sort on `[b]`, and a requirement that mentions `a` can drop it (see -/// [Normal form]). -/// -/// A constant is *uniform* if it has the same value in every output -/// partition of the node, and *heterogeneous* if it has one value within each -/// partition but may differ between partitions (see [`AcrossPartitions`]). A -/// literal and the `a` of the filter `a = 5` are uniform. In the output of a -/// `UNION ALL` whose inputs each have their own constant value of `c`, `c` is -/// heterogeneous: each partition comes from one input, so it has one value, -/// but the inputs differ. A `ConstExpr` created from a non-literal expression -/// is heterogeneous unless stated otherwise, which is the safe assumption. -/// -/// The difference matters once partitions are merged. Within a partition, a -/// heterogeneous constant never changes the order of rows, but an operator -/// that merges partitions, such as a sort preserving merge, sees several -/// values, so the constant still discriminates the order and must be kept in -/// an ordering. Only uniform constants are removed from orderings and -/// requirements. +/// node. A constant is *uniform* if it has the same value in every output +/// partition, and *heterogeneous* if it has one value within each output +/// partition but may differ between output partitions (see +/// [`AcrossPartitions`]). /// /// ## Example: Uniform Constant from a Filter /// @@ -148,21 +133,14 @@ use itertools::{Either, Itertools}; /// └───┴───┘ /// ``` /// -/// Observations: -/// - Every row that passes the filter has `a = 5`, in every partition, so `a` -/// is a uniform constant. -/// - The rows are in the same order whether sorted by `[b ASC]`, -/// `[a ASC, b ASC]` or `[b ASC, a DESC]`. -/// -/// Given those observations, the filter adds `a` as a uniform [`ConstExpr`]. -/// A requirement such as `[a ASC, b ASC]` normalizes to `[b ASC]`, which the -/// input ordering satisfies, so no sort is needed. +/// The filter adds `a` as a uniform [`ConstExpr`]. If the query requests +/// sorting on `[a ASC, b ASC]` DataFusion knows it can skip sorting. /// /// ## Example: Heterogeneous Constant from `UNION ALL` /// /// Consider `SELECT 1 AS c, x FROM t1 UNION ALL SELECT 2 AS c, x FROM t2`, -/// where each input is sorted by `x ASC` and has one partition. The union has -/// two output partitions, one from each input: +/// where each input is sorted by `x ASC` and has one output partition. The +/// union has two output partitions, one from each input: /// /// ```text /// Partition 0 Partition 1 @@ -175,49 +153,29 @@ use itertools::{Either, Itertools}; /// ``` /// /// Observations: -/// - Within each partition `c` has one value, but the values differ between -/// partitions, so `c` is a heterogeneous constant. -/// - Each partition is sorted by `[x ASC]`, and therefore also by +/// - Within each output partition `c` has one value, but the values differ +/// between output partitions, so `c` is an +/// [`AcrossPartitions::Heterogeneous`] constant. +/// - Each output partition is sorted by `[x ASC]`, and therefore also by /// `[c ASC, x ASC]`. -/// - Merging the partitions by `[x ASC]` alone gives `c` values `1, 2, 1, 2`, -/// which is not sorted by `c`. -/// -/// Given those observations, `c` stays in the ordering `[c ASC, x ASC]` -/// rather than being removed as a constant would be. A sort preserving merge -/// that needs `[c ASC, x ASC]` then compares `c` first and produces -/// `1, 1, 2, 2`, which is correct, whereas merging on `[x ASC]` would not be. -/// -/// # Per-partition constants from window functions -/// -/// Here *partition* means a window partition: the rows with equal -/// `PARTITION BY` keys, not an execution partition. A whole-partition window -/// aggregate such as `SUM(x) OVER (PARTITION BY k) AS w` has one value per -/// `k`. This is not a constant, since `w` differs across the output, and it -/// is not an ordering on its own. But within each group of rows with the same -/// `k`, `w` has one value, so if the input is sorted by `[k]`, the output is -/// sorted by `[k, w]` in *every* direction of `w`: ascending or descending, -/// nulls first or last. -/// -/// The window operator records this by adding `[k, w]` once for each of the -/// four directions of `w`. The encoding is read back in two places: -/// -/// - [`OrderingEquivalenceClass::is_expr_partial_const`] reports that `w` -/// has a single value when one ordering leads with `w` in some direction -/// and another leads with it in the exact reverse, since rows sorted both -/// ways all have the same value (or are all null). -/// - When a requirement is checked (see [Checking whether a requirement is satisfied]), -/// pinning `k` makes `w` the leading expression of orderings in opposite -/// directions, so `w` is a singleton among the rows that tie on `k`. -/// -/// This encoding may become unnecessary if `SortOptions` can express that an -/// expression is constant, or if `EquivalenceProperties` gains functional -/// dependency awareness, which would also remove the need for constants and -/// constraints. +/// - However, when the output partitions are merged by `[x ASC]` alone, the +/// result is NOT sorted by `c`, unlike with an [`AcrossPartitions::Uniform`] +/// constant, which would have the same value in every output partition and +/// thus be sorted by `[c ASC, x ASC]` as well. +/// +/// Given those observations, `c` must stay in the ordering `[c ASC, x ASC]` +/// rather than being removed as a uniform constant would be. +/// +/// # Constants within a window partition +/// +/// NOTE: it is important to distinguish between *window partition*, the rows +/// with equal `PARTITION BY` keys, and *output partition*, how the data is +/// divided up across execution units during execution. /// /// ## Example: Window Functions with `PARTITION BY` /// /// Consider an input sorted by `k ASC` and the window expression -/// `SUM(x) OVER (PARTITION BY k) AS w`, which produces the following table: +/// `SUM(x) OVER (PARTITION BY k) AS w`, which produces the following rows: /// /// ```text /// ┌───┬───┬────┐ @@ -232,11 +190,16 @@ use itertools::{Either, Itertools}; /// ``` /// /// Observations: -/// - `w` is not a constant because it differs between partitions. -/// - The output is sorted by `k` (the partitioning), because DataFusion's -/// window implementation keeps the input order. +/// - `w` is not a constant because it differs between window partitions. +/// - The output is sorted by `k` (the window partition key), because +/// DataFusion's window implementation keeps the input order. /// - Within each run of equal `k`, `w` has one value. /// +/// `w` is not a constant, since `w` differs across the output. However, +/// within each group of rows with the same `k`, `w` has one value, so if the +/// input is sorted by `[k]`, the output is sorted by `[k, w]` in *every* +/// direction of `w`: ascending or descending, nulls first or last. +/// /// Given those observations, the output sorting is described by all of: /// /// - `[k ASC, w ASC NULLS FIRST]` @@ -244,10 +207,11 @@ use itertools::{Either, Itertools}; /// - `[k ASC, w DESC NULLS FIRST]` /// - `[k ASC, w DESC NULLS LAST]` /// +/// /// ## Example: Window Functions without `PARTITION BY` /// /// Consider the same input and the window expression `SUM(x) OVER () AS w`, -/// which produces the following table: +/// which produces the following rows: /// /// ```text /// ┌───┬───┬────┐ @@ -270,8 +234,8 @@ use itertools::{Either, Itertools}; /// /// ## Example: Set-Monotonic Window Functions /// -/// Consider an input sorted by `ts ASC` and the window expression -/// `MAX(x) OVER (ORDER BY ts) AS m`, which produces the following table: +/// Consider an input sorted by `[ts ASC]` and the window function +/// `MAX(x) OVER (ORDER BY ts) AS m`, which produces the following rows: /// /// ```text /// ┌────┬───┬───┐ @@ -285,18 +249,20 @@ use itertools::{Either, Itertools}; /// ``` /// /// Observations: -/// - The default frame with an `ORDER BY` is `UNBOUNDED PRECEDING` to -/// `CURRENT ROW`, so the frame of each row contains the frames of all -/// earlier rows. +/// - When written with explicit defaults, this query is equivalent to +/// `SELECT ts, x, MAX(x) OVER (ORDER BY ts RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS m`. /// - `MAX` is set-monotonically increasing: adding rows to its input can only /// raise its value or leave it unchanged (see [`SetMonotonicity`]). /// /// Given those observations, `m` never decreases as `ts` increases, so the -/// output sorting is described by `[m ASC NULLS LAST]`, and with a -/// `PARTITION BY k` it would be `[k ASC, m ASC NULLS LAST]`. A decreasing -/// aggregate such as `MIN` yields `DESC NULLS LAST`, and a frame that shrinks -/// instead, such as `CURRENT ROW` to `UNBOUNDED FOLLOWING`, reverses the -/// direction. +/// output sorting is described by `[ts ASC]` AND `[m ASC NULLS LAST]`, +/// +/// If the query had a `PARTITION BY k` the ordering would be `[k ASC, m ASC NULLS LAST]`. +/// +/// A decreasing aggregate such as `MIN` results in `DESC NULLS LAST` +/// +/// A window frame that shrinks the rows, such as `CURRENT ROW` to `UNBOUNDED +/// FOLLOWING`, reverses the direction. /// /// [`SetMonotonicity`]: datafusion_expr::SetMonotonicity /// @@ -612,8 +578,9 @@ impl EquivalenceProperties { self.geq_cache.clear(); } - /// Removes constant expressions that may change across partitions. - /// This method should be used when merging data from different partitions. + /// Removes constant expressions that may change across output partitions. + /// This method should be used when merging data from different output + /// partitions. pub fn clear_per_partition_constants(&mut self) { if self.eq_group.clear_per_partition_constants() { // Renormalize orderings if the equivalence group changes: @@ -1673,7 +1640,7 @@ impl EquivalenceProperties { /// Returns a `Some` value if the expression is constant according to /// equivalence group, and `None` otherwise. The `Some` variant contains /// an `AcrossPartitions` value indicating whether the expression is - /// constant across partitions, and its actual value (if available). + /// constant across output partitions, and its actual value (if available). pub fn is_expr_constant( &self, expr: &Arc, diff --git a/datafusion/physical-expr/src/equivalence/properties/walk.rs b/datafusion/physical-expr/src/equivalence/properties/walk.rs index 6ac7a69615538..88b8da8c60374 100644 --- a/datafusion/physical-expr/src/equivalence/properties/walk.rs +++ b/datafusion/physical-expr/src/equivalence/properties/walk.rs @@ -302,12 +302,12 @@ impl<'a> SortKeyWalk<'a> { // strict: node.data.strictly_order_preserving = true; } - // Now, check what we know about orderings. A whole-partition window - // such as `SUM(x) OVER (PARTITION BY k)` adds `[k, w]` in all four - // directions of `w` (see "Per-partition constants from window - // functions" on `EquivalenceProperties`). Once `k` is pinned, `w` - // heads orderings in opposite directions, so it has one value among - // the rows that tie on `k`: + // Now, check what we know about orderings. A window aggregate over a + // whole window partition, such as `SUM(x) OVER (PARTITION BY k)`, adds + // `[k, w]` in all four directions of `w` (see "Constants within a + // window partition" on `EquivalenceProperties`). Once `k` is pinned, + // `w` heads orderings in opposite directions, so it has one value + // among the rows that tie on `k`: let normal_expr = self.props.eq_group.normalize_expr(Arc::clone(&node.expr)); if self.is_fixed(&normal_expr) || is_partial_const(self.heads(), &normal_expr) { node.data.sort_properties = SortProperties::Singleton; @@ -480,8 +480,8 @@ mod tests { Ok(()) } - /// A window over a whole partition, such as `SUM(..) OVER (PARTITION BY - /// k)`, adds `[k, x]` in all four directions of `x`. Pinning `k` leaves `x` + /// A window over a whole window partition, such as `SUM(..) OVER + /// (PARTITION BY k)`, adds `[k, x]` in all four directions of `x`. Pinning `k` leaves `x` /// heading orderings in opposite directions, so `x` has one value, and so /// does `CAST(x AS BIGINT)`: `[k, CAST(x AS BIGINT)]` holds in every /// direction. Without the partial constant check, `x` would only be diff --git a/datafusion/physical-plan/src/windows/mod.rs b/datafusion/physical-plan/src/windows/mod.rs index b6ff93b2682d7..dc792984e4f6a 100644 --- a/datafusion/physical-plan/src/windows/mod.rs +++ b/datafusion/physical-plan/src/windows/mod.rs @@ -366,10 +366,10 @@ pub(crate) fn get_partition_by_sort_exprs( /// Computes the [`EquivalenceProperties`] of a window operator's output. /// -/// See [Per-partition constants from window functions] for what this computes -/// along with examples. +/// See [Constants within a window partition] for what this computes along with +/// examples. /// -/// [Per-partition constants from window functions]: EquivalenceProperties#per-partition-constants-from-window-functions +/// [Constants within a window partition]: EquivalenceProperties#constants-within-a-window-partition pub(crate) fn window_equivalence_properties( schema: &SchemaRef, input: &Arc, From 2c00a22677e09b288cdf50bce57c486bfcd20c1a Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sun, 8 Nov 2026 11:01:33 -0500 Subject: [PATCH 10/16] clean --- .../physical-expr/src/equivalence/properties/mod.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rs index ec7e6c60b9ea5..0b6847b5f31fe 100644 --- a/datafusion/physical-expr/src/equivalence/properties/mod.rs +++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs @@ -250,7 +250,8 @@ use itertools::{Either, Itertools}; /// /// Observations: /// - When written with explicit defaults, this query is equivalent to -/// `SELECT ts, x, MAX(x) OVER (ORDER BY ts RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS m`. +/// `SELECT ts, x, MAX(x) OVER (ORDER BY ts RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS m`, +/// so each row's frame contains every earlier row's frame. /// - `MAX` is set-monotonically increasing: adding rows to its input can only /// raise its value or leave it unchanged (see [`SetMonotonicity`]). /// @@ -271,9 +272,12 @@ use itertools::{Either, Itertools}; /// Orderings and requirements are compared in *normal form*: every /// expression is replaced by the first member of its equivalence class, /// uniform constants are removed, and so are the expressions that repeat an -/// earlier one. If `a` and `b` are equal and `d` is a uniform constant, the -/// requirement `[b ASC, d DESC, c ASC, a ASC]` normalizes to -/// `[a ASC, c ASC]`. Orderings are kept in normal form in a cache along with +/// earlier one. +/// +/// For example, `a` and `b` are equal and `d` is a uniform constant, the +/// requirement `[b ASC, d DESC, c ASC, a ASC]` normalizes to `[a ASC, c ASC]`. +/// +/// Orderings are kept in normal form in a cache along with /// a map from each leading expression to the orderings it leads, so that a /// requirement is checked against orderings in the same form. /// From 29638f3f2303bdea7f3b7413e1ff020be69aef04 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sun, 8 Nov 2026 11:19:30 -0500 Subject: [PATCH 11/16] expr properties --- .../physical-expr-common/src/sort_expr.rs | 15 +++ .../src/equivalence/properties/mod.rs | 91 +++++++++++++------ .../physical-plan/src/execution_plan.rs | 5 + 3 files changed, 81 insertions(+), 30 deletions(-) diff --git a/datafusion/physical-expr-common/src/sort_expr.rs b/datafusion/physical-expr-common/src/sort_expr.rs index 8af850e759c3f..7f61536f31379 100644 --- a/datafusion/physical-expr-common/src/sort_expr.rs +++ b/datafusion/physical-expr-common/src/sort_expr.rs @@ -344,6 +344,11 @@ pub fn options_compatible( /// * `ORDER BY A DESC NULLS FIRST` matches /// * `ORDER BY A ASC NULLS FIRST` matches (`ASC` and `NULL` options ignored) /// * `ORDER BY B DESC NULLS FIRST` does not match (different expr) +/// +/// See [Required orderings] for where requirements come from and how they +/// are checked against the orderings an input provides. +/// +/// [Required orderings]: https://docs.rs/datafusion-physical-expr/latest/datafusion_physical_expr/equivalence/struct.EquivalenceProperties.html#required-orderings #[derive(Clone, Debug)] pub struct PhysicalSortRequirement { /// Physical expression representing the column to sort @@ -697,6 +702,11 @@ impl From for Vec { /// expressions after construction like a `LexOrdering` does, because it provides /// no mutation methods. If such methods become necessary, we will need to /// enforce uniqueness like the latter object. +/// +/// See [Required orderings] for where requirements come from and how they +/// are checked against the orderings an input provides. +/// +/// [Required orderings]: https://docs.rs/datafusion-physical-expr/latest/datafusion_physical_expr/equivalence/struct.EquivalenceProperties.html#required-orderings #[derive(Debug, Clone, PartialEq)] pub struct LexRequirement { reqs: Vec, @@ -807,6 +817,11 @@ impl From for LexOrdering { /// 1. It is non-degenerate, meaning it contains at least one ordering. The /// absence of an input ordering requirement is represented by a `None` value /// in `ExecutionPlan` APIs, which return an `Option`. +/// +/// See [Required orderings] for where requirements come from and how they +/// are checked against the orderings an input provides. +/// +/// [Required orderings]: https://docs.rs/datafusion-physical-expr/latest/datafusion_physical_expr/equivalence/struct.EquivalenceProperties.html#required-orderings #[derive(Debug, Clone, PartialEq)] pub enum OrderingRequirements { /// The operator is not able to work without one of these requirements. diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rs index 0b6847b5f31fe..db875e9682f0e 100644 --- a/datafusion/physical-expr/src/equivalence/properties/mod.rs +++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs @@ -66,6 +66,8 @@ use itertools::{Either, Itertools}; /// /// [`EquivalenceClass`]: crate::equivalence::EquivalenceClass /// [Normal form]: #normal-form +/// [Required orderings]: #required-orderings +/// [`ExecutionPlan::required_input_ordering`]: https://docs.rs/datafusion-physical-plan/latest/datafusion_physical_plan/trait.ExecutionPlan.html#method.required_input_ordering /// [Checking whether a requirement is satisfied]: #checking-whether-a-requirement-is-satisfied /// /// # Equivalent sort expressions @@ -267,48 +269,73 @@ use itertools::{Either, Itertools}; /// /// [`SetMonotonicity`]: datafusion_expr::SetMonotonicity /// +/// # Required orderings +/// +/// Some operators only work on sorted input. A sort merge join needs +/// inputs sorted on the join keys, and a streaming window needs its input +/// sorted by its `PARTITION BY` and then its `ORDER BY` expressions. +/// +/// Each operator states this through [`ExecutionPlan::required_input_ordering`]. +/// +/// Orderings describe what the data *is*; requirements describe what an +/// operator *needs*. The planner compares them with +/// [`Self::ordering_satisfy_requirement`] and inserts / removes sorts so they +/// match [Checking whether a requirement is satisfied]). +/// /// # Normal form /// -/// Orderings and requirements are compared in *normal form*: every +/// Orderings and requirements (see [Required orderings]) are compared in +/// *normal form*: every /// expression is replaced by the first member of its equivalence class, /// uniform constants are removed, and so are the expressions that repeat an /// earlier one. /// -/// For example, `a` and `b` are equal and `d` is a uniform constant, the +/// For example, if `a` and `b` are equal and `d` is a uniform constant, the /// requirement `[b ASC, d DESC, c ASC, a ASC]` normalizes to `[a ASC, c ASC]`. /// -/// Orderings are kept in normal form in a cache along with -/// a map from each leading expression to the orderings it leads, so that a -/// requirement is checked against orderings in the same form. -/// /// # Checking whether a requirement is satisfied /// /// The requirement `[a, b]` holds if the rows are sorted by `a`, and the rows -/// that tie on `a` are sorted by `b`. So each key only has to be ordered -/// among the rows that tie on the keys before it. [`Self::ordering_satisfy_requirement`] -/// checks the keys in order. Once a key is satisfied, it is *pinned*: it is -/// treated like a constant for the rest of the keys, so every known ordering -/// can skip it. With orderings `[a, b]` and `[a, c]`, the requirement -/// `[a, c]` holds: pinning `a` leaves `c` as the first unpinned expression of -/// the second ordering. Pinning is kept in a `SortKeyWalk` (see -/// `properties/walk.rs`), which defines its terms; the properties being read -/// never change. -/// -/// A key need not appear in an ordering itself. Its order can follow from an -/// expression that does, as [`PhysicalExpr::get_properties`] reports: -/// `CAST(a AS BIGINT)` is sorted when `a` is, and -/// [`Self::get_expr_properties`] exposes this. +/// that tie on `a` are sorted by `b`. +/// +/// [`Self::ordering_satisfy_requirement`] checks whether a requirement is +/// satisfied by an ordering, and is quite complex. /// /// # Discovering orderings /// -/// Adding an ordering, an equality or a constant can imply orderings that -/// were not stated. If the rows are sorted by `[c, a, b]` and `c = f(a, b)` -/// for a function `f` that preserves the lexicographical order of its -/// arguments, such as `c = CAST(a AS BIGINT)`, the rows are also sorted by -/// `[a, b]`: rows with different `(a, b)` are in the order of `f(a, b)`, -/// which is the order of `c`, and rows with equal `c` are sorted by `[a, b]` -/// anyway. [`Self::add_ordering`], [`Self::add_equal_conditions`] and -/// [`Self::add_constants`] look for such orderings. +/// `EquivalenceProperties` discovers and accounts for additional orderings that +/// may not be obvious from various additional information such as constants and +/// equality. +/// +/// ## Example: Ordering Implied by a `CAST` +/// +/// Consider `SELECT a, b, CAST(a AS BIGINT) AS c FROM t ORDER BY c, b`: +/// +/// ```text +/// ┌───┬───┬───┐ +/// │ a │ b │ c │ +/// ├───┼───┼───┤ +/// │ 1 │ 2 │ 1 │ +/// │ 1 │ 5 │ 1 │ +/// │ 2 │ 1 │ 2 │ +/// │ 3 │ 4 │ 3 │ +/// └───┴───┴───┘ +/// ``` +/// +/// Observations: +/// - Casting an `INT` to a `BIGINT` preserves order and never maps two +/// different values to the same one, so rows with different `a` are in the +/// order of `c`, which is the order of `a`. +/// - Rows with equal `c` have equal `a` +/// +/// Given those observations, the rows are also sorted by `[a ASC, b ASC]`, +/// even though no operator sorted on `a` +/// +/// This property holds in general for a function `f` that preserves the +/// lexicographical order of its arguments (that set [`ExprProperties::preserves_lex_ordering`]). +/// +/// See [`Self::add_ordering`], [`Self::add_equal_conditions`] and +/// [`Self::add_constants`] for more details. /// /// # Code Example /// ``` @@ -325,10 +352,10 @@ use itertools::{Either, Itertools}; /// # let col_a = col("a", &schema).unwrap(); /// # let col_b = col("b", &schema).unwrap(); /// # let col_c = col("c", &schema).unwrap(); -/// // This object represents data that is sorted by a ASC, c DESC -/// // with a single constant value of b /// let mut eq_properties = EquivalenceProperties::new(schema); +/// // Single constant value of b /// eq_properties.add_constants(vec![ConstExpr::from(col_b)]); +/// // ORDER BY a ASC, c DESC /// eq_properties.add_ordering([ /// PhysicalSortExpr::new_default(col_a).asc(), /// PhysicalSortExpr::new_default(col_c).desc(), @@ -973,6 +1000,10 @@ impl EquivalenceProperties { /// tie on `a`. /// 3. The orderings now start at `[b, c]` and `[d]`. /// 4. The function returns `true` as `[d]` satisfies `d`. + /// + /// See [Required orderings] for what a requirement is. + /// + /// [Required orderings]: EquivalenceProperties#required-orderings pub fn ordering_satisfy_requirement( &self, given: impl IntoIterator, diff --git a/datafusion/physical-plan/src/execution_plan.rs b/datafusion/physical-plan/src/execution_plan.rs index b236cc2f7ce00..fce0a8391b173 100644 --- a/datafusion/physical-plan/src/execution_plan.rs +++ b/datafusion/physical-plan/src/execution_plan.rs @@ -205,6 +205,11 @@ pub trait ExecutionPlan: Any + Debug + DisplayAs + Send + Sync { /// NOTE that checking `!is_empty()` does **not** check for a /// required input ordering. Instead, the correct check is that at /// least one entry must be `Some` + /// + /// See [Required orderings] for how the planner checks these requirements + /// against the orderings a child provides. + /// + /// [Required orderings]: EquivalenceProperties#required-orderings fn required_input_ordering(&self) -> Vec> { vec![None; self.children().len()] } From 5817741acc461cadee6f91175f2bc6cef595966a Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sun, 8 Nov 2026 16:29:46 -0500 Subject: [PATCH 12/16] more --- .../src/equivalence/properties/mod.rs | 63 ++++++++++--------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rs index db875e9682f0e..d8efbc5e7b916 100644 --- a/datafusion/physical-expr/src/equivalence/properties/mod.rs +++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs @@ -59,7 +59,7 @@ use itertools::{Either, Itertools}; /// | Concept | Example | Stored in | /// |------------------------|-------------------------------------------|---------------------------------------------------------------------| /// | Equivalent expressions | `a` and `b` after the filter `a = b` | An [`EquivalenceGroup`] of [`EquivalenceClass`]es | -/// | Constants | `a` after the filter `a = 5` | A [`ConstExpr`] of [`EquivalenceClass`]es | +/// | Constants | `a` after the filter `a = 5` | The `constant` of an [`EquivalenceClass`], added as a [`ConstExpr`] | /// | Orderings | `[a ASC, b DESC]` after a sort | An [`OrderingEquivalenceClass`] | /// | Groupings | `[a, b]` after a hash aggregate on `a, b` | A [`GroupingEquivalenceClass`] | /// | Constraints | A primary key or unique constraint | [`Constraints`] | @@ -197,19 +197,14 @@ use itertools::{Either, Itertools}; /// DataFusion's window implementation keeps the input order. /// - Within each run of equal `k`, `w` has one value. /// -/// `w` is not a constant, since `w` differs across the output. However, -/// within each group of rows with the same `k`, `w` has one value, so if the -/// input is sorted by `[k]`, the output is sorted by `[k, w]` in *every* -/// direction of `w`: ascending or descending, nulls first or last. -/// -/// Given those observations, the output sorting is described by all of: +/// Given those observations, the output is sorted by `[k, w]` in *every* +/// direction of `w`: ascending or descending, nulls first or last: /// /// - `[k ASC, w ASC NULLS FIRST]` /// - `[k ASC, w ASC NULLS LAST]` /// - `[k ASC, w DESC NULLS FIRST]` /// - `[k ASC, w DESC NULLS LAST]` /// -/// /// ## Example: Window Functions without `PARTITION BY` /// /// Consider the same input and the window expression `SUM(x) OVER () AS w`, @@ -258,14 +253,16 @@ use itertools::{Either, Itertools}; /// raise its value or leave it unchanged (see [`SetMonotonicity`]). /// /// Given those observations, `m` never decreases as `ts` increases, so the -/// output sorting is described by `[ts ASC]` AND `[m ASC NULLS LAST]`, +/// output sorting is described by both `[ts ASC]` and `[m ASC NULLS LAST]`. /// -/// If the query had a `PARTITION BY k` the ordering would be `[k ASC, m ASC NULLS LAST]`. +/// If the query had a `PARTITION BY k`, with the input sorted by `k` first, +/// the added ordering would be `[k, m ASC NULLS LAST]`. /// -/// A decreasing aggregate such as `MIN` results in `DESC NULLS LAST` +/// A decreasing aggregate such as `MIN` results in `DESC NULLS LAST`. /// -/// A window frame that shrinks the rows, such as `CURRENT ROW` to `UNBOUNDED -/// FOLLOWING`, reverses the direction. +/// A frame that shrinks as rows advance, such as `CURRENT ROW` to +/// `UNBOUNDED FOLLOWING`, reverses the direction: `MAX` then results in +/// `DESC NULLS FIRST`. /// /// [`SetMonotonicity`]: datafusion_expr::SetMonotonicity /// @@ -275,20 +272,21 @@ use itertools::{Either, Itertools}; /// inputs sorted on the join keys, and a streaming window needs its input /// sorted by its `PARTITION BY` and then its `ORDER BY` expressions. /// -/// Each operator states this through [`ExecutionPlan::required_input_ordering`]. +/// Each operator describes its requirements via +/// [`ExecutionPlan::required_input_ordering`]. /// -/// Orderings describe what the data *is*; requirements describe what an -/// operator *needs*. The planner compares them with -/// [`Self::ordering_satisfy_requirement`] and inserts / removes sorts so they -/// match [Checking whether a requirement is satisfied]). +/// **Orderings** describe how the data is actually distributed. +/// **Requirements** describe what an operator *needs*. The planner compares +/// orderings to requirements with [`Self::ordering_satisfy_requirement`] and +/// inserts or removes Sort operations so that they match (see [Checking whether +/// a requirement is satisfied]). /// /// # Normal form /// /// Orderings and requirements (see [Required orderings]) are compared in -/// *normal form*: every -/// expression is replaced by the first member of its equivalence class, -/// uniform constants are removed, and so are the expressions that repeat an -/// earlier one. +/// *normal form*: every expression is replaced by the first member of its +/// equivalence class, uniform constants and expressions that repeat an earlier +/// one are removed. /// /// For example, if `a` and `b` are equal and `d` is a uniform constant, the /// requirement `[b ASC, d DESC, c ASC, a ASC]` normalizes to `[a ASC, c ASC]`. @@ -309,7 +307,9 @@ use itertools::{Either, Itertools}; /// /// ## Example: Ordering Implied by a `CAST` /// -/// Consider `SELECT a, b, CAST(a AS BIGINT) AS c FROM t ORDER BY c, b`: +/// Consider a table `t` sorted by `[c ASC, b ASC]` and the query +/// `SELECT a, b, c FROM t WHERE c = CAST(a AS BIGINT)`, which produces the +/// following rows: /// /// ```text /// ┌───┬───┬───┐ @@ -323,19 +323,24 @@ use itertools::{Either, Itertools}; /// ``` /// /// Observations: +/// - The filter keeps the input order, so the rows are still sorted by +/// `[c ASC, b ASC]` /// - Casting an `INT` to a `BIGINT` preserves order and never maps two /// different values to the same one, so rows with different `a` are in the /// order of `c`, which is the order of `a`. -/// - Rows with equal `c` have equal `a` +/// - Rows with equal `c` have equal `a`, and are also still sorted by `b`. /// /// Given those observations, the rows are also sorted by `[a ASC, b ASC]`, -/// even though no operator sorted on `a` +/// even though no operator sorted on `a`. `EquivalenceProperties` derives +/// this ordering when the equality is added, so a later `ORDER BY a, b` +/// needs no re-sorting. /// -/// This property holds in general for a function `f` that preserves the -/// lexicographical order of its arguments (that set [`ExprProperties::preserves_lex_ordering`]). +/// This holds in general for a function `f` whose output follows the +/// lexicographical order of its arguments, which `f` declares by setting +/// [`ExprProperties::preserves_lex_ordering`]. /// -/// See [`Self::add_ordering`], [`Self::add_equal_conditions`] and -/// [`Self::add_constants`] for more details. +/// [`Self::add_equal_conditions`] and [`Self::add_constants`] look for such +/// orderings whenever they learn something new. /// /// # Code Example /// ``` From 60a244ea8af0791671e6569719bd64e2f2f20d00 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 9 Nov 2026 08:23:53 -0500 Subject: [PATCH 13/16] updates --- .../src/equivalence/properties/walk.rs | 199 +++++++++++++----- 1 file changed, 144 insertions(+), 55 deletions(-) diff --git a/datafusion/physical-expr/src/equivalence/properties/walk.rs b/datafusion/physical-expr/src/equivalence/properties/walk.rs index 88b8da8c60374..9053694468cf2 100644 --- a/datafusion/physical-expr/src/equivalence/properties/walk.rs +++ b/datafusion/physical-expr/src/equivalence/properties/walk.rs @@ -22,11 +22,125 @@ use itertools::Itertools; /// The state of one walk along a sequence of sort keys, such as the keys of a /// required ordering. /// -/// [`EquivalenceProperties`] uses a `SortKeyWalk` to decide whether the -/// orderings it already knows satisfy a requirement, one key at a time. The -/// walk reads the properties but never changes them: everything it learns -/// along the way lives in the walk itself and is dropped with it. For the -/// concepts, see [Checking whether a requirement is satisfied]. +/// [`EquivalenceProperties`] uses a `SortKeyWalk` to decide if a set of +/// orderings satisfy a requirement. See [Checking whether a requirement is +/// satisfied] for more details. +/// +/// # Intuition and Introduction +/// +/// An ordering `[x, y, z]` is a chain of claims about the rows: they are sorted +/// by x; among rows with equal x, sorted by y; among rows with equal x and y, +/// sorted by z. +/// +/// During a walk, the code scans the ordering requirement left to right. When +/// it finds key `k` satisfied, it *pins* `k`, meaning that for the next steps +/// in the walk, `k` is effectively a constant across the remaining ordering +/// requirement. +/// +/// An expression is *fixed* if it has one value for all the rows that tie on the +/// currently pinned expressions. Fixed expressions include literals, constants, +/// pinned expressions, and singleton leaf expressions (such as an +/// uncorrelated scalar subquery). See [`Self::is_fixed`]. +/// +/// +/// Fixed is relative to the walk: the expression has one value among the rows +/// that tie on what the walk has pinned. That is a smaller set of rows than the +/// whole output, so more things qualify. Every constant is fixed, but a pinned +/// key is fixed without being a constant. k takes many values across the +/// output, yet among rows tied on k it has one. +/// +/// +/// For each known ordering, the head is the first expression of that ordering +/// that still carries information given what is pinned. +/// +/// # Example: Walking a Requirement +/// +/// The walk checks the keys of a requirement one at a time. Each known +/// ordering has a *head*, its first expression that the walk has not +/// yet pinned. A key is satisfied when it heads some ordering, and is then +/// *pinned*, which moves the heads past it. The walk stops at the first key +/// that is not satisfied. +/// +/// **Orderings `[a ASC, b ASC, c ASC]` and `[d ASC]`, requirement +/// `[a ASC, b ASC]`**: satisfied, since the requirement is a prefix of the +/// first ordering. The second ordering plays no part: its head stays `d`. +/// +/// ```text +/// Step Key Pinned Head [a, b, c] Head [d] Outcome +/// 1 a {} a d a heads the first: pin a +/// 2 b {a} b d b heads the first: done +/// ``` +/// +/// The remaining cases use the requirement `[a ASC, b ASC, c ASC]` against +/// two different sets of known orderings, and then a shorter requirement. +/// +/// **Orderings `[a ASC, b ASC]` and `[b ASC, c ASC]`**: the requirement is +/// satisfied, even though neither ordering contains all three keys. +/// +/// ```text +/// Step Key Pinned Head [a, b] Head [b, c] Outcome +/// 1 a {} a b a heads the first: pin a +/// 2 b {a} b b b heads both: pin b +/// 3 c {a, b} - c c heads the second: done +/// ``` +/// +/// After step 2, `[a, b]` is used up: both of its expressions are pinned, so +/// it has no head. Among the rows that tie on `a` and `b`, the second +/// ordering says the rows are sorted by `c`, which is what the third key +/// asks. All three keys are satisfied, so no sort is needed. +/// +/// To see why this is right, take a table sorted by both orderings: +/// +/// ```text +/// ┌───┬───┬───┐ +/// │ a │ b │ c │ +/// ├───┼───┼───┤ +/// │ 1 │ 1 │ 5 │ +/// │ 1 │ 2 │ 3 │ +/// │ 1 │ 2 │ 4 │ +/// │ 2 │ 2 │ 6 │ +/// │ 2 │ 3 │ 1 │ +/// └───┴───┴───┘ +/// ``` +/// +/// The `(a, b)` pairs run `(1,1), (1,2), (1,2), (2,2), (2,3)`, so the table +/// is sorted by `[a, b]`. The `b` values run `1, 2, 2, 2, 3` and within +/// `b = 2` the `c` values run `3, 4, 6`, so it is sorted by `[b, c]` too. +/// Now check `[a, b, c]`: +/// +/// 1. `a` is sorted over all rows, from `[a, b]`. +/// 2. Among rows that tie on `a`, `b` is sorted, also from `[a, b]`. +/// 3. Among rows that tie on both `a` and `b`, `c` is sorted: those rows are +/// a subset of the rows that tie on `b`, and `[b, c]` says every run of +/// equal `b` is sorted by `c`. A subset of a sorted run, in the same +/// relative order, is still sorted. Rows 2 and 3 tie on `(1, 2)` and +/// their `c` values are `3, 4`. +/// +/// A head is a promise that the rows tied on everything pinned are sorted by +/// that expression, which is what step 3 relies on. +/// +/// **Orderings `[a ASC, c ASC]` and `[c ASC, b ASC]`**: the requirement is +/// not satisfied. +/// +/// ```text +/// Step Key Pinned Head [a, c] Head [c, b] Outcome +/// 1 a {} a c a heads the first: pin a +/// 2 b {a} c c b heads nothing: stop +/// ``` +/// +/// After pinning `a`, both orderings are headed by `c`. Nothing says the rows +/// that tie on `a` are sorted by `b`, so the walk stops. It reports that only +/// the first key holds, and the planner has to sort. +/// +/// **Orderings `[a ASC, b ASC]` and `[a ASC, c ASC]`, requirement +/// `[a ASC, c ASC]`**: satisfied. Pinning `a` moves the head of *both* +/// orderings, and the second one then supplies `c`. +/// +/// ```text +/// Step Key Pinned Head [a, b] Head [a, c] Outcome +/// 1 a {} a a a heads both: pin a +/// 2 c {a} b c c heads the second: done +/// ``` /// /// # Why a walk /// @@ -38,13 +152,6 @@ use itertools::Itertools; /// /// # Terms /// -/// - An expression is *pinned* if the walk has pinned it. Pinning a member of -/// an equivalence class pins all of its members. -/// - An expression is *fixed* if it has one value among the rows that tie on -/// the pinned expressions: it is a literal, a constant, pinned, a leaf that -/// reports itself as a singleton (such as an uncorrelated scalar subquery), -/// or not volatile and computed only from fixed expressions. See -/// [`Self::is_fixed`]. /// - The *head* of an ordering is its first expression that is not fixed. An /// ordering whose expressions are all fixed has no head. See /// [`Self::heads`]. @@ -53,38 +160,6 @@ use itertools::Itertools; /// collects implied orderings as it pins, and treats them like the /// orderings of the properties from then on. /// -/// A key is satisfied if it is fixed, if it is the head of some ordering in -/// the required direction, or if its order follows from the heads (for -/// example, `CAST(a AS BIGINT)` when `a` is a head). See -/// [`Self::sorted_prefix_len`]. -/// -/// # Example -/// -/// With orderings `[a, b]` and `[a, c]`, the requirement `[a, c]` is -/// satisfied: -/// -/// 1. Nothing is pinned, so `a` heads both orderings. `a` is satisfied and -/// pinned. -/// 2. `a` is now fixed, so the heads move to `b` and `c`. `c` heads the second -/// ordering, so it is satisfied. -/// -/// With orderings `[a, b]` and `[b, c]`, the requirement `[a, b, c]` is -/// satisfied too: pinning `a` moves the head of the first ordering to `b`, -/// which now heads both orderings. Pinning `b` uses up the first ordering and -/// moves the head of the second to `c`. -/// -/// # Where it is used -/// -/// - [`EquivalenceProperties::ordering_satisfy_requirement`] and -/// [`EquivalenceProperties::common_sort_prefix_length`] walk the keys of a -/// requirement with [`Self::sorted_prefix_len`]. -/// - [`EquivalenceProperties::find_longest_permutation`] pins each expression -/// it finds with [`Self::pin`], so that the next expression is checked among -/// the rows that tie on the ones found so far. -/// - [`EquivalenceProperties::get_expr_properties`] reads the properties of -/// an expression through a walk with nothing pinned, via -/// [`Self::expr_properties`]. -/// /// [Checking whether a requirement is satisfied]: EquivalenceProperties#checking-whether-a-requirement-is-satisfied pub(super) struct SortKeyWalk<'a> { props: &'a EquivalenceProperties, @@ -94,9 +169,21 @@ pub(super) struct SortKeyWalk<'a> { /// Orderings that hold among the rows that tie on the pinned expressions, /// found by [`EquivalenceProperties::implied_ordering`] as the walk goes. implied: Vec, - /// For each ordering (the normal orderings of `props`, then `implied`), - /// the index of its head. Empty until the first pin, so that a walk with - /// nothing pinned allocates nothing; until then every head is at index 0. + /// The position of the head of each ordering. + /// + /// Empty until the first pin. + /// + /// `head_idxs[i]` tracks the position in the `LexOrdering` at index `i`, + /// as returned by [`Self::ordering`]. The ordering at index `i` is: + /// * if `i < self.props.normal_cls.len()` then `self.props.normal_cls[i]` + /// * else, `self.implied[i - self.props.normal_cls.len()]` otherwise. See + /// + /// The value is either an index into that ordering's sort expressions, or + /// its length when it has no head. + /// + /// For example, with orderings `[a, b]` and `[b, c]`, head_idxs is `[0, 0]` + /// at the start, `[1, 0]` after pinning `a`, and `[2, 1]` after pinning + /// `b`, when `[a, b]` is used up. head_idxs: Vec, } @@ -184,10 +271,13 @@ impl<'a> SortKeyWalk<'a> { Ok(()) } - /// Pins `expr` and the rest of its equivalence class (but, unlike - /// [`Self::pin_key`], not its children), then moves the head of each - /// ordering past the expressions that are now fixed, and adds the - /// orderings that the heads imply, until no new ordering is found. + /// Pins `expr` . + /// + /// See [`SortKeyWalk`] for an explanation of pinning. + /// + /// + /// This method pins `expr` (and thus by extension all members of its equivalence class), + /// but not its children (see [`Self::pin_key`']. pub(super) fn pin(&mut self, expr: Arc) -> Result<()> { match self.props.eq_group.get_equivalence_class(&expr) { Some(class) => self.pinned.extend(class.iter().cloned()), @@ -317,10 +407,9 @@ impl<'a> SortKeyWalk<'a> { Ok(Transformed::yes(node)) } - /// Whether `expr` is fixed (see [`SortKeyWalk`]). Callers pass `expr` in - /// normal form. Pins are stored as given, so a pinned expression that is - /// not in normal form is found only through its equivalence class, whose - /// members are all pinned. + /// Whether `expr` is fixed (see [`SortKeyWalk`]). + /// + /// `expr` must be in normal form. fn is_fixed(&self, expr: &Arc) -> bool { if expr.is::() || self.pinned.contains(expr) { return true; From 428434fa9a0fd9dd3357e7e42953c61347e96772 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 9 Nov 2026 08:54:05 -0500 Subject: [PATCH 14/16] updates --- .../src/equivalence/properties/walk.rs | 171 ++++++++---------- 1 file changed, 71 insertions(+), 100 deletions(-) diff --git a/datafusion/physical-expr/src/equivalence/properties/walk.rs b/datafusion/physical-expr/src/equivalence/properties/walk.rs index 9053694468cf2..ef89ba833c836 100644 --- a/datafusion/physical-expr/src/equivalence/properties/walk.rs +++ b/datafusion/physical-expr/src/equivalence/properties/walk.rs @@ -26,139 +26,110 @@ use itertools::Itertools; /// orderings satisfy a requirement. See [Checking whether a requirement is /// satisfied] for more details. /// -/// # Intuition and Introduction +/// # Intuition and Introduction, and Definitions /// /// An ordering `[x, y, z]` is a chain of claims about the rows: they are sorted /// by x; among rows with equal x, sorted by y; among rows with equal x and y, /// sorted by z. /// +/// ## Pinning and Fixed Expressions +/// /// During a walk, the code scans the ordering requirement left to right. When /// it finds key `k` satisfied, it *pins* `k`, meaning that for the next steps /// in the walk, `k` is effectively a constant across the remaining ordering /// requirement. /// -/// An expression is *fixed* if it has one value for all the rows that tie on the -/// currently pinned expressions. Fixed expressions include literals, constants, -/// pinned expressions, and singleton leaf expressions (such as an -/// uncorrelated scalar subquery). See [`Self::is_fixed`]. -/// -/// -/// Fixed is relative to the walk: the expression has one value among the rows -/// that tie on what the walk has pinned. That is a smaller set of rows than the -/// whole output, so more things qualify. Every constant is fixed, but a pinned -/// key is fixed without being a constant. k takes many values across the -/// output, yet among rows tied on k it has one. -/// -/// -/// For each known ordering, the head is the first expression of that ordering -/// that still carries information given what is pinned. -/// -/// # Example: Walking a Requirement +/// An expression is *fixed* if it has one value for all rows that tie on the +/// currently pinned expressions. Fixed expressions the pinned expressions +/// themselves, as well as literals, constants, and singleton leaf expressions +/// (such as an uncorrelated scalar subquery). See [`Self::is_fixed`] for more +/// details. +/// +/// ## Implied Orderings /// -/// The walk checks the keys of a requirement one at a time. Each known -/// ordering has a *head*, its first expression that the walk has not -/// yet pinned. A key is satisfied when it heads some ordering, and is then -/// *pinned*, which moves the heads past it. The walk stops at the first key -/// that is not satisfied. -/// -/// **Orderings `[a ASC, b ASC, c ASC]` and `[d ASC]`, requirement -/// `[a ASC, b ASC]`**: satisfied, since the requirement is a prefix of the -/// first ordering. The second ordering plays no part: its head stays `d`. -/// -/// ```text -/// Step Key Pinned Head [a, b, c] Head [d] Outcome -/// 1 a {} a d a heads the first: pin a -/// 2 b {a} b d b heads the first: done -/// ``` +/// - An ordering is *implied* if it follows from the heads through a known +/// equality, as [`EquivalenceProperties::implied_ordering`] finds. The walk +/// collects implied orderings as it pins, and treats them like the +/// orderings of the properties from then on. /// -/// The remaining cases use the requirement `[a ASC, b ASC, c ASC]` against -/// two different sets of known orderings, and then a shorter requirement. +/// ## Heads /// -/// **Orderings `[a ASC, b ASC]` and `[b ASC, c ASC]`**: the requirement is -/// satisfied, even though neither ordering contains all three keys. +/// For each equivalent ordering, the walk tracks its current *head* expression, +/// which is the current claim that is valid given the currently pinned +/// expressions. When an expression is pinned, it advances the head of any +/// existing claim about that key. An ordering with nothing left to promise is +/// used up. /// -/// ```text -/// Step Key Pinned Head [a, b] Head [b, c] Outcome -/// 1 a {} a b a heads the first: pin a -/// 2 b {a} b b b heads both: pin b -/// 3 c {a, b} - c c heads the second: done -/// ``` +/// For example, if we have data ordered by `[a, b]` as well as `[c]`, and we +/// walk the requirement `[a, c]`, pinning `a` after the first step results in: /// -/// After step 2, `[a, b]` is used up: both of its expressions are pinned, so -/// it has no head. Among the rows that tie on `a` and `b`, the second -/// ordering says the rows are sorted by `c`, which is what the third key -/// asks. All three keys are satisfied, so no sort is needed. +/// | Step | Requirement Key | Pinned | Head `[a, b]` | Head `[c]` | Outcome | +/// |------|-----------------|--------|---------------|------------|-----------| +/// | 1 | a | {} | a | c | pin a | +/// | 2 | c | {a} | b | c | satisfied | /// -/// To see why this is right, take a table sorted by both orderings: +/// It may be confusing that the head of `[c]` does not advance. The intuition +/// is that `[c]` still promises the data is sorted by `c` for all equal values +/// of `a`. This is true because the data was entirely sorted by `[c]` to begin +/// with, and pinning `a` does not change the order of the rows that tie on `a`, +/// as illustrated in this example: /// /// ```text /// ┌───┬───┬───┐ /// │ a │ b │ c │ /// ├───┼───┼───┤ -/// │ 1 │ 1 │ 5 │ -/// │ 1 │ 2 │ 3 │ -/// │ 1 │ 2 │ 4 │ -/// │ 2 │ 2 │ 6 │ -/// │ 2 │ 3 │ 1 │ +/// │ 1 │ 1 │ 2 │ +/// │ 1 │ 3 │ 5 │ +/// │ 2 │ 1 │ 7 │ +/// │ 2 │ 2 │ 9 │ /// └───┴───┴───┘ /// ``` /// -/// The `(a, b)` pairs run `(1,1), (1,2), (1,2), (2,2), (2,3)`, so the table -/// is sorted by `[a, b]`. The `b` values run `1, 2, 2, 2, 3` and within -/// `b = 2` the `c` values run `3, 4, 6`, so it is sorted by `[b, c]` too. -/// Now check `[a, b, c]`: +/// # Example: Walking a Requirement /// -/// 1. `a` is sorted over all rows, from `[a, b]`. -/// 2. Among rows that tie on `a`, `b` is sorted, also from `[a, b]`. -/// 3. Among rows that tie on both `a` and `b`, `c` is sorted: those rows are -/// a subset of the rows that tie on `b`, and `[b, c]` says every run of -/// equal `b` is sorted by `c`. A subset of a sorted run, in the same -/// relative order, is still sorted. Rows 2 and 3 tie on `(1, 2)` and -/// their `c` values are `3, 4`. +/// * Orderings: `[a ASC, b ASC, c ASC]` and `[d ASC]` +/// * Requirement: `[a ASC, b ASC]` /// -/// A head is a promise that the rows tied on everything pinned are sorted by -/// that expression, which is what step 3 relies on. +/// | Step | Requirement Key | Pinned | Head `[a, b, c]` | Head `[d]` | Outcome | +/// |------|-----------------|--------|------------------|------------|-----------| +/// | 1 | a | {} | a | d | pin a | +/// | 2 | b | {a} | b | d | satisfied | /// -/// **Orderings `[a ASC, c ASC]` and `[c ASC, b ASC]`**: the requirement is -/// not satisfied. +/// This ordering is satisfied, (intuitively since the requirement is a prefix +/// of the first ordering). The second ordering plays no part. /// -/// ```text -/// Step Key Pinned Head [a, c] Head [c, b] Outcome -/// 1 a {} a c a heads the first: pin a -/// 2 b {a} c c b heads nothing: stop -/// ``` +/// * Orderings: `[a ASC, b ASC]` and `[b ASC, c ASC]` +/// * Requirement: `[a ASC, b ASC, c ASC]` /// -/// After pinning `a`, both orderings are headed by `c`. Nothing says the rows -/// that tie on `a` are sorted by `b`, so the walk stops. It reports that only -/// the first key holds, and the planner has to sort. +/// | Step | Requirement Key | Pinned | Head `[a, b]` | Head `[b, c]` | Outcome | +/// |------|-----------------|--------|---------------|---------------|-----------| +/// | 1 | a | {} | a | b | pin a | +/// | 2 | b | {a} | b | b | pin b | +/// | 3 | c | {a, b} | - | c | satisfied | /// -/// **Orderings `[a ASC, b ASC]` and `[a ASC, c ASC]`, requirement -/// `[a ASC, c ASC]`**: satisfied. Pinning `a` moves the head of *both* -/// orderings, and the second one then supplies `c`. +/// Note that After step 2, `[a, b]` is used up as both of its expressions are +/// pinned. Among the rows that tie on `a` and `b`, the second ordering says the +/// rows are sorted by `c`, which can satisfy the third key. /// -/// ```text -/// Step Key Pinned Head [a, b] Head [a, c] Outcome -/// 1 a {} a a a heads both: pin a -/// 2 c {a} b c c heads the second: done -/// ``` +/// * Orderings: `[a ASC, c ASC]` and `[c ASC, b ASC]` +/// * Requirement: `[a ASC, b ASC, c ASC]` /// -/// # Why a walk +/// | Step | Requirement Key | Pinned | Head `[a, c]` | Head `[c, b]` | Outcome | +/// |------|-----------------|--------|---------------|---------------|-------------| +/// | 1 | a | {} | a | c | pin a | +/// | 2 | b | {a} | c | c | unsatisfied | /// -/// A sort key only matters among the rows that tie on the keys before it. For -/// the requirement `[a, b]`, `a` must be sorted over all rows, but `b` only -/// has to be sorted within each group of rows with the same `a`. So once a -/// key is satisfied, the walk *pins* it: for the rest of the walk, the key has -/// one value, like a constant, and every known ordering can skip it. +/// After pinning `a`, both orderings are headed by `c`. Since nothing says the +/// rows that tie on `a` are sorted by `b`, the walk stops, and reports that the +/// requiremet is not satisfied. /// -/// # Terms +/// * Orderings: `[a ASC, b ASC]` and `[a ASC, c ASC]` +/// * Requirement: `[a ASC, c ASC]` /// -/// - The *head* of an ordering is its first expression that is not fixed. An -/// ordering whose expressions are all fixed has no head. See -/// [`Self::heads`]. -/// - An ordering is *implied* if it follows from the heads through a known -/// equality, as [`EquivalenceProperties::implied_ordering`] finds. The walk -/// collects implied orderings as it pins, and treats them like the -/// orderings of the properties from then on. +/// | Step | Requirement Key | Pinned | Head `[a, b]` | Head `[a, c]` | Outcome | +/// |------|-----------------|--------|---------------|---------------|-----------| +/// | 1 | a | {} | a | a | pin a | +/// | 2 | c | {a} | b | c | satisfied | /// /// [Checking whether a requirement is satisfied]: EquivalenceProperties#checking-whether-a-requirement-is-satisfied pub(super) struct SortKeyWalk<'a> { @@ -276,8 +247,8 @@ impl<'a> SortKeyWalk<'a> { /// See [`SortKeyWalk`] for an explanation of pinning. /// /// - /// This method pins `expr` (and thus by extension all members of its equivalence class), - /// but not its children (see [`Self::pin_key`']. + /// This method pins `expr` (and thus by extension all members of its + /// equivalence class), but not its children (see [`Self::pin_key`]). pub(super) fn pin(&mut self, expr: Arc) -> Result<()> { match self.props.eq_group.get_equivalence_class(&expr) { Some(class) => self.pinned.extend(class.iter().cloned()), From 0c9847c37aee8540e69cddbc44a9a9432bd12916 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 9 Nov 2026 09:41:49 -0500 Subject: [PATCH 15/16] updates --- .../physical-expr/src/equivalence/properties/walk.rs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/datafusion/physical-expr/src/equivalence/properties/walk.rs b/datafusion/physical-expr/src/equivalence/properties/walk.rs index ef89ba833c836..bc1428a29ad1b 100644 --- a/datafusion/physical-expr/src/equivalence/properties/walk.rs +++ b/datafusion/physical-expr/src/equivalence/properties/walk.rs @@ -44,13 +44,6 @@ use itertools::Itertools; /// themselves, as well as literals, constants, and singleton leaf expressions /// (such as an uncorrelated scalar subquery). See [`Self::is_fixed`] for more /// details. -/// -/// ## Implied Orderings -/// -/// - An ordering is *implied* if it follows from the heads through a known -/// equality, as [`EquivalenceProperties::implied_ordering`] finds. The walk -/// collects implied orderings as it pins, and treats them like the -/// orderings of the properties from then on. /// /// ## Heads /// @@ -151,10 +144,6 @@ pub(super) struct SortKeyWalk<'a> { /// /// The value is either an index into that ordering's sort expressions, or /// its length when it has no head. - /// - /// For example, with orderings `[a, b]` and `[b, c]`, head_idxs is `[0, 0]` - /// at the start, `[1, 0]` after pinning `a`, and `[2, 1]` after pinning - /// `b`, when `[a, b]` is used up. head_idxs: Vec, } From d8496cfdf3d45ebc49b601bad3a20775b0d6f4b3 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 9 Nov 2026 09:47:25 -0500 Subject: [PATCH 16/16] updates --- .../src/equivalence/properties/walk.rs | 41 +++++++++++-------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/datafusion/physical-expr/src/equivalence/properties/walk.rs b/datafusion/physical-expr/src/equivalence/properties/walk.rs index bc1428a29ad1b..5b8b602626cbc 100644 --- a/datafusion/physical-expr/src/equivalence/properties/walk.rs +++ b/datafusion/physical-expr/src/equivalence/properties/walk.rs @@ -114,7 +114,7 @@ use itertools::Itertools; /// /// After pinning `a`, both orderings are headed by `c`. Since nothing says the /// rows that tie on `a` are sorted by `b`, the walk stops, and reports that the -/// requiremet is not satisfied. +/// requirement is not satisfied. /// /// * Orderings: `[a ASC, b ASC]` and `[a ASC, c ASC]` /// * Requirement: `[a ASC, c ASC]` @@ -127,23 +127,22 @@ use itertools::Itertools; /// [Checking whether a requirement is satisfied]: EquivalenceProperties#checking-whether-a-requirement-is-satisfied pub(super) struct SortKeyWalk<'a> { props: &'a EquivalenceProperties, - /// Pinned expressions. Pinning a member of an equivalence class pins all - /// of its members. + /// Pinned expressions. pinned: HashSet>, /// Orderings that hold among the rows that tie on the pinned expressions, /// found by [`EquivalenceProperties::implied_ordering`] as the walk goes. implied: Vec, - /// The position of the head of each ordering. - /// - /// Empty until the first pin. + /// The position of the head of each ordering. Empty until the first pin. /// /// `head_idxs[i]` tracks the position in the `LexOrdering` at index `i`, - /// as returned by [`Self::ordering`]. The ordering at index `i` is: + /// as returned by [`Self::ordering`]. + /// + /// The ordering at index `i` is: /// * if `i < self.props.normal_cls.len()` then `self.props.normal_cls[i]` /// * else, `self.implied[i - self.props.normal_cls.len()]` otherwise. See /// /// The value is either an index into that ordering's sort expressions, or - /// its length when it has no head. + /// its length when the ordering has been exhausted. head_idxs: Vec, } @@ -231,13 +230,13 @@ impl<'a> SortKeyWalk<'a> { Ok(()) } - /// Pins `expr` . + /// Pins `expr`. /// /// See [`SortKeyWalk`] for an explanation of pinning. /// /// - /// This method pins `expr` (and thus by extension all members of its - /// equivalence class), but not its children (see [`Self::pin_key`]). + /// This method pins `expr` and thus by extension all members of its + /// equivalence class), but not its children. See [`Self::pin_key`] for more details. pub(super) fn pin(&mut self, expr: Arc) -> Result<()> { match self.props.eq_group.get_equivalence_class(&expr) { Some(class) => self.pinned.extend(class.iter().cloned()), @@ -354,10 +353,10 @@ impl<'a> SortKeyWalk<'a> { } // Now, check what we know about orderings. A window aggregate over a // whole window partition, such as `SUM(x) OVER (PARTITION BY k)`, adds - // `[k, w]` in all four directions of `w` (see "Constants within a - // window partition" on `EquivalenceProperties`). Once `k` is pinned, - // `w` heads orderings in opposite directions, so it has one value - // among the rows that tie on `k`: + // `[k, w]` in all four directions of `w` (see `EquivalenceProperties` + // documentation for more details). Once `k` is pinned, `w` heads + // orderings in opposite directions, so it has one value among the rows + // that tie on `k`: let normal_expr = self.props.eq_group.normalize_expr(Arc::clone(&node.expr)); if self.is_fixed(&normal_expr) || is_partial_const(self.heads(), &normal_expr) { node.data.sort_properties = SortProperties::Singleton; @@ -367,9 +366,13 @@ impl<'a> SortKeyWalk<'a> { Ok(Transformed::yes(node)) } - /// Whether `expr` is fixed (see [`SortKeyWalk`]). + /// Whether `expr` is fixed + /// + /// Note `expr` must be in [normal form]. /// - /// `expr` must be in normal form. + /// See [`SortKeyWalk`] for more details + /// + /// [normal form]: EquivalenceProperties#normal-form fn is_fixed(&self, expr: &Arc) -> bool { if expr.is::() || self.pinned.contains(expr) { return true; @@ -530,7 +533,9 @@ mod tests { } /// A window over a whole window partition, such as `SUM(..) OVER - /// (PARTITION BY k)`, adds `[k, x]` in all four directions of `x`. Pinning `k` leaves `x` + /// (PARTITION BY k)`, adds `[k, x]` in all four directions of `x`. + /// + /// Pinning `k` leaves `x` /// heading orderings in opposite directions, so `x` has one value, and so /// does `CAST(x AS BIGINT)`: `[k, CAST(x AS BIGINT)]` holds in every /// direction. Without the partial constant check, `x` would only be