From 9bfc9d43166914c06db6e97d2f6d3d87872fc64b Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sun, 8 Nov 2026 16:32:25 -0500 Subject: [PATCH 01/10] docs: explain the concepts behind EquivalenceProperties Co-Authored-By: Claude Fable 5.1 --- .../physical-expr-common/src/sort_expr.rs | 15 + .../src/equivalence/properties/mod.rs | 287 +++++++++++++++++- .../physical-plan/src/execution_plan.rs | 5 + datafusion/physical-plan/src/windows/mod.rs | 6 + 4 files changed, 297 insertions(+), 16 deletions(-) diff --git a/datafusion/physical-expr-common/src/sort_expr.rs b/datafusion/physical-expr-common/src/sort_expr.rs index 8af850e759c3f..7f61536f31379 100644 --- a/datafusion/physical-expr-common/src/sort_expr.rs +++ b/datafusion/physical-expr-common/src/sort_expr.rs @@ -344,6 +344,11 @@ pub fn options_compatible( /// * `ORDER BY A DESC NULLS FIRST` matches /// * `ORDER BY A ASC NULLS FIRST` matches (`ASC` and `NULL` options ignored) /// * `ORDER BY B DESC NULLS FIRST` does not match (different expr) +/// +/// See [Required orderings] for where requirements come from and how they +/// are checked against the orderings an input provides. +/// +/// [Required orderings]: https://docs.rs/datafusion-physical-expr/latest/datafusion_physical_expr/equivalence/struct.EquivalenceProperties.html#required-orderings #[derive(Clone, Debug)] pub struct PhysicalSortRequirement { /// Physical expression representing the column to sort @@ -697,6 +702,11 @@ impl From for Vec { /// expressions after construction like a `LexOrdering` does, because it provides /// no mutation methods. If such methods become necessary, we will need to /// enforce uniqueness like the latter object. +/// +/// See [Required orderings] for where requirements come from and how they +/// are checked against the orderings an input provides. +/// +/// [Required orderings]: https://docs.rs/datafusion-physical-expr/latest/datafusion_physical_expr/equivalence/struct.EquivalenceProperties.html#required-orderings #[derive(Debug, Clone, PartialEq)] pub struct LexRequirement { reqs: Vec, @@ -807,6 +817,11 @@ impl From for LexOrdering { /// 1. It is non-degenerate, meaning it contains at least one ordering. The /// absence of an input ordering requirement is represented by a `None` value /// in `ExecutionPlan` APIs, which return an `Option`. +/// +/// See [Required orderings] for where requirements come from and how they +/// are checked against the orderings an input provides. +/// +/// [Required orderings]: https://docs.rs/datafusion-physical-expr/latest/datafusion_physical_expr/equivalence/struct.EquivalenceProperties.html#required-orderings #[derive(Debug, Clone, PartialEq)] pub enum OrderingRequirements { /// The operator is not able to work without one of these requirements. diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rs index 1206315bbb274..d8efbc5e7b916 100644 --- a/datafusion/physical-expr/src/equivalence/properties/mod.rs +++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs @@ -38,10 +38,12 @@ use datafusion_expr::sort_properties::{ExprProperties, SortProperties}; use indexmap::IndexSet; use itertools::{Either, Itertools}; -/// `EquivalenceProperties` stores information about the output of a plan node -/// that can be used to optimize the plan. Currently, it keeps track of: +/// `EquivalenceProperties` stores information about how values are distributed +/// across the output of a plan node that can be used to optimize the plan. +/// +/// Currently, it tracks: /// - Sort expressions (orderings), -/// - Complete expression tuples whose values are contiguous, +/// - When all rows of equal values are contiguous for certain tuples (groupings) /// - Equivalent expressions; i.e. expressions known to have the same value. /// - Constants expressions; i.e. expressions known to contain a single constant /// value. @@ -50,7 +52,25 @@ use itertools::{Either, Itertools}; /// /// [Using Ordering for Better Plans]: https://datafusion.apache.org/blog/2025/03/11/ordering-analysis/ /// -/// # Example equivalent sort expressions +/// This document explains the concepts and how the pieces fit together. +/// +/// # Overview +/// +/// | Concept | Example | Stored in | +/// |------------------------|-------------------------------------------|---------------------------------------------------------------------| +/// | Equivalent expressions | `a` and `b` after the filter `a = b` | An [`EquivalenceGroup`] of [`EquivalenceClass`]es | +/// | Constants | `a` after the filter `a = 5` | The `constant` of an [`EquivalenceClass`], added as a [`ConstExpr`] | +/// | Orderings | `[a ASC, b DESC]` after a sort | An [`OrderingEquivalenceClass`] | +/// | Groupings | `[a, b]` after a hash aggregate on `a, b` | A [`GroupingEquivalenceClass`] | +/// | Constraints | A primary key or unique constraint | [`Constraints`] | +/// +/// [`EquivalenceClass`]: crate::equivalence::EquivalenceClass +/// [Normal form]: #normal-form +/// [Required orderings]: #required-orderings +/// [`ExecutionPlan::required_input_ordering`]: https://docs.rs/datafusion-physical-plan/latest/datafusion_physical_plan/trait.ExecutionPlan.html#method.required_input_ordering +/// [Checking whether a requirement is satisfied]: #checking-whether-a-requirement-is-satisfied +/// +/// # Equivalent sort expressions /// /// Consider table below: /// @@ -67,11 +87,11 @@ use itertools::{Either, Itertools}; /// /// In this case, both `a ASC` and `b DESC` can describe the table ordering. /// `EquivalenceProperties` tracks these different valid sort expressions and -/// treat `a ASC` and `b DESC` on an equal footing. For example, if the query +/// treats `a ASC` and `b DESC` on an equal footing. For example, if the query /// specifies the output sorted by EITHER `a ASC` or `b DESC`, the sort can be /// avoided. /// -/// # Example equivalent expressions +/// # Equivalent expressions /// /// Similarly, consider the table below: /// @@ -86,11 +106,241 @@ use itertools::{Either, Itertools}; /// └---┴---┘ /// ``` /// -/// In this case, columns `a` and `b` always have the same value. With this -/// information, Datafusion can optimize various operations. For example, if -/// the partition requirement is `Hash(a)` and output partitioning is -/// `Hash(b)`, then DataFusion avoids repartitioning the data as the existing -/// partitioning satisfies the requirement. +/// In this case, columns `a` and `b` always have the same value. DataFusion +/// uses this information to optimize various operations. For example, if the +/// partition requirement is `Hash(a)` and output partitioning is `Hash(b)`, +/// then DataFusion avoids repartitioning the data as the existing partitioning +/// satisfies the requirement. +/// +/// # Constants +/// +/// A constant is an expression with a single value in the output of a plan +/// node. A constant is *uniform* if it has the same value in every output +/// partition, and *heterogeneous* if it has one value within each output +/// partition but may differ between output partitions (see +/// [`AcrossPartitions`]). +/// +/// ## Example: Uniform Constant from a Filter +/// +/// Consider an input sorted by `b ASC` and the filter `a = 5`, which produces +/// the following table: +/// +/// ```text +/// ┌───┬───┐ +/// │ a │ b │ +/// ├───┼───┤ +/// │ 5 │ 1 │ +/// │ 5 │ 2 │ +/// │ 5 │ 4 │ +/// └───┴───┘ +/// ``` +/// +/// The filter adds `a` as a uniform [`ConstExpr`]. If the query requests +/// sorting on `[a ASC, b ASC]` DataFusion knows it can skip sorting. +/// +/// ## Example: Heterogeneous Constant from `UNION ALL` +/// +/// Consider `SELECT 1 AS c, x FROM t1 UNION ALL SELECT 2 AS c, x FROM t2`, +/// where each input is sorted by `x ASC` and has one output partition. The +/// union has two output partitions, one from each input: +/// +/// ```text +/// Partition 0 Partition 1 +/// ┌───┬───┐ ┌───┬───┐ +/// │ c │ x │ │ c │ x │ +/// ├───┼───┤ ├───┼───┤ +/// │ 1 │ 1 │ │ 2 │ 2 │ +/// │ 1 │ 3 │ │ 2 │ 5 │ +/// └───┴───┘ └───┴───┘ +/// ``` +/// +/// Observations: +/// - Within each output partition `c` has one value, but the values differ +/// between output partitions, so `c` is an +/// [`AcrossPartitions::Heterogeneous`] constant. +/// - Each output partition is sorted by `[x ASC]`, and therefore also by +/// `[c ASC, x ASC]`. +/// - However, when the output partitions are merged by `[x ASC]` alone, the +/// result is NOT sorted by `c`, unlike with an [`AcrossPartitions::Uniform`] +/// constant, which would have the same value in every output partition and +/// thus be sorted by `[c ASC, x ASC]` as well. +/// +/// Given those observations, `c` must stay in the ordering `[c ASC, x ASC]` +/// rather than being removed as a uniform constant would be. +/// +/// # Constants within a window partition +/// +/// NOTE: it is important to distinguish between *window partition*, the rows +/// with equal `PARTITION BY` keys, and *output partition*, how the data is +/// divided up across execution units during execution. +/// +/// ## Example: Window Functions with `PARTITION BY` +/// +/// Consider an input sorted by `k ASC` and the window expression +/// `SUM(x) OVER (PARTITION BY k) AS w`, which produces the following rows: +/// +/// ```text +/// ┌───┬───┬────┐ +/// │ k │ x │ w │ +/// ├───┼───┼────┤ +/// │ 1 │ 3 │ 5 │ +/// │ 1 │ 2 │ 5 │ +/// │ 2 │ 4 │ 10 │ +/// │ 2 │ 6 │ 10 │ +/// │ 3 │ 1 │ 1 │ +/// └───┴───┴────┘ +/// ``` +/// +/// Observations: +/// - `w` is not a constant because it differs between window partitions. +/// - The output is sorted by `k` (the window partition key), because +/// DataFusion's window implementation keeps the input order. +/// - Within each run of equal `k`, `w` has one value. +/// +/// Given those observations, the output is sorted by `[k, w]` in *every* +/// direction of `w`: ascending or descending, nulls first or last: +/// +/// - `[k ASC, w ASC NULLS FIRST]` +/// - `[k ASC, w ASC NULLS LAST]` +/// - `[k ASC, w DESC NULLS FIRST]` +/// - `[k ASC, w DESC NULLS LAST]` +/// +/// ## Example: Window Functions without `PARTITION BY` +/// +/// Consider the same input and the window expression `SUM(x) OVER () AS w`, +/// which produces the following rows: +/// +/// ```text +/// ┌───┬───┬────┐ +/// │ k │ x │ w │ +/// ├───┼───┼────┤ +/// │ 1 │ 3 │ 16 │ +/// │ 1 │ 2 │ 16 │ +/// │ 2 │ 4 │ 16 │ +/// │ 2 │ 6 │ 16 │ +/// │ 3 │ 1 │ 16 │ +/// └───┴───┴────┘ +/// ``` +/// +/// Observations: +/// - The frame of every row is the whole table, so `w` has one value. +/// - The output is still sorted by `k`, as the input was. +/// +/// Given those observations, `w` is added as a constant (a [`ConstExpr`]) +/// rather than as orderings. +/// +/// ## Example: Set-Monotonic Window Functions +/// +/// Consider an input sorted by `[ts ASC]` and the window function +/// `MAX(x) OVER (ORDER BY ts) AS m`, which produces the following rows: +/// +/// ```text +/// ┌────┬───┬───┐ +/// │ ts │ x │ m │ +/// ├────┼───┼───┤ +/// │ 1 │ 3 │ 3 │ +/// │ 2 │ 1 │ 3 │ +/// │ 3 │ 5 │ 5 │ +/// │ 4 │ 2 │ 5 │ +/// └────┴───┴───┘ +/// ``` +/// +/// Observations: +/// - When written with explicit defaults, this query is equivalent to +/// `SELECT ts, x, MAX(x) OVER (ORDER BY ts RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS m`, +/// so each row's frame contains every earlier row's frame. +/// - `MAX` is set-monotonically increasing: adding rows to its input can only +/// raise its value or leave it unchanged (see [`SetMonotonicity`]). +/// +/// Given those observations, `m` never decreases as `ts` increases, so the +/// output sorting is described by both `[ts ASC]` and `[m ASC NULLS LAST]`. +/// +/// If the query had a `PARTITION BY k`, with the input sorted by `k` first, +/// the added ordering would be `[k, m ASC NULLS LAST]`. +/// +/// A decreasing aggregate such as `MIN` results in `DESC NULLS LAST`. +/// +/// A frame that shrinks as rows advance, such as `CURRENT ROW` to +/// `UNBOUNDED FOLLOWING`, reverses the direction: `MAX` then results in +/// `DESC NULLS FIRST`. +/// +/// [`SetMonotonicity`]: datafusion_expr::SetMonotonicity +/// +/// # Required orderings +/// +/// Some operators only work on sorted input. A sort merge join needs +/// inputs sorted on the join keys, and a streaming window needs its input +/// sorted by its `PARTITION BY` and then its `ORDER BY` expressions. +/// +/// Each operator describes its requirements via +/// [`ExecutionPlan::required_input_ordering`]. +/// +/// **Orderings** describe how the data is actually distributed. +/// **Requirements** describe what an operator *needs*. The planner compares +/// orderings to requirements with [`Self::ordering_satisfy_requirement`] and +/// inserts or removes Sort operations so that they match (see [Checking whether +/// a requirement is satisfied]). +/// +/// # Normal form +/// +/// Orderings and requirements (see [Required orderings]) are compared in +/// *normal form*: every expression is replaced by the first member of its +/// equivalence class, uniform constants and expressions that repeat an earlier +/// one are removed. +/// +/// For example, if `a` and `b` are equal and `d` is a uniform constant, the +/// requirement `[b ASC, d DESC, c ASC, a ASC]` normalizes to `[a ASC, c ASC]`. +/// +/// # Checking whether a requirement is satisfied +/// +/// The requirement `[a, b]` holds if the rows are sorted by `a`, and the rows +/// that tie on `a` are sorted by `b`. +/// +/// [`Self::ordering_satisfy_requirement`] checks whether a requirement is +/// satisfied by an ordering, and is quite complex. +/// +/// # Discovering orderings +/// +/// `EquivalenceProperties` discovers and accounts for additional orderings that +/// may not be obvious from various additional information such as constants and +/// equality. +/// +/// ## Example: Ordering Implied by a `CAST` +/// +/// Consider a table `t` sorted by `[c ASC, b ASC]` and the query +/// `SELECT a, b, c FROM t WHERE c = CAST(a AS BIGINT)`, which produces the +/// following rows: +/// +/// ```text +/// ┌───┬───┬───┐ +/// │ a │ b │ c │ +/// ├───┼───┼───┤ +/// │ 1 │ 2 │ 1 │ +/// │ 1 │ 5 │ 1 │ +/// │ 2 │ 1 │ 2 │ +/// │ 3 │ 4 │ 3 │ +/// └───┴───┴───┘ +/// ``` +/// +/// Observations: +/// - The filter keeps the input order, so the rows are still sorted by +/// `[c ASC, b ASC]` +/// - Casting an `INT` to a `BIGINT` preserves order and never maps two +/// different values to the same one, so rows with different `a` are in the +/// order of `c`, which is the order of `a`. +/// - Rows with equal `c` have equal `a`, and are also still sorted by `b`. +/// +/// Given those observations, the rows are also sorted by `[a ASC, b ASC]`, +/// even though no operator sorted on `a`. `EquivalenceProperties` derives +/// this ordering when the equality is added, so a later `ORDER BY a, b` +/// needs no re-sorting. +/// +/// This holds in general for a function `f` whose output follows the +/// lexicographical order of its arguments, which `f` declares by setting +/// [`ExprProperties::preserves_lex_ordering`]. +/// +/// [`Self::add_equal_conditions`] and [`Self::add_constants`] look for such +/// orderings whenever they learn something new. /// /// # Code Example /// ``` @@ -107,10 +357,10 @@ use itertools::{Either, Itertools}; /// # let col_a = col("a", &schema).unwrap(); /// # let col_b = col("b", &schema).unwrap(); /// # let col_c = col("c", &schema).unwrap(); -/// // This object represents data that is sorted by a ASC, c DESC -/// // with a single constant value of b /// let mut eq_properties = EquivalenceProperties::new(schema); +/// // Single constant value of b /// eq_properties.add_constants(vec![ConstExpr::from(col_b)]); +/// // ORDER BY a ASC, c DESC /// eq_properties.add_ordering([ /// PhysicalSortExpr::new_default(col_a).asc(), /// PhysicalSortExpr::new_default(col_c).desc(), @@ -364,8 +614,9 @@ impl EquivalenceProperties { self.geq_cache.clear(); } - /// Removes constant expressions that may change across partitions. - /// This method should be used when merging data from different partitions. + /// Removes constant expressions that may change across output partitions. + /// This method should be used when merging data from different output + /// partitions. pub fn clear_per_partition_constants(&mut self) { if self.eq_group.clear_per_partition_constants() { // Renormalize orderings if the equivalence group changes: @@ -754,6 +1005,10 @@ impl EquivalenceProperties { /// tie on `a`. /// 3. The orderings now start at `[b, c]` and `[d]`. /// 4. The function returns `true` as `[d]` satisfies `d`. + /// + /// See [Required orderings] for what a requirement is. + /// + /// [Required orderings]: EquivalenceProperties#required-orderings pub fn ordering_satisfy_requirement( &self, given: impl IntoIterator, @@ -1425,7 +1680,7 @@ impl EquivalenceProperties { /// Returns a `Some` value if the expression is constant according to /// equivalence group, and `None` otherwise. The `Some` variant contains /// an `AcrossPartitions` value indicating whether the expression is - /// constant across partitions, and its actual value (if available). + /// constant across output partitions, and its actual value (if available). pub fn is_expr_constant( &self, expr: &Arc, diff --git a/datafusion/physical-plan/src/execution_plan.rs b/datafusion/physical-plan/src/execution_plan.rs index b236cc2f7ce00..fce0a8391b173 100644 --- a/datafusion/physical-plan/src/execution_plan.rs +++ b/datafusion/physical-plan/src/execution_plan.rs @@ -205,6 +205,11 @@ pub trait ExecutionPlan: Any + Debug + DisplayAs + Send + Sync { /// NOTE that checking `!is_empty()` does **not** check for a /// required input ordering. Instead, the correct check is that at /// least one entry must be `Some` + /// + /// See [Required orderings] for how the planner checks these requirements + /// against the orderings a child provides. + /// + /// [Required orderings]: EquivalenceProperties#required-orderings fn required_input_ordering(&self) -> Vec> { vec![None; self.children().len()] } diff --git a/datafusion/physical-plan/src/windows/mod.rs b/datafusion/physical-plan/src/windows/mod.rs index 1e5eea0b01285..dc792984e4f6a 100644 --- a/datafusion/physical-plan/src/windows/mod.rs +++ b/datafusion/physical-plan/src/windows/mod.rs @@ -364,6 +364,12 @@ pub(crate) fn get_partition_by_sort_exprs( } } +/// Computes the [`EquivalenceProperties`] of a window operator's output. +/// +/// See [Constants within a window partition] for what this computes along with +/// examples. +/// +/// [Constants within a window partition]: EquivalenceProperties#constants-within-a-window-partition pub(crate) fn window_equivalence_properties( schema: &SchemaRef, input: &Arc, From 92cf6f2093dbdeb6c26d81ad7aa6ebb27822e997 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sun, 8 Nov 2026 16:39:33 -0500 Subject: [PATCH 02/10] docs: link library user guide to EquivalenceProperties rustdoc Co-Authored-By: Claude Fable 5.1 --- docs/source/library-user-guide/custom-table-providers.md | 4 +++- docs/source/library-user-guide/query-optimizer.md | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/source/library-user-guide/custom-table-providers.md b/docs/source/library-user-guide/custom-table-providers.md index 0c6b5d1aa688c..f96715ba06ae2 100644 --- a/docs/source/library-user-guide/custom-table-providers.md +++ b/docs/source/library-user-guide/custom-table-providers.md @@ -43,6 +43,7 @@ actually produced during execution. [listingtable]: https://docs.rs/datafusion/latest/datafusion/datasource/listing/struct.ListingTable.html [viewtable]: https://docs.rs/datafusion/latest/datafusion/datasource/view/struct.ViewTable.html [planproperties]: https://docs.rs/datafusion/latest/datafusion/physical_plan/struct.PlanProperties.html +[equivalenceproperties]: https://docs.rs/datafusion/latest/datafusion/physical_expr/struct.EquivalenceProperties.html [streamingtableexec]: https://docs.rs/datafusion/latest/datafusion/physical_plan/streaming/struct.StreamingTableExec.html [`optimizer`]: https://docs.rs/datafusion/latest/datafusion/optimizer/struct.Optimizer.html#built-in-optimizer-rules [`physicaloptimizer`]: https://docs.rs/datafusion/latest/datafusion/physical_optimizer/optimizer/struct.PhysicalOptimizer.html#built-in-physical-optimizer-rules @@ -278,7 +279,8 @@ by file or by shard), expose that here. **Output ordering** declares whether your data is naturally sorted. This enables the optimizer to avoid inserting a `SortExec` when a query requires -ordered data. Getting this right can be a significant performance win. +ordered data. Getting this right can be a significant performance win. See +[EquivalenceProperties], for more detailed information. ### Partitioning Strategies diff --git a/docs/source/library-user-guide/query-optimizer.md b/docs/source/library-user-guide/query-optimizer.md index a9415fd89cbee..9eb148c51ee05 100644 --- a/docs/source/library-user-guide/query-optimizer.md +++ b/docs/source/library-user-guide/query-optimizer.md @@ -23,6 +23,10 @@ For a deeper background on optimizer architecture and rule types and predicates, [Using Ordering for Better Plans in Apache DataFusion], and [Dynamic Filters: Passing Information Between Operators During Execution for 25x Faster Queries]. +The physical optimizer's reasoning about sort orders, equivalent expressions, +and constants is implemented by [`EquivalenceProperties`], which explains more +details. + [`optimizerrule`]: https://docs.rs/datafusion/latest/datafusion/optimizer/trait.OptimizerRule.html [`physicaloptimizerrule`]: https://docs.rs/datafusion/latest/datafusion/physical_optimizer/trait.PhysicalOptimizerRule.html [optimizing sql (and dataframes) in datafusion, part 1]: https://datafusion.apache.org/blog/2025/06/15/optimizing-sql-dataframes-part-one @@ -32,6 +36,7 @@ For a deeper background on optimizer architecture and rule types and predicates, [`analyzer`]: https://docs.rs/datafusion/latest/datafusion/optimizer/struct.Analyzer.html#built-in-analyzer-rules [`optimizer`]: https://docs.rs/datafusion/latest/datafusion/optimizer/struct.Optimizer.html#built-in-optimizer-rules [`physicaloptimizer`]: https://docs.rs/datafusion/latest/datafusion/physical_optimizer/optimizer/struct.PhysicalOptimizer.html#built-in-physical-optimizer-rules +[`equivalenceproperties`]: https://docs.rs/datafusion/latest/datafusion/physical_expr/struct.EquivalenceProperties.html [`logicalplan`]: https://docs.rs/datafusion/latest/datafusion/logical_expr/enum.LogicalPlan.html ## Running the Optimizer From dd69b5b0c83a454801a96a6f3318d709a5205121 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 9 Nov 2026 10:08:58 -0500 Subject: [PATCH 03/10] Apply batched suggestions from code review Co-authored-by: Jay Zhan --- datafusion/physical-expr/src/equivalence/properties/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rs index d8efbc5e7b916..740fa8ea20d55 100644 --- a/datafusion/physical-expr/src/equivalence/properties/mod.rs +++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs @@ -65,7 +65,6 @@ use itertools::{Either, Itertools}; /// | Constraints | A primary key or unique constraint | [`Constraints`] | /// /// [`EquivalenceClass`]: crate::equivalence::EquivalenceClass -/// [Normal form]: #normal-form /// [Required orderings]: #required-orderings /// [`ExecutionPlan::required_input_ordering`]: https://docs.rs/datafusion-physical-plan/latest/datafusion_physical_plan/trait.ExecutionPlan.html#method.required_input_ordering /// [Checking whether a requirement is satisfied]: #checking-whether-a-requirement-is-satisfied @@ -276,7 +275,8 @@ use itertools::{Either, Itertools}; /// [`ExecutionPlan::required_input_ordering`]. /// /// **Orderings** describe how the data is actually distributed. -/// **Requirements** describe what an operator *needs*. The planner compares +/// **Requirements** describe what an operator *needs*. The physical optimizer +/// (`EnforceSorting`) compares /// orderings to requirements with [`Self::ordering_satisfy_requirement`] and /// inserts or removes Sort operations so that they match (see [Checking whether /// a requirement is satisfied]). From 9ac748999dea627a5b69fdb005353da36eba1e69 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 9 Nov 2026 10:10:42 -0500 Subject: [PATCH 04/10] Apply suggestion from @jayzhan211 Co-authored-by: Jay Zhan --- datafusion/physical-expr/src/equivalence/properties/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rs index 740fa8ea20d55..1d9413d601d93 100644 --- a/datafusion/physical-expr/src/equivalence/properties/mod.rs +++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs @@ -61,7 +61,7 @@ use itertools::{Either, Itertools}; /// | Equivalent expressions | `a` and `b` after the filter `a = b` | An [`EquivalenceGroup`] of [`EquivalenceClass`]es | /// | Constants | `a` after the filter `a = 5` | The `constant` of an [`EquivalenceClass`], added as a [`ConstExpr`] | /// | Orderings | `[a ASC, b DESC]` after a sort | An [`OrderingEquivalenceClass`] | -/// | Groupings | `[a, b]` after a hash aggregate on `a, b` | A [`GroupingEquivalenceClass`] | +/// | Groupings | `[a, b]` declared by a custom data source | A [`GroupingEquivalenceClass`] | /// | Constraints | A primary key or unique constraint | [`Constraints`] | /// /// [`EquivalenceClass`]: crate::equivalence::EquivalenceClass From c954d145007d56efe04fa9458eafe25cdabede25 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 9 Nov 2026 10:13:37 -0500 Subject: [PATCH 05/10] Udpate headings --- datafusion/physical-expr/src/equivalence/properties/mod.rs | 2 +- datafusion/physical-plan/src/windows/mod.rs | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rs index 1d9413d601d93..83972c689c701 100644 --- a/datafusion/physical-expr/src/equivalence/properties/mod.rs +++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs @@ -167,7 +167,7 @@ use itertools::{Either, Itertools}; /// Given those observations, `c` must stay in the ordering `[c ASC, x ASC]` /// rather than being removed as a uniform constant would be. /// -/// # Constants within a window partition +/// # Window functions /// /// NOTE: it is important to distinguish between *window partition*, the rows /// with equal `PARTITION BY` keys, and *output partition*, how the data is diff --git a/datafusion/physical-plan/src/windows/mod.rs b/datafusion/physical-plan/src/windows/mod.rs index dc792984e4f6a..404f2253874ed 100644 --- a/datafusion/physical-plan/src/windows/mod.rs +++ b/datafusion/physical-plan/src/windows/mod.rs @@ -366,10 +366,9 @@ pub(crate) fn get_partition_by_sort_exprs( /// Computes the [`EquivalenceProperties`] of a window operator's output. /// -/// See [Constants within a window partition] for what this computes along with -/// examples. +/// See [Window functions] for what this computes along with examples. /// -/// [Constants within a window partition]: EquivalenceProperties#constants-within-a-window-partition +/// [Window functions]: EquivalenceProperties#window-functions pub(crate) fn window_equivalence_properties( schema: &SchemaRef, input: &Arc, From 78301d83e31869c0d74977eb8c43d4545342f8fa Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 9 Nov 2026 10:20:31 -0500 Subject: [PATCH 06/10] Apply suggestion from @jayzhan211 Co-authored-by: Jay Zhan --- .../src/equivalence/properties/mod.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rs index 1d9413d601d93..4c08cc76c10da 100644 --- a/datafusion/physical-expr/src/equivalence/properties/mod.rs +++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs @@ -307,7 +307,7 @@ use itertools::{Either, Itertools}; /// /// ## Example: Ordering Implied by a `CAST` /// -/// Consider a table `t` sorted by `[c ASC, b ASC]` and the query +/// Consider a table `t` sorted by `[c ASC, a ASC, b ASC]` and the query /// `SELECT a, b, c FROM t WHERE c = CAST(a AS BIGINT)`, which produces the /// following rows: /// @@ -324,16 +324,18 @@ use itertools::{Either, Itertools}; /// /// Observations: /// - The filter keeps the input order, so the rows are still sorted by -/// `[c ASC, b ASC]` +/// `[c ASC, a ASC, b ASC]`. /// - Casting an `INT` to a `BIGINT` preserves order and never maps two /// different values to the same one, so rows with different `a` are in the /// order of `c`, which is the order of `a`. -/// - Rows with equal `c` have equal `a`, and are also still sorted by `b`. +/// - Rows with equal `c` are sorted by `[a, b]`. /// /// Given those observations, the rows are also sorted by `[a ASC, b ASC]`, -/// even though no operator sorted on `a`. `EquivalenceProperties` derives -/// this ordering when the equality is added, so a later `ORDER BY a, b` -/// needs no re-sorting. +/// even though `a` is not the leading sort key. `EquivalenceProperties` +/// derives this ordering when the equality is added, so a later +/// `ORDER BY a, b` needs no re-sorting. The arguments of the cast must +/// directly follow `c` in the ordering: from `[c ASC, b ASC]` alone, the +/// ordering `[a ASC, b ASC]` is not derived. /// /// This holds in general for a function `f` whose output follows the /// lexicographical order of its arguments, which `f` declares by setting From 9a5db64b2b3bd8addaa310e168b5e1a924ae72f9 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 9 Nov 2026 10:22:38 -0500 Subject: [PATCH 07/10] clarify and trim --- .../physical-expr/src/equivalence/properties/mod.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rs index ec8537ae9035c..1478f8ab779bd 100644 --- a/datafusion/physical-expr/src/equivalence/properties/mod.rs +++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs @@ -307,8 +307,8 @@ use itertools::{Either, Itertools}; /// /// ## Example: Ordering Implied by a `CAST` /// -/// Consider a table `t` sorted by `[c ASC, a ASC, b ASC]` and the query -/// `SELECT a, b, c FROM t WHERE c = CAST(a AS BIGINT)`, which produces the +/// Consider a table `t` sorted by `[c ASC, a ASC, b ASC]` and a query `SELECT +/// a, b, c FROM t WHERE c = CAST(a AS BIGINT) ORDER BY a, b` that produces the /// following rows: /// /// ```text @@ -332,10 +332,8 @@ use itertools::{Either, Itertools}; /// /// Given those observations, the rows are also sorted by `[a ASC, b ASC]`, /// even though `a` is not the leading sort key. `EquivalenceProperties` -/// derives this ordering when the equality is added, so a later -/// `ORDER BY a, b` needs no re-sorting. The arguments of the cast must -/// directly follow `c` in the ordering: from `[c ASC, b ASC]` alone, the -/// ordering `[a ASC, b ASC]` is not derived. +/// derives this ordering when the equality is added, so the `ORDER BY a, b` +/// needs no sort. /// /// This holds in general for a function `f` whose output follows the /// lexicographical order of its arguments, which `f` declares by setting From 8342eeba18df3f84db7e9bed839429a770dc5909 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 9 Nov 2026 10:23:23 -0500 Subject: [PATCH 08/10] Apply suggestion from @jayzhan211 Co-authored-by: Jay Zhan --- docs/source/library-user-guide/custom-table-providers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/library-user-guide/custom-table-providers.md b/docs/source/library-user-guide/custom-table-providers.md index f96715ba06ae2..6e6ce5ab2f974 100644 --- a/docs/source/library-user-guide/custom-table-providers.md +++ b/docs/source/library-user-guide/custom-table-providers.md @@ -280,7 +280,7 @@ by file or by shard), expose that here. **Output ordering** declares whether your data is naturally sorted. This enables the optimizer to avoid inserting a `SortExec` when a query requires ordered data. Getting this right can be a significant performance win. See -[EquivalenceProperties], for more detailed information. +[EquivalenceProperties] for more details. ### Partitioning Strategies From 0741e914bbf3c11bdec45da8c9643b3a02b9620d Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 9 Nov 2026 10:24:30 -0500 Subject: [PATCH 09/10] cleanups --- .../physical-expr/src/equivalence/properties/mod.rs | 10 +++++----- .../library-user-guide/custom-table-providers.md | 2 +- docs/source/library-user-guide/query-optimizer.md | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rs index 1478f8ab779bd..27b6665a55e2e 100644 --- a/datafusion/physical-expr/src/equivalence/properties/mod.rs +++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs @@ -43,7 +43,7 @@ use itertools::{Either, Itertools}; /// /// Currently, it tracks: /// - Sort expressions (orderings), -/// - When all rows of equal values are contiguous for certain tuples (groupings) +/// - When all rows of equal values are contiguous for certain tuples (groupings), /// - Equivalent expressions; i.e. expressions known to have the same value. /// - Constants expressions; i.e. expressions known to contain a single constant /// value. @@ -276,10 +276,10 @@ use itertools::{Either, Itertools}; /// /// **Orderings** describe how the data is actually distributed. /// **Requirements** describe what an operator *needs*. The physical optimizer -/// (`EnforceSorting`) compares -/// orderings to requirements with [`Self::ordering_satisfy_requirement`] and -/// inserts or removes Sort operations so that they match (see [Checking whether -/// a requirement is satisfied]). +/// (`EnforceSorting`) compares orderings to requirements with +/// [`Self::ordering_satisfy_requirement`] and inserts or removes Sort +/// operations so that they match (see [Checking whether a requirement is +/// satisfied]). /// /// # Normal form /// diff --git a/docs/source/library-user-guide/custom-table-providers.md b/docs/source/library-user-guide/custom-table-providers.md index f96715ba06ae2..6e6ce5ab2f974 100644 --- a/docs/source/library-user-guide/custom-table-providers.md +++ b/docs/source/library-user-guide/custom-table-providers.md @@ -280,7 +280,7 @@ by file or by shard), expose that here. **Output ordering** declares whether your data is naturally sorted. This enables the optimizer to avoid inserting a `SortExec` when a query requires ordered data. Getting this right can be a significant performance win. See -[EquivalenceProperties], for more detailed information. +[EquivalenceProperties] for more details. ### Partitioning Strategies diff --git a/docs/source/library-user-guide/query-optimizer.md b/docs/source/library-user-guide/query-optimizer.md index 9eb148c51ee05..b939333c5f222 100644 --- a/docs/source/library-user-guide/query-optimizer.md +++ b/docs/source/library-user-guide/query-optimizer.md @@ -24,8 +24,8 @@ For a deeper background on optimizer architecture and rule types and predicates, [Dynamic Filters: Passing Information Between Operators During Execution for 25x Faster Queries]. The physical optimizer's reasoning about sort orders, equivalent expressions, -and constants is implemented by [`EquivalenceProperties`], which explains more -details. +and constants is implemented by [`EquivalenceProperties`], whose documentation +explains the details. [`optimizerrule`]: https://docs.rs/datafusion/latest/datafusion/optimizer/trait.OptimizerRule.html [`physicaloptimizerrule`]: https://docs.rs/datafusion/latest/datafusion/physical_optimizer/trait.PhysicalOptimizerRule.html From c2c09b3a32dfbc86b691422a7bd9ce3578a9a134 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 9 Nov 2026 10:37:14 -0500 Subject: [PATCH 10/10] remove explicit NULLs --- .../physical-expr/src/equivalence/properties/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rs index 27b6665a55e2e..b45ee313ca313 100644 --- a/datafusion/physical-expr/src/equivalence/properties/mod.rs +++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs @@ -252,16 +252,16 @@ use itertools::{Either, Itertools}; /// raise its value or leave it unchanged (see [`SetMonotonicity`]). /// /// Given those observations, `m` never decreases as `ts` increases, so the -/// output sorting is described by both `[ts ASC]` and `[m ASC NULLS LAST]`. +/// output sorting is described by both `[ts ASC]` and `[m ASC]`. /// /// If the query had a `PARTITION BY k`, with the input sorted by `k` first, -/// the added ordering would be `[k, m ASC NULLS LAST]`. +/// the added ordering would be `[k, m ASC]`. /// -/// A decreasing aggregate such as `MIN` results in `DESC NULLS LAST`. +/// A decreasing aggregate such as `MIN` results in `DESC`. /// /// A frame that shrinks as rows advance, such as `CURRENT ROW` to /// `UNBOUNDED FOLLOWING`, reverses the direction: `MAX` then results in -/// `DESC NULLS FIRST`. +/// `DESC`. /// /// [`SetMonotonicity`]: datafusion_expr::SetMonotonicity ///