diff --git a/Cargo.lock b/Cargo.lock index e3bfa7f5959b1..f332d5ca26690 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -159,8 +159,7 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d965156f2220e94dcf676e04322b7b0b41c97e85cb3e1c365b8e56d0f70be999" +source = "git+https://github.com/adriangb/arrow-rs.git?rev=585ab29ab94504ae48ae431298503b107601696a#585ab29ab94504ae48ae431298503b107601696a" dependencies = [ "arrow-arith", "arrow-array", @@ -182,8 +181,7 @@ dependencies = [ [[package]] name = "arrow-arith" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca16fdd1daeb1eccefb42df52ec65448ef33eec72488598675d46dfa9ce58923" +source = "git+https://github.com/adriangb/arrow-rs.git?rev=585ab29ab94504ae48ae431298503b107601696a#585ab29ab94504ae48ae431298503b107601696a" dependencies = [ "arrow-array", "arrow-buffer", @@ -196,8 +194,7 @@ dependencies = [ [[package]] name = "arrow-array" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f3d22a41d3df78b71df0b685af219ae3a94188ab98d6fb29c958eb55c3247f4" +source = "git+https://github.com/adriangb/arrow-rs.git?rev=585ab29ab94504ae48ae431298503b107601696a#585ab29ab94504ae48ae431298503b107601696a" dependencies = [ "ahash", "arrow-buffer", @@ -216,8 +213,7 @@ dependencies = [ [[package]] name = "arrow-avro" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd40a71a38e4366b28742cc46912b9289d89e0c858895c8276ac0fc83dacdc6" +source = "git+https://github.com/adriangb/arrow-rs.git?rev=585ab29ab94504ae48ae431298503b107601696a#585ab29ab94504ae48ae431298503b107601696a" dependencies = [ "arrow-array", "arrow-buffer", @@ -240,8 +236,7 @@ dependencies = [ [[package]] name = "arrow-buffer" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd46a530bc1a8e763102fc36c1dc0f51f3ffa150c78e7f5658ccb54b1458ade" +source = "git+https://github.com/adriangb/arrow-rs.git?rev=585ab29ab94504ae48ae431298503b107601696a#585ab29ab94504ae48ae431298503b107601696a" dependencies = [ "bytes", "half", @@ -252,8 +247,7 @@ dependencies = [ [[package]] name = "arrow-cast" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baa83a0752814a1922d5ee75032524ca9bbfb7f60ee35a67ed4ab8b275260aaf" +source = "git+https://github.com/adriangb/arrow-rs.git?rev=585ab29ab94504ae48ae431298503b107601696a#585ab29ab94504ae48ae431298503b107601696a" dependencies = [ "arrow-array", "arrow-buffer", @@ -274,8 +268,7 @@ dependencies = [ [[package]] name = "arrow-cmp" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1913e57a64fbf2c4502fa6d54b547d6a1ae59e3146cbc15321d47468c937abd1" +source = "git+https://github.com/adriangb/arrow-rs.git?rev=585ab29ab94504ae48ae431298503b107601696a#585ab29ab94504ae48ae431298503b107601696a" dependencies = [ "arrow-array", "arrow-buffer", @@ -285,8 +278,7 @@ dependencies = [ [[package]] name = "arrow-csv" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4572af05fadde76a063139805c089d0915b7eb64ed5bef14456c6e458a80b28" +source = "git+https://github.com/adriangb/arrow-rs.git?rev=585ab29ab94504ae48ae431298503b107601696a#585ab29ab94504ae48ae431298503b107601696a" dependencies = [ "arrow-array", "arrow-cast", @@ -300,8 +292,7 @@ dependencies = [ [[package]] name = "arrow-data" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "415b7fbc4cf56b080bb5677a4ae74211b5c79e571ed37a1d547633ec07f3b43b" +source = "git+https://github.com/adriangb/arrow-rs.git?rev=585ab29ab94504ae48ae431298503b107601696a#585ab29ab94504ae48ae431298503b107601696a" dependencies = [ "arrow-buffer", "arrow-schema", @@ -313,8 +304,7 @@ dependencies = [ [[package]] name = "arrow-flight" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8709a798bd0825cb6207861aad09093133d64314c25569132dbc7f9f083879a5" +source = "git+https://github.com/adriangb/arrow-rs.git?rev=585ab29ab94504ae48ae431298503b107601696a#585ab29ab94504ae48ae431298503b107601696a" dependencies = [ "arrow-arith", "arrow-array", @@ -339,8 +329,7 @@ dependencies = [ [[package]] name = "arrow-ipc" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45f3e766bccf5a0cb57a5de52a4cba450dea5171547c2cb5939b486d08887cba" +source = "git+https://github.com/adriangb/arrow-rs.git?rev=585ab29ab94504ae48ae431298503b107601696a#585ab29ab94504ae48ae431298503b107601696a" dependencies = [ "arrow-array", "arrow-buffer", @@ -355,8 +344,7 @@ dependencies = [ [[package]] name = "arrow-json" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d843cca4b5361d7667564083b151a30d816d2b7e7ce2baa53948e2188f087f06" +source = "git+https://github.com/adriangb/arrow-rs.git?rev=585ab29ab94504ae48ae431298503b107601696a#585ab29ab94504ae48ae431298503b107601696a" dependencies = [ "arrow-array", "arrow-buffer", @@ -380,8 +368,7 @@ dependencies = [ [[package]] name = "arrow-ord" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c58f2ae6dd1480fc413f7320a547e39dcc499c23d4f36ca19572b685811e6e4" +source = "git+https://github.com/adriangb/arrow-rs.git?rev=585ab29ab94504ae48ae431298503b107601696a#585ab29ab94504ae48ae431298503b107601696a" dependencies = [ "arrow-array", "arrow-buffer", @@ -394,8 +381,7 @@ dependencies = [ [[package]] name = "arrow-row" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f9db8345b0638416d3b56d117d262d13070fdef788c6d75b53a180215c477e" +source = "git+https://github.com/adriangb/arrow-rs.git?rev=585ab29ab94504ae48ae431298503b107601696a#585ab29ab94504ae48ae431298503b107601696a" dependencies = [ "arrow-array", "arrow-buffer", @@ -407,8 +393,7 @@ dependencies = [ [[package]] name = "arrow-schema" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de2c8e714b466404cdc1189e89f892426efea614d938e16c15518c340cd43ba3" +source = "git+https://github.com/adriangb/arrow-rs.git?rev=585ab29ab94504ae48ae431298503b107601696a#585ab29ab94504ae48ae431298503b107601696a" dependencies = [ "bitflags", "serde", @@ -419,8 +404,7 @@ dependencies = [ [[package]] name = "arrow-select" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5af3f6656c5f2c6944f93d9a357a7f204c93d5f349385a44cf46c4a26fbcf30" +source = "git+https://github.com/adriangb/arrow-rs.git?rev=585ab29ab94504ae48ae431298503b107601696a#585ab29ab94504ae48ae431298503b107601696a" dependencies = [ "ahash", "arrow-array", @@ -434,8 +418,7 @@ dependencies = [ [[package]] name = "arrow-string" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab1be9306b9a413b5b004e2a5a19caac728870d76e8891b351f040d554c93862" +source = "git+https://github.com/adriangb/arrow-rs.git?rev=585ab29ab94504ae48ae431298503b107601696a#585ab29ab94504ae48ae431298503b107601696a" dependencies = [ "arrow-array", "arrow-buffer", @@ -2585,6 +2568,7 @@ dependencies = [ "loom", "num-traits", "parking_lot 0.13.0", + "parquet", "pin-project-lite", "rand 0.9.4", "rstest", @@ -2674,7 +2658,6 @@ dependencies = [ "insta", "itertools 0.15.0", "log", - "rstest", ] [[package]] @@ -4685,8 +4668,7 @@ dependencies = [ [[package]] name = "parquet" version = "60.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806eaa0f1c0436dcb43bee8e6a82bf2efff90521a60172f8281cb9ef577b43a4" +source = "git+https://github.com/adriangb/arrow-rs.git?rev=585ab29ab94504ae48ae431298503b107601696a#585ab29ab94504ae48ae431298503b107601696a" dependencies = [ "ahash", "arrow-array", diff --git a/Cargo.toml b/Cargo.toml index bfc4b28c4d52e..c87c652ef5a6c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -400,3 +400,25 @@ incremental = false [profile.ci-bench] inherits = "ci" opt-level = 1 + +# TODO: remove once arrow-rs releases the `Sbbf` APIs of +# (branch sbbf-batch-union-apis-60 = arrow-rs 60.1.0 + the commit of) +# https://github.com/apache/arrow-rs/pull/11641 (used by the hash join bloom filter) +[patch.crates-io] +arrow = { git = "https://github.com/adriangb/arrow-rs.git", rev = "585ab29ab94504ae48ae431298503b107601696a" } +arrow-arith = { git = "https://github.com/adriangb/arrow-rs.git", rev = "585ab29ab94504ae48ae431298503b107601696a" } +arrow-array = { git = "https://github.com/adriangb/arrow-rs.git", rev = "585ab29ab94504ae48ae431298503b107601696a" } +arrow-avro = { git = "https://github.com/adriangb/arrow-rs.git", rev = "585ab29ab94504ae48ae431298503b107601696a" } +arrow-buffer = { git = "https://github.com/adriangb/arrow-rs.git", rev = "585ab29ab94504ae48ae431298503b107601696a" } +arrow-cast = { git = "https://github.com/adriangb/arrow-rs.git", rev = "585ab29ab94504ae48ae431298503b107601696a" } +arrow-csv = { git = "https://github.com/adriangb/arrow-rs.git", rev = "585ab29ab94504ae48ae431298503b107601696a" } +arrow-data = { git = "https://github.com/adriangb/arrow-rs.git", rev = "585ab29ab94504ae48ae431298503b107601696a" } +arrow-flight = { git = "https://github.com/adriangb/arrow-rs.git", rev = "585ab29ab94504ae48ae431298503b107601696a" } +arrow-ipc = { git = "https://github.com/adriangb/arrow-rs.git", rev = "585ab29ab94504ae48ae431298503b107601696a" } +arrow-json = { git = "https://github.com/adriangb/arrow-rs.git", rev = "585ab29ab94504ae48ae431298503b107601696a" } +arrow-ord = { git = "https://github.com/adriangb/arrow-rs.git", rev = "585ab29ab94504ae48ae431298503b107601696a" } +arrow-row = { git = "https://github.com/adriangb/arrow-rs.git", rev = "585ab29ab94504ae48ae431298503b107601696a" } +arrow-schema = { git = "https://github.com/adriangb/arrow-rs.git", rev = "585ab29ab94504ae48ae431298503b107601696a" } +arrow-select = { git = "https://github.com/adriangb/arrow-rs.git", rev = "585ab29ab94504ae48ae431298503b107601696a" } +arrow-string = { git = "https://github.com/adriangb/arrow-rs.git", rev = "585ab29ab94504ae48ae431298503b107601696a" } +parquet = { git = "https://github.com/adriangb/arrow-rs.git", rev = "585ab29ab94504ae48ae431298503b107601696a" } diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 63a7797e396ef..0ec655e09f7d4 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -1899,6 +1899,43 @@ config_namespace! { /// See: pub hash_join_inlist_pushdown_max_distinct_values: usize, default = 150 + /// Maximum number of distinct build side keys, over all partitions of a hash join, + /// for the global dynamic filter to test membership with one InList of all keys. + /// Above this limit the dynamic filter uses a bloom filter + /// (see `hash_join_bloom_filter_fpp`). + /// + /// Every partition must also use an InList by itself + /// (`hash_join_inlist_pushdown_max_distinct_values` and `hash_join_inlist_pushdown_max_size`). + /// For larger sets, a bloom filter is cheaper per probe row than an InList. + pub hash_join_dynamic_filter_inlist_max_distinct_values: usize, default = 20 + + /// Target false positive probability of the bloom filter in the dynamic + /// filter of a hash join. + /// + /// When the build side of a hash join has more distinct keys than + /// `hash_join_inlist_pushdown_max_distinct_values` (over all partitions), + /// the dynamic filter tests probe rows against one bloom filter of all + /// build side keys. Each build partition fills a filter of its keys, sized + /// for the estimated number of distinct keys of all partitions at this + /// probability. Then one task ORs the partition filters and folds the + /// result to this probability. + pub hash_join_bloom_filter_fpp: f64, default = 0.01 + + /// Maximum size, in bytes, of the bloom filter in the dynamic filter of a + /// hash join (and of the filter of each build partition). A build side that needs a larger + /// filter at `hash_join_bloom_filter_fpp` gets no bloom filter: such a + /// large build side is rarely selective, so the filter is too big to be + /// useful. The dynamic filter then tests only the min/max bounds (and + /// keeps key range pruning). + pub hash_join_bloom_filter_max_bytes: usize, default = 8 * 1024 * 1024 + + /// The bloom filter of a hash join dynamic filter is not used when its + /// estimated false positive probability is above this value (the filter + /// is saturated, a safety net). Then the dynamic filter tests only the + /// min/max bounds of the build keys, because a saturated filter removes few rows and costs + /// CPU time for each probe row. + pub hash_join_bloom_filter_max_fpp: f64, default = 0.15 + /// The default filter selectivity used by Filter Statistics /// when an exact selectivity cannot be determined. Valid values are /// between 0 (no selectivity) and 100 (all rows are selected). diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs index 6b4f47227c427..ebb19311eb98d 100644 --- a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs +++ b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs @@ -1145,7 +1145,7 @@ async fn test_hashjoin_dynamic_filter_pushdown_partitioned() { #[cfg(not(feature = "force_hash_collisions"))] insta::assert_snapshot!( format_plan_for_test(&plan), - @r" + @" - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false] - CoalescePartitionsExec - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, a@0), (b@1, b@1)] @@ -1361,7 +1361,7 @@ async fn test_hashjoin_dynamic_filter_pushdown_range_partitioned() { // Now check what our filter looks like insta::assert_snapshot!( format_plan_for_test(&plan), - @r" + @" - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false] - CoalescePartitionsExec - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, a@0), (b@1, b@1)] @@ -3928,11 +3928,11 @@ async fn test_hashjoin_hash_table_pushdown_partitioned() { config.optimizer.enable_dynamic_filter_pushdown = true; let (plan, batches) = optimize_and_collect_pushdown_plan(plan, config).await; - // Verify that hash_lookup is used instead of IN (SET) + // Verify that a bloom filter is used instead of IN (SET) let plan_str = format_plan_for_test(&plan).to_string(); assert!( - plan_str.contains("hash_lookup"), - "Expected hash_lookup in plan but got: {plan_str}" + plan_str.contains("bloom_filter"), + "Expected bloom_filter in plan but got: {plan_str}" ); assert!( !plan_str.contains("IN (SET)"), @@ -3962,7 +3962,7 @@ async fn test_hashjoin_hash_table_pushdown_partitioned() { // Ported to push_down_filter_parquet.slt (`hl_build`/`hl_probe` fixture). // Rust version retained only because the slt port cannot hand-wire the -// RepartitionExec-above-probe shape this test uses; the hash_lookup vs +// RepartitionExec-above-probe shape this test uses; the bloom_filter vs // IN (SET) invariant is captured in the slt port. #[tokio::test] async fn test_hashjoin_hash_table_pushdown_collect_left() { @@ -4029,11 +4029,11 @@ async fn test_hashjoin_hash_table_pushdown_collect_left() { config.optimizer.enable_dynamic_filter_pushdown = true; let (plan, batches) = optimize_and_collect_pushdown_plan(plan, config).await; - // Verify that hash_lookup is used instead of IN (SET) + // Verify that a bloom filter is used instead of IN (SET) let plan_str = format_plan_for_test(&plan).to_string(); assert!( - plan_str.contains("hash_lookup"), - "Expected hash_lookup in plan but got: {plan_str}" + plan_str.contains("bloom_filter"), + "Expected bloom_filter in plan but got: {plan_str}" ); assert!( !plan_str.contains("IN (SET)"), diff --git a/datafusion/physical-plan/Cargo.toml b/datafusion/physical-plan/Cargo.toml index 25f8e437407a1..66307ec7f3d5c 100644 --- a/datafusion/physical-plan/Cargo.toml +++ b/datafusion/physical-plan/Cargo.toml @@ -74,6 +74,7 @@ itertools = { workspace = true, features = ["use_std"] } log = { workspace = true } num-traits = { workspace = true } parking_lot = { workspace = true } +parquet = { workspace = true } pin-project-lite = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/datafusion/physical-plan/src/joins/hash_join/bloom_filter.rs b/datafusion/physical-plan/src/joins/hash_join/bloom_filter.rs new file mode 100644 index 0000000000000..9818b7c47b94f --- /dev/null +++ b/datafusion/physical-plan/src/joins/hash_join/bloom_filter.rs @@ -0,0 +1,671 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Bloom filter membership test over the build side keys of a hash join, for +//! the join's dynamic filter. +//! +//! Each build partition inserts the hash join hashes of its keys into its own +//! bloom filter ([`partition_filter`]), in its own build task. Once every +//! partition is done, one task ORs the partition filters into one filter +//! ([`Sbbf::union`]) and folds it to the target false positive probability. +//! Folding and union commute, so this is the filter of all build keys. + +use std::fmt::{self, Display}; +use std::hash::Hash; +use std::sync::Arc; + +use arrow::array::{Array, ArrayRef, BooleanArray}; +use arrow::buffer::NullBuffer; +use arrow::datatypes::{DataType, Schema}; +use arrow::record_batch::RecordBatch; +use datafusion_common::hash_utils::create_hashes; +use datafusion_common::{DataFusionError, Result, ScalarValue}; +use datafusion_execution::memory_pool::{MemoryConsumer, MemoryPool, MemoryReservation}; +use datafusion_expr::ColumnarValue; +use datafusion_expr_common::dyn_eq::DynHash; +use datafusion_physical_expr_common::physical_expr::{PhysicalExpr, PhysicalExprRef}; +use parquet::bloom_filter::Sbbf; + +use crate::joins::hash_join::exec::HASH_JOIN_SEED; +use crate::joins::key_range_bitmap::KeyRangeBitmap; + +/// Sizing options of a [`BloomFilterExpr`]. +#[derive(Debug, Clone, Copy)] +pub(crate) struct BloomFilterOptions { + /// Target false positive probability. + pub(crate) fpp: f64, + /// Largest filter, in bytes. A build side that needs a larger filter at + /// `fpp` gets no bloom filter. + pub(crate) max_bytes: usize, + /// Do not use the filter when its estimated false positive probability is + /// above this value. + pub(crate) max_fpp: f64, +} + +impl BloomFilterOptions { + /// Bytes of a filter for `num_distinct` keys at `fpp`, or `None` when + /// that is more than `max_bytes`. + fn bytes(&self, num_distinct: usize) -> Option { + let bits = -8.0 * num_distinct as f64 / (1.0 - self.fpp.powf(1.0 / 8.0)).ln(); + let bytes = (bits / 8.0) as usize; + (bytes <= self.max_bytes).then_some(bytes) + } +} + +fn parquet_err(e: parquet::errors::ParquetError) -> DataFusionError { + DataFusionError::External(Box::new(e)) +} + +/// Rows hashed at a time when filling a filter. +const HASH_BATCH_SIZE: usize = 8192; + +/// Hashes `keys` like the hash join and calls `f` with the hashes of each +/// batch of rows. A row with a NULL key gets no hash. +fn for_each_hash_batch( + keys: &[ArrayRef], + mut f: impl FnMut(&[u64]) -> Result<()>, +) -> Result<()> { + let num_rows = keys.first().map_or(0, |k| k.len()); + let mut hashes = Vec::with_capacity(HASH_BATCH_SIZE.min(num_rows)); + for offset in (0..num_rows).step_by(HASH_BATCH_SIZE) { + let len = HASH_BATCH_SIZE.min(num_rows - offset); + let batch: Vec = keys.iter().map(|k| k.slice(offset, len)).collect(); + hashes.clear(); + hashes.resize(len, 0); + create_hashes(&batch, HASH_JOIN_SEED.random_state(), &mut hashes)?; + if let Some(valid) = valid_rows(&batch) { + let mut num_valid = 0; + for i in valid.valid_indices() { + hashes[num_valid] = hashes[i]; + num_valid += 1; + } + hashes.truncate(num_valid); + } + f(&hashes)?; + } + Ok(()) +} + +/// The rows where no key is NULL, or `None` when no key has NULLs. +fn valid_rows(keys: &[ArrayRef]) -> Option { + let nulls: Vec<_> = keys.iter().map(|k| k.logical_nulls()).collect(); + NullBuffer::union_many(nulls.iter().map(Option::as_ref)) +} + +/// The bloom filter of the keys of one build partition, sized for +/// `num_distinct` keys (the estimated number of distinct keys of all +/// partitions, so that the partition filters can be ORed). `None` when that +/// filter would be larger than `options.max_bytes`. +/// +/// The filter memory is charged to `reservation`. `None` also when it cannot +/// grow. +pub(crate) fn partition_filter( + keys: &[ArrayRef], + num_distinct: usize, + options: BloomFilterOptions, + reservation: &MemoryReservation, +) -> Result> { + let Some(bytes) = options.bytes(num_distinct) else { + return Ok(None); + }; + // The size that `Sbbf::new_with_num_of_bytes` allocates + if reservation + .try_grow(bytes.max(32).next_power_of_two()) + .is_err() + { + return Ok(None); + } + let mut filter = Sbbf::new_with_num_of_bytes(bytes); + for_each_hash_batch(keys, |hashes| { + filter.insert_hashes(hashes); + Ok(()) + })?; + Ok(Some(filter)) +} + +/// The bloom filter of a [`BloomFilterExpr`] and the reservation for its +/// memory, which is released when the filter is dropped. +struct Filter { + filter: Sbbf, + _reservation: MemoryReservation, +} + +/// ORs the filters of all build partitions and folds the result. Returns +/// `None` (no bloom filter) when `pool` cannot hold the filter or it is +/// saturated. +fn merge( + partitions: &[&Sbbf], + options: BloomFilterOptions, + pool: &Arc, +) -> Result> { + // Partitions estimate the number of keys of all partitions from their own, + // so their filters can differ in size: fold the larger ones into the + // smallest. + let Some((smallest, first)) = partitions + .iter() + .enumerate() + .min_by_key(|(_, f)| f.num_blocks()) + .map(|(i, f)| (i, *f)) + else { + return Ok(None); + }; + let reservation = MemoryConsumer::new("HashJoinBloomFilter").register(pool); + if reservation.try_grow(first.num_blocks() * 32).is_err() { + return Ok(None); + } + let mut filter = first.clone(); + for (i, partition) in partitions.iter().enumerate() { + if i != smallest { + filter.union(partition).map_err(parquet_err)?; + } + } + filter.fold_to_target_fpp(options.fpp); + // A saturated filter costs CPU time and removes almost no rows + if filter.estimated_fpp() > options.max_fpp { + return Ok(None); + } + reservation.resize(filter.num_blocks() * 32); + Ok(Some(Filter { + filter, + _reservation: reservation, + })) +} + +/// Physical expression that tests join keys against a bloom filter of the +/// build side keys of a hash join. +/// +/// The result is `true` for a row whose keys may be on the build side, and +/// `false` for a row whose keys are definitely not on the build side (or are +/// NULL). The filter holds the hash join hashes of the keys (all key columns +/// together), so it supports every key type that the hash join supports. +pub struct BloomFilterExpr { + /// Probe side join key expressions + on_columns: Vec, + /// `None` when no bloom filter was built + filter: Option>, + /// Which parts of the build side's key range hold keys, for container pruning + pruning_bitmap: Option>, +} + +impl BloomFilterExpr { + /// ORs the bloom filters of every build partition ([`partition_filter`]) + /// into one filter, charged to `pool`. + /// + /// When no useful filter is possible (`partitions` is `None` because a + /// partition has no filter, `pool` cannot hold the filter, or the filter + /// is saturated), the expression has no filter: it is `true` for every + /// row and only carries `pruning_bitmap` (see [`Self::has_filter`]). + pub(crate) fn try_new( + on_columns: Vec, + partitions: Option<&[&Sbbf]>, + options: BloomFilterOptions, + pool: &Arc, + pruning_bitmap: Option>, + ) -> Result { + let filter = match partitions { + Some(partitions) => merge(partitions, options, pool)?.map(Arc::new), + None => None, + }; + Ok(Self { + on_columns, + filter, + pruning_bitmap, + }) + } + + /// Returns `true` if the expression tests rows against a bloom filter. + /// Without a filter it only carries the pruning bitmap. + pub fn has_filter(&self) -> bool { + self.filter.is_some() + } + + /// Which parts of the build side's key range hold keys, for container pruning. + pub fn pruning_bitmap(&self) -> Option<&Arc> { + self.pruning_bitmap.as_ref() + } + + /// Bytes of the bloom filter, 0 without a filter. + #[cfg(test)] + pub(crate) fn filter_bytes(&self) -> usize { + self.filter + .as_ref() + .map_or(0, |filter| filter.filter.num_blocks() * 32) + } + + /// The probe side key expressions. + pub fn on_columns(&self) -> &[PhysicalExprRef] { + &self.on_columns + } +} + +impl fmt::Debug for BloomFilterExpr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let cols = self + .on_columns + .iter() + .map(|e| e.to_string()) + .collect::>() + .join(", "); + write!(f, "bloom_filter({cols})") + } +} + +impl Hash for BloomFilterExpr { + fn hash(&self, state: &mut H) { + self.on_columns.dyn_hash(state); + self.filter.as_ref().map(Arc::as_ptr).hash(state); + } +} + +impl PartialEq for BloomFilterExpr { + fn eq(&self, other: &Self) -> bool { + self.on_columns == other.on_columns + && match (&self.filter, &other.filter) { + (Some(a), Some(b)) => Arc::ptr_eq(a, b), + (None, None) => true, + _ => false, + } + } +} + +impl Eq for BloomFilterExpr {} + +impl Display for BloomFilterExpr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.has_filter() { + write!(f, "bloom_filter") + } else { + write!(f, "key_range_pruning") + } + } +} + +impl PhysicalExpr for BloomFilterExpr { + fn children(&self) -> Vec<&Arc> { + self.on_columns.iter().collect() + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + Ok(Arc::new(Self { + on_columns: children, + filter: self.filter.clone(), + pruning_bitmap: self.pruning_bitmap.clone(), + })) + } + + fn data_type(&self, _input_schema: &Schema) -> Result { + Ok(DataType::Boolean) + } + + fn nullable(&self, _input_schema: &Schema) -> Result { + Ok(false) + } + + fn evaluate(&self, batch: &RecordBatch) -> Result { + let Some(filter) = &self.filter else { + // No filter: only the pruning bitmap, which is not tested per row + return Ok(ColumnarValue::Scalar(ScalarValue::Boolean(Some(true)))); + }; + let num_rows = batch.num_rows(); + let keys = self + .on_columns + .iter() + .map(|c| c.evaluate(batch)?.into_array(num_rows)) + .collect::>>()?; + let mut hashes = vec![0; num_rows]; + create_hashes(&keys, HASH_JOIN_SEED.random_state(), &mut hashes)?; + let matches = filter.filter.check_hashes(&hashes); + // A NULL key matches no build key + let matches = match valid_rows(&keys) { + Some(valid) => &matches & valid.inner(), + None => matches, + }; + Ok(ColumnarValue::Array(Arc::new(BooleanArray::new( + matches, None, + )))) + } + + #[cfg(feature = "proto")] + fn try_to_proto( + &self, + _ctx: &datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>, + ) -> Result> { + use datafusion_proto_models::protobuf; + use datafusion_proto_models::protobuf::physical_expr_node::ExprType; + + // The expression holds runtime build side + // state that is not serialized. It is replaced with lit(true), which + // keeps every row: the dynamic filter is only an optimization. A plan + // that is executed again calls reset_state(), and the join builds a new + // dynamic filter at runtime. + let Self { + on_columns: _, + filter: _, + pruning_bitmap: _, + } = self; + let value = datafusion_proto_common::ScalarValue { + value: Some(datafusion_proto_common::scalar_value::Value::BoolValue( + true, + )), + }; + Ok(Some(protobuf::PhysicalExprNode { + expr_id: None, + expr_type: Some(ExprType::Literal(value)), + })) + } + + fn fmt_sql(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{self}") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{Float64Array, Int64Array, StringArray}; + use arrow::datatypes::Field; + use datafusion_execution::memory_pool::{GreedyMemoryPool, UnboundedMemoryPool}; + use datafusion_physical_expr::expressions::Column; + + const OPTIONS: BloomFilterOptions = BloomFilterOptions { + fpp: 0.01, + max_bytes: 8 * 1024 * 1024, + max_fpp: 0.15, + }; + + fn unbounded() -> Arc { + Arc::new(UnboundedMemoryPool::default()) + } + + /// The filters of `partitions`, each sized for the keys of all partitions. + fn partition_filters( + partitions: &[Vec], + options: BloomFilterOptions, + ) -> Option> { + let num_distinct = partitions.iter().map(|p| p[0].len()).sum(); + partitions + .iter() + .map(|keys| filter_of(keys, num_distinct, options)) + .collect() + } + + fn filter_of( + keys: &[ArrayRef], + num_distinct: usize, + options: BloomFilterOptions, + ) -> Option { + let reservation = MemoryConsumer::new("test").register(&unbounded()); + partition_filter(keys, num_distinct, options, &reservation).unwrap() + } + + fn expr_with( + columns: Vec, + partitions: &[Vec], + options: BloomFilterOptions, + pool: &Arc, + ) -> Option { + let filters = partition_filters(partitions, options); + let refs = filters.as_ref().map(|f| f.iter().collect::>()); + let expr = + BloomFilterExpr::try_new(columns, refs.as_deref(), options, pool, None) + .unwrap(); + expr.has_filter().then_some(expr) + } + + fn int_keys(keys: Vec) -> Vec { + vec![Arc::new(Int64Array::from(keys))] + } + + fn int_expr_with( + partitions: Vec>, + options: BloomFilterOptions, + pool: &Arc, + ) -> Option { + let partitions: Vec<_> = partitions.into_iter().map(int_keys).collect(); + expr_with( + vec![Arc::new(Column::new("k", 0))], + &partitions, + options, + pool, + ) + } + + fn int_expr(partitions: Vec>) -> BloomFilterExpr { + int_expr_with(partitions, OPTIONS, &unbounded()).unwrap() + } + + fn int_batch(keys: Vec>) -> RecordBatch { + let schema = Schema::new(vec![Field::new("k", DataType::Int64, true)]); + RecordBatch::try_new(Arc::new(schema), vec![Arc::new(Int64Array::from(keys))]) + .unwrap() + } + + fn eval(expr: &BloomFilterExpr, batch: &RecordBatch) -> Vec { + let result = expr + .evaluate(batch) + .unwrap() + .into_array(batch.num_rows()) + .unwrap(); + let result = result.as_any().downcast_ref::().unwrap(); + result.iter().map(|v| v.unwrap()).collect() + } + + #[test] + fn finds_build_keys_and_rejects_others() { + let expr = int_expr(vec![(0..1000).collect(), (5000..6000).collect()]); + let batch = int_batch((0..10_000).map(Some).collect()); + let result = eval(&expr, &batch); + // No false negatives + for k in (0..1000).chain(5000..6000) { + assert!(result[k], "key {k} dropped"); + } + // Few false positives + let false_positives = result.iter().filter(|v| **v).count() - 2000; + assert!(false_positives < 200, "{false_positives} false positives"); + } + + #[test] + fn null_keys_do_not_match() { + // A NULL build key is not inserted, a NULL probe key does not match + let build: ArrayRef = Arc::new(Int64Array::from(vec![Some(1), None, Some(3)])); + let expr = expr_with( + vec![Arc::new(Column::new("k", 0))], + &[vec![build]], + OPTIONS, + &unbounded(), + ) + .unwrap(); + let batch = int_batch(vec![Some(1), None, Some(3)]); + assert_eq!(eval(&expr, &batch), vec![true, false, true]); + } + + /// ORing the partition filters gives the filter of all keys. + #[test] + fn keys_with_null_buffers_but_no_nulls() { + // A null buffer can have no NULLs, e.g. after slicing off the NULL rows + let key: ArrayRef = + Arc::new(Int64Array::from(vec![None, Some(1), Some(2)]).slice(1, 2)); + assert!(key.logical_nulls().is_some()); + assert_eq!(valid_rows(&[Arc::clone(&key), Arc::clone(&key)]), None); + } + + #[test] + fn merged_filter_matches_one_filter_of_all_keys() { + let partitions: Vec> = (0..8) + .map(|p| (0..10_000).map(|i| (i * 7919 + p) % 1_000_000).collect()) + .collect(); + let expr = int_expr(partitions.clone()); + let all: Vec = partitions.into_iter().flatten().collect(); + let mut expected = filter_of(&int_keys(all.clone()), all.len(), OPTIONS).unwrap(); + expected.fold_to_target_fpp(OPTIONS.fpp); + let (mut a, mut b) = (vec![], vec![]); + expected.write_bitset(&mut a).unwrap(); + let filter = &expr.filter.as_ref().unwrap().filter; + filter.write_bitset(&mut b).unwrap(); + assert_eq!(a, b); + } + + /// Partition filters of different sizes (from different estimates of the + /// number of keys) are folded into the smallest. + #[test] + fn partition_filters_of_different_sizes() { + let small = filter_of(&int_keys((0..100).collect()), 1000, OPTIONS).unwrap(); + let large = + filter_of(&int_keys((1000..1100).collect()), 100_000, OPTIONS).unwrap(); + assert!(large.num_blocks() > small.num_blocks()); + let expr = BloomFilterExpr::try_new( + vec![Arc::new(Column::new("k", 0))], + Some(&[&large, &small]), + OPTIONS, + &unbounded(), + None, + ) + .unwrap(); + let result = eval(&expr, &int_batch((0..1100).map(Some).collect())); + assert!(result[..100].iter().all(|v| *v)); + assert!(result[1000..].iter().all(|v| *v)); + } + + /// A build side that needs a filter larger than `max_bytes` at the target + /// fpp gets no bloom filter: it is too big to be useful. + #[test] + fn too_big_build_side_has_no_filter() { + let options = BloomFilterOptions { + max_bytes: 1024, + ..OPTIONS + }; + assert!( + int_expr_with(vec![(0..10_000).collect()], options, &unbounded()).is_none() + ); + assert!(int_expr_with(vec![(0..100).collect()], options, &unbounded()).is_some()); + } + + /// A saturated filter is not used. + #[test] + fn saturated_filter_is_not_used() { + // Sized for 10 keys, filled with 10k keys + let keys = int_keys((0..10_000).collect()); + let filter = filter_of(&keys, 10, OPTIONS).unwrap(); + let expr = BloomFilterExpr::try_new( + vec![Arc::new(Column::new("k", 0))], + Some(&[&filter]), + OPTIONS, + &unbounded(), + None, + ) + .unwrap(); + assert!(!expr.has_filter()); + assert_eq!(expr.to_string(), "key_range_pruning"); + let batch = int_batch(vec![Some(-1)]); + assert_eq!(eval(&expr, &batch), vec![true]); + } + + /// The filter memory is charged to the pool and released on drop. + #[test] + fn memory_is_reserved_and_released() { + let pool: Arc = Arc::new(GreedyMemoryPool::new(64 << 20)); + let expr = int_expr_with(vec![(0..100_000).collect()], OPTIONS, &pool).unwrap(); + let built = &expr.filter.as_ref().unwrap().filter; + assert_eq!(pool.reserved(), built.num_blocks() * 32); + drop(expr); + assert_eq!(pool.reserved(), 0); + } + + /// When the pool cannot hold the filter, no filter is built. The query + /// does not fail. + #[test] + fn small_memory_pool_skips_the_filter() { + let pool: Arc = Arc::new(GreedyMemoryPool::new(1024)); + assert!(int_expr_with(vec![(0..100_000).collect()], OPTIONS, &pool).is_none()); + assert_eq!(pool.reserved(), 0); + } + + #[test] + fn multi_column_keys() { + let keys = vec![ + Arc::new(Int64Array::from(vec![1, 2])) as ArrayRef, + Arc::new(StringArray::from(vec!["a", "b"])) as ArrayRef, + ]; + let expr = expr_with( + vec![Arc::new(Column::new("k", 0)), Arc::new(Column::new("s", 1))], + &[keys], + OPTIONS, + &unbounded(), + ) + .unwrap(); + let schema = Schema::new(vec![ + Field::new("k", DataType::Int64, true), + Field::new("s", DataType::Utf8, true), + ]); + let batch = RecordBatch::try_new( + Arc::new(schema), + vec![ + Arc::new(Int64Array::from(vec![1, 2, 1, 99])), + Arc::new(StringArray::from(vec!["a", "b", "zzz", "a"])), + ], + ) + .unwrap(); + assert_eq!(eval(&expr, &batch), vec![true, true, false, false]); + } + + /// Any key type the hash join hashes is supported, floats included. + #[test] + fn float_keys() { + let keys: ArrayRef = Arc::new(Float64Array::from(vec![1.5, 2.5])); + let expr = expr_with( + vec![Arc::new(Column::new("f", 0))], + &[vec![keys]], + OPTIONS, + &unbounded(), + ) + .unwrap(); + let schema = Schema::new(vec![Field::new("f", DataType::Float64, true)]); + let batch = RecordBatch::try_new( + Arc::new(schema), + vec![Arc::new(Float64Array::from(vec![1.5, 2.5, 3.5]))], + ) + .unwrap(); + assert_eq!(eval(&expr, &batch), vec![true, true, false]); + } + + /// The filter holds runtime build side state, so it serializes as lit(true). + #[cfg(feature = "proto")] + #[test] + fn serializes_as_literal_true() { + use datafusion_common::internal_datafusion_err; + use datafusion_physical_expr_common::physical_expr::proto_encode::{ + PhysicalExprEncode, PhysicalExprEncodeCtx, + }; + use datafusion_proto_models::protobuf; + + struct TestEncoder; + impl PhysicalExprEncode for TestEncoder { + fn encode( + &self, + expr: &Arc, + ) -> Result { + let ctx = PhysicalExprEncodeCtx::new(self); + expr.try_to_proto(&ctx)? + .ok_or_else(|| internal_datafusion_err!("cannot encode {expr:?}")) + } + } + + let expr: Arc = Arc::new(int_expr(vec![(0..1000).collect()])); + let proto = TestEncoder.encode(&expr).unwrap(); + let Some(protobuf::physical_expr_node::ExprType::Literal(value)) = + proto.expr_type + else { + panic!("expected a literal, got {proto:?}"); + }; + assert_eq!( + value.value, + Some(datafusion_proto_common::scalar_value::Value::BoolValue( + true + )) + ); + } +} diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs index ffa77f55892ee..2025dbaf3827b 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs @@ -18,10 +18,12 @@ use crate::filter_pushdown::{ }; use crate::joins::Map; use crate::joins::array_map::ArrayMap; +use crate::joins::hash_join::bloom_filter; use crate::joins::hash_join::inlist_builder::build_struct_inlist_values; use crate::joins::hash_join::not_in::{NotIn, NotInBuild}; use crate::joins::hash_join::shared_bounds::{ - ColumnBounds, PartitionBounds, PushdownStrategy, SharedBuildAccumulator, + ColumnBounds, GlobalFilterOptions, PartitionBounds, PushdownStrategy, + SharedBuildAccumulator, }; use crate::joins::hash_join::stream::{ BuildSide, BuildSideInitialState, HashJoinStream, HashJoinStreamState, @@ -40,7 +42,6 @@ use crate::projection::{ EmbeddedProjection, JoinData, ProjectionExec, try_embed_projection, try_pushdown_through_join_with_column_indices, }; -use crate::repartition::REPARTITION_RANDOM_STATE; use crate::statistics::{ChildStats, StatisticsArgs, with_per_partition_fetch}; use crate::{ ChildrenPropertiesMode, ExecutionPlanProperties, ReplaceChildrenOptions, @@ -58,6 +59,7 @@ use crate::{ }, metrics::{ExecutionPlanMetricsSet, MetricsSet}, }; +use parquet::bloom_filter::Sbbf; use arrow::array::{Array, ArrayData, ArrayRef, BooleanBufferBuilder}; use arrow::compute::concat_batches; @@ -257,6 +259,9 @@ struct JoinBuildData { bounds: Option, /// IN-list values or a hash-table reference used for filter pushdown. membership: PushdownStrategy, + /// Bloom filter of the build keys, for the global dynamic filter. `None` + /// when no bloom filter is possible for this build. + bloom_filter: Option>, // Keep the reservation after the allocations it accounts for. reservation: MemoryReservation, } @@ -324,6 +329,11 @@ impl JoinLeftData { &self.build.membership } + /// Bloom filter of the build keys, for the dynamic filter. + pub(super) fn bloom_filter(&self) -> Option<&Arc> { + self.build.bloom_filter.as_ref() + } + /// Borrow bounds without sharing a consuming join's dynamic-filter state. pub(super) fn bounds(&self) -> Option<&PartitionBounds> { self.build.bounds.as_ref() @@ -1103,57 +1113,12 @@ impl HashJoinExec { return false; } - // Under `preserve_file_partitions`, file groups can report Hash without following - // the dynamic filter hash router and look like a real hash repartition, so reject - // Hash. Range inputs with matching split points are safe. - // Follow-up work: enable dynamic filtering for preserve_file_partitioned scans (issue #20195). - // https://github.com/apache/datafusion/issues/20195 - if config.optimizer.preserve_file_partitions > 0 - && self.mode == PartitionMode::Partitioned - && matches!( - ( - self.left.output_partitioning(), - self.right.output_partitioning() - ), - (Partitioning::Hash(_, _), Partitioning::Hash(_, _)) - ) - { - return false; - } - - if self.mode == PartitionMode::Partitioned - && !self.has_partitioned_dynamic_filter_routing() - { - return false; - } - + // The partitioning of the inputs does not matter. The filter holds the + // keys of all build partitions, so it passes every probe row that any + // build partition can match. true } - fn has_partitioned_dynamic_filter_routing(&self) -> bool { - match ( - self.left.output_partitioning(), - self.right.output_partitioning(), - ) { - ( - Partitioning::Hash(_, left_partition_count), - Partitioning::Hash(_, right_partition_count), - ) => left_partition_count == right_partition_count, - (Partitioning::Range(_), Partitioning::Range(_)) => { - let children = [self.left.as_ref(), self.right.as_ref()]; - matches!( - self.input_distribution_requirements() - .unsatisfied_co_partitioned_children(self.name(), &children), - Ok(unsatisfied) if unsatisfied.is_empty() - ) - } - (left_partitioning, right_partitioning) => { - left_partitioning.partition_count() == 1 - && right_partitioning.partition_count() == 1 - } - } - } - /// left (build) side which gets hashed pub fn left(&self) -> &Arc { &self.left @@ -1725,8 +1690,6 @@ impl ExecutionPlan for HashJoinExec { .counter(ARRAY_MAP_CREATED_COUNT_METRIC_NAME, partition); // Initialize build_accumulator lazily with runtime partition counts (only if enabled) - // Use RepartitionExec's random state (seeds: 0,0,0,0) for partition routing - let repartition_random_state = REPARTITION_RANDOM_STATE; let build_accumulator = enable_dynamic_filter_pushdown .then(|| { self.dynamic_filter.as_ref().map(|df| { @@ -1743,10 +1706,10 @@ impl ExecutionPlan for HashJoinExec { self.right.as_ref(), filter, on_right, - repartition_random_state, self.null_equality, not_in, - &context.session_config().options().optimizer, + GlobalFilterOptions::new(context.session_config().options()) + .with_memory_pool(Arc::clone(context.memory_pool())), )) }))) }) @@ -1783,6 +1746,7 @@ impl ExecutionPlan for HashJoinExec { not_in, array_map_created_count, BuildMode::Ordinary, + 1, )) })?, (None, PartitionMode::Partitioned) => { @@ -1805,6 +1769,7 @@ impl ExecutionPlan for HashJoinExec { not_in, array_map_created_count, BuildMode::Ordinary, + self.left.output_partitioning().partition_count(), )) } (None, PartitionMode::Auto) => { @@ -3127,6 +3092,7 @@ async fn collect_left_input( not_in: Option, array_map_created_count: Count, mode: BuildMode, + num_build_partitions: usize, ) -> Result { let schema = left_stream.schema(); let prepared = mode == BuildMode::Prepared; @@ -3220,6 +3186,7 @@ async fn collect_left_input( not_in, &array_map_created_count, prepared, + num_build_partitions, ) } @@ -3251,6 +3218,7 @@ fn build_in_memory( not_in: Option, array_map_created_count: &Count, prepared: bool, + num_build_partitions: usize, ) -> Result { let is_phj_candidate = is_perfect_hash_join_candidate(on_left, schema)?; @@ -3399,7 +3367,6 @@ fn build_in_memory( PushdownStrategy::Empty } else { // If the build side is small enough we can use IN list pushdown. - // If it's too big we fall back to pushing down a reference to the hash table. // See `PushdownStrategy` for more details. let estimated_size = left_values .iter() @@ -3443,7 +3410,7 @@ fn build_in_memory( } ok }); - PushdownStrategy::Map(Arc::clone(&map), pruning_bitmap) + PushdownStrategy::Large(pruning_bitmap) } }; @@ -3451,6 +3418,29 @@ fn build_in_memory( bounds = None; } + // Fill the bloom filter of this partition's keys here, in its build task, + // so the finalizing task only ORs the partition filters. Partitions hold + // disjoint keys of similar counts, so the filter is sized for the keys of + // all partitions estimated from this one. + let bloom_filter = if !should_compute_dynamic_filters + || matches!(membership, PushdownStrategy::Empty) + || left_values.is_empty() + { + None + } else { + let filter = bloom_filter::partition_filter( + &left_values, + map.num_of_distinct_key() + .saturating_mul(num_build_partitions), + GlobalFilterOptions::new(config).bloom_filter, + &reservation, + )?; + if let Some(filter) = &filter { + metrics.build_mem_used.add(filter.num_blocks() * 32); + } + filter.map(Arc::new) + }; + if prepared { batches.clear(); // Prepared keys are direct columns. IN-list arrays share these batch @@ -3471,6 +3461,7 @@ fn build_in_memory( values: normalized_values, bounds, membership, + bloom_filter, reservation, }), not_in, @@ -3704,148 +3695,51 @@ mod tests { Ok(()) } - /// The dynamic filter shares the build map and can outlive the join's - /// stream and plan, so the map stays reserved until the last holder drops - /// it, not only until the partition's stream ends. - #[rstest] - #[tokio::test] - async fn build_map_stays_reserved_with_dynamic_filter( - #[values(PartitionMode::CollectLeft, PartitionMode::Partitioned)] - mode: PartitionMode, - ) -> Result<()> { - let SparseKeyJoinInputs { build, probe, on } = sparse_key_join_inputs()?; - let (join, filter) = hash_join_with_dynamic_filter_and_mode( - build, - probe, - on, - JoinType::Inner, - mode, - )?; - let join: Arc = Arc::new(join); - - let pool: Arc = Arc::new(GreedyMemoryPool::new(1 << 20)); - let runtime = RuntimeEnvBuilder::new() - .with_memory_pool(Arc::clone(&pool)) - .build_arc()?; - let mut config = SessionConfig::new(); - config - .options_mut() - .optimizer - .hash_join_inlist_pushdown_max_size = 0; - let task_ctx = Arc::new( - TaskContext::default() - .with_runtime(runtime) - .with_session_config(config), - ); - let batches = common::collect(join.execute(0, task_ctx)?).await?; - assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 3); - - let build_mem_used = join - .metrics() - .unwrap() - .sum_by_name("build_mem_used") - .unwrap() - .as_usize(); - let keys = sparse_keys(); - let map_bytes = - estimate_memory_size::<(u32, u64)>(keys.len(), size_of::())? - + keys.len() * size_of::(); - let keys_array: ArrayRef = Arc::new(Int64Array::from(keys.clone())); - let bitmap_bytes = KeyRangeBitmap::try_new( - &keys_array, - &ScalarValue::Int64(Some(0)), - &ScalarValue::Int64(Some(keys[keys.len() - 1])), - keys.len(), - ) - .map_or(0, |bitmap| bitmap.size()); - let expected = match mode { - // The stream is gone; only the map and its bitmap are still held. - PartitionMode::Partitioned => map_bytes + bitmap_bytes, - PartitionMode::CollectLeft => build_mem_used, - PartitionMode::Auto => unreachable!(), - }; - assert_eq!(pool.reserved(), expected); - - // The filter is the last holder. - drop(join); - assert_eq!(pool.reserved(), map_bytes + bitmap_bytes); - drop(filter); - assert_eq!(pool.reserved(), 0); - Ok(()) + /// Bytes of the bloom filters in `expr`. + fn bloom_filter_bytes(expr: &Arc) -> Result { + let mut bytes = 0; + expr.apply(|e| { + if let Some(bloom) = e.downcast_ref::() { + bytes += bloom.filter_bytes(); + } + Ok(TreeNodeRecursion::Continue) + })?; + Ok(bytes) } - /// Dense keys build an `ArrayMap`, which is charged to the map's own - /// reservation in `array_map_key_range` rather than the build's. + /// The dynamic filter can outlive the join's streams and plan. It holds + /// its own bloom filter, not the build map, so after the plan is dropped + /// the pool holds only the bloom filter, until the filter is dropped. #[rstest] + #[case::collect_left(PartitionMode::CollectLeft, 1)] + #[case::partitioned(PartitionMode::Partitioned, 1)] + #[case::two_hash_partitions(PartitionMode::Partitioned, 2)] #[tokio::test] - async fn array_map_stays_reserved_with_dynamic_filter( - #[values(PartitionMode::CollectLeft, PartitionMode::Partitioned)] - mode: PartitionMode, + async fn dynamic_filter_reserves_only_its_bloom_filter( + #[case] mode: PartitionMode, + #[case] partitions: usize, ) -> Result<()> { - let schema = Arc::new(Schema::new(vec![Field::new("k", DataType::Int64, false)])); - let batch = |keys: Vec| { - let column = Arc::new(Int64Array::from(keys)) as ArrayRef; - let batch = RecordBatch::try_new(Arc::clone(&schema), vec![column])?; - TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None) - }; - let on = vec![( - Arc::new(Column::new_with_schema("k", &schema)?) as _, - Arc::new(Column::new_with_schema("k", &schema)?) as _, - )]; - let (join, filter) = hash_join_with_dynamic_filter_and_mode( - batch((0..20_000).collect())?, - batch(vec![0, 5, 15])?, + let SparseKeyJoinInputs { + mut build, + mut probe, on, - JoinType::Inner, - mode, - )?; - let join: Arc = Arc::new(join); - - let pool: Arc = Arc::new(GreedyMemoryPool::new(1 << 22)); - let runtime = RuntimeEnvBuilder::new() - .with_memory_pool(Arc::clone(&pool)) - .build_arc()?; - let mut config = SessionConfig::new(); - config - .options_mut() - .optimizer - .hash_join_inlist_pushdown_max_size = 0; - let task_ctx = Arc::new( - TaskContext::default() - .with_runtime(runtime) - .with_session_config(config), - ); - let batches = common::collect(join.execute(0, task_ctx)?).await?; - assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 3); - assert_phj_used(&join.metrics().unwrap(), true); - - drop(join); - assert_eq!( - pool.reserved(), - ArrayMap::estimate_memory_size(0, 19_999, 20_000) - ); - drop(filter); - assert_eq!(pool.reserved(), 0); - Ok(()) - } - - /// With rows in more than one build partition, the filter is a routed lookup - /// that holds every partition's map, so the maps stay reserved until it is - /// dropped. - #[cfg(not(feature = "force_hash_collisions"))] - #[tokio::test] - async fn routed_maps_stay_reserved_with_dynamic_filter() -> Result<()> { - let SparseKeyJoinInputs { build, probe, on } = sparse_key_join_inputs()?; - let repartition = |input: Arc, key: &PhysicalExprRef| { - let partitioning = Partitioning::Hash(vec![Arc::clone(key)], 2); - RepartitionExec::try_new(input, partitioning) - }; + } = sparse_key_join_inputs()?; + if partitions > 1 { + let repartition = |input, key: &PhysicalExprRef| { + let partitioning = Partitioning::Hash(vec![Arc::clone(key)], partitions); + Ok::<_, datafusion_common::DataFusionError>(Arc::new( + RepartitionExec::try_new(input, partitioning)?, + ) as _) + }; + build = repartition(build, &on[0].0)?; + probe = repartition(probe, &on[0].1)?; + } let (join, filter) = hash_join_with_dynamic_filter_and_mode( - Arc::new(repartition(build, &on[0].0)?), - Arc::new(repartition(probe, &on[0].1)?), + build, + probe, on, JoinType::Inner, - PartitionMode::Partitioned, + mode, )?; let join: Arc = Arc::new(join); @@ -3865,14 +3759,11 @@ mod tests { ); let batches = crate::execution_plan::collect(Arc::clone(&join), task_ctx).await?; assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 3); - assert!(format!("{:?}", filter.current()?).contains("routed by")); - // Every stream is gone; only the maps and their bitmaps are still held. - let reserved = pool.reserved(); - assert!(reserved > 0); - // The filter is the last holder. + let bloom_bytes = bloom_filter_bytes(&filter.current()?)?; + assert!(bloom_bytes > 0); drop(join); - assert_eq!(pool.reserved(), reserved); + assert_eq!(pool.reserved(), bloom_bytes); drop(filter); assert_eq!(pool.reserved(), 0); Ok(()) @@ -4550,6 +4441,7 @@ mod tests { None, Count::new(), BuildMode::Ordinary, + 1, ) .await?; Ok((left_data, metrics)) @@ -9373,7 +9265,7 @@ mod tests { &child_right_schema, )?) as _, )]; - let (child_join, _child_dynamic_filter) = hash_join_with_dynamic_filter_and_mode( + let (child_join, child_dynamic_filter) = hash_join_with_dynamic_filter_and_mode( child_left, child_right, child_on, @@ -9416,6 +9308,8 @@ mod tests { ], &batches ); + // The parent drops child partitions 1 and 3, so their keys are unknown. + assert_eq!(child_dynamic_filter.current()?.to_string(), "true"); Ok(()) } @@ -10546,9 +10440,7 @@ mod tests { /// /// Here the build-side logical NULL lives in exactly one build partition, so /// the `keys_have_null` flag has to survive the per-partition aggregation in - /// `build_partitioned_filter` and widen the *routed* filter as a whole: the - /// probe NULL is routed to a single `CASE` branch, and OR-ing `IS NULL` into - /// that one branch would still drop it for every other route. + /// `build_partitioned_filter` and widen the global filter as a whole. #[tokio::test] async fn test_partitioned_null_equal_dynamic_filter_keeps_probe_nulls_for_build_logical_null() -> Result<()> { @@ -10601,7 +10493,7 @@ mod tests { // The FilterExec sits above the repartition so each probe partition is only // read after the join has waited for every partition's build report, which - // makes the finalized (routed and widened) filter the one actually applied. + // makes the finalized (widened) filter the one actually applied. let dynamic_filter = HashJoinExec::create_dynamic_filter(&on); let consumer: Arc = Arc::clone(&dynamic_filter) as _; let right_filtered: Arc = @@ -10627,11 +10519,11 @@ mod tests { dynamic_filter.wait_complete().await; let filter = dynamic_filter.current()?.to_string(); - // The aggregated build-side NULL must widen the whole merged filter once, - // at the top: `c2 IS NULL OR `. + // The aggregated build-side NULL must widen the global filter once, at the + // top: `c2 IS NULL OR `. assert!( filter.starts_with("c2@0 IS NULL OR"), - "expected the IS NULL disjunct to wrap the whole routed filter, got: {filter}" + "expected the IS NULL disjunct to wrap the whole filter, got: {filter}" ); // The NULL-NULL row (d1 = 200, d2 = 20) is only produced if the probe @@ -12309,52 +12201,32 @@ mod tests { .build() } + /// The filter holds the keys of all build partitions, so it does not + /// depend on how the inputs are partitioned. #[test] - fn test_partitioned_dynamic_filter_pushdown_allows_supported_partitioning() - -> Result<()> { - let (range_join, on) = range_partitioned_dynamic_filter_test_join(10, 10)?; - let hash_join = with_hash_partitioned_children(&range_join, &on)?; - let mut session_config = SessionConfig::default(); - session_config - .options_mut() - .optimizer - .enable_join_dynamic_filter_pushdown = true; - - assert!(range_join.allow_join_dynamic_filter_pushdown(session_config.options())); - assert!(hash_join.allow_join_dynamic_filter_pushdown(session_config.options())); - - session_config - .options_mut() - .optimizer - .preserve_file_partitions = 1; - assert!(range_join.allow_join_dynamic_filter_pushdown(session_config.options())); - - Ok(()) - } - - #[test] - fn test_partitioned_dynamic_filter_pushdown_rejects_unsupported_partitioning() - -> Result<()> { + fn test_partitioned_dynamic_filter_pushdown_allows_any_partitioning() -> Result<()> { let (range_join, on) = range_partitioned_dynamic_filter_test_join(10, 10)?; let hash_join = with_hash_partitioned_children(&range_join, &on)?; let (mismatched_range_join, _) = range_partitioned_dynamic_filter_test_join(10, 11)?; + let joins = [&range_join, &hash_join, &mismatched_range_join]; let mut session_config = SessionConfig::default(); session_config .options_mut() .optimizer .enable_join_dynamic_filter_pushdown = true; - assert!( - !mismatched_range_join - .allow_join_dynamic_filter_pushdown(session_config.options()) - ); - - session_config - .options_mut() - .optimizer - .preserve_file_partitions = 1; - assert!(!hash_join.allow_join_dynamic_filter_pushdown(session_config.options())); + for preserve_file_partitions in [0, 1] { + session_config + .options_mut() + .optimizer + .preserve_file_partitions = preserve_file_partitions; + for join in joins { + assert!( + join.allow_join_dynamic_filter_pushdown(session_config.options()) + ); + } + } Ok(()) } diff --git a/datafusion/physical-plan/src/joins/hash_join/exec/prepared.rs b/datafusion/physical-plan/src/joins/hash_join/exec/prepared.rs index 70bdfdd0988b7..8c819a3314ab9 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec/prepared.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec/prepared.rs @@ -183,6 +183,7 @@ impl HashJoinExec { None, count, BuildMode::Prepared, + 1, ) .await?; Ok(Arc::new(PreparedHashJoinBuild { diff --git a/datafusion/physical-plan/src/joins/hash_join/exec/prepared/tests.rs b/datafusion/physical-plan/src/joins/hash_join/exec/prepared/tests.rs index 4f874f8c95af2..e90c3b384240d 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec/prepared/tests.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec/prepared/tests.rs @@ -784,8 +784,8 @@ async fn prepared_inlist_shares_accounted_build_buffers() -> Result<()> { Arc::new(config), ) .await?; - let PushdownStrategy::Map(_, bitmap) = &map.build.membership else { - panic!("expected hash-map membership"); + let PushdownStrategy::Large(bitmap) = &map.build.membership else { + panic!("expected large membership"); }; let bitmap_bytes = bitmap.as_ref().map_or(0, |bitmap| bitmap.size()); assert_eq!(map.reserved_bytes(), bytes + bitmap_bytes); @@ -880,7 +880,7 @@ async fn prepared_byte_keys_use_hash_membership() -> Result<()> { assert!(prepared.build.bounds.is_none()); assert!(matches!( &prepared.build.membership, - PushdownStrategy::Map(_, _) + PushdownStrategy::Large(_) )); let task = with_probe_filter(base)? .builder() diff --git a/datafusion/physical-plan/src/joins/hash_join/mod.rs b/datafusion/physical-plan/src/joins/hash_join/mod.rs index 2254b38200242..34b3c26d4b05c 100644 --- a/datafusion/physical-plan/src/joins/hash_join/mod.rs +++ b/datafusion/physical-plan/src/joins/hash_join/mod.rs @@ -4,16 +4,17 @@ //! [`HashJoinExec`] Partitioned Hash Join Operator -pub use exec::{HashJoinExec, HashJoinExecBuilder, PreparedHashJoinBuild}; /// # Public Only for Internal Use: /// /// This is not a public API and is for internal use only; see [API policy] for details. /// /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis #[doc(hidden)] -pub use partitioned_hash_eval::HashTableLookupExpr; +pub use bloom_filter::BloomFilterExpr; +pub use exec::{HashJoinExec, HashJoinExecBuilder, PreparedHashJoinBuild}; pub use partitioned_hash_eval::{HashExpr, SeededRandomState}; +mod bloom_filter; mod exec; mod inlist_builder; mod not_in; diff --git a/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rs b/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rs index ad52f23a52189..caf9e3e1a481e 100644 --- a/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rs +++ b/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rs @@ -2,26 +2,22 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -//! Hash computation and hash table lookup expressions for dynamic filtering +//! Hash computation expression for dynamic filtering use std::{fmt::Display, hash::Hash, sync::Arc}; use arrow::{ - array::{ArrayRef, BooleanArray, BooleanBufferBuilder, UInt64Array}, + array::{ArrayRef, UInt64Array}, datatypes::{DataType, Schema}, record_batch::RecordBatch, }; +use datafusion_common::Result; use datafusion_common::hash_utils::RandomState; -use datafusion_common::hash_utils::{create_hashes, with_hashes}; -use datafusion_common::{Result, internal_err}; +use datafusion_common::hash_utils::create_hashes; use datafusion_expr::ColumnarValue; use datafusion_expr_common::dyn_eq::DynHash; use datafusion_physical_expr_common::physical_expr::{PhysicalExpr, PhysicalExprRef}; -use crate::joins::key_range_bitmap::KeyRangeBitmap; -use crate::joins::{Map, ReservedMap}; -use crate::repartition::RangeExpr; - /// RandomState wrapper that preserves the seed used to create it. /// /// This is needed because `RandomState` doesn't expose its seed after creation, @@ -239,7 +235,9 @@ impl HashExpr { let Some(protobuf::physical_expr_node::ExprType::HashExpr(hash_expr)) = &node.expr_type else { - return internal_err!("PhysicalExprNode is not a HashExpr"); + return datafusion_common::internal_err!( + "PhysicalExprNode is not a HashExpr" + ); }; // Destructure exhaustively (no `..`) so that a newly added proto field // is a compile error here instead of being silently ignored. @@ -258,402 +256,6 @@ impl HashExpr { } } -/// Physical expression that checks join keys in a [`Map`] (hash table or array map). -/// -/// Returns a [`BooleanArray`] indicating if join keys (from `on_columns`) exist in the map. -// TODO: rename to MapLookupExpr -pub struct HashTableLookupExpr { - /// Columns in the ON clause used to compute the join key for lookups - on_columns: Vec, - /// Random state for hashing (with seeds preserved for serialization) - random_state: SeededRandomState, - /// Map(s) to check against, with their reservations - lookup: Lookup, - /// Description for display - description: String, - /// Shared pruning-only view of the build side's key range - pruning_bitmap: Option>, -} - -/// The build side a [`HashTableLookupExpr`] checks probe keys against. -#[derive(Clone)] -enum Lookup { - /// One map holding every build key, probed directly with no routing. - Single(Arc), - /// One hash table per build partition of a `Partitioned` join, `None` where - /// the partition is empty. A key is only ever built into the partition its - /// key routes to, so each probe row is looked up in that partition's table - /// alone: no `CASE` over per-partition filters, and no merged table. - Routed { - router: PartitionRouter, - maps: Arc<[Option>]>, - }, -} - -/// Assigns probe rows to the build partition their key was repartitioned to. -/// -/// # Public Only for Internal Use: -/// Public only because [`HashTableLookupExpr::new`] takes it; not part of the -/// supported public API. -#[doc(hidden)] -#[derive(Clone)] -pub enum PartitionRouter { - /// `hash(keys) % partitions`, as hash repartitioning routes rows. - Hash(SeededRandomState), - /// The probe side's range partitioning. - Range(Arc), -} - -impl PartitionRouter { - /// Writes the partition of each row of `keys` to `partition_ids`, which - /// holds one slot per row. - fn route( - &self, - keys: &[ArrayRef], - partitions: usize, - partition_ids: &mut Vec, - ) -> Result<()> { - match self { - PartitionRouter::Hash(random_state) => { - partition_ids.resize(keys[0].len(), 0); - create_hashes(keys, random_state.random_state(), partition_ids)?; - let partitions = partitions as u64; - for id in partition_ids.iter_mut() { - *id %= partitions; - } - Ok(()) - } - PartitionRouter::Range(range) => { - range.route_partition_ids(keys, partition_ids) - } - } - } - - fn seed(&self) -> Option { - match self { - PartitionRouter::Hash(random_state) => Some(random_state.seed()), - PartitionRouter::Range(_) => None, - } - } -} - -impl HashTableLookupExpr { - /// Create a new HashTableLookupExpr that looks each probe row up in the - /// map of the build partition `router` assigns it to. - /// - /// # Arguments - /// * `on_columns` - Columns in the ON clause used to compute the join key - /// * `random_state` - SeededRandomState for hashing - /// * `router` - Routes each probe row to the partition its key was - /// repartitioned to; `None` when `maps` is a single map - /// * `maps` - One entry per build partition, `None` for an empty partition - /// * `description` - Description for debugging - /// * `pruning_bitmap` - key-range summary covering every partition's keys - /// for pruning only, or `None` - /// - /// A single non-empty map is probed directly without routing, so it may be - /// any [`Map`] and `router` is ignored. Otherwise `router` is required and - /// every map must be a [`Map::HashMap`]. - /// - /// # Public Only for Internal Use: - /// `datafusion-proto` tests require this constructor, but it is not part of - /// the supported public API. - #[doc(hidden)] - pub fn new( - on_columns: Vec, - random_state: SeededRandomState, - router: Option, - maps: Vec>>, - description: String, - pruning_bitmap: Option>, - ) -> Result { - let maps = maps - .into_iter() - .map(|map| map.map(ReservedMap::unreserved).map(Arc::new)) - .collect(); - Self::new_reserved( - on_columns, - random_state, - router, - maps, - description, - pruning_bitmap, - ) - } - - /// [`Self::new`] for a join's maps, whose memory stays reserved while the - /// lookup holds them. - pub(crate) fn new_reserved( - on_columns: Vec, - random_state: SeededRandomState, - router: Option, - maps: Vec>>, - description: String, - pruning_bitmap: Option>, - ) -> Result { - let lookup = match (router, maps.as_slice()) { - // Every row routes to the only partition, so skip routing. - (_, [Some(map)]) => Lookup::Single(Arc::clone(map)), - (None, _) => { - return internal_err!( - "a lookup over anything but one non-empty map needs a partition router" - ); - } - (Some(router), _) => { - if maps.iter().flatten().any(|map| { - let map: &Map = map; - !matches!(map, Map::HashMap(_)) - }) { - return internal_err!( - "routed lookups only support hash table partitions" - ); - } - Lookup::Routed { - router, - maps: maps.into(), - } - } - }; - Ok(Self { - on_columns, - random_state, - lookup, - description, - pruning_bitmap, - }) - } - - /// Which parts of this build side's key range hold keys, for container pruning. - pub fn pruning_bitmap(&self) -> Option<&Arc> { - self.pruning_bitmap.as_ref() - } - - /// Membership of each row of `keys` in the hash table of its routed partition. - fn evaluate_routed( - &self, - keys: &[ArrayRef], - router: &PartitionRouter, - maps: &[Option>], - ) -> Result { - let num_rows = keys[0].len(); - let mut partition_ids = Vec::with_capacity(num_rows); - router.route(keys, maps.len(), &mut partition_ids)?; - - with_hashes(keys, self.random_state.random_state(), |hashes| { - // Counting sort of the rows by partition, so each table is probed - // once per batch over a contiguous run of its rows' hashes. - let mut starts = vec![0usize; maps.len() + 1]; - for &id in &partition_ids { - starts[id as usize + 1] += 1; - } - for i in 0..maps.len() { - starts[i + 1] += starts[i]; - } - let mut next = starts.clone(); - let mut rows = vec![0u32; num_rows]; - let mut sorted_hashes = vec![0u64; num_rows]; - for (row, (&id, &hash)) in partition_ids.iter().zip(hashes).enumerate() { - let slot = &mut next[id as usize]; - rows[*slot] = row as u32; - sorted_hashes[*slot] = hash; - *slot += 1; - } - - let mut matched = BooleanBufferBuilder::new(num_rows); - matched.append_n(num_rows, false); - for (partition, map) in maps.iter().enumerate() { - // An empty partition holds no key, so its rows stay unmatched. - let Some(map) = map else { continue }; - let map: &Map = map; - let Map::HashMap(map) = map else { - return internal_err!( - "routed lookups only support hash table partitions" - ); - }; - let run = starts[partition]..starts[partition + 1]; - if run.is_empty() { - continue; - } - let hits = map.contain_hashes(&sorted_hashes[run.clone()]); - for hit in hits.values().set_indices() { - matched.set_bit(rows[run.start + hit] as usize, true); - } - } - Ok(BooleanArray::new(matched.finish(), None)) - }) - } -} -impl std::fmt::Debug for HashTableLookupExpr { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let cols = self - .on_columns - .iter() - .map(|e| e.to_string()) - .collect::>() - .join(", "); - let seed = self.random_state.seed(); - match &self.lookup { - Lookup::Single(_) => write!(f, "{}({cols}, [{seed}])", self.description), - Lookup::Routed { router, maps } => { - let router = match router.seed() { - Some(routing_seed) => format!("hash [{routing_seed}]"), - None => "range".to_string(), - }; - write!( - f, - "{}({cols}, [{seed}], routed by {router} over {} partitions)", - self.description, - maps.len() - ) - } - } - } -} - -impl Hash for HashTableLookupExpr { - fn hash(&self, state: &mut H) { - self.on_columns.dyn_hash(state); - self.description.hash(state); - self.random_state.seed().hash(state); - // Hash immutable map identity, consistently with pointer equality below. - self.lookup.identity().hash(state); - } -} - -impl PartialEq for HashTableLookupExpr { - fn eq(&self, other: &Self) -> bool { - // Sharing an immutable map implies equal contents, including across - // prepared-build consumers. Distinct maps need no expensive deep comparison. - self.on_columns == other.on_columns - && self.description == other.description - && self.random_state.seed() == other.random_state.seed() - && self.lookup.identity() == other.lookup.identity() - } -} - -impl Eq for HashTableLookupExpr {} - -impl Lookup { - /// Identity of the immutable build-side maps. A routed lookup's router is - /// created together with its maps, so the maps identify it as well. - fn identity(&self) -> *const () { - match self { - Lookup::Single(map) => std::ptr::from_ref::(map).cast(), - Lookup::Routed { maps, .. } => Arc::as_ptr(maps).cast(), - } - } -} - -impl Display for HashTableLookupExpr { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.description) - } -} - -impl PhysicalExpr for HashTableLookupExpr { - fn children(&self) -> Vec<&Arc> { - self.on_columns.iter().collect() - } - - fn with_new_children( - self: Arc, - children: Vec>, - ) -> Result> { - Ok(Arc::new(HashTableLookupExpr { - on_columns: children, - random_state: self.random_state.clone(), - lookup: self.lookup.clone(), - description: self.description.clone(), - pruning_bitmap: self.pruning_bitmap.clone(), - })) - } - - fn data_type(&self, _input_schema: &Schema) -> Result { - Ok(DataType::Boolean) - } - - fn nullable(&self, _input_schema: &Schema) -> Result { - Ok(false) - } - - fn evaluate(&self, batch: &RecordBatch) -> Result { - // Evaluate columns - let join_keys = evaluate_columns(&self.on_columns, batch)?; - - match &self.lookup { - Lookup::Single(map) => { - let map: &Map = map; - match map { - Map::HashMap(map) => with_hashes( - &join_keys, - self.random_state.random_state(), - |hashes| { - let array = map.contain_hashes(hashes); - Ok(ColumnarValue::Array(Arc::new(array))) - }, - ), - Map::ArrayMap(map) => { - let array = map.contain_keys(&join_keys)?; - Ok(ColumnarValue::Array(Arc::new(array))) - } - } - } - Lookup::Routed { router, maps } => { - let array = self.evaluate_routed(&join_keys, router, maps)?; - Ok(ColumnarValue::Array(Arc::new(array))) - } - } - } - #[cfg(feature = "proto")] - fn try_to_proto( - &self, - _ctx: &datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>, - ) -> Result> { - use datafusion_proto_models::protobuf; - use datafusion_proto_models::protobuf::physical_expr_node::ExprType; - - // Destructure exhaustively (no `..`) so that a newly added field is a - // compile error here, forcing a decision about whether the lit(true) - // replacement below is still the right thing to emit. - let Self { - // deliberately not serialized: the whole expression is replaced - // with lit(true), see the comment below - on_columns: _, - random_state: _, - lookup: _, - description: _, - pruning_bitmap: _, - } = self; - - // HashTableLookupExpr holds a runtime Arc (the build-side hash - // table) that cannot be serialized, so it is replaced with lit(true). - // - // Dynamic filtering is a performance optimisation only — replacing the - // lookup with lit(true) preserves correctness by allowing all rows - // through. - // - // If a plan is serialized before execution, HashTableLookupExpr is not - // yet present in the dynamic filter expression. - // - // If a plan is serialized after execution, any runtime-created - // HashTableLookupExpr is replaced during serialization. Re-executing - // the plan requires reset_state(), after which HashJoinExec rebuilds - // fresh dynamic filters at runtime. - let value = datafusion_proto_common::ScalarValue { - value: Some(datafusion_proto_common::scalar_value::Value::BoolValue( - true, - )), - }; - Ok(Some(protobuf::PhysicalExprNode { - expr_id: None, - expr_type: Some(ExprType::Literal(value)), - })) - } - fn fmt_sql(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.description) - } -} - fn evaluate_columns( columns: &[PhysicalExprRef], batch: &RecordBatch, @@ -668,8 +270,6 @@ fn evaluate_columns( #[cfg(test)] mod tests { use super::*; - use crate::joins::join_hash_map::JoinHashMapU32; - use datafusion_common::ScalarValue; use datafusion_physical_expr::expressions::Column; use std::collections::hash_map::DefaultHasher; use std::hash::Hasher; @@ -680,42 +280,6 @@ mod tests { hasher.finish() } - /// The pruning bitmap must survive `with_new_children`, which a scan calls - /// once per file. - #[test] - fn test_pruning_bitmap_shared_with_derived_children() -> Result<()> { - let keys: ArrayRef = Arc::new(arrow::array::Int64Array::from(vec![0, 1_000_000])); - let bitmap = Arc::new( - KeyRangeBitmap::try_new( - &keys, - &ScalarValue::Int64(Some(0)), - &ScalarValue::Int64(Some(1_000_000)), - 2, - ) - .ok_or_else(|| { - datafusion_common::internal_datafusion_err!( - "two keys a million apart leave prunable gaps" - ) - })?, - ); - let expr = Arc::new(HashTableLookupExpr::new( - vec![Arc::new(Column::new("a", 0))], - SeededRandomState::with_seed(1), - None, - vec![Some(Arc::new(Map::HashMap(Box::new( - JoinHashMapU32::with_capacity(10), - ))))], - "hash_lookup".to_string(), - Some(Arc::clone(&bitmap)), - )?); - - let derived = expr.with_new_children(vec![Arc::new(Column::new("a", 7))])?; - let derived = derived.downcast_ref::().unwrap(); - assert_eq!(derived.children()[0].to_string(), "a@7"); - assert!(Arc::ptr_eq(&bitmap, derived.pruning_bitmap().unwrap())); - Ok(()) - } - #[test] fn test_hash_expr_eq_same() { let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0)); @@ -857,7 +421,9 @@ mod tests { Some(protobuf::physical_expr_node::ExprType::Column(_)) => { Column::try_from_proto(node, &ctx) } - _ => internal_err!("test decoder cannot decode {node:?}"), + _ => datafusion_common::internal_err!( + "test decoder cannot decode {node:?}" + ), } } } @@ -982,270 +548,4 @@ mod tests { ); } } - - #[test] - fn test_hash_table_lookup_expr_eq_same() -> Result<()> { - let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0)); - let hash_map = - Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(10)))); - - let expr1 = HashTableLookupExpr::new( - vec![Arc::clone(&col_a)], - SeededRandomState::with_seed(1), - None, - vec![Some(Arc::clone(&hash_map))], - "lookup".to_string(), - None, - )?; - - let expr2 = HashTableLookupExpr::new( - vec![Arc::clone(&col_a)], - SeededRandomState::with_seed(1), - None, - vec![Some(Arc::clone(&hash_map))], - "lookup".to_string(), - None, - )?; - - assert_eq!(expr1, expr2); - Ok(()) - } - - #[test] - fn test_hash_table_lookup_expr_eq_different_columns() -> Result<()> { - let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0)); - let col_b: PhysicalExprRef = Arc::new(Column::new("b", 1)); - - let hash_map = - Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(10)))); - - let expr1 = HashTableLookupExpr::new( - vec![Arc::clone(&col_a)], - SeededRandomState::with_seed(1), - None, - vec![Some(Arc::clone(&hash_map))], - "lookup".to_string(), - None, - )?; - - let expr2 = HashTableLookupExpr::new( - vec![Arc::clone(&col_b)], - SeededRandomState::with_seed(1), - None, - vec![Some(Arc::clone(&hash_map))], - "lookup".to_string(), - None, - )?; - - assert_ne!(expr1, expr2); - Ok(()) - } - - #[test] - fn test_hash_table_lookup_expr_eq_different_description() -> Result<()> { - let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0)); - let hash_map = - Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(10)))); - - let expr1 = HashTableLookupExpr::new( - vec![Arc::clone(&col_a)], - SeededRandomState::with_seed(1), - None, - vec![Some(Arc::clone(&hash_map))], - "lookup_one".to_string(), - None, - )?; - - let expr2 = HashTableLookupExpr::new( - vec![Arc::clone(&col_a)], - SeededRandomState::with_seed(1), - None, - vec![Some(Arc::clone(&hash_map))], - "lookup_two".to_string(), - None, - )?; - - assert_ne!(expr1, expr2); - Ok(()) - } - - #[test] - fn test_hash_table_lookup_expr_eq_different_hash_map() -> Result<()> { - let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0)); - - // Two different Arc pointers (even with same content) should not be equal - let hash_map1 = - Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(10)))); - let hash_map2 = - Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(10)))); - let expr1 = HashTableLookupExpr::new( - vec![Arc::clone(&col_a)], - SeededRandomState::with_seed(1), - None, - vec![Some(hash_map1)], - "lookup".to_string(), - None, - )?; - - let expr2 = HashTableLookupExpr::new( - vec![Arc::clone(&col_a)], - SeededRandomState::with_seed(1), - None, - vec![Some(hash_map2)], - "lookup".to_string(), - None, - )?; - - // Different Arc pointers means not equal (uses Arc::ptr_eq) - assert_ne!(expr1, expr2); - Ok(()) - } - - #[test] - fn test_hash_table_lookup_expr_hash_consistency() -> Result<()> { - let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0)); - let hash_map = - Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(10)))); - - let expr1 = HashTableLookupExpr::new( - vec![Arc::clone(&col_a)], - SeededRandomState::with_seed(1), - None, - vec![Some(Arc::clone(&hash_map))], - "lookup".to_string(), - None, - )?; - - let expr2 = HashTableLookupExpr::new( - vec![Arc::clone(&col_a)], - SeededRandomState::with_seed(1), - None, - vec![Some(Arc::clone(&hash_map))], - "lookup".to_string(), - None, - )?; - - // Equal expressions should have equal hashes - assert_eq!(expr1, expr2); - assert_eq!(compute_hash(&expr1), compute_hash(&expr2)); - Ok(()) - } - - /// Probes `probe` against `expr`, one row per key of column `a`. - fn lookup(expr: &HashTableLookupExpr, probe: &ArrayRef) -> Result { - let schema = Arc::new(Schema::new(vec![arrow::datatypes::Field::new( - "a", - DataType::Int32, - true, - )])); - let batch = RecordBatch::try_new(schema, vec![Arc::clone(probe)])?; - let matched = expr.evaluate(&batch)?.into_array(batch.num_rows())?; - Ok(arrow::array::AsArray::as_boolean(&matched).clone()) - } - - /// A hash table of `keys`, hashed with `random_state` as the build side - /// hashes them. - fn hash_table(keys: &ArrayRef, random_state: &SeededRandomState) -> Result> { - use crate::joins::join_hash_map::JoinHashMapType; - let mut hashes = vec![0; keys.len()]; - create_hashes([keys], random_state.random_state(), &mut hashes)?; - let mut map = JoinHashMapU32::with_capacity(keys.len()); - map.update_from_iter(Box::new(hashes.iter().enumerate()), 0); - Ok(Arc::new(Map::HashMap(Box::new(map)))) - } - - /// One randomly generated scenario, see [`routed_lookup_fuzz`]. - fn run_routed_lookup_scenario(seed: u64) -> Result<()> { - use crate::repartition::{BatchPartitioner, REPARTITION_RANDOM_STATE}; - use rand::rngs::StdRng; - use rand::{Rng, SeedableRng}; - - let mut rng = StdRng::seed_from_u64(seed); - let num_partitions = rng.random_range(1..8usize); - // A key domain near the row counts, so probes both hit and miss. - let build: Vec = (0..rng.random_range(0..128)) - .map(|_| rng.random_range(-64..64)) - .collect(); - let probe: Vec> = (0..rng.random_range(0..128)) - .map(|_| rng.random_bool(0.9).then(|| rng.random_range(-80..80))) - .collect(); - let context = format!( - "seed {seed}: {num_partitions} partitions, {} build rows, {} probe rows", - build.len(), - probe.len() - ); - - let column: PhysicalExprRef = Arc::new(Column::new("a", 0)); - let schema = Arc::new(Schema::new(vec![arrow::datatypes::Field::new( - "a", - DataType::Int32, - false, - )])); - let build: ArrayRef = Arc::new(arrow::array::Int32Array::from(build)); - let probe: ArrayRef = Arc::new(arrow::array::Int32Array::from(probe)); - let random_state = SeededRandomState::with_seed(1); - - // Split the build side exactly as hash repartitioning does. - let mut maps = vec![None; num_partitions]; - BatchPartitioner::new_hash_partitioner( - vec![Arc::clone(&column)], - num_partitions, - crate::metrics::Time::new(), - )? - .partition( - RecordBatch::try_new(schema, vec![Arc::clone(&build)])?, - |partition, batch| { - maps[partition] = Some(hash_table(batch.column(0), &random_state)?); - Ok(()) - }, - )?; - let routed = HashTableLookupExpr::new( - vec![Arc::clone(&column)], - random_state.clone(), - Some(PartitionRouter::Hash(REPARTITION_RANDOM_STATE)), - maps, - "routed".to_string(), - None, - )?; - // Reference: the single-table lookup over the whole build side. - let single = HashTableLookupExpr::new( - vec![column], - random_state.clone(), - None, - vec![Some(hash_table(&build, &random_state)?)], - "single".to_string(), - None, - )?; - - assert_eq!( - lookup(&routed, &probe)?, - lookup(&single, &probe)?, - "{context}" - ); - Ok(()) - } - - /// Routing each probe row to its build partition's table matches one table - /// over the whole build side, when the build side is split the way hash - /// repartitioning splits it. - /// - /// `DATAFUSION_ROUTED_LOOKUP_FUZZ_ITERATIONS` and - /// `DATAFUSION_ROUTED_LOOKUP_FUZZ_SEED` select how many scenarios run and - /// from which seed; a failure names its seed so it can be replayed. - #[test] - fn routed_lookup_fuzz() -> Result<()> { - let env_u64 = |name: &str, default: u64| { - std::env::var(name) - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(default) - }; - let first_seed = env_u64("DATAFUSION_ROUTED_LOOKUP_FUZZ_SEED", 0); - let iterations = env_u64("DATAFUSION_ROUTED_LOOKUP_FUZZ_ITERATIONS", 256); - // `saturating_add` so a replay seed near `u64::MAX` cannot overflow. - for seed in first_seed..first_seed.saturating_add(iterations) { - run_routed_lookup_scenario(seed)?; - } - Ok(()) - } } diff --git a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs index 395e8c92f8bc4..6293b1c321ec4 100644 --- a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs +++ b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs @@ -5,52 +5,61 @@ //! Utilities for shared build-side information. Used in dynamic filter pushdown in Hash Joins. // TODO: include the link to the Dynamic Filter blog post. +use std::collections::HashSet; use std::fmt; use std::sync::Arc; use crate::ExecutionPlan; use crate::ExecutionPlanProperties; -use crate::Partitioning; -use crate::joins::Map; use crate::joins::PartitionMode; -use crate::joins::ReservedMap; -use crate::joins::hash_join::exec::HASH_JOIN_SEED; +use crate::joins::hash_join::bloom_filter::{BloomFilterExpr, BloomFilterOptions}; use crate::joins::hash_join::inlist_builder::build_struct_fields; use crate::joins::hash_join::not_in::NotIn; -use crate::joins::hash_join::partitioned_hash_eval::{ - HashExpr, HashTableLookupExpr, PartitionRouter, SeededRandomState, -}; -use crate::joins::join_hash_map::{JoinHashMapType, JoinHashMapU32}; use crate::joins::key_range_bitmap::{KeyRangeBitmap, KeyRangePart}; -use crate::repartition::RangeExpr; -use arrow::array::{ArrayRef, AsArray}; -use arrow::compute::concat; +use arrow::array::{Array, ArrayRef, UInt32Array}; use arrow::datatypes::{DataType, Field, Schema}; -use datafusion_common::config::{ConfigOptions, OptimizerOptions}; -use datafusion_common::hash_utils::create_hashes; +use arrow::row::{RowConverter, SortField}; +use datafusion_common::config::ConfigOptions; use datafusion_common::{ DataFusionError, NullEquality, Result, ScalarValue, SharedResult, - assert_or_internal_err, internal_err, + assert_or_internal_err, }; +use datafusion_execution::memory_pool::{MemoryPool, UnboundedMemoryPool}; use datafusion_expr::Operator; use datafusion_functions::core::r#struct as struct_func; use datafusion_physical_expr::expressions::{ - BinaryExpr, CaseExpr, DynamicFilterPhysicalExpr, InListExpr, IsNullExpr, lit, -}; -use datafusion_physical_expr::{ - PhysicalExpr, PhysicalExprRef, RangePartitioning, ScalarFunctionExpr, + BinaryExpr, DynamicFilterPhysicalExpr, InListExpr, IsNullExpr, lit, }; +use datafusion_physical_expr::{PhysicalExpr, PhysicalExprRef, ScalarFunctionExpr}; +use parquet::bloom_filter::Sbbf; use parking_lot::Mutex; use tokio::sync::Notify; +/// The distinct values of `values`, in the order of their first occurrence. +/// +/// The InList of a partition holds every build row, so the concatenated lists +/// of all partitions can repeat few keys many times. +fn distinct_values(values: &ArrayRef) -> Result { + let converter = RowConverter::new(vec![SortField::new(values.data_type().clone())])?; + let rows = converter.convert_columns(std::slice::from_ref(values))?; + let mut seen = HashSet::with_capacity(rows.num_rows()); + let indices: UInt32Array = (0..rows.num_rows() as u32) + .filter(|&i| seen.insert(rows.row(i as usize))) + .collect(); + if indices.len() == values.len() { + return Ok(Arc::clone(values)); + } + Ok(arrow::compute::take(values.as_ref(), &indices, None)?) +} + /// Represents the minimum and maximum values for a specific column. /// Used in dynamic filter pushdown to establish value boundaries. #[derive(Debug, Clone, PartialEq)] pub(crate) struct ColumnBounds { /// The minimum value observed for this column pub(crate) min: ScalarValue, - /// The maximum value observed for this column + /// The maximum value observed for this column pub(crate) max: ScalarValue, } @@ -78,101 +87,132 @@ impl PartitionBounds { self.column_bounds.get(index) } - /// Bounds covering both `self` and `other`, which must describe the same - /// columns. A NULL bound means the partition holds no non-null key in that - /// column, so it never widens the other side's bound. - fn union(self, other: &PartitionBounds) -> PartitionBounds { - // Every non-empty partition of a join reports one bound per join key. - debug_assert_eq!( - self.column_bounds.len(), - other.column_bounds.len(), - "partitions report bounds for different numbers of columns" - ); - let column_bounds = self - .column_bounds - .into_iter() - .zip(&other.column_bounds) - .map(|(a, b)| { - let min = if !b.min.is_null() && (a.min.is_null() || b.min < a.min) { - b.min.clone() - } else { - a.min - }; - let max = if !b.max.is_null() && (a.max.is_null() || b.max > a.max) { - b.max.clone() - } else { - a.max - }; - ColumnBounds::new(min, max) - }) - .collect(); - PartitionBounds::new(column_bounds) + /// Bounds that hold the keys of every partition: the smallest minimum and the + /// largest maximum of each column. + /// + /// `None` when a partition has no bounds or a column's values cannot be + /// compared. Then a bounds predicate is not possible. + fn merge<'a>( + mut partitions: impl Iterator, + ) -> Option { + let mut merged = partitions.next()?.clone(); + if merged.column_bounds.is_empty() { + return None; + } + for bounds in partitions { + if bounds.column_bounds.len() != merged.column_bounds.len() { + return None; + } + for (acc, b) in merged.column_bounds.iter_mut().zip(&bounds.column_bounds) { + // A NULL bound comes from a partition with no non-NULL key in this + // column. It must not widen (or, as a NULL comparison, disable) the + // bounds of the keys of other partitions. + if acc.min.is_null() + || (!b.min.is_null() && b.min.partial_cmp(&acc.min)?.is_lt()) + { + acc.min = b.min.clone(); + } + if acc.max.is_null() + || (!b.max.is_null() && b.max.partial_cmp(&acc.max)?.is_gt()) + { + acc.max = b.max.clone(); + } + } + } + Some(merged) } } -/// Creates a membership predicate for filter pushdown. -/// -/// If `inlist_values` is provided (for small build sides), creates an InList expression. -/// Otherwise, creates a HashTableLookup expression (for large build sides). -/// -/// Supports both single-column and multi-column joins using struct expressions. -fn create_membership_predicate( - on_right: &[PhysicalExprRef], - pushdown: PushdownStrategy, - random_state: &SeededRandomState, - schema: &Schema, -) -> Result>> { - match pushdown { - // Use InList expression for small build sides - PushdownStrategy::InList(in_list_array) => { - // Build the expression to compare against - let expr = if on_right.len() == 1 { - // Single column: col IN (val1, val2, ...) - Arc::clone(&on_right[0]) - } else { - let fields = build_struct_fields( - on_right - .iter() - .map(|r| r.data_type(schema)) - .collect::>>()? - .as_ref(), - )?; +/// The build side key columns of one partition, for the global dynamic filter. +#[derive(Clone, Default)] +pub(crate) struct BuildKeys { + /// One array per join key + pub(crate) keys: Vec, + /// Bloom filter of the keys. `None` when this partition has no filter, + /// then the global filter has no bloom filter. + pub(crate) bloom_filter: Option>, + /// Number of distinct key tuples + pub(crate) num_distinct: usize, +} - // The return field name and the function field name don't really matter here. - let return_field = - Arc::new(Field::new("struct", DataType::Struct(fields), true)); - - Arc::new(ScalarFunctionExpr::new( - "struct", - struct_func(), - on_right.to_vec(), - return_field, - Arc::new(ConfigOptions::default()), - )) as Arc - }; +/// Options for the global dynamic filter of a hash join, from the session +/// configuration. +#[derive(Debug, Clone)] +pub(crate) struct GlobalFilterOptions { + /// Use an InList when all partitions together have at most this many + /// distinct keys (`hash_join_dynamic_filter_inlist_max_distinct_values`). + pub(crate) inlist_max_distinct_values: usize, + pub(crate) bloom_filter: BloomFilterOptions, + /// Pool that the bloom filter memory is charged to. `None` (tests only) + /// does not track the memory. + pub(crate) memory_pool: Option>, +} - // Use InListExpr::try_new_from_array() to build an InList with static_filter optimization (hash-based lookup) - Ok(Some(Arc::new(InListExpr::try_new_from_array( - expr, - in_list_array, - false, - schema, - )?))) - } - // Use hash table lookup for large build sides - PushdownStrategy::Map(hash_map, pruning_bitmap) => { - Ok(Some(Arc::new(HashTableLookupExpr::new_reserved( - on_right.to_vec(), - random_state.clone(), - None, - vec![Some(hash_map)], - "hash_lookup".to_string(), - pruning_bitmap, - )?) as Arc)) +impl GlobalFilterOptions { + pub(crate) fn new(config: &ConfigOptions) -> Self { + let optimizer = &config.optimizer; + Self { + inlist_max_distinct_values: optimizer + .hash_join_dynamic_filter_inlist_max_distinct_values, + bloom_filter: BloomFilterOptions { + fpp: optimizer.hash_join_bloom_filter_fpp, + max_bytes: optimizer.hash_join_bloom_filter_max_bytes, + max_fpp: optimizer.hash_join_bloom_filter_max_fpp, + }, + memory_pool: None, } - // Empty partition - should not create a filter for this - PushdownStrategy::Empty => Ok(None), } + + /// Charge the bloom filter memory to `pool`. + pub(crate) fn with_memory_pool(mut self, pool: Arc) -> Self { + self.memory_pool = Some(pool); + self + } +} + +impl Default for GlobalFilterOptions { + fn default() -> Self { + Self::new(&ConfigOptions::default()) + } +} + +/// Creates `key IN (values)` for a single key, or +/// `struct(keys) IN (values)` for several keys. +fn create_in_list_predicate( + on_right: &[PhysicalExprRef], + in_list_array: ArrayRef, + schema: &Schema, +) -> Result> { + let expr = if on_right.len() == 1 { + Arc::clone(&on_right[0]) + } else { + let fields = build_struct_fields( + on_right + .iter() + .map(|r| r.data_type(schema)) + .collect::>>()? + .as_ref(), + )?; + + // The return field name and the function field name don't really matter here. + let return_field = Arc::new(Field::new("struct", DataType::Struct(fields), true)); + + Arc::new(ScalarFunctionExpr::new( + "struct", + struct_func(), + on_right.to_vec(), + return_field, + Arc::new(ConfigOptions::default()), + )) as Arc + }; + + // `try_new_from_array` builds the InList with its hash-based static filter. + Ok(Arc::new(InListExpr::try_new_from_array( + expr, + in_list_array, + false, + schema, + )?)) } /// Creates a bounds predicate from partition bounds. @@ -221,8 +261,7 @@ fn create_bounds_predicate( /// Combines a membership predicate and a bounds predicate with logical AND. /// -/// Returns `None` when neither is available; callers decide the fallback (e.g. -/// skip updating the filter vs. emit a `lit(true)` branch inside a CASE). +/// Returns `None` when neither is available. fn combine_membership_and_bounds( membership_expr: Option>, bounds_expr: Option>, @@ -240,14 +279,14 @@ fn combine_membership_and_bounds( /// Coordinates build-side information collection across multiple partitions /// -/// This structure collects information from the build side (hash tables and/or bounds) and +/// This structure collects information from the build side (keys and bounds) and /// ensures that dynamic filters are built with complete information from all relevant /// partitions before being applied to probe-side scans. Incomplete filters would /// incorrectly eliminate valid join results. /// /// ## Synchronization Strategy /// -/// 1. Each partition computes information from its build-side data (hash maps and/or bounds) +/// 1. Each partition computes information from its build-side data (keys and bounds) /// 2. Information is stored in the shared state, which tracks how many partitions have reported /// 3. When the last partition reports, one waiter is elected as the finalizer; it merges the /// collected information, updates the dynamic filter exactly once, and publishes the @@ -255,11 +294,12 @@ fn combine_membership_and_bounds( /// 4. A [`tokio::sync::Notify`] wakes any other partitions parked in `wait_for_completion`, /// which then observe the `Ready` state under the mutex and return immediately /// -/// ## Hash Map vs Bounds +/// ## Filter /// -/// - **Hash Maps (Partitioned mode)**: Collects Arc references to hash tables from each partition. -/// Creates a `PartitionedHashLookupPhysicalExpr` that routes rows to the correct partition's hash table. -/// - **Bounds (CollectLeft mode)**: Collects min/max bounds and creates range predicates. +/// The filter merges the bounds and keys of all partitions into one global +/// filter: global min/max bounds AND an InList (few keys) or a bloom filter of +/// all build keys. See `build_global_filter`. When a partition is canceled, +/// the filter is `true`. /// /// ## Partition Counting /// @@ -285,25 +325,16 @@ pub(crate) struct SharedBuildAccumulator { dynamic_filter: Arc, /// Right side join expressions needed for creating filter expressions on_right: Vec, - /// Random state for partitioning (RepartitionExec's hash function with 0,0,0,0 seeds) - /// Used for PartitionedHashLookupPhysicalExpr - repartition_random_state: SeededRandomState, /// Schema of the probe (right) side for evaluating filter expressions probe_schema: Arc, - /// Probe-side Range routing metadata for partitioned dynamic filters. - probe_range_partitioning: Option, /// Null equality of the join. Under `NullEqualsNull` a probe-side NULL can match a /// build-side NULL, so the pushed filter must keep NULL rows here too. null_equality: NullEquality, /// Null-aware anti join (`NOT IN`). A probe-side NULL must reach the join so its /// three-valued logic can collapse the result, so the pushed filter keeps NULL rows. not_in: Option, - /// Largest merged IN list, in bytes, that partitioned filters keep as an IN - /// list: `hash_join_inlist_pushdown_max_size`, as for a single build. - in_list_max_size: usize, - /// Largest merged IN list, in values, that partitioned filters keep as an IN - /// list: `hash_join_inlist_pushdown_max_distinct_values`, as for a single build. - in_list_max_values: usize, + /// Options for the global filter + global_filter_options: GlobalFilterOptions, } /// Strategy for filter pushdown (decided at collection time) @@ -311,9 +342,10 @@ pub(crate) struct SharedBuildAccumulator { pub(crate) enum PushdownStrategy { /// Use InList for small build sides (< 128MB) InList(ArrayRef), - /// Use map lookup for large build sides. If `Some` the second field - /// represents the build side's keys using a bucket bitmap. - Map(Arc, Option>), + /// The build side has too many keys for an InList. The global filter + /// uses the bloom filter of [`BuildKeys`]. `Some` holds a bucket bitmap of + /// the keys, for pruning. + Large(Option>), /// There was no data in this partition, do not build a dynamic filter for it Empty, } @@ -325,11 +357,13 @@ pub(crate) enum PartitionBuildData { pushdown: PushdownStrategy, bounds: PartitionBounds, keys_have_null: bool, + build_keys: BuildKeys, }, CollectLeft { pushdown: PushdownStrategy, bounds: PartitionBounds, keys_have_null: bool, + build_keys: BuildKeys, }, } @@ -341,6 +375,8 @@ struct PartitionData { /// Whether any build key of this partition is NULL. Decides whether the pushed /// filter must keep probe-side NULL rows for a null-equal join to match them. keys_have_null: bool, + /// The build side keys, for the global filter + build_keys: BuildKeys, } /// Build-side data organized by partition mode @@ -413,10 +449,9 @@ impl SharedBuildAccumulator { right_child: &dyn ExecutionPlan, dynamic_filter: Arc, on_right: Vec, - repartition_random_state: SeededRandomState, null_equality: NullEquality, not_in: Option, - optimizer: &OptimizerOptions, + global_filter_options: GlobalFilterOptions, ) -> Self { // Troubleshooting: If partition counts are incorrect, verify this logic matches // the actual execution pattern in collect_build_side() @@ -453,14 +488,6 @@ impl SharedBuildAccumulator { ), }; - let probe_range_partitioning = - match (partition_mode, right_child.output_partitioning()) { - (PartitionMode::Partitioned, Partitioning::Range(range)) => { - Some(range.clone()) - } - _ => None, - }; - Self { inner: Mutex::new(AccumulatorState { data: mode_data, @@ -469,13 +496,10 @@ impl SharedBuildAccumulator { completion_notify: Notify::new(), dynamic_filter, on_right, - repartition_random_state, probe_schema: right_child.schema(), - probe_range_partitioning, null_equality, not_in, - in_list_max_size: optimizer.hash_join_inlist_pushdown_max_size, - in_list_max_values: optimizer.hash_join_inlist_pushdown_max_distinct_values, + global_filter_options, } } @@ -483,11 +507,10 @@ impl SharedBuildAccumulator { /// /// This unified method handles both CollectLeft and Partitioned modes. When all partitions /// have reported (barrier wait), the leader builds the appropriate filter expression: - /// - CollectLeft: Simple conjunction of bounds and membership check - /// - Partitioned: CASE expression routing to per-partition filters + /// the global filter of all partitions, or `true` when a partition is canceled. /// /// # Arguments - /// * `data` - Build data including hash map, pushdown strategy, and bounds + /// * `data` - Build data including keys, pushdown strategy, and bounds /// /// # Returns /// * `Result<()>` - Ok if successful, Err if filter update failed or mode mismatch @@ -529,6 +552,7 @@ impl SharedBuildAccumulator { pushdown, bounds, keys_have_null, + build_keys, }, AccumulatedBuildData::Partitioned { partitions, @@ -542,6 +566,7 @@ impl SharedBuildAccumulator { bounds, pushdown, keys_have_null, + build_keys, }); } ( @@ -549,6 +574,7 @@ impl SharedBuildAccumulator { pushdown, bounds, keys_have_null, + build_keys, }, AccumulatedBuildData::CollectLeft { data, @@ -561,6 +587,7 @@ impl SharedBuildAccumulator { bounds, pushdown, keys_have_null, + build_keys, }); } *reported_count += 1; @@ -653,7 +680,7 @@ impl SharedBuildAccumulator { self.build_collect_left_filter(partition) } FinalizeInput::Partitioned(partitions) => { - self.build_partitioned_filter(partitions) + self.build_partitioned_filter(&partitions) } } } @@ -661,28 +688,14 @@ impl SharedBuildAccumulator { /// Builds the single global filter used by a collect-left join. fn build_collect_left_filter(&self, partition: PartitionStatus) -> Result<()> { match partition { + // An empty build side keeps the filter unchanged. PartitionStatus::Reported(PartitionData { - bounds, - pushdown, - keys_have_null, - }) => { - let membership_expr = create_membership_predicate( - &self.on_right, - pushdown, - &HASH_JOIN_SEED, - self.probe_schema.as_ref(), - )?; - let bounds_expr = create_bounds_predicate(&self.on_right, &bounds); - - if let Some(filter_expr) = - combine_membership_and_bounds(membership_expr, bounds_expr) - { - self.dynamic_filter.update( - self.preserve_probe_nulls(filter_expr, keys_have_null)?, - )?; - } - Ok(()) - } + pushdown: PushdownStrategy::Empty, + .. + }) => Ok(()), + PartitionStatus::Reported(data) => self + .dynamic_filter + .update(self.build_global_filter(&[&data])?), PartitionStatus::Pending => datafusion_common::internal_err!( "attempted to finalize collect-left dynamic filter without reported build data" ), @@ -692,73 +705,18 @@ impl SharedBuildAccumulator { } } - /// Builds one routed probe-side filter from finalized partitioned build data. - /// Empty partitions reject their routed rows, while canceled partitions stay - /// permissive because their build contents are unknown. - fn build_partitioned_filter(&self, partitions: Vec) -> Result<()> { - let mut partition_filters = Vec::with_capacity(partitions.len()); - let mut real_partition_ids = Vec::new(); - let mut empty_partition_ids = Vec::new(); - let mut has_canceled_unknown = false; - let mut keys_have_null = false; - // Every non-empty partition's membership strategy and bounds, merged into - // one membership filter below; `None` once a partition cannot be merged. - let mut members = Some(Vec::with_capacity(partitions.len())); - let mut merged_bounds: Option = None; - - for (partition_id, partition) in partitions.into_iter().enumerate() { + /// Builds the global filter from finalized partitioned build data. + /// + /// A canceled partition's keys are unknown, so then the filter is `true`. + /// `true` is correct for any consumer that reads the filter, including the + /// rows of the canceled partition. + fn build_partitioned_filter(&self, partitions: &[PartitionStatus]) -> Result<()> { + let mut reported = Vec::with_capacity(partitions.len()); + for partition in partitions { match partition { - PartitionStatus::Reported(PartitionData { - pushdown: PushdownStrategy::Empty, - .. - }) => { - empty_partition_ids.push(partition_id); - partition_filters.push(lit(false)); - } - PartitionStatus::Reported(PartitionData { - bounds, - pushdown, - keys_have_null: partition_keys_have_null, - }) => { - real_partition_ids.push(partition_id); - keys_have_null |= partition_keys_have_null; - let mergeable = match &pushdown { - PushdownStrategy::Map(map, _) => { - let map: &Map = map; - matches!(map, Map::HashMap(_)) - } - PushdownStrategy::InList(_) => true, - PushdownStrategy::Empty => false, - }; - members = members.filter(|_| mergeable).map(|mut members| { - members.push((partition_id, pushdown.clone(), bounds.clone())); - members - }); - merged_bounds = Some(match merged_bounds { - None => bounds.clone(), - Some(merged) => merged.union(&bounds), - }); - - let membership_expr = create_membership_predicate( - &self.on_right, - pushdown, - &HASH_JOIN_SEED, - self.probe_schema.as_ref(), - )?; - let bounds_expr = create_bounds_predicate(&self.on_right, &bounds); - let then_expr = - combine_membership_and_bounds(membership_expr, bounds_expr) - .unwrap_or_else(|| lit(true)); - partition_filters.push(then_expr); - } + PartitionStatus::Reported(data) => reported.push(data), PartitionStatus::CanceledUnknown => { - has_canceled_unknown = true; - partition_filters.push(lit(true)); - // Rows routed here must pass, which a merged filter cannot express. - members = None; - // A canceled partition's build content is unknown, so it - // may hold a NULL key. - keys_have_null = true; + return self.dynamic_filter.update(lit(true)); } PartitionStatus::Pending => { return datafusion_common::internal_err!( @@ -767,245 +725,127 @@ impl SharedBuildAccumulator { } } } - - let all_partitions_canceled = has_canceled_unknown - && real_partition_ids.is_empty() - && empty_partition_ids.is_empty(); - let all_partitions_empty = !has_canceled_unknown && real_partition_ids.is_empty(); - let one_non_empty_partition = - !has_canceled_unknown && real_partition_ids.len() == 1; - - let filter_expr = if all_partitions_canceled { - // No build data is known, so filtering any probe row could discard a match. - lit(true) - } else if all_partitions_empty { - // No build row exists, so no probe row can match. - lit(false) - } else if one_non_empty_partition { - // Only one build partition contains rows, so its filter covers every - // possible probe match without routing. - Arc::clone(&partition_filters[real_partition_ids[0]]) - } else if let Some(members) = members { - self.merged_membership_filter( - &members, - merged_bounds, - partition_filters.len(), - )? - } else { - // Builds the shared sparse `CASE` for partition filter routing. - // Without cancellation, omitted branches are known empty and safely fall - // through to `ELSE false`. With cancellation, omitted canceled partitions - // have unknown contents and must fall through to `ELSE true`, so known-empty - // partitions are emitted explicitly as false branches. - let mut branches = if has_canceled_unknown { - empty_partition_ids - .iter() - .map(|&partition_id| (lit(partition_id as u64), lit(false))) - .collect::>() - } else { - vec![] - }; - branches.extend(real_partition_ids.iter().map(|&partition_id| { - ( - lit(partition_id as u64), - Arc::clone(&partition_filters[partition_id]), - ) - })); - - let routing_expr = if let Some(range_partitioning) = - &self.probe_range_partitioning - { - // Routes probe rows with the partition id selected by [`RangeExpr`]. - // CASE range_partition(keys) - // WHEN empty_partition_id THEN false -- only when cancellation exists - // WHEN real_partition_id THEN F(real_partition_id) - // ... - // ELSE has_canceled_unknown - // END - assert_or_internal_err!( - partition_filters.len() == range_partitioning.partition_count(), - "Dynamic filter partition count {} does not match Range partition count {}", - partition_filters.len(), - range_partitioning.partition_count() - ); - Arc::new(RangeExpr::try_new_with_schema( - self.on_right.clone(), - range_partitioning, - &self.probe_schema, - )?) as Arc - } else { - // Routes probe rows with the same `hash(keys) % partition_count` expression used - // by Hash repartitioning. - // CASE hash(keys) % partition_count - // WHEN empty_partition_id THEN false -- only when cancellation exists - // WHEN real_partition_id THEN F(real_partition_id) - // ... - // ELSE has_canceled_unknown - // END - let routing_hash_expr = Arc::new(HashExpr::new( - self.on_right.clone(), - self.repartition_random_state.clone(), - "hash_repartition".to_string(), - )) as Arc; - Arc::new(BinaryExpr::new( - routing_hash_expr, - Operator::Modulo, - lit(partition_filters.len() as u64), - )) as Arc - }; - - Arc::new(CaseExpr::try_new( - Some(routing_expr), - branches, - Some(lit(has_canceled_unknown)), - )?) - }; - self.dynamic_filter - .update(self.preserve_probe_nulls(filter_expr, keys_have_null)?) + .update(self.build_global_filter(&reported)?) } - /// `members` holds each non-empty partition's id, strategy (an IN list or a - /// hash table) and bounds, out of `num_partitions` partitions. - fn merged_membership_filter( + /// Builds one filter for all probe rows from the build data of every partition, + /// with no routing to partitions: + /// + /// ```text + /// [key IS NULL OR] key >= min AND key <= max AND membership(key) + /// ``` + /// + /// `min` and `max` are the global bounds. `membership` is an InList of the + /// keys of all partitions when they have at most + /// `hash_join_dynamic_filter_inlist_max_distinct_values` distinct keys together, + /// and a [`BloomFilterExpr`] otherwise. The filter is true for every build + /// key, whatever the partitioning of the build and probe sides, so it is + /// correct for hash, range and collect-left joins. + fn build_global_filter( &self, - members: &[(usize, PushdownStrategy, PartitionBounds)], - merged_bounds: Option, - num_partitions: usize, + partitions: &[&PartitionData], ) -> Result> { - let lists = members + let non_empty: Vec<&PartitionData> = partitions + .iter() + .copied() + .filter(|p| !matches!(p.pushdown, PushdownStrategy::Empty)) + .collect(); + if non_empty.is_empty() { + // No build row exists, so no probe row can match. + return Ok(lit(false)); + } + let keys_have_null = non_empty.iter().any(|p| p.keys_have_null); + let bounds = PartitionBounds::merge(non_empty.iter().map(|p| &p.bounds)); + let num_distinct: usize = + non_empty.iter().map(|p| p.build_keys.num_distinct).sum(); + + let in_lists = non_empty .iter() - .filter_map(|(_, pushdown, _)| match pushdown { - PushdownStrategy::InList(values) => Some(values), + .map(|p| match &p.pushdown { + PushdownStrategy::InList(values) => Some(Arc::clone(values)), _ => None, }) - .collect::>(); - - // Every partition is an IN list, and together they still fit the limits a - // single build uses to pick an IN list: one combined list keeps the - // statistics and bloom filter pruning an IN list allows. Past those - // limits, a hash lookup is cheaper per row and carries a pruning bitmap. - let combined_in_list = lists.len() == members.len() && { - let values = lists.iter().map(|list| list.len()).sum::(); - let bytes = lists - .iter() - .map(|list| list.get_array_memory_size()) - .sum::(); - values <= self.in_list_max_values && bytes <= self.in_list_max_size - }; - let membership = if combined_in_list { - let values = concat(&lists.iter().map(|l| l.as_ref()).collect::>())?; - create_membership_predicate( - &self.on_right, - PushdownStrategy::InList(values), - &HASH_JOIN_SEED, - self.probe_schema.as_ref(), - )? - .expect("an IN list always has a membership predicate") - } else { - // Each probe row is looked up in the table of the partition its key - // routes to, so the partitions' tables are shared as they are: no - // merged table to build while every probe stream waits on the filter. - let mut maps = vec![None; num_partitions]; - for (partition_id, pushdown, _) in members { - maps[*partition_id] = Some(match pushdown { - PushdownStrategy::Map(map, _) => Arc::clone(map), - PushdownStrategy::InList(values) => self.in_list_hash_map(values)?, - PushdownStrategy::Empty => { - return internal_err!("empty partitions have no members"); - } - }); + .collect::>>(); + let membership_expr = match in_lists { + // The partitions hold disjoint keys, so the sum of their distinct + // counts is the number of distinct keys of the build side. + Some(in_lists) + if num_distinct + <= self.global_filter_options.inlist_max_distinct_values => + { + let arrays: Vec<&dyn Array> = + in_lists.iter().map(|a| a.as_ref()).collect(); + let values = distinct_values(&arrow::compute::concat(&arrays)?)?; + Some(create_in_list_predicate( + &self.on_right, + values, + self.probe_schema.as_ref(), + )?) } - Arc::new(HashTableLookupExpr::new_reserved( - self.on_right.clone(), - HASH_JOIN_SEED, - Some(self.partition_router(num_partitions)?), - maps, - String::from("hash_lookup"), - self.merged_pruning_bitmap(members, merged_bounds.as_ref()), - )?) - }; - - let bounds = merged_bounds - .and_then(|bounds| create_bounds_predicate(&self.on_right, &bounds)); - Ok(combine_membership_and_bounds(Some(membership), bounds) - .expect("membership predicate is always present")) - } - - /// How probe rows find their build partition: as the probe side was - /// partitioned. - fn partition_router(&self, num_partitions: usize) -> Result { - Ok(match &self.probe_range_partitioning { - Some(range_partitioning) => { - assert_or_internal_err!( - num_partitions == range_partitioning.partition_count(), - "Dynamic filter partition count {} does not match Range partition count {}", - num_partitions, - range_partitioning.partition_count() - ); - PartitionRouter::Range(Arc::new(RangeExpr::try_new_with_schema( + _ => { + let pruning_bitmap = + self.merge_pruning_bitmaps(&non_empty, bounds.as_ref()); + // Every partition must have its filter. A partition has none + // when the build side is too big for a bloom filter, or its + // memory reservation failed. + let filters = non_empty + .iter() + .map(|p| p.build_keys.bloom_filter.as_deref()) + .collect::>>(); + let pool = self + .global_filter_options + .memory_pool + .clone() + .unwrap_or_else(|| Arc::new(UnboundedMemoryPool::default())); + let bloom = BloomFilterExpr::try_new( self.on_right.clone(), - range_partitioning, - &self.probe_schema, - )?)) + filters.as_deref(), + self.global_filter_options.bloom_filter, + &pool, + pruning_bitmap, + )?; + // Without a bloom filter, keep the expression only for its + // pruning bitmap + (bloom.has_filter() || bloom.pruning_bitmap().is_some()) + .then(|| Arc::new(bloom) as Arc) } - None => PartitionRouter::Hash(self.repartition_random_state.clone()), - }) - } - - /// A hash table of an IN list partition's keys, hashed as the probe side - /// hashes its join keys: one column per join key. - /// - /// The table is not reserved, same as the IN list it is built from. Its - /// size follows that list, which `hash_join_inlist_pushdown_max_size` caps. - fn in_list_hash_map(&self, values: &ArrayRef) -> Result> { - let columns = match values.as_struct_opt() { - Some(columns) if self.on_right.len() > 1 => columns.columns(), - _ => std::slice::from_ref(values), }; - let mut hashes = vec![0; values.len()]; - create_hashes(columns, HASH_JOIN_SEED.random_state(), &mut hashes)?; - let mut map = JoinHashMapU32::with_capacity(hashes.len()); - map.update_from_iter(Box::new(hashes.iter().enumerate()), 0); - let map = Arc::new(Map::HashMap(Box::new(map))); - Ok(Arc::new(ReservedMap::unreserved(map))) - } - - /// Container pruning reads one bitmap, so merge the partitions' key ranges: - /// a hash table's bitmap, an IN list's keys, or else a partition's whole - /// bounds. `None` when no partition narrows its bounds, as the bounds - /// predicate already prunes that much. - fn merged_pruning_bitmap( + let bounds_expr = bounds + .as_ref() + .and_then(|b| create_bounds_predicate(&self.on_right, b)); + let filter_expr = combine_membership_and_bounds(membership_expr, bounds_expr) + .unwrap_or_else(|| lit(true)); + self.preserve_probe_nulls(filter_expr, keys_have_null) + } + + /// One [`KeyRangeBitmap`] for the keys of all partitions, so that scans can + /// prune containers (for example parquet row groups and pages) with the + /// global filter. Only for a single integer key. + fn merge_pruning_bitmaps( &self, - members: &[(usize, PushdownStrategy, PartitionBounds)], - merged_bounds: Option<&PartitionBounds>, + partitions: &[&PartitionData], + bounds: Option<&PartitionBounds>, ) -> Option> { if self.on_right.len() != 1 { return None; } - let merged = merged_bounds?.get_column_bounds(0)?; - let parts = members + let column_bounds = bounds?.get_column_bounds(0)?; + let parts = partitions .iter() - .map(|(_, pushdown, bounds)| match pushdown { - PushdownStrategy::Map(_, Some(bitmap)) => { - Some(KeyRangePart::Bitmap(bitmap)) + .map(|p| match &p.pushdown { + PushdownStrategy::Large(Some(bitmap)) => { + Some(KeyRangePart::Bitmap(bitmap.as_ref())) } - PushdownStrategy::InList(keys) => Some(KeyRangePart::Keys(keys.as_ref())), - _ => bounds - .get_column_bounds(0) - .map(|bounds| KeyRangePart::Range { - min: &bounds.min, - max: &bounds.max, - }), + _ => p + .build_keys + .keys + .first() + .map(|keys| KeyRangePart::Keys(keys.as_ref())), }) .collect::>>()?; - if parts - .iter() - .all(|part| matches!(part, KeyRangePart::Range { .. })) - { - return None; - } - KeyRangeBitmap::union(&merged.min, &merged.max, &parts).map(Arc::new) + KeyRangeBitmap::union(&column_bounds.min, &column_bounds.max, &parts) + .map(Arc::new) } /// Keeps probe rows with a NULL key when the join semantics need them. @@ -1099,13 +939,10 @@ pub(super) fn make_partitioned_accumulator_for_test( completion_notify: Notify::new(), dynamic_filter, on_right: vec![], - repartition_random_state: SeededRandomState::with_seed(1), probe_schema, - probe_range_partitioning: None, null_equality: NullEquality::NullEqualsNothing, not_in: None, - in_list_max_size: 128 * 1024, - in_list_max_values: 150, + global_filter_options: GlobalFilterOptions::default(), } } @@ -1127,15 +964,26 @@ mod tests { use super::*; use crate::joins::hash_join::not_in::{NotInOutput, NotInScope}; - use crate::joins::join_hash_map::JoinHashMapU32; + /// Adds the bloom filter of `build_keys.keys`, sized for 4 times its + /// keys (the keys of a few partitions). + fn test_build_keys(mut build_keys: BuildKeys) -> BuildKeys { + let reservation = datafusion_execution::memory_pool::MemoryConsumer::new("test") + .register(&(Arc::new(UnboundedMemoryPool::default()) as Arc)); + build_keys.bloom_filter = + crate::joins::hash_join::bloom_filter::partition_filter( + &build_keys.keys, + build_keys.num_distinct * 4, + GlobalFilterOptions::default().bloom_filter, + &reservation, + ) + .unwrap() + .map(Arc::new); + build_keys + } + use arrow::array::{ArrayRef, BooleanArray, Float64Array, Int32Array}; - use arrow::compute::SortOptions; use arrow::record_batch::RecordBatch; - use datafusion_common::SplitPoint; - use datafusion_physical_expr::{ - PhysicalSortExpr, - expressions::{Column, Literal}, - }; + use datafusion_physical_expr::expressions::{Column, Literal}; fn test_on_right() -> Vec { vec![Arc::new(Column::new("probe_key", 0))] @@ -1168,13 +1016,10 @@ mod tests { completion_notify: Notify::new(), dynamic_filter, on_right, - repartition_random_state: SeededRandomState::with_seed(1), probe_schema: test_probe_schema(), - probe_range_partitioning: None, null_equality: NullEquality::NullEqualsNothing, not_in: None, - in_list_max_size: 128 * 1024, - in_list_max_values: 150, + global_filter_options: GlobalFilterOptions::default(), } } @@ -1221,6 +1066,7 @@ mod tests { pushdown, bounds, keys_have_null: false, + build_keys: BuildKeys::default(), }) } @@ -1270,11 +1116,6 @@ mod tests { .expect("expected BinaryExpr dynamic filter") } - fn case_expr(expr: &PhysicalExprRef) -> &CaseExpr { - expr.downcast_ref::() - .expect("expected CaseExpr dynamic filter") - } - fn assert_literal_bool(expr: &PhysicalExprRef, expected: bool) { let literal = expr .downcast_ref::() @@ -1298,439 +1139,351 @@ mod tests { (partitions.clone(), *completed_partitions) } - #[test] - fn collect_left_updates_with_membership_only() { - let acc = make_collect_left_accumulator_for_test(); + /// A partition with more keys than the InList limit. + fn reported_map(keys: Vec) -> PartitionStatus { + let (min, max) = (*keys.iter().min().unwrap(), *keys.iter().max().unwrap()); + let num_distinct = keys.len(); + PartitionStatus::Reported(PartitionData { + pushdown: PushdownStrategy::Large(None), + bounds: bounds(min, max), + keys_have_null: false, + build_keys: test_build_keys(BuildKeys { + bloom_filter: None, + keys: vec![Arc::new(Int32Array::from(keys))], + num_distinct, + }), + }) + } - acc.build_filter(FinalizeInput::CollectLeft(reported( - in_list(&[1, 2, 3]), - no_bounds(), - ))) + fn eval_bool(expr: &PhysicalExprRef, keys: Vec) -> Vec { + let batch = RecordBatch::try_new( + test_probe_schema(), + vec![Arc::new(Int32Array::from(keys))], + ) .unwrap(); - - let expr = current_expr(&acc); - assert_in_list_column_values(&expr, "probe_key", 0, &[1, 2, 3]); + let result = expr + .evaluate(&batch) + .unwrap() + .into_array(batch.num_rows()) + .unwrap(); + let result = result.as_any().downcast_ref::().unwrap(); + result.iter().map(|v| v.unwrap_or(false)).collect() } + /// Hash partitions with many keys get one unrouted filter: + /// `key >= global_min AND key <= global_max AND bloom_filter`. #[test] - fn collect_left_updates_with_bounds_only() { - let acc = make_collect_left_accumulator_for_test(); - - acc.build_filter(FinalizeInput::CollectLeft(reported( - PushdownStrategy::Empty, - bounds(10, 20), - ))) + fn partitioned_global_bloom_filter() { + let acc = make_partitioned_expr_accumulator_for_test(3); + acc.build_filter(FinalizeInput::Partitioned(vec![ + reported_map((0..1000).map(|k| k * 3).collect()), + reported(PushdownStrategy::Empty, no_bounds()), + reported_map((0..1000).map(|k| k * 3 + 10_000).collect()), + ])) .unwrap(); let expr = current_expr(&acc); - assert_top_binary_op(&expr, Operator::And); + let text = format!("{expr}"); + assert_eq!( + text, "probe_key@0 >= 0 AND probe_key@0 <= 12997 AND bloom_filter", + "{text}" + ); + assert!(!text.contains("CASE")); + + let probe: Vec = (-10..13_010).collect(); + let result = eval_bool(&expr, probe.clone()); + let mut false_positives = 0; + for (k, pass) in probe.iter().zip(result) { + let is_key = (*k >= 0 && *k < 3000 && k % 3 == 0) + || (*k >= 10_000 && *k < 13_000 && (k - 10_000) % 3 == 0); + assert!(pass || !is_key, "build key {k} dropped"); + false_positives += (pass && !is_key) as usize; + } + // Bounds remove everything outside [0, 12997]; the bloom filter removes + // most other keys + assert!(false_positives < 300, "{false_positives} false positives"); } + /// Partitions that each fit an InList but together exceed the InList limit get + /// a bloom filter of all their keys. #[test] - fn collect_left_empty_build_data_does_not_update_filter() { - let acc = make_collect_left_accumulator_for_test(); - let initial_generation = acc.dynamic_filter.snapshot_generation(); + fn partitioned_inlists_over_global_limit_use_bloom_filter() { + let mut acc = make_partitioned_expr_accumulator_for_test(2); + acc.global_filter_options.inlist_max_distinct_values = 3; + let in_list_partition = |keys: Vec| { + PartitionStatus::Reported(PartitionData { + pushdown: in_list(&keys), + bounds: bounds(*keys.iter().min().unwrap(), *keys.iter().max().unwrap()), + keys_have_null: false, + build_keys: test_build_keys(BuildKeys { + bloom_filter: None, + num_distinct: keys.len(), + keys: vec![Arc::new(Int32Array::from(keys))], + }), + }) + }; + acc.build_filter(FinalizeInput::Partitioned(vec![ + in_list_partition(vec![1, 3]), + in_list_partition(vec![5, 7]), + ])) + .unwrap(); + let expr = current_expr(&acc); + assert!(format!("{expr}").ends_with("AND bloom_filter"), "{expr}"); + let result = eval_bool(&expr, (0..10).collect()); + for k in [1, 3, 5, 7] { + assert!(result[k], "build key {k} dropped"); + } + assert!(!result[0] && !result[8] && !result[9]); - acc.build_filter(FinalizeInput::CollectLeft(reported( - PushdownStrategy::Empty, - no_bounds(), - ))) + // At the limit: one InList with the keys of both partitions + let mut acc = make_partitioned_expr_accumulator_for_test(2); + acc.global_filter_options.inlist_max_distinct_values = 4; + acc.build_filter(FinalizeInput::Partitioned(vec![ + in_list_partition(vec![1, 3]), + in_list_partition(vec![5, 7]), + ])) .unwrap(); + let expr = current_expr(&acc); + let and = binary_expr(&expr); + assert_in_list_column_values(and.right(), "probe_key", 0, &[1, 3, 5, 7]); + } - assert_eq!( - acc.dynamic_filter.snapshot_generation(), - initial_generation, - "empty CollectLeft input must not update with a no-op filter" - ); + /// A partition InList holds every build row; the global InList holds each + /// key once. + #[test] + fn global_inlist_has_distinct_keys() { + let acc = make_partitioned_expr_accumulator_for_test(2); + let partition = |rows: Vec, num_distinct: usize| { + PartitionStatus::Reported(PartitionData { + pushdown: in_list(&rows), + bounds: bounds(*rows.iter().min().unwrap(), *rows.iter().max().unwrap()), + keys_have_null: false, + build_keys: test_build_keys(BuildKeys { + bloom_filter: None, + num_distinct, + keys: vec![Arc::new(Int32Array::from(rows))], + }), + }) + }; + acc.build_filter(FinalizeInput::Partitioned(vec![ + partition(vec![1, 1, 3, 1, 3], 2), + partition(vec![5, 5, 5], 1), + ])) + .unwrap(); let expr = current_expr(&acc); - assert_literal_bool(&expr, true); + let and = binary_expr(&expr); + assert_in_list_column_values(and.right(), "probe_key", 0, &[1, 3, 5]); } - fn hash_map() -> PushdownStrategy { - let map = Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(1)))); - PushdownStrategy::Map(Arc::new(ReservedMap::unreserved(map)), None) + /// A partition whose only key is NULL has NULL bounds. The merged bounds + /// ignore them, so the filter keeps the non-NULL keys of the other partitions. + #[test] + fn merged_bounds_ignore_null_bounds() { + let null_bounds = PartitionBounds::new(vec![ColumnBounds::new( + ScalarValue::Int32(None), + ScalarValue::Int32(None), + )]); + let merged = PartitionBounds::merge( + [&null_bounds, &bounds(11, 11), &null_bounds].into_iter(), + ) + .unwrap(); + assert_eq!( + merged.get_column_bounds(0).unwrap(), + &ColumnBounds::new( + ScalarValue::Int32(Some(11)), + ScalarValue::Int32(Some(11)) + ) + ); + let merged = + PartitionBounds::merge([&bounds(5, 6), &null_bounds].into_iter()).unwrap(); + assert_eq!( + merged.get_column_bounds(0).unwrap(), + &ColumnBounds::new(ScalarValue::Int32(Some(5)), ScalarValue::Int32(Some(6))) + ); } + /// A canceled partition's keys are unknown, so the filter keeps every row. #[test] - fn partitioned_hash_map_partitions_route_without_case() { + fn partitioned_canceled_partition_sets_filter_to_true() { let acc = make_partitioned_expr_accumulator_for_test(3); - acc.build_filter(FinalizeInput::Partitioned(vec![ - reported(hash_map(), bounds(5, 10)), + reported_map((0..1000).collect()), + PartitionStatus::CanceledUnknown, reported(PushdownStrategy::Empty, no_bounds()), - reported(hash_map(), bounds(1, 7)), ])) .unwrap(); - - let expr = current_expr(&acc); - assert!(expr.downcast_ref::().is_none()); - assert_eq!( - expr.to_string(), - "probe_key@0 >= 1 AND probe_key@0 <= 10 AND hash_lookup" - ); + assert_eq!(current_expr(&acc).to_string(), "true"); } - /// A hash table holding `keys`, hashed as the build side hashes them. - #[cfg(not(feature = "force_hash_collisions"))] - fn hash_map_of(keys: &[i32]) -> PushdownStrategy { - use crate::joins::join_hash_map::JoinHashMapType; - let keys: ArrayRef = Arc::new(Int32Array::from(keys.to_vec())); - let mut hashes = vec![0; keys.len()]; - create_hashes([&keys], HASH_JOIN_SEED.random_state(), &mut hashes).unwrap(); - let mut map = JoinHashMapU32::with_capacity(keys.len()); - map.update_from_iter(Box::new(hashes.iter().enumerate()), 0); - let map = Arc::new(Map::HashMap(Box::new(map))); - PushdownStrategy::Map(Arc::new(ReservedMap::unreserved(map)), None) - } - - /// Splits `keys` across `num_partitions` build partitions as hash - /// repartitioning would, reporting each non-empty partition with the - /// strategy `strategy` picks for it. - #[cfg(not(feature = "force_hash_collisions"))] - fn routed_partitions( - acc: &SharedBuildAccumulator, - num_partitions: usize, - keys: &[i32], - strategy: impl Fn(usize, &[i32]) -> PushdownStrategy, - ) -> Vec { - let array: ArrayRef = Arc::new(Int32Array::from(keys.to_vec())); - let mut hashes = vec![0; keys.len()]; - create_hashes( - [&array], - acc.repartition_random_state.random_state(), - &mut hashes, - ) + /// Only canceled partitions: the filter is `true`, not the `false` of an + /// empty build side. + #[test] + fn partitioned_all_canceled_partitions_set_filter_to_true() { + let acc = make_partitioned_expr_accumulator_for_test(2); + acc.build_filter(FinalizeInput::Partitioned(vec![ + PartitionStatus::CanceledUnknown, + PartitionStatus::CanceledUnknown, + ])) .unwrap(); - (0..num_partitions) - .map(|partition| { - let held: Vec = keys - .iter() - .zip(&hashes) - .filter(|(_, hash)| *hash % num_partitions as u64 == partition as u64) - .map(|(key, _)| *key) - .collect(); - if held.is_empty() { - reported(PushdownStrategy::Empty, no_bounds()) - } else { - reported(strategy(partition, &held), no_bounds()) - } - }) - .collect() + assert_eq!(current_expr(&acc).to_string(), "true"); } - /// Partitions that chose an IN list and partitions that chose a hash table - /// merge into one lookup without a `CASE`: a probe row passes exactly when - /// the partition its key routes to holds the key. + /// All partitions empty: no probe row can match. #[test] - #[cfg(not(feature = "force_hash_collisions"))] - fn partitioned_in_list_and_hash_map_partitions_merge() { - let acc = make_partitioned_expr_accumulator_for_test(4); - let build: Vec = (0..40).step_by(2).collect(); - - acc.build_filter(FinalizeInput::Partitioned(routed_partitions( - &acc, - 4, - &build, - |partition, keys| { - if partition % 2 == 0 { - hash_map_of(keys) - } else { - in_list(keys) - } - }, - ))) + fn partitioned_all_empty_partitions_set_filter_to_false() { + let acc = make_partitioned_expr_accumulator_for_test(2); + acc.build_filter(FinalizeInput::Partitioned(vec![ + reported(PushdownStrategy::Empty, no_bounds()), + reported(PushdownStrategy::Empty, no_bounds()), + ])) .unwrap(); - - assert!( - current_expr(&acc).downcast_ref::().is_none(), - "expected a merged lookup, got: {}", - current_expr(&acc) - ); - let probe: Vec = (0..40).collect(); - let expected: Vec = probe.iter().map(|key| key % 2 == 0).collect(); - assert_matches(&acc, &probe, &expected); + assert_eq!(current_expr(&acc).to_string(), "false"); } - /// IN list partitions whose combined list passes the IN list limits merge - /// into a hash lookup rather than one oversized IN list, which must still - /// accept every partition's keys. + /// Float keys get the global bloom filter too. #[test] - #[cfg(not(feature = "force_hash_collisions"))] - fn partitioned_in_lists_past_limit_merge_into_hash_lookup() { - let mut acc = make_partitioned_expr_accumulator_for_test(3); - acc.in_list_max_values = 2; - let build: Vec = (1..=9).collect(); - - acc.build_filter(FinalizeInput::Partitioned(routed_partitions( - &acc, - 3, - &build, - |_, keys| in_list(keys), - ))) + fn float_keys_use_global_bloom_filter() { + let mut acc = make_partitioned_expr_accumulator_for_test(2); + acc.probe_schema = Arc::new(Schema::new(vec![Field::new( + "probe_key", + DataType::Float64, + false, + )])); + let float_partition = |keys: Vec| { + PartitionStatus::Reported(PartitionData { + pushdown: PushdownStrategy::Large(None), + bounds: no_bounds(), + keys_have_null: false, + build_keys: test_build_keys(BuildKeys { + bloom_filter: None, + num_distinct: keys.len(), + keys: vec![Arc::new(Float64Array::from(keys))], + }), + }) + }; + acc.build_filter(FinalizeInput::Partitioned(vec![ + float_partition((0..1000).map(f64::from).collect()), + float_partition((1000..2000).map(f64::from).collect()), + ])) .unwrap(); - let expr = current_expr(&acc); - assert!( - expr.downcast_ref::().is_none() - && expr.downcast_ref::().is_none(), - "expected a hash lookup, got: {expr}" - ); - let probe: Vec = (0..20).collect(); - let expected: Vec = probe.iter().map(|key| (1..=9).contains(key)).collect(); - assert_matches(&acc, &probe, &expected); + assert_eq!(format!("{expr}"), "bloom_filter"); } - /// Evaluates the current filter over probe keys `keys`. - #[cfg(not(feature = "force_hash_collisions"))] - fn assert_matches(acc: &SharedBuildAccumulator, keys: &[i32], expected: &[bool]) { - let probe = RecordBatch::try_new( - test_probe_schema(), - vec![Arc::new(Int32Array::from(keys.to_vec()))], - ) + /// With a memory pool too small for the bloom filter, the global filter + /// keeps every build key: it falls back to the bounds (and the pruning + /// bitmap), and the join results stay correct. + #[test] + fn small_memory_pool_falls_back_to_bounds() { + use datafusion_execution::memory_pool::GreedyMemoryPool; + let mut acc = make_partitioned_expr_accumulator_for_test(2); + let pool: Arc = Arc::new(GreedyMemoryPool::new(1024)); + acc.global_filter_options.memory_pool = Some(Arc::clone(&pool)); + acc.build_filter(FinalizeInput::Partitioned(vec![ + reported_map((0..1000).map(|k| k * 4).collect()), + reported_map((0..1000).map(|k| k * 4 + 1).collect()), + ])) .unwrap(); - let matched = current_expr(acc) - .evaluate(&probe) - .unwrap() - .into_array(keys.len()) - .unwrap(); - assert_eq!(matched.as_boolean(), &BooleanArray::from(expected.to_vec())); + let expr = current_expr(&acc); + let result = eval_bool(&expr, (-10..4010).collect()); + for (k, pass) in (-10..4010).zip(result) { + // Bounds only: every key in [0, 3997] passes, nothing outside + assert_eq!(pass, (0..=3997).contains(&k), "key {k}"); + } + assert_eq!(pool.reserved(), 0); } + /// A partition without hashes (too big alone, or its memory reservation + /// failed) means no bloom filter: the filter keeps only the bounds and the + /// pruning bitmap. #[test] - fn partitioned_one_real_partition_with_rest_empty_skips_case() { - let acc = make_partitioned_expr_accumulator_for_test(3); - + fn partition_without_hashes_skips_bloom_filter() { + let acc = make_partitioned_expr_accumulator_for_test(2); + let mut no_hashes = reported_map((1_000_000..1_000_100).collect()); + if let PartitionStatus::Reported(data) = &mut no_hashes { + data.build_keys.bloom_filter = None; + } acc.build_filter(FinalizeInput::Partitioned(vec![ - reported(PushdownStrategy::Empty, no_bounds()), - reported(in_list(&[2]), no_bounds()), - reported(PushdownStrategy::Empty, no_bounds()), + reported_map((0..100).collect()), + no_hashes, ])) .unwrap(); - let expr = current_expr(&acc); - in_list_expr(&expr); - assert!(expr.downcast_ref::().is_none()); + assert_eq!( + expr.to_string(), + "probe_key@0 >= 0 AND probe_key@0 <= 1000099 AND key_range_pruning" + ); } + /// The bloom filter carries one key-range bitmap for all partitions, so scans + /// can prune containers with it. #[test] - fn partitioned_canceled_unknown_partitions_keep_unknown_routes_permissive() { + fn global_bloom_filter_has_merged_pruning_bitmap() { let acc = make_partitioned_expr_accumulator_for_test(2); - acc.build_filter(FinalizeInput::Partitioned(vec![ - PartitionStatus::CanceledUnknown, - reported(PushdownStrategy::Empty, no_bounds()), + reported_map((0..100).collect()), + reported_map((1_000_000..1_000_100).collect()), ])) .unwrap(); - let expr = current_expr(&acc); - let case = case_expr(&expr); - assert_eq!(case.when_then_expr().len(), 1); - assert_literal_bool(&case.when_then_expr()[0].1, false); - assert_literal_bool( - case.else_expr().expect("expected permissive fallback"), - true, - ); + let bloom = binary_expr(&expr) + .right() + .downcast_ref::() + .expect("expected a bloom filter"); + let bitmap = bloom.pruning_bitmap().expect("expected a pruning bitmap"); + let may = |lo: i32, hi: i32| bitmap.may_contain_range(lo as u64, hi as u64); + assert!(may(50, 60)); + assert!(may(1_000_050, 1_000_060)); + assert!(!may(400_000, 600_000)); } #[test] - fn partitioned_range_dynamic_filter_routes_with_range_expr() -> Result<()> { - let mut acc = make_partitioned_expr_accumulator_for_test(4); - acc.probe_range_partitioning = Some(RangePartitioning::try_new( - [PhysicalSortExpr::new( - Arc::clone(&acc.on_right[0]), - Default::default(), - )] - .into(), - vec![ - SplitPoint::new(vec![ScalarValue::Int32(Some(10))]), - SplitPoint::new(vec![ScalarValue::Int32(Some(20))]), - SplitPoint::new(vec![ScalarValue::Int32(Some(30))]), - ], - )?); + fn collect_left_updates_with_membership_only() { + let acc = make_collect_left_accumulator_for_test(); - acc.build_filter(FinalizeInput::Partitioned(vec![ - reported(PushdownStrategy::Empty, no_bounds()), - PartitionStatus::CanceledUnknown, - reported(in_list(&[20, 29]), no_bounds()), - reported(in_list(&[30]), no_bounds()), - ]))?; + acc.build_filter(FinalizeInput::CollectLeft(reported( + in_list(&[1, 2, 3]), + no_bounds(), + ))) + .unwrap(); let expr = current_expr(&acc); - let case = case_expr(&expr); - assert!( - case.expr() - .and_then(|expr| expr.downcast_ref::()) - .is_some(), - "Range routing must use RangeExpr" - ); - assert_eq!(case.when_then_expr().len(), 3); - assert_literal_bool(&case.when_then_expr()[0].1, false); - assert_literal_bool( - case.else_expr().expect("expected permissive fallback"), - true, - ); - - let batch = RecordBatch::try_new( - test_probe_schema(), - vec![Arc::new(Int32Array::from(vec![ - 9, 10, 19, 20, 21, 29, 30, 31, - ]))], - )?; - let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?; - let result = result - .as_any() - .downcast_ref::() - .expect("dynamic filter should evaluate to BooleanArray"); - assert_eq!( - result, - &BooleanArray::from(vec![false, true, true, true, false, true, true, false,]) - ); - - Ok(()) + assert_in_list_column_values(&expr, "probe_key", 0, &[1, 2, 3]); } #[test] - fn partitioned_range_dynamic_filter_routes_compound_nullable_keys() -> Result<()> { - let probe_schema = Arc::new(Schema::new(vec![ - Field::new("probe_key", DataType::Int32, true), - Field::new("probe_tie", DataType::Int32, true), - ])); - let on_right: Vec = vec![ - Arc::new(Column::new("probe_key", 0)), - Arc::new(Column::new("probe_tie", 1)), - ]; - let mut acc = make_accumulator_for_test( - AccumulatedBuildData::Partitioned { - partitions: vec![PartitionStatus::Pending; 4], - completed_partitions: 0, - }, - on_right, - ); - acc.probe_schema = Arc::clone(&probe_schema); - acc.probe_range_partitioning = Some(RangePartitioning::try_new( - [ - PhysicalSortExpr::new( - Arc::clone(&acc.on_right[0]), - SortOptions::new(false, true), - ), - PhysicalSortExpr::new( - Arc::clone(&acc.on_right[1]), - SortOptions::new(false, false), - ), - ] - .into(), - vec![ - SplitPoint::new(vec![ - ScalarValue::Int32(None), - ScalarValue::Int32(Some(10)), - ]), - SplitPoint::new(vec![ScalarValue::Int32(None), ScalarValue::Int32(None)]), - SplitPoint::new(vec![ - ScalarValue::Int32(Some(10)), - ScalarValue::Int32(None), - ]), - ], - )?); - - acc.build_filter(FinalizeInput::Partitioned(vec![ - reported(PushdownStrategy::Empty, no_bounds()), - PartitionStatus::CanceledUnknown, - reported(PushdownStrategy::Empty, no_bounds()), - PartitionStatus::CanceledUnknown, - ]))?; + fn collect_left_empty_build_data_does_not_update_filter() { + let acc = make_collect_left_accumulator_for_test(); + let initial_generation = acc.dynamic_filter.snapshot_generation(); - let expr = current_expr(&acc); - let case = case_expr(&expr); - assert!(case.expr().is_some()); - assert_eq!(case.when_then_expr().len(), 2); - assert_literal_bool( - case.else_expr().expect("expected permissive fallback"), - true, - ); + acc.build_filter(FinalizeInput::CollectLeft(reported( + PushdownStrategy::Empty, + no_bounds(), + ))) + .unwrap(); - let batch = RecordBatch::try_new( - probe_schema, - vec![ - Arc::new(Int32Array::from(vec![ - None, - None, - None, - None, - Some(9), - Some(10), - Some(10), - Some(11), - ])), - Arc::new(Int32Array::from(vec![ - Some(9), - Some(10), - Some(11), - None, - None, - Some(9), - None, - None, - ])), - ], - )?; - let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?; - let result = result - .as_any() - .downcast_ref::() - .expect("dynamic filter should evaluate to BooleanArray"); assert_eq!( - result, - &BooleanArray::from( - vec![false, true, true, false, false, false, true, true,] - ) + acc.dynamic_filter.snapshot_generation(), + initial_generation, + "empty CollectLeft input must not update with a no-op filter" ); - - Ok(()) + let expr = current_expr(&acc); + assert_literal_bool(&expr, true); } #[test] - fn partitioned_range_dynamic_filter_preserves_signed_zero_routing() -> Result<()> { - let probe_schema = Arc::new(Schema::new(vec![Field::new( - "probe_key", - DataType::Float64, - false, - )])); - let on_right: Vec = vec![Arc::new(Column::new("probe_key", 0))]; - let mut acc = make_accumulator_for_test( - AccumulatedBuildData::Partitioned { - partitions: vec![PartitionStatus::Pending; 2], - completed_partitions: 0, - }, - on_right, - ); - acc.probe_schema = Arc::clone(&probe_schema); - acc.probe_range_partitioning = Some(RangePartitioning::try_new( - [PhysicalSortExpr::new( - Arc::clone(&acc.on_right[0]), - SortOptions::default(), - )] - .into(), - vec![SplitPoint::new(vec![ScalarValue::Float64(Some(0.0))])], - )?); + fn partitioned_one_real_partition_with_rest_empty_uses_its_in_list() { + let acc = make_partitioned_expr_accumulator_for_test(3); acc.build_filter(FinalizeInput::Partitioned(vec![ - PartitionStatus::CanceledUnknown, reported(PushdownStrategy::Empty, no_bounds()), - ]))?; + reported(in_list(&[2]), no_bounds()), + reported(PushdownStrategy::Empty, no_bounds()), + ])) + .unwrap(); let expr = current_expr(&acc); - let batch = RecordBatch::try_new( - probe_schema, - vec![Arc::new(Float64Array::from(vec![-0.0, 0.0]))], - )?; - let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?; - let result = result - .as_any() - .downcast_ref::() - .expect("dynamic filter should evaluate to BooleanArray"); - assert_eq!(result, &BooleanArray::from(vec![true, false])); - - Ok(()) + assert_in_list_column_values(&expr, "probe_key", 0, &[2]); } // Regression guard for the build-report lifecycle fix: on `Drop`, a stream @@ -1754,6 +1507,7 @@ mod tests { pushdown: PushdownStrategy::Empty, bounds: PartitionBounds::new(vec![]), keys_have_null: false, + build_keys: BuildKeys::default(), }, ) .unwrap(); @@ -1791,6 +1545,34 @@ mod tests { assert_eq!(completed, 1); } + /// A partition that waits for the filter completes when another partition + /// is canceled, and the filter is `true`. + #[tokio::test] + async fn canceled_partition_completes_waiting_partition() { + let acc = make_partitioned_expr_accumulator_for_test(2); + let report = acc.report_build_data(PartitionBuildData::Partitioned { + partition_id: 0, + pushdown: in_list(&[1, 2]), + bounds: bounds(1, 2), + keys_have_null: false, + build_keys: BuildKeys::default(), + }); + let mut report = std::pin::pin!(report); + let mut cx = std::task::Context::from_waker(futures::task::noop_waker_ref()); + assert!(report.as_mut().poll(&mut cx).is_pending()); + + acc.report_canceled_partition(1); + + tokio::time::timeout(std::time::Duration::from_secs(5), report) + .await + .expect("the waiting partition must complete") + .unwrap(); + assert_eq!(current_expr(&acc).to_string(), "true"); + assert!( + futures::poll!(std::pin::pin!(acc.dynamic_filter.wait_complete())).is_ready() + ); + } + fn null_semantics_accumulator( probe_schema: Arc, on_right: Vec, @@ -1808,16 +1590,13 @@ mod tests { completion_notify: Notify::new(), dynamic_filter: Arc::new(DynamicFilterPhysicalExpr::new(vec![], lit(true))), on_right, - repartition_random_state: SeededRandomState::with_seed(1), probe_schema, - probe_range_partitioning: None, null_equality, not_in: null_aware.then_some(NotIn::BuildRows { scope: NotInScope::Uncorrelated, output: NotInOutput::Rows, }), - in_list_max_size: 128 * 1024, - in_list_max_values: 150, + global_filter_options: GlobalFilterOptions::default(), } } @@ -1841,6 +1620,7 @@ mod tests { pushdown, bounds, keys_have_null: true, + build_keys: BuildKeys::default(), }) } @@ -1865,11 +1645,10 @@ mod tests { acc } - /// The IS NULL disjunct must wrap the merged filter as a whole. A probe NULL - /// belongs to exactly one partition, so widening only the filter of the - /// partition that reported the NULL would still drop it everywhere else. + /// The IS NULL disjunct must wrap the global filter as a whole when any + /// partition reports a NULL key. #[test] - fn partitioned_null_keys_in_one_partition_widen_whole_routed_filter() { + fn partitioned_null_keys_in_one_partition_widen_whole_global_filter() { let acc = null_equal_partitioned_accumulator(2); acc.build_filter(FinalizeInput::Partitioned(vec![ @@ -1886,33 +1665,22 @@ mod tests { "expected the IS NULL disjunct first, got: {expr}" ); assert_in_list_column_values(widened.right(), "probe_key", 0, &[1, 2]); - assert_eq!( - format!("{expr}").matches("IS NULL").count(), - 1, - "the routed filter must be widened once, not per branch" - ); + assert_eq!(format!("{expr}").matches("IS NULL").count(), 1); } - /// A canceled partition's build content is unknown, so it may hold a NULL key: - /// the aggregated flag must be permissive even when no partition reported one. + /// A canceled partition gives `true` without an IS NULL disjunct, also for a + /// null-equal join. #[test] - fn partitioned_canceled_partition_widens_routed_filter() { + fn partitioned_canceled_partition_null_equal_filter_is_true() { let acc = null_equal_partitioned_accumulator(2); acc.build_filter(FinalizeInput::Partitioned(vec![ PartitionStatus::CanceledUnknown, - reported(in_list(&[2]), no_bounds()), + reported_with_null_keys(in_list(&[2]), no_bounds()), ])) .unwrap(); - let expr = current_expr(&acc); - assert_top_binary_op(&expr, Operator::Or); - assert!( - binary_expr(&expr) - .left() - .downcast_ref::() - .is_some() - ); + assert_eq!(current_expr(&acc).to_string(), "true"); } #[test] @@ -2025,16 +1793,13 @@ mod tests { completion_notify: Notify::new(), dynamic_filter, on_right, - repartition_random_state: SeededRandomState::with_seed(1), probe_schema: Arc::new(Schema::new(fields)), - probe_range_partitioning: None, null_equality: NullEquality::NullEqualsNothing, not_in: Some(NotIn::BuildRows { scope: NotInScope::Correlated { value_keys }, output: NotInOutput::Mark, }), - in_list_max_size: 128 * 1024, - in_list_max_values: 150, + global_filter_options: GlobalFilterOptions::default(), }; let bounds = PartitionBounds::new( @@ -2048,7 +1813,7 @@ mod tests { .collect(), ); acc.build_filter(FinalizeInput::CollectLeft(reported( - PushdownStrategy::Empty, + PushdownStrategy::Large(None), bounds, ))) .unwrap(); diff --git a/datafusion/physical-plan/src/joins/hash_join/stream.rs b/datafusion/physical-plan/src/joins/hash_join/stream.rs index 38da5166e1efe..d48d7949f124c 100644 --- a/datafusion/physical-plan/src/joins/hash_join/stream.rs +++ b/datafusion/physical-plan/src/joins/hash_join/stream.rs @@ -16,7 +16,7 @@ use crate::joins::PartitionMode; use crate::joins::hash_join::exec::JoinLeftData; use crate::joins::hash_join::not_in::NotInProbe; use crate::joins::hash_join::shared_bounds::{ - PartitionBounds, PartitionBuildData, SharedBuildAccumulator, + BuildKeys, PartitionBounds, PartitionBuildData, SharedBuildAccumulator, }; use crate::joins::join_hash_map::ProbeOffset; use crate::joins::join_hash_map::ProbeOffset::AtRow; @@ -638,17 +638,25 @@ impl HashJoinStream { .iter() .any(|array| array.logical_null_count() > 0); + let build_keys = BuildKeys { + keys: left_data.values().to_vec(), + bloom_filter: left_data.bloom_filter().cloned(), + num_distinct: left_data.map().num_of_distinct_key(), + }; + let build_data = match self.mode { PartitionMode::Partitioned => PartitionBuildData::Partitioned { partition_id: self.partition, pushdown, bounds, keys_have_null, + build_keys, }, PartitionMode::CollectLeft => PartitionBuildData::CollectLeft { pushdown, bounds, keys_have_null, + build_keys, }, PartitionMode::Auto => unreachable!( "PartitionMode::Auto should not be present at execution time. This is a bug in DataFusion, please report it!" @@ -1238,6 +1246,7 @@ mod tests { pushdown: PushdownStrategy::Empty, bounds: PartitionBounds::new(vec![]), keys_have_null: false, + build_keys: Default::default(), } } diff --git a/datafusion/physical-plan/src/joins/key_range_bitmap.rs b/datafusion/physical-plan/src/joins/key_range_bitmap.rs index 47030f5cadc4a..9b519adec44d8 100644 --- a/datafusion/physical-plan/src/joins/key_range_bitmap.rs +++ b/datafusion/physical-plan/src/joins/key_range_bitmap.rs @@ -22,6 +22,8 @@ pub(crate) enum KeyRangePart<'a> { /// The partition's build keys, which must lie within its `[min, max]`. Keys(&'a dyn Array), /// The partition has no bitmap, so any key in `[min, max]` may be present. + // Unused since the global dynamic filter merges keys rather than ranges. + #[cfg_attr(not(test), expect(dead_code))] Range { min: &'a ScalarValue, max: &'a ScalarValue, diff --git a/datafusion/physical-plan/src/joins/mod.rs b/datafusion/physical-plan/src/joins/mod.rs index c4549d847f554..dfd7b299b8dd1 100644 --- a/datafusion/physical-plan/src/joins/mod.rs +++ b/datafusion/physical-plan/src/joins/mod.rs @@ -12,7 +12,7 @@ use std::sync::Arc; use arrow::array::BooleanBufferBuilder; pub use asof_join::{AsOfJoinExec, AsOfMatchExpr}; pub use cross_join::CrossJoinExec; -use datafusion_common::{Result, internal_err}; +use datafusion_common::Result; use datafusion_execution::memory_pool::MemoryReservation; use datafusion_physical_expr::PhysicalExprRef; /// # Public Only for Internal Use: @@ -21,7 +21,7 @@ use datafusion_physical_expr::PhysicalExprRef; /// /// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis #[doc(hidden)] -pub use hash_join::HashTableLookupExpr; +pub use hash_join::BloomFilterExpr; pub use hash_join::{ HashExpr, HashJoinExec, HashJoinExecBuilder, PreparedHashJoinBuild, SeededRandomState, }; @@ -98,16 +98,13 @@ impl Map { } } -/// A [`Map`] and the reservation for its memory, which the dynamic filter -/// shares for as long as the plan lives. Held together, the memory stays -/// reserved until the last holder drops the map. +/// A [`Map`] and the reservation for its memory. Held together, the memory +/// stays reserved until the last holder drops the map. pub(crate) struct ReservedMap { - /// Shared, as [`HashTableLookupExpr::new`] takes one and compares by it. map: Arc, /// Also charged for data kept with the map, such as the pruning bitmap. - /// `None` for a map built outside a join; see [`HashTableLookupExpr::new`]. // Keep the reservation after the map it accounts for. - reservation: Option, + reservation: MemoryReservation, } impl ReservedMap { @@ -115,28 +112,17 @@ impl ReservedMap { pub(crate) fn new(map: Map, reservation: MemoryReservation) -> Self { Self { map: Arc::new(map), - reservation: Some(reservation), - } - } - - /// Wraps a map that was not charged to a pool. - pub(crate) fn unreserved(map: Arc) -> Self { - Self { - map, - reservation: None, + reservation, } } /// Charges `bytes` for data kept with the map. pub(crate) fn try_grow(&self, bytes: usize) -> Result<()> { - match &self.reservation { - Some(reservation) => reservation.try_grow(bytes), - None => internal_err!("ReservedMap has no reservation to grow"), - } + self.reservation.try_grow(bytes) } pub(crate) fn reserved_bytes(&self) -> usize { - self.reservation.as_ref().map_or(0, MemoryReservation::size) + self.reservation.size() } } diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rs index 13274d26e1fb4..6705d7d5c76e8 100644 --- a/datafusion/physical-plan/src/repartition/mod.rs +++ b/datafusion/physical-plan/src/repartition/mod.rs @@ -783,20 +783,6 @@ impl RangeExpr { pub fn sort_options(&self) -> &[SortOptions] { self.router.sort_options() } - - /// Appends the partition of each row of `arrays`, the evaluated - /// `on_columns`, to `partition_ids`. - pub(crate) fn route_partition_ids( - &self, - arrays: &[arrow::array::ArrayRef], - partition_ids: &mut Vec, - ) -> Result<()> { - if self.router.split_points().is_empty() { - partition_ids.resize(partition_ids.len() + arrays[0].len(), 0); - return Ok(()); - } - self.router.route_partition_ids(arrays, partition_ids) - } } impl Display for RangeExpr { diff --git a/datafusion/proto/tests/cases/plans/exprs.rs b/datafusion/proto/tests/cases/plans/exprs.rs index 8c193283ad917..386d6c8764312 100644 --- a/datafusion/proto/tests/cases/plans/exprs.rs +++ b/datafusion/proto/tests/cases/plans/exprs.rs @@ -426,58 +426,6 @@ fn roundtrip_like() -> Result<()> { roundtrip_test(plan) } -/// Test that HashTableLookupExpr serializes to lit(true) -/// -/// HashTableLookupExpr contains a runtime hash table that cannot be serialized. -/// The serialization code replaces it with lit(true) which is safe because -/// it's a performance optimization filter, not a correctness requirement. -#[test] -fn roundtrip_hash_table_lookup_expr_to_lit() -> Result<()> { - use datafusion::physical_plan::joins::join_hash_map::JoinHashMapU32; - use datafusion::physical_plan::joins::{HashTableLookupExpr, Map}; - - // Create a simple schema and input plan - let schema = Arc::new(Schema::new(vec![Field::new("col", DataType::Int64, false)])); - let input = Arc::new(EmptyExec::new(schema.clone())); - - // Create a HashTableLookupExpr - it will be replaced with lit(true) during serialization - let hash_map = Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(0)))); - let on_columns = vec![col("col", &schema)?]; - let lookup_expr: Arc = Arc::new(HashTableLookupExpr::new( - on_columns, - datafusion::physical_plan::joins::SeededRandomState::with_seed(0), - None, - vec![Some(hash_map)], - "test_lookup".to_string(), - None, - )?); - - // Create a filter with the lookup expression - let filter = Arc::new(FilterExec::try_new(lookup_expr, input)?); - - // Serialize - let ctx = SessionContext::new(); - let codec = DefaultPhysicalExtensionCodec {}; - - let proto: PhysicalPlanNode = - PhysicalPlanNode::try_from_physical_plan(filter.clone(), &codec) - .expect("serialization should succeed"); - - // Deserialize - let result: Arc = proto - .try_into_physical_plan(&ctx.task_ctx(), &codec) - .expect("deserialization should succeed"); - - // The deserialized plan should have lit(true) instead of HashTableLookupExpr - // Verify the filter predicate is a Literal(true) - let result_filter = result.downcast_ref::().unwrap(); - let predicate = result_filter.predicate(); - let literal = predicate.downcast_ref::().unwrap(); - assert_eq!(*literal.value(), ScalarValue::Boolean(Some(true))); - - Ok(()) -} - #[test] fn roundtrip_hash_expr() -> Result<()> { use datafusion::physical_plan::joins::{HashExpr, SeededRandomState}; diff --git a/datafusion/pruning/Cargo.toml b/datafusion/pruning/Cargo.toml index 0d6e6e49c01d0..08b72f5932a1c 100644 --- a/datafusion/pruning/Cargo.toml +++ b/datafusion/pruning/Cargo.toml @@ -35,7 +35,6 @@ datafusion-expr = { workspace = true } datafusion-functions-nested = { workspace = true } insta = { workspace = true } itertools = { workspace = true } -rstest = { workspace = true } [[bench]] harness = false diff --git a/datafusion/pruning/src/pruning_predicate.rs b/datafusion/pruning/src/pruning_predicate.rs index b7325d90d5672..612d89c9410e8 100644 --- a/datafusion/pruning/src/pruning_predicate.rs +++ b/datafusion/pruning/src/pruning_predicate.rs @@ -38,7 +38,8 @@ use datafusion_expr_common::operator::Operator; use datafusion_physical_expr::utils::{Guarantee, LiteralGuarantee}; use datafusion_physical_expr::{PhysicalExprRef, expressions as phys_expr}; use datafusion_physical_expr_common::physical_expr::snapshot_physical_expr_opt; -use datafusion_physical_plan::joins::HashTableLookupExpr; +use datafusion_physical_plan::joins::BloomFilterExpr; +use datafusion_physical_plan::joins::key_range_bitmap::KeyRangeBitmap; use datafusion_physical_plan::{ColumnarValue, PhysicalExpr}; /// Used to prove that arbitrary predicates (boolean expression) can not @@ -1530,18 +1531,18 @@ fn min_max_non_null_exprs( statistics } -/// Builds a "may match" expression for a [`HashTableLookupExpr`] (a large join -/// build side pushed down as an opaque hash-table lookup): tests container -/// min/max stats against the build side's key-range bitmap, which is fixed size -/// and built once with the join's hash table. -fn build_hash_lookup_pruning_expr( - lookup: &HashTableLookupExpr, +/// Builds a "may match" expression for a [`BloomFilterExpr`] (a large join +/// build side pushed down as an opaque membership test) with key-range bitmap +/// `bitmap`: tests container min/max stats against the build side's key-range +/// bitmap, which is fixed size and built once with the join's build side. +fn build_key_range_bitmap_pruning_expr( + bitmap: &Arc, + on_columns: &[&Arc], schema: &Schema, required_columns: &mut RequiredColumns, ) -> Option> { - let bitmap = Arc::clone(lookup.pruning_bitmap()?); - let on_columns = lookup.children(); - let [column_expr] = on_columns[..] else { + let bitmap = Arc::clone(bitmap); + let [column_expr] = on_columns else { return None; }; // The schema adapter often wraps the column in a same-type or widening cast @@ -1848,14 +1849,22 @@ fn build_predicate_expression( return unhandled_hook.handle(expr); } } - if let Some(lookup) = expr.downcast_ref::() { - return build_hash_lookup_pruning_expr(lookup, schema, required_columns) + if let Some(bloom) = expr.downcast_ref::() { + return bloom + .pruning_bitmap() + .and_then(|bitmap| { + build_key_range_bitmap_pruning_expr( + bitmap, + &bloom.children(), + schema, + required_columns, + ) + }) .unwrap_or_else(|| unhandled_hook.handle(expr)); } - // A partitioned hash join hides its per-partition filters under a `CASE` on the - // repartition hash. A row takes exactly one branch, so a container may match - // only if some branch may: the branches' disjunction is a sound relaxation, and - // the `WHEN`s (a hash, which no statistics describe) can be dropped. + // A row takes exactly one branch of a `CASE`, so a container may match only + // if some branch may: the branches' disjunction is a sound relaxation, and + // the `WHEN`s can be dropped. if let Some(case) = expr.downcast_ref::() { // Only a Boolean `CASE` is a predicate; anything else is a value for // whatever compares it to handle. @@ -2478,11 +2487,7 @@ mod tests { self as phys_expr, DynamicFilterPhysicalExpr, }; use datafusion_physical_expr::planner::logical2physical; - use datafusion_physical_plan::joins::join_hash_map::JoinHashMapU32; - use datafusion_physical_plan::joins::key_range_bitmap::KeyRangeBitmap; - use datafusion_physical_plan::joins::{Map, SeededRandomState}; use itertools::Itertools; - use rstest::rstest; #[derive(Debug, Default)] /// Mock statistic provider for tests @@ -7546,129 +7551,34 @@ mod tests { assert_eq!(res.to_string(), expected); } - /// A hash-table lookup over `column` carrying a bitmap over `keys`. - fn hash_lookup( - column: &Arc, - keys: &[i64], - ) -> Result> { - let array: ArrayRef = Arc::new(Int64Array::from(keys.to_vec())); - let bitmap = KeyRangeBitmap::try_new( - &array, - &ScalarValue::Int64(Some(*keys.iter().min().unwrap())), - &ScalarValue::Int64(Some(*keys.iter().max().unwrap())), - keys.len(), - ); - Ok(Arc::new(HashTableLookupExpr::new( - vec![Arc::clone(column)], - SeededRandomState::with_seed(1), - None, - vec![Some(Arc::new(Map::HashMap(Box::new( - JoinHashMapU32::with_capacity(1), - ))))], - "hash_lookup".to_string(), - bitmap.map(Arc::new), - )?)) - } - - #[test] - fn test_hash_lookup_pruning_via_min_max() -> Result<()> { - let schema = Arc::new(Schema::new(vec![Field::new("b", DataType::Int64, true)])); - let column: Arc = Arc::new(phys_expr::Column::new("b", 0)); - // Gapped keys: 10,20,30 then a far-away 100000, so the bitmap is sparse. - let lookup = hash_lookup(&column, &[10, 20, 30, 100_000])?; - - let predicate = PruningPredicateBuilder::new() - .with_file_schema(Arc::clone(&schema)) - .try_build(lookup)?; - - // An opaque lookup yields no literal guarantees, so `contained()` is never - // consulted and every exclusion below comes from the min/max rewrite. - assert!(predicate.literal_guarantees().is_empty()); - - let statistics = TestStatistics::new().with( - "b", - ContainerStats::new_i64( - vec![Some(5000), Some(15), Some(50000)], - vec![Some(8000), Some(25), Some(60000)], - ), - ); - - let result = predicate.prune(&statistics)?; - // Containers 0 and 2 sit in the gap between 30 and 100000; container 1 - // holds 20. The bitmap excludes the first and third. - assert_eq!(result, vec![false, true, false]); - Ok(()) - } - - #[rstest] - #[case::widening_i32_to_i64(DataType::Int32, DataType::Int64, true)] - #[case::widening_u32_to_i64(DataType::UInt32, DataType::Int64, true)] - #[case::widening_u16_to_i64(DataType::UInt16, DataType::Int64, true)] - #[case::same_type_i64_to_i64(DataType::Int64, DataType::Int64, true)] - #[case::narrowing_i64_to_i32(DataType::Int64, DataType::Int32, false)] - #[case::narrowing_u32_to_i16(DataType::UInt32, DataType::Int16, false)] - fn test_hash_lookup_pruning_through_cast( - #[case] file_type: DataType, - #[case] cast_type: DataType, - #[case] pruned: bool, - ) -> Result<()> { - let schema = - Arc::new(Schema::new(vec![Field::new("b", file_type.clone(), true)])); - let column: Arc = Arc::new(phys_expr::Column::new("b", 0)); - let cast_column: Arc = - Arc::new(phys_expr::CastExpr::new(column, cast_type, None)); - let lookup = hash_lookup(&cast_column, &[10, 20, 30, 100_000])?; - - let predicate = PruningPredicateBuilder::new() - .with_file_schema(Arc::clone(&schema)) - .try_build(lookup)?; - - let to_file_type = |values: Vec>| -> ArrayRef { - let values: ArrayRef = Arc::new(values.into_iter().collect::()); - arrow::compute::cast(&values, &file_type).unwrap() - }; - let statistics = TestStatistics::new().with( - "b", - ContainerStats::new() - .with_min(to_file_type(vec![Some(5000), Some(15), Some(50000)])) - .with_max(to_file_type(vec![Some(8000), Some(25), Some(60000)])), - ); - - let result = predicate.prune(&statistics)?; - // Containers 0 and 2 sit in the gap between 30 and 100000; container 1 - // holds 20. A working cast excludes the first and third; a declined one - // falls back to keeping every container. - let expected = if pruned { - vec![false, true, false] - } else { - vec![true, true, true] - }; - assert_eq!(result, expected); - Ok(()) - } - #[test] - fn test_partition_routed_hash_lookup_pruning() -> Result<()> { + fn test_case_pruning_uses_disjunction_of_branches() -> Result<()> { let schema = Arc::new(Schema::new(vec![Field::new("b", DataType::Int32, true)])); let column: Arc = Arc::new(phys_expr::Column::new("b", 0)); let literal = |value: ScalarValue| -> Arc { Arc::new(phys_expr::Literal::new(value)) }; - // One branch per build partition, each holding its own slice of the build - // side, as `build_partitioned_filter` produces them. + let between = |min: i32, max: i32| -> Arc { + Arc::new(phys_expr::BinaryExpr::new( + Arc::new(phys_expr::BinaryExpr::new( + Arc::clone(&column), + Operator::GtEq, + literal(ScalarValue::Int32(Some(min))), + )), + Operator::And, + Arc::new(phys_expr::BinaryExpr::new( + Arc::clone(&column), + Operator::LtEq, + literal(ScalarValue::Int32(Some(max))), + )), + )) + }; let case: Arc = Arc::new(phys_expr::CaseExpr::try_new( Some(literal(ScalarValue::UInt64(Some(0)))), vec![ - ( - literal(ScalarValue::UInt64(Some(0))), - hash_lookup(&column, &[10, 20, 30])?, - ), - ( - literal(ScalarValue::UInt64(Some(1))), - hash_lookup(&column, &[40, 50, 60])?, - ), + (literal(ScalarValue::UInt64(Some(0))), between(18, 22)), + (literal(ScalarValue::UInt64(Some(1))), between(48, 52)), ], - // Partitions with no build rows reject everything routed to them. Some(literal(ScalarValue::Boolean(Some(false)))), )?); @@ -7685,9 +7595,9 @@ mod tests { ); let result = predicate.prune(&statistics)?; - // Container 1 ([15,25]) holds 20 from the first branch and container 2 - // ([45,55]) holds 50 from the second, so a branch may match in each. The - // other two intersect neither branch, nor the `ELSE`. + // Container 1 ([15,25]) overlaps the first branch and container 2 + // ([45,55]) overlaps the second. The other two overlap neither branch, + // nor the `ELSE`. assert_eq!(result, vec![false, true, true, false]); Ok(()) } diff --git a/datafusion/sqllogictest/test_files/dynamic_row_group_pruning.slt b/datafusion/sqllogictest/test_files/dynamic_row_group_pruning.slt index a51ee03c38737..8bce1626198d7 100644 --- a/datafusion/sqllogictest/test_files/dynamic_row_group_pruning.slt +++ b/datafusion/sqllogictest/test_files/dynamic_row_group_pruning.slt @@ -327,7 +327,7 @@ explain analyze select rgsel.b from dim join rgsel on dim.column1 = rgsel.b; Plan with Metrics 01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(column1@0, b@0)], projection=[b@1], metrics=[output_rows=3, elapsed_compute=, output_bytes=, output_batches=1, build_mem_used=, array_map_created_count=0, build_input_batches=1, build_input_rows=3, input_batches=2, input_rows=3, build_time=, join_time=, avg_fanout=100% (3/3), output_rows_skew=0%, probe_hit_rate=100% (3/3)] 02)--DataSourceExec: partitions=1, partition_sizes=[1], metrics=[] -03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_row_group_pruning/rgsel.parquet]]}, projection=[b], file_type=parquet, predicate=DynamicFilter [ b@1 >= 1000 AND b@1 <= 3050 AND hash_lookup ], dynamic_rg_pruning=eligible, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 >= 1000 AND b_null_count@1 != row_count@2 AND b_min@3 <= 3050 AND b_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(b_min@3, b_max@0, 3/4096 buckets), required_guarantees=[], metrics=[output_rows=3, elapsed_compute=, output_bytes=, output_batches=2, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 2 matched, page_index_pages_pruned=20 total → 3 matched, page_index_rows_pruned=200 total → 30 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, bytes_processed=, bytes_scanned=, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=27, row_groups_pruned_dynamic_filter=0, predicate_cache_inner_records=200, predicate_cache_records=13, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=, output_rows_skew=, scan_efficiency_ratio=] +03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_row_group_pruning/rgsel.parquet]]}, projection=[b], file_type=parquet, predicate=DynamicFilter [ b@1 >= 1000 AND b@1 <= 3050 AND bloom_filter ], dynamic_rg_pruning=eligible, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 >= 1000 AND b_null_count@1 != row_count@2 AND b_min@3 <= 3050 AND b_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(b_min@3, b_max@0, 3/4096 buckets), required_guarantees=[], metrics=[output_rows=3, elapsed_compute=, output_bytes=, output_batches=2, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 2 matched, page_index_pages_pruned=20 total → 3 matched, page_index_rows_pruned=200 total → 30 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, bytes_processed=, bytes_scanned=, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=27, row_groups_pruned_dynamic_filter=0, predicate_cache_inner_records=200, predicate_cache_records=13, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=, output_rows_skew=, scan_efficiency_ratio=] statement ok RESET datafusion.optimizer.hash_join_inlist_pushdown_max_distinct_values; @@ -346,3 +346,66 @@ set datafusion.execution.target_partitions = 4; statement ok RESET datafusion.execution.parquet.pushdown_filters; + +# Partitioned hash join with filter pushdown off. The dynamic filter is one +# global filter. The probe scan uses it +# only for statistics pruning, so the bloom filter is never built, but the +# key-range bitmap merged from all build partitions prunes row groups 1 and 2, +# which the [5, 390] bounds alone cannot prune. +statement ok +set datafusion.execution.parquet.pushdown_filters = false; + +statement ok +set datafusion.optimizer.hash_join_single_partition_threshold = 0; + +statement ok +set datafusion.optimizer.hash_join_single_partition_threshold_rows = 0; + +statement ok +set datafusion.optimizer.hash_join_inlist_pushdown_max_distinct_values = 0; + +statement ok +COPY (SELECT CAST(i AS BIGINT) AS k FROM generate_series(0, 399) AS t(i)) +TO 'test_files/scratch/dynamic_row_group_pruning/pdoff_probe.parquet' +STORED AS PARQUET +OPTIONS ('format.max_row_group_size' '100'); + +statement ok +CREATE EXTERNAL TABLE pdoff_probe STORED AS PARQUET +LOCATION 'test_files/scratch/dynamic_row_group_pruning/pdoff_probe.parquet'; + +statement ok +CREATE TABLE pdoff_build AS SELECT CAST(column1 AS BIGINT) AS k FROM (VALUES (5), (17), (390)); + +query TT +explain analyze select pdoff_probe.k from pdoff_build join pdoff_probe on pdoff_build.k = pdoff_probe.k; +---- +Plan with Metrics +01)HashJoinExec: mode=Partitioned, join_type=Inner, on=[(k@0, k@0)], projection=[k@1], metrics=[output_rows=3, elapsed_compute=, output_bytes=, output_batches=1, build_mem_used=, array_map_created_count=1, build_input_batches=1, build_input_rows=3, input_batches=2, input_rows=200, build_time=, join_time=, avg_fanout=100% (3/3), output_rows_skew=, probe_hit_rate=1.5% (3/200)] +02)--DataSourceExec: partitions=1, partition_sizes=[1], metrics=[] +03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_row_group_pruning/pdoff_probe.parquet]]}, projection=[k], output_ordering=[k@0 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ k@0 >= 5 AND k@0 <= 390 AND bloom_filter ], dynamic_rg_pruning=eligible, pruning_predicate=k_null_count@1 != row_count@2 AND k_max@0 >= 5 AND k_null_count@1 != row_count@2 AND k_min@3 <= 390 AND k_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(k_min@3, k_max@0, 3/512 buckets), required_guarantees=[], metrics=[output_rows=200, elapsed_compute=, output_bytes=, output_batches=2, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 2 matched, page_index_pages_pruned=2 total → 2 matched, page_index_rows_pruned=200 total → 200 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, bytes_processed=, bytes_scanned=, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, row_groups_pruned_dynamic_filter=0, predicate_cache_inner_records=0, predicate_cache_records=0, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=, output_rows_skew=, scan_efficiency_ratio=] + +query I rowsort +select pdoff_probe.k from pdoff_build join pdoff_probe on pdoff_build.k = pdoff_probe.k; +---- +17 +390 +5 + +statement ok +drop table pdoff_build; + +statement ok +drop table pdoff_probe; + +statement ok +RESET datafusion.optimizer.hash_join_inlist_pushdown_max_distinct_values; + +statement ok +RESET datafusion.optimizer.hash_join_single_partition_threshold; + +statement ok +RESET datafusion.optimizer.hash_join_single_partition_threshold_rows; + +statement ok +RESET datafusion.execution.parquet.pushdown_filters; diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index c1aabc2946ea6..c818d0bd0bffa 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -320,6 +320,10 @@ datafusion.optimizer.enable_window_limits true datafusion.optimizer.enable_window_topn false datafusion.optimizer.expand_views_at_output false datafusion.optimizer.filter_null_join_keys false +datafusion.optimizer.hash_join_bloom_filter_fpp 0.01 +datafusion.optimizer.hash_join_bloom_filter_max_bytes 8388608 +datafusion.optimizer.hash_join_bloom_filter_max_fpp 0.15 +datafusion.optimizer.hash_join_dynamic_filter_inlist_max_distinct_values 20 datafusion.optimizer.hash_join_inlist_pushdown_max_distinct_values 150 datafusion.optimizer.hash_join_inlist_pushdown_max_size 131072 datafusion.optimizer.hash_join_single_partition_threshold 4194304 @@ -489,6 +493,10 @@ datafusion.optimizer.enable_window_limits true When set to true, the optimizer w datafusion.optimizer.enable_window_topn false When set to true, the optimizer will replace Filter(rn<=K) → Window(ROW_NUMBER) → Sort patterns with a PartitionedTopKExec that maintains per-partition heaps, avoiding a full sort of the input. When the window partition key has low cardinality, enabling this optimization can improve performance. However, for high cardinality keys, it may cause regressions in both memory usage and runtime. datafusion.optimizer.expand_views_at_output false When set to true, if the returned type is a view type then the output will be coerced to a non-view. Coerces `Utf8View` to `LargeUtf8`, and `BinaryView` to `LargeBinary`. datafusion.optimizer.filter_null_join_keys false When set to true, the optimizer will insert filters before a join between a nullable and non-nullable column to filter out nulls on the nullable side. This filter can add additional overhead when the file format does not fully support predicate push down. +datafusion.optimizer.hash_join_bloom_filter_fpp 0.01 Target false positive probability of the bloom filter in the dynamic filter of a hash join. When the build side of a hash join has more distinct keys than `hash_join_inlist_pushdown_max_distinct_values` (over all partitions), the dynamic filter tests probe rows against one bloom filter of all build side keys. Each build partition fills a filter of its keys, sized for the estimated number of distinct keys of all partitions at this probability. Then one task ORs the partition filters and folds the result to this probability. +datafusion.optimizer.hash_join_bloom_filter_max_bytes 8388608 Maximum size, in bytes, of the bloom filter in the dynamic filter of a hash join (and of the filter of each build partition). A build side that needs a larger filter at `hash_join_bloom_filter_fpp` gets no bloom filter: such a large build side is rarely selective, so the filter is too big to be useful. The dynamic filter then tests only the min/max bounds (and keeps key range pruning). +datafusion.optimizer.hash_join_bloom_filter_max_fpp 0.15 The bloom filter of a hash join dynamic filter is not used when its estimated false positive probability is above this value (the filter is saturated, a safety net). Then the dynamic filter tests only the min/max bounds of the build keys, because a saturated filter removes few rows and costs CPU time for each probe row. +datafusion.optimizer.hash_join_dynamic_filter_inlist_max_distinct_values 20 Maximum number of distinct build side keys, over all partitions of a hash join, for the global dynamic filter to test membership with one InList of all keys. Above this limit the dynamic filter uses a bloom filter (see `hash_join_bloom_filter_fpp`). Every partition must also use an InList by itself (`hash_join_inlist_pushdown_max_distinct_values` and `hash_join_inlist_pushdown_max_size`). For larger sets, a bloom filter is cheaper per probe row than an InList. datafusion.optimizer.hash_join_inlist_pushdown_max_distinct_values 150 Maximum number of distinct values (rows) in the build side of a hash join to be pushed down as an InList expression for dynamic filtering. Build sides with more rows than this will use hash table lookups instead. Set to 0 to always use hash table lookups. This provides an additional limit beyond `hash_join_inlist_pushdown_max_size` to prevent very large IN lists that might not provide much benefit over hash table lookups. This uses the deduplicated row count once the build side has been evaluated. The default is 150 values per partition. This is inspired by Trino's `max-filter-keys-per-column` setting. See: datafusion.optimizer.hash_join_inlist_pushdown_max_size 131072 Maximum size in bytes for the build side of a hash join to be pushed down as an InList expression for dynamic filtering. Build sides larger than this will use hash table lookups instead. Set to 0 to always use hash table lookups. InList pushdown can be more efficient for small build sides because it can result in better statistics pruning as well as use any bloom filters present on the scan side. InList expressions are also more transparent and easier to serialize over the network in distributed uses of DataFusion. On the other hand InList pushdown requires making a copy of the data and thus adds some overhead to the build side and uses more memory. This setting is per-partition, so we may end up using `hash_join_inlist_pushdown_max_size` * `target_partitions` memory. The default is 128kB per partition. This should allow point lookup joins (e.g. joining on a unique primary key) to use InList pushdown in most cases but avoids excessive memory usage or overhead for larger joins. datafusion.optimizer.hash_join_single_partition_threshold 4194304 The maximum estimated size in bytes for one input side of a HashJoin will be collected into a single partition diff --git a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt index 92fac9936d0fc..70059f1d34ca0 100644 --- a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt +++ b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt @@ -478,7 +478,7 @@ Plan with Metrics 02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, b@0)], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=1, input_rows=2, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)] 03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/nested_t1.parquet]]}, projection=[a, x], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=18.08% (132/730)] 04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/nested_t2.parquet]]}, projection=[b, c, y], file_type=parquet, predicate=DynamicFilter [ b@0 >= aa AND b@0 <= ab AND b@0 IN (SET) ([aa, ab]) ], dynamic_rg_pruning=eligible, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 >= aa AND b_null_count@1 != row_count@2 AND b_min@3 <= ab AND (b_null_count@1 != row_count@2 AND b_min@3 <= aa AND aa <= b_max@0 OR b_null_count@1 != row_count@2 AND b_min@3 <= ab AND ab <= b_max@0), required_guarantees=[b in (aa, ab)], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, page_index_pages_pruned=1 total → 1 matched, page_index_rows_pruned=5 total → 5 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=3, predicate_cache_inner_records=5, predicate_cache_records=2, scan_efficiency_ratio=23.12% (234/1.01 K)] -05)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/nested_t3.parquet]]}, projection=[d, z], file_type=parquet, predicate=DynamicFilter [ d@0 >= ca AND d@0 <= cb AND hash_lookup ], dynamic_rg_pruning=eligible, pruning_predicate=d_null_count@1 != row_count@2 AND d_max@0 >= ca AND d_null_count@1 != row_count@2 AND d_min@3 <= cb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, page_index_pages_pruned=1 total → 1 matched, page_index_rows_pruned=8 total → 8 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=6, predicate_cache_inner_records=8, predicate_cache_records=2, scan_efficiency_ratio=22.28% (172/772)] +05)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/nested_t3.parquet]]}, projection=[d, z], file_type=parquet, predicate=DynamicFilter [ d@0 >= ca AND d@0 <= cb AND bloom_filter ], dynamic_rg_pruning=eligible, pruning_predicate=d_null_count@1 != row_count@2 AND d_max@0 >= ca AND d_null_count@1 != row_count@2 AND d_min@3 <= cb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, page_index_pages_pruned=1 total → 1 matched, page_index_rows_pruned=8 total → 8 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=6, predicate_cache_inner_records=8, predicate_cache_records=2, scan_efficiency_ratio=22.28% (172/772)] statement ok reset datafusion.explain.analyze_categories; @@ -910,7 +910,7 @@ drop table lj_probe; # test_hashjoin_hash_table_pushdown_integer_keys # # With `hash_join_inlist_pushdown_max_size = 1`, the HashJoin dynamic filter -# should use the HashTable strategy (`hash_lookup`) instead of the InList +# should use the bloom filter strategy (`bloom_filter`) instead of the InList # strategy (`IN (SET) ([...])`), even when the build side is tiny. ######## @@ -952,7 +952,7 @@ LOCATION 'test_files/scratch/push_down_filter_parquet/hl_probe.parquet'; statement ok set datafusion.explain.analyze_categories = 'rows'; -# probe scan filter should contain `hash_lookup`, not `IN (SET)`. +# probe scan filter should contain `bloom_filter`, not `IN (SET)`. query TT EXPLAIN ANALYZE SELECT p.a, p.b, build.c, p.e @@ -962,7 +962,7 @@ FROM hl_probe p INNER JOIN hl_build AS build Plan with Metrics 01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0), (b@1, b@1)], projection=[a@3, b@4, c@2, e@5], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=1, input_rows=2, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)] 02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/hl_build.parquet]]}, projection=[a, b, c], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=20.19% (196/971)] -03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/hl_probe.parquet]]}, projection=[a, b, e], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND hash_lookup ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=22.71% (228/1.00 K)] +03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/hl_probe.parquet]]}, projection=[a, b, e], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND bloom_filter ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=22.71% (228/1.00 K)] statement ok drop table hl_build; @@ -1011,7 +1011,7 @@ FROM int_build b INNER JOIN int_probe p Plan with Metrics 01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id1@0, id1@0), (id2@1, id2@1)], projection=[id1@0, id2@1, value@2, data@5], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=1, input_rows=2, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)] 02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/int_build.parquet]]}, projection=[id1, id2, value], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=18.73% (204/1.09 K)] -03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/int_probe.parquet]]}, projection=[id1, id2, data], file_type=parquet, predicate=DynamicFilter [ id1@0 >= 1 AND id1@0 <= 2 AND id2@1 >= 10 AND id2@1 <= 20 AND hash_lookup ], dynamic_rg_pruning=eligible, pruning_predicate=id1_null_count@1 != row_count@2 AND id1_max@0 >= 1 AND id1_null_count@1 != row_count@2 AND id1_min@3 <= 2 AND id2_null_count@5 != row_count@2 AND id2_max@4 >= 10 AND id2_null_count@5 != row_count@2 AND id2_min@6 <= 20, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=20.97% (221/1.05 K)] +03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/int_probe.parquet]]}, projection=[id1, id2, data], file_type=parquet, predicate=DynamicFilter [ id1@0 >= 1 AND id1@0 <= 2 AND id2@1 >= 10 AND id2@1 <= 20 AND bloom_filter ], dynamic_rg_pruning=eligible, pruning_predicate=id1_null_count@1 != row_count@2 AND id1_max@0 >= 1 AND id1_null_count@1 != row_count@2 AND id1_min@3 <= 2 AND id2_null_count@5 != row_count@2 AND id2_max@4 >= 10 AND id2_null_count@5 != row_count@2 AND id2_min@6 <= 20, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=20.97% (221/1.05 K)] statement ok reset datafusion.explain.analyze_categories; @@ -1200,8 +1200,8 @@ statement ok drop table nnp_probe; -# Partitioned mode: the per-partition CASE filter gets the same IS NULL widening, so a -# probe NULL routed to a pruning branch still reaches the join. +# Partitioned mode: the global filter gets the same IS NULL widening, so a +# probe NULL still reaches the join. statement ok set datafusion.execution.target_partitions = 4; diff --git a/docs/source/contributor-guide/api-health.md b/docs/source/contributor-guide/api-health.md index d26f25abaeb34..ecd78df99f33a 100644 --- a/docs/source/contributor-guide/api-health.md +++ b/docs/source/contributor-guide/api-health.md @@ -54,7 +54,7 @@ For APIs intended only for internal use, add `#[doc(hidden)]` and use the following doc comment format, including the API policy link: ```txt -impl HashTableLookupExpr { +impl MyInternalExpr { /// ... /// /// # Public Only for Internal Use: diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index 2c3ca887d4a38..694ce233066ea 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -50,164 +50,168 @@ SET datafusion.execution.target_partitions = '1'; The following configuration settings are available: -| key | default | description | -| ----------------------------------------------------------------------- | ----------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| datafusion.catalog.create_default_catalog_and_schema | true | Whether the default catalog and schema should be created automatically. | -| datafusion.catalog.default_catalog | datafusion | The default catalog name - this impacts what SQL queries use if not specified | -| datafusion.catalog.default_schema | public | The default schema name - this impacts what SQL queries use if not specified | -| datafusion.catalog.information_schema | false | Should DataFusion provide access to `information_schema` virtual tables for displaying schema information | -| datafusion.catalog.partition_information_schema | false | When set together with `information_schema`, the `information_schema` of each catalog only describes the objects in that catalog. When not set, the `information_schema` of each catalog describes all catalogs. | -| datafusion.catalog.location | NULL | Location scanned to load tables for `default` schema | -| datafusion.catalog.format | NULL | Type of `TableProvider` to use when loading `default` schema | -| datafusion.catalog.has_header | true | Default value for `format.has_header` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. | -| datafusion.catalog.newlines_in_values | false | Specifies whether newlines in (quoted) CSV values are supported. This is the default value for `format.newlines_in_values` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. Parsing newlines in quoted values may be affected by execution behaviour such as parallel file scanning. Setting this to `true` ensures that newlines in values are parsed successfully, which may reduce performance. | -| datafusion.execution.batch_size | 8192 | Default batch size while creating new batches, it's especially useful for buffer-in-memory batches since creating tiny batches would result in too much metadata memory consumption | -| datafusion.execution.perfect_hash_join_small_build_threshold | 1024 | A perfect hash join (see `HashJoinExec` for more details) will be considered if the range of keys (max - min) on the build side is < this threshold. This provides a fast path for joins with very small key ranges, bypassing the density check. Currently only supports cases where build_side.num_rows() < u32::MAX. Support for build_side.num_rows() >= u32::MAX will be added in the future. | -| datafusion.execution.perfect_hash_join_min_key_density | 0.15 | The minimum required density of join keys on the build side to consider a perfect hash join (see `HashJoinExec` for more details). Density is calculated as: `(number of rows) / (max_key - min_key + 1)`. A perfect hash join may be used if the actual key density > this value. Currently only supports cases where build_side.num_rows() < u32::MAX. Support for build_side.num_rows() >= u32::MAX will be added in the future. | -| datafusion.execution.coalesce_batches | true | When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting | -| datafusion.execution.collect_statistics | true | Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Defaults to true. | -| datafusion.execution.target_partitions | 0 | Number of partitions for query execution. Increasing partitions can increase concurrency. Defaults to the number of CPU cores on the system | -| datafusion.execution.time_zone | NULL | The default time zone Some functions, e.g. `now`, return timestamps in this time zone. This is also used to interpret timezone-naive timestamps in comparisons and subtraction with timezone-aware timestamps. | -| datafusion.execution.parquet.enable_page_index | true | (reading) If true, reads the Parquet data page level metadata (the Page Index), if present, to reduce the I/O and number of rows decoded. | -| datafusion.execution.parquet.pruning | true | (reading) If true, the parquet reader attempts to skip entire row groups based on the predicate in the query and the metadata (min/max values) stored in the parquet file | -| datafusion.execution.parquet.skip_metadata | true | (reading) If true, the parquet reader skip the optional embedded metadata that may be in the file Schema. This setting can help avoid schema conflicts when querying multiple parquet files with schemas containing compatible types but different metadata. Arrow extension types (the `ARROW:extension:name` and `ARROW:extension:metadata` field metadata) are kept, because they are part of the type of a column | -| datafusion.execution.parquet.metadata_size_hint | 524288 | (reading) If specified, the parquet reader will try and fetch the last `size_hint` bytes of the parquet file optimistically. If not specified, two reads are required: One read to fetch the 8-byte parquet footer and another to fetch the metadata length encoded in the footer Default setting to 512 KiB, which should be sufficient for most parquet files, it can reduce one I/O operation per parquet file. If the metadata is larger than the hint, two reads will still be performed. | -| datafusion.execution.parquet.pushdown_filters | false | (reading) If true, filter expressions are be applied during the parquet decoding operation to reduce the number of rows decoded. This optimization is sometimes called "late materialization". | -| datafusion.execution.parquet.reorder_filters | false | (reading) If true, filter expressions evaluated during the parquet decoding operation will be reordered heuristically to minimize the cost of evaluation. If false, the filters are applied in the same order as written in the query | -| datafusion.execution.parquet.force_filter_selections | false | (reading) Force the use of RowSelections for filter results, when pushdown_filters is enabled. If false, the reader will automatically choose between a RowSelection and a Bitmap based on the number and pattern of selected rows. | -| datafusion.execution.parquet.schema_force_view_types | true | (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, and `Binary/BinaryLarge` with `BinaryView`. The parquet reader is optimized for reading `Utf8View` and `BinaryView`, so such queries are significantly faster than reading `Utf8`/`Binary` and then casting to the view types. | -| datafusion.execution.parquet.binary_as_string | false | (reading) If true, parquet reader will read columns of `Binary/LargeBinary` with `Utf8`, and `BinaryView` with `Utf8View`. Parquet files generated by some legacy writers do not correctly set the UTF8 flag for strings, causing string columns to be loaded as BLOB instead. The parquet reader has special optimizations for `Utf8` validation, so reading such columns as strings is significantly faster than reading them as binary and then casting to string. | -| datafusion.execution.parquet.coerce_int96 | NULL | (reading) If set, parquet reader will read columns of physical type int96 as originating from this resolution instead of nanosecond. Valid values are: ns, us, ms, and s. This is useful for reading data from systems like Spark which store microsecond resolution timestamps in an int96, allowing them to write values with a larger date range than 64-bit timestamps with nanosecond resolution. | -| datafusion.execution.parquet.coerce_int96_tz | NULL | (reading) Optional timezone applied to INT96 columns when `coerce_int96` is set. When `Some`, INT96 columns coerce to `Timestamp(, Some())` instead of the default `Timestamp(, None)`. Spark and other systems write INT96 values as UTC-adjusted instants, so callers that need the resulting Arrow type to be timezone-aware (e.g. for Spark `TimestampType` semantics) should set this to `"UTC"`. No effect when `coerce_int96` is `None`, but the value is still validated when it is set. | -| datafusion.execution.parquet.bloom_filter_on_read | true | (reading) Use any available bloom filters when reading parquet files | -| datafusion.execution.parquet.max_predicate_cache_size | NULL | (reading) The maximum predicate cache size, in bytes. When `pushdown_filters` is enabled, sets the maximum memory used to cache the results of predicate evaluation between filter evaluation and output generation. Decreasing this value will reduce memory usage, but may increase IO and CPU usage. None means use the default parquet reader setting. 0 means no caching. | -| datafusion.execution.parquet.max_in_list_size | 20 | Maximum number of input values in an `IN (...)` list eligible for min/max pruning. Lists above this cap, or a cap of 0, skip this rewrite; other predicates and Bloom-filter pruning remain available. Within the cap, nonempty lists of at most 20 values use the existing per-value rewrite. Larger literal lists use a compact representation when the column type is string, variable-length binary, integer, decimal, date, time, timestamp, or duration. This applies to both `IN` and `NOT IN`, including lists with NULL members. `NOT IN` with NULL and all-NULL `IN` lists cannot match any rows. Compact lists containing NULL do not use the fully-matched-row-group optimization. Floating-point and other lists retain the existing per-value rewrite, so raising the cap can make those predicates expensive to build and evaluate. Defaults to 20. | -| datafusion.execution.parquet.enable_rle_to_dictionary | false | (reading) If true, top-level string and binary Parquet columns with dictionary pages are inferred and scanned as `Dictionary` / `Dictionary` instead of their plain value type. This applies only when DataFusion infers the table schema. Tables with a user-supplied schema are not promoted because the Parquet footer is not read at DDL time, so dictionary pages cannot be detected per column. See | -| datafusion.execution.parquet.row_group_range_assignment | `start_offset` | (reading) Which byte range of a split file reads each row group. `start_offset` picks the range containing the row group's start. `midpoint` picks the range containing its midpoint, as Spark does, which spreads large row groups more evenly across ranges. | -| datafusion.execution.parquet.data_pagesize_limit | 1048576 | (writing) Sets best effort maximum size of data page in bytes | -| datafusion.execution.parquet.write_batch_size | 1024 | (writing) Sets write_batch_size in rows | -| datafusion.execution.parquet.writer_version | 1.0 | (writing) Sets parquet writer version valid values are "1.0" and "2.0" | -| datafusion.execution.parquet.skip_arrow_metadata | false | (writing) Skip encoding the embedded arrow metadata in the KV_meta This is analogous to the `ArrowWriterOptions::with_skip_arrow_metadata`. Refer to | -| datafusion.execution.parquet.compression | zstd(3) | (writing) Sets default parquet compression codec. Valid values are: uncompressed, snappy, gzip(level), brotli(level), lz4, zstd(level), and lz4_raw. These values are not case sensitive. If NULL, uses default parquet writer setting Note that this default setting is not the same as the default parquet writer setting. | -| datafusion.execution.parquet.dictionary_enabled | true | (writing) Sets if dictionary encoding is enabled. If NULL, uses default parquet writer setting | -| datafusion.execution.parquet.dictionary_page_size_limit | 1048576 | (writing) Sets best effort maximum dictionary page size, in bytes | -| datafusion.execution.parquet.statistics_enabled | page | (writing) Sets if statistics are enabled for any column Valid values are: "none", "chunk", and "page" These values are not case sensitive. If NULL, uses default parquet writer setting | -| datafusion.execution.parquet.max_row_group_size | 1048576 | (writing) Target maximum number of rows in each row group (defaults to 1M rows). Writing larger row groups requires more memory to write, but can get better compression and be faster to read. When `max_row_group_bytes` is also set, the writer flushes a row group when either limit is reached, whichever comes first. | -| datafusion.execution.parquet.max_row_group_bytes | NULL | (writing) Target maximum size of each row group in bytes. When set, the writer flushes whenever either this limit or `max_row_group_size` is reached, whichever comes first. Useful for bounding writer memory on wide schemas where a row-count limit can map to very different byte sizes. Matches the behavior of `parquet.block.size` in parquet-mr. If `None` (the default), only the row-count limit applies. Currently only honored when `allow_single_file_parallelism` is `false`; by default the parallel file writer ignores this limit. | -| datafusion.execution.parquet.created_by | datafusion version 57.0.0-dev | (writing) Sets "created by" property | -| datafusion.execution.parquet.column_index_truncate_length | 64 | (writing) Sets column index truncate length | -| datafusion.execution.parquet.statistics_truncate_length | 64 | (writing) Sets statistics truncate length. If NULL, uses default parquet writer setting | -| datafusion.execution.parquet.data_page_row_count_limit | 20000 | (writing) Sets best effort maximum number of rows in data page | -| datafusion.execution.parquet.encoding | NULL | (writing) Sets default encoding for any column. Valid values are: plain, plain_dictionary, rle, bit_packed, delta_binary_packed, delta_length_byte_array, delta_byte_array, rle_dictionary, byte_stream_split, and alp. These values are not case sensitive. If NULL, uses default parquet writer setting | -| datafusion.execution.parquet.bloom_filter_on_write | false | (writing) Write bloom filters for all columns when creating parquet files | -| datafusion.execution.parquet.bloom_filter_fpp | NULL | (writing) Sets bloom filter false positive probability. If NULL, uses default parquet writer setting | -| datafusion.execution.parquet.bloom_filter_ndv | NULL | (writing) Sets bloom filter number of distinct values. If NULL, uses default parquet writer setting | -| datafusion.execution.parquet.write_row_group_number_distinct_values | false | (writing) Write the number of distinct values (NDV) for each column in row group statistics when creating parquet files. Enabling this improves NDV-based query optimizations at the cost of hashing every non-null value during write. | -| datafusion.execution.parquet.allow_single_file_parallelism | true | (writing) Controls whether DataFusion will attempt to speed up writing parquet files by serializing them in parallel. Each column in each row group in each output file are serialized in parallel leveraging a maximum possible core count of n_files\*n_row_groups\*n_columns. | -| datafusion.execution.parquet.maximum_parallel_row_group_writers | 1 | (writing) By default parallel parquet writer is tuned for minimum memory usage in a streaming execution plan. You may see a performance benefit when writing large parquet files by increasing maximum_parallel_row_group_writers and maximum_buffered_record_batches_per_stream if your system has idle cores and can tolerate additional memory usage. Boosting these values is likely worthwhile when writing out already in-memory data, such as from a cached data frame. | -| datafusion.execution.parquet.maximum_buffered_record_batches_per_stream | 2 | (writing) By default parallel parquet writer is tuned for minimum memory usage in a streaming execution plan. You may see a performance benefit when writing large parquet files by increasing maximum_parallel_row_group_writers and maximum_buffered_record_batches_per_stream if your system has idle cores and can tolerate additional memory usage. Boosting these values is likely worthwhile when writing out already in-memory data, such as from a cached data frame. | -| datafusion.execution.parquet.content_defined_chunking.enabled | false | (writing) EXPERIMENTAL: Enable content-defined chunking (CDC) when writing parquet files. When enabled, parallel writing is automatically disabled since the chunker state must persist across row groups. | -| datafusion.execution.parquet.content_defined_chunking.min_chunk_size | 262144 | Minimum chunk size in bytes. The rolling hash will not trigger a split until this many bytes have been accumulated. Default is 256 KiB. | -| datafusion.execution.parquet.content_defined_chunking.max_chunk_size | 1048576 | Maximum chunk size in bytes. A split is forced when the accumulated size exceeds this value. Default is 1 MiB. | -| datafusion.execution.parquet.content_defined_chunking.norm_level | 0 | Normalization level. Increasing this improves deduplication ratio but increases fragmentation. Recommended range is [-3, 3], default is 0. | -| datafusion.execution.planning_concurrency | 0 | Fan-out during initial physical planning. This is mostly use to plan `UNION` children in parallel. Defaults to the number of CPU cores on the system | -| datafusion.execution.skip_physical_aggregate_schema_check | false | When set to true, skips verifying that the schema produced by planning the input of `LogicalPlan::Aggregate` exactly matches the schema of the input plan. When set to false, if the schema does not match exactly (including nullability and metadata), a planning error will be raised. This is used to workaround bugs in the planner that are now caught by the new schema verification step. | -| datafusion.execution.enable_migration_aggregate | true | Whether aggregation uses the implementation from the major refactor completed in the 56.0.0 release. When set to `false`, aggregation falls back to the implementation used before 55.0.0. The fallback exists only as a workaround for bugs in the new implementation and will be removed, together with this option, after the 56.0.0 release. See for details. | -| datafusion.execution.spill_compression | uncompressed | Sets the compression codec used when spilling data to disk. Since datafusion writes spill files using the Arrow IPC Stream format, only codecs supported by the Arrow IPC Stream Writer are allowed. Valid values are: uncompressed, lz4_frame, zstd. Note: lz4_frame offers faster (de)compression, but typically results in larger spill files. In contrast, zstd achieves higher compression ratios at the cost of slower (de)compression speed. | -| datafusion.execution.sort_spill_reservation_bytes | 10485760 | Specifies the reserved memory for each spillable sort operation to facilitate an in-memory merge. When a sort operation spills to disk, the in-memory data must be sorted and merged before being written to a file. This setting reserves a specific amount of memory for that in-memory sort/merge process. Note: This setting is irrelevant if the sort operation cannot spill (i.e., if there's no `DiskManager` configured). | -| datafusion.execution.sort_in_place_threshold_bytes | 1048576 | When sorting, below what size should data be concatenated and sorted in a single RecordBatch rather than sorted in batches and merged. | -| datafusion.execution.sort_pushdown_buffer_capacity | 1073741824 | Maximum buffer capacity (in bytes) per partition for BufferExec inserted during sort pushdown optimization. When PushdownSort eliminates a SortExec under SortPreservingMergeExec, a BufferExec is inserted to replace SortExec's buffering role. This prevents I/O stalls by allowing the scan to run ahead of the merge. This uses strictly less memory than the SortExec it replaces (which buffers the entire partition). The buffer respects the global memory pool limit. Setting this to a large value is safe — actual memory usage is bounded by partition size and global memory limits. | -| datafusion.execution.max_spill_file_size_bytes | 134217728 | Maximum size in bytes for individual spill files before rotating to a new file. When operators spill data to disk (e.g., RepartitionExec), they write multiple batches to the same file until this size limit is reached, then rotate to a new file. This reduces syscall overhead compared to one-file-per-batch while preventing files from growing too large. A larger value reduces file creation overhead but may hold more disk space. A smaller value creates more files but allows finer-grained space reclamation as files can be deleted once fully consumed. Now only `RepartitionExec` supports this spill file rotation feature, other spilling operators may create spill files larger than the limit. Default: 128 MB | -| datafusion.execution.enable_nlj_coordinated_fallback | true | Enables the memory-limited fallback for `NestedLoopJoinExec` join types that emit unmatched left rows in the final output (LEFT, LEFT SEMI, LEFT ANTI, LEFT MARK, FULL) when the right side has multiple partitions. This fallback shares left-side state (the current left chunk, the visited bitmap and the probe-thread counter) across all right-side partitions, which assumes every partition runs in the same process. Distributed engines that execute each output partition as an independent task (e.g. Ballista, datafusion-distributed) give each task its own copy of this state and poll only one partition, so the cross-partition counter never reaches zero and the fallback would stall. Such engines should set this to `false`: the coordinated fallback is then disabled for left-emitting multi-partition joins, which instead fail with a resource-exhaustion error under memory pressure rather than deadlocking. Single-partition and non-left-emitting joins are unaffected and always keep the fallback. | -| datafusion.execution.meta_fetch_concurrency | 32 | Number of files to read in parallel when inferring schema and statistics | -| datafusion.execution.minimum_parallel_output_files | 4 | Guarantees a minimum level of output files running in parallel. RecordBatches will be distributed in round robin fashion to each parallel writer. Each writer is closed and a new file opened once soft_max_rows_per_output_file or soft_max_bytes_per_output_file is reached. | -| datafusion.execution.soft_max_rows_per_output_file | 50000000 | Target number of rows in output files when writing multiple. This is a soft max, so it can be exceeded slightly. There also will be one file smaller than the limit if the total number of rows written is not roughly divisible by the soft max | -| datafusion.execution.soft_max_bytes_per_output_file | 4294967295 | Target encoded size in bytes of output files when writing multiple. Writers asynchronously report the cumulative encoded size as they process RecordBatches. The final file size may exceed this limit due to batches buffered before the limit is observed, the size of a batch, and file metadata written when the file is finalized. | -| datafusion.execution.max_buffered_batches_per_output_file | 2 | This is the maximum number of RecordBatches buffered for each output file being worked. Higher values can potentially give faster write performance at the cost of higher peak memory consumption. This budget is split evenly between two independent points in the write pipeline (see the demuxer diagram in #7791): how many files can be in flight from the demuxer to a writer task, and how many RecordBatches are buffered for a single file's writer. Must be at least 2 so each half gets at least 1 unit of buffering - 0 or 1 would leave one side with a zero-capacity channel and panic at write time. | -| datafusion.execution.listing_table_ignore_subdirectory | true | Should sub directories be ignored when scanning directories for data files. Defaults to true (ignores subdirectories), consistent with Hive. Note that this setting does not affect reading partitioned tables (e.g. `/table/year=2021/month=01/data.parquet`). | -| datafusion.execution.listing_table_factory_infer_partitions | true | Should a `ListingTable` created through the `ListingTableFactory` infer table partitions from Hive compliant directories. Defaults to true (partition columns are inferred and will be represented in the table schema). | -| datafusion.execution.listing_table_hive_default_partition_name | `__HIVE_DEFAULT_PARTITION__` | The directory name Hive compliant writers use in place of a null partition value, e.g. `/table/date=__HIVE_DEFAULT_PARTITION__/data.parquet`. Partition values equal to this name are read as NULL, as Hive, Spark and Impala do, and partitioned writes reject them. Corresponds to Hive's `hive.exec.default.partition.name`. Set this to the empty string to read and write such values as the literal directory name instead. | -| datafusion.execution.enable_recursive_ctes | true | Should DataFusion support recursive CTEs | -| datafusion.execution.split_file_groups_by_statistics | false | Attempt to eliminate sorts by packing & sorting files with non-overlapping statistics into the same file groups. Currently experimental | -| datafusion.execution.keep_partition_by_columns | false | Should DataFusion keep the columns used for partition_by in the output RecordBatches | -| datafusion.execution.enable_file_stream_work_stealing | true | When `true` (the default), DataFusion's built-in file scans dynamically rebalance files across partitions at query execution time: a partition that goes idle reads files (or byte-range morsels) originally assigned to a sibling partition, which keeps all partitions busy in a single process. Executors that depend on the plan-time partition assignment — such as Ballista and datafusion-distributed, which run each partition as an isolated task and never poll the siblings — should set this to `false` so each partition reads only its own file group and no runtime reassignment occurs. | -| datafusion.execution.skip_partial_aggregation_probe_ratio_threshold | 0.8 | Aggregation ratio (number of distinct groups / number of input rows) threshold for skipping partial aggregation. If the value is greater then partial aggregation will skip aggregation for further input | -| datafusion.execution.skip_partial_aggregation_probe_rows_threshold | 100000 | Number of input rows partial aggregation partition should process, before aggregation ratio check and trying to switch to skipping aggregation mode | -| datafusion.execution.use_row_number_estimates_to_optimize_partitioning | false | Should DataFusion use row number estimates at the input to decide whether increasing parallelism is beneficial or not. By default, only exact row numbers (not estimates) are used for this decision. Setting this flag to `true` will likely produce better plans. if the source of statistics is accurate. We plan to make this the default in the future. | -| datafusion.execution.enforce_batch_size_in_joins | false | Should DataFusion enforce batch size in joins or not. By default, DataFusion will not enforce batch size in joins. Enforcing batch size in joins can reduce memory usage when joining large tables with a highly-selective join filter, but is also slightly slower. Note: this option currently only applies to the symmetric hash join. | -| datafusion.execution.objectstore_writer_buffer_size | 10485760 | Size (bytes) of data buffer DataFusion uses when writing output files. This affects the size of the data chunks that are uploaded to remote object stores (e.g. AWS S3). If very large (>= 100 GiB) output files are being written, it may be necessary to increase this size to avoid errors from the remote end point. | -| datafusion.execution.enable_ansi_mode | false | Whether to enable ANSI SQL mode. The flag is experimental and relevant only for DataFusion Spark built-in functions When `enable_ansi_mode` is set to `true`, the query engine follows ANSI SQL semantics for expressions, casting, and error handling. This means: - **Strict type coercion rules:** implicit casts between incompatible types are disallowed. - **Standard SQL arithmetic behavior:** operations such as division by zero, numeric overflow, or invalid casts raise runtime errors rather than returning `NULL` or adjusted values. - **Consistent ANSI behavior** for string concatenation, comparisons, and `NULL` handling. When `enable_ansi_mode` is `false` (the default), the engine uses a more permissive, non-ANSI mode designed for user convenience and backward compatibility. In this mode: - Implicit casts between types are allowed (e.g., string to integer when possible). - Arithmetic operations are more lenient — for example, `abs()` on the minimum representable integer value returns the input value instead of raising overflow. - Division by zero or invalid casts may return `NULL` instead of failing. # Default `false` — ANSI SQL mode is disabled by default. | -| datafusion.execution.hash_join_buffering_capacity | 0 | How many bytes to buffer in the probe side of hash joins while the build side is concurrently being built. Without this, hash joins will wait until the full materialization of the build side before polling the probe side. This is useful in scenarios where the query is not completely CPU bounded, allowing to do some early work concurrently and reducing the latency of the query. Note that when hash join buffering is enabled, the probe side will start eagerly polling data, not giving time for the producer side of dynamic filters to produce any meaningful predicate. Queries with dynamic filters might see performance degradation. Disabled by default, set to a number greater than 0 for enabling it. | -| datafusion.optimizer.enable_distinct_aggregation_soft_limit | true | When set to true, the optimizer will push a limit operation into grouped aggregations which have no aggregate expressions, as a soft limit, emitting groups once the limit is reached, before all rows in the group are read. | -| datafusion.optimizer.enable_round_robin_repartition | true | When set to true, the physical plan optimizer will try to add round robin repartitioning to increase parallelism to leverage more CPU cores | -| datafusion.optimizer.enable_topk_aggregation | true | When set to true, the optimizer will attempt to perform limit operations during aggregations, if possible | -| datafusion.optimizer.enable_window_limits | true | When set to true, the optimizer will attempt to push limit operations past window functions, if possible | -| datafusion.optimizer.enable_window_topn | false | When set to true, the optimizer will replace Filter(rn<=K) → Window(ROW_NUMBER) → Sort patterns with a PartitionedTopKExec that maintains per-partition heaps, avoiding a full sort of the input. When the window partition key has low cardinality, enabling this optimization can improve performance. However, for high cardinality keys, it may cause regressions in both memory usage and runtime. | -| datafusion.optimizer.enable_row_number_to_aggregate | false | When set to true, the optimizer will rewrite a "top-1 per group" pattern of the form `Filter(row_number() = 1)` over a `PARTITION BY` window into an `Aggregate(first_value(... ORDER BY ...) GROUP BY partition)`. This avoids buffering / fully sorting the input, which is especially beneficial on wide payloads. Disabled by default because on nested/wide value types the aggregate only becomes memory efficient once the columnar nested-type `GroupsAccumulator` fast path is available; enabling it without that support can route wide payloads onto the slow per-group accumulator path. | -| datafusion.optimizer.enable_topk_repartition | true | When set to true, the optimizer will push TopK (Sort with fetch) below hash repartition when the partition key is a prefix of the sort key, reducing data volume before the shuffle. | -| datafusion.optimizer.enable_topk_dynamic_filter_pushdown | true | When set to true, the optimizer will attempt to push down TopK dynamic filters into the file scan phase. | -| datafusion.optimizer.enable_physical_uncorrelated_scalar_subquery | true | When set to true, uncorrelated scalar subqueries are left in the logical plan and executed by `ScalarSubqueryExec` during physical execution. When set to false, all scalar subqueries (including uncorrelated ones) are rewritten to left joins by the `ScalarSubqueryToJoin` optimizer rule. Note disabling this option is not recommended. It restores pre behavior, which silently produces incorrect results for multi-row subqueries and does not support scalar subqueries in ORDER BY / JOIN ON / aggregate-function arguments. This option is intended as a temporary escape hatch for distributed execution frameworks and is planned to be removed in a future DataFusion release. | -| datafusion.optimizer.enable_join_dynamic_filter_pushdown | true | When set to true, the optimizer will attempt to push down Join dynamic filters into the file scan phase. | -| datafusion.optimizer.enable_aggregate_dynamic_filter_pushdown | true | When set to true, the optimizer will attempt to push down Aggregate dynamic filters into the file scan phase. | -| datafusion.optimizer.enable_dynamic_filter_pushdown | true | When set to true attempts to push down dynamic filters generated by operators (TopK, Join & Aggregate) into the file scan phase. For example, for a query such as `SELECT * FROM t ORDER BY timestamp DESC LIMIT 10`, the optimizer will attempt to push down the current top 10 timestamps that the TopK operator references into the file scans. This means that if we already have 10 timestamps in the year 2025 any files that only have timestamps in the year 2024 can be skipped / pruned at various stages in the scan. The config will suppress `enable_join_dynamic_filter_pushdown`, `enable_topk_dynamic_filter_pushdown` & `enable_aggregate_dynamic_filter_pushdown` So if you disable `enable_topk_dynamic_filter_pushdown`, then enable `enable_dynamic_filter_pushdown`, the `enable_topk_dynamic_filter_pushdown` will be overridden. | -| datafusion.optimizer.filter_null_join_keys | false | When set to true, the optimizer will insert filters before a join between a nullable and non-nullable column to filter out nulls on the nullable side. This filter can add additional overhead when the file format does not fully support predicate push down. | -| datafusion.optimizer.repartition_aggregations | true | Should DataFusion repartition data using the aggregate keys to execute aggregates in parallel using the provided `target_partitions` level | -| datafusion.optimizer.repartition_file_min_size | 1048576 | Minimum total file size in bytes for file-group byte-range splitting to fire. Files (or merged file groups) smaller than this stay as one partition. Lower values produce more, smaller partitions — better at filling `target_partitions` worth of cores when files are modestly sized, at the cost of slightly more per-partition open / metadata-load overhead. | -| datafusion.optimizer.repartition_joins | true | Should DataFusion repartition data using the join keys to execute joins in parallel using the provided `target_partitions` level | -| datafusion.optimizer.allow_symmetric_joins_without_pruning | true | Should DataFusion allow symmetric hash joins for unbounded data sources even when its inputs do not have any ordering or filtering If the flag is not enabled, the SymmetricHashJoin operator will be unable to prune its internal buffers, resulting in certain join types - such as Full, Left, LeftAnti, LeftSemi, Right, RightAnti, and RightSemi - being produced only at the end of the execution. This is not typical in stream processing. Additionally, without proper design for long runner execution, all types of joins may encounter out-of-memory errors. | -| datafusion.optimizer.repartition_file_scans | true | When set to `true`, datasource partitions will be repartitioned to achieve maximum parallelism. This applies to both in-memory partitions and FileSource's file groups (1 group is 1 partition). For FileSources, only Parquet and CSV formats are currently supported. If set to `true` for a FileSource, all files will be repartitioned evenly (i.e., a single large file might be partitioned into smaller chunks) for parallel scanning. If set to `false` for a FileSource, different files will be read in parallel, but repartitioning won't happen within a single file. If set to `true` for an in-memory source, all memtable's partitions will have their batches repartitioned evenly to the desired number of `target_partitions`. Repartitioning can change the total number of partitions and batches per partition, but does not slice the initial record tables provided to the MemTable on creation. | -| datafusion.optimizer.preserve_file_partitions | 0 | Minimum number of distinct partition values required to group files by their Hive partition column values (enabling output partitioning declaration). How the option is used: - preserve_file_partitions=0: Disable it. - preserve_file_partitions=1: Always enable it. - preserve_file_partitions=N, actual file partitions=M: Only enable when M >= N. This threshold preserves I/O parallelism when file partitioning is below it. Note: This may reduce parallelism, rooting from the I/O level, if the number of distinct partitions is less than the target_partitions. | -| datafusion.optimizer.repartition_windows | true | Should DataFusion repartition data using the partitions keys to execute window functions in parallel using the provided `target_partitions` level | -| datafusion.optimizer.repartition_sorts | true | Should DataFusion execute sorts in a per-partition fashion and merge afterwards instead of coalescing first and sorting globally. With this flag is enabled, plans in the form below `text "SortExec: [a@0 ASC]", " CoalescePartitionsExec", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ` would turn into the plan below which performs better in multithreaded environments `text "SortPreservingMergeExec: [a@0 ASC]", " SortExec: [a@0 ASC]", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ` | -| datafusion.optimizer.subset_repartition_threshold | 4 | Partition count threshold for subset satisfaction optimization. When the current partition count is >= this threshold, DataFusion will skip repartitioning if the required partitioning expression is a subset of the current partition expression such as Hash(a) satisfies Hash(a, b). When the current partition count is < this threshold, DataFusion will repartition to increase parallelism even when subset satisfaction applies. Set to 0 to always repartition (disable subset satisfaction optimization). Set to a high value to always use subset satisfaction. Example (subset_repartition_threshold = 4): `text Hash([a]) satisfies Hash([a, b]) because (Hash([a, b]) is subset of Hash([a]) If current partitions (3) < threshold (4), repartition: AggregateExec: mode=FinalPartitioned, gby=[a, b], aggr=[SUM(x)] RepartitionExec: partitioning=Hash([a, b], 8), input_partitions=3 AggregateExec: mode=Partial, gby=[a, b], aggr=[SUM(x)] DataSourceExec: file_groups={...}, output_partitioning=Hash([a], 3) If current partitions (8) >= threshold (4), use subset satisfaction: AggregateExec: mode=SinglePartitioned, gby=[a, b], aggr=[SUM(x)] DataSourceExec: file_groups={...}, output_partitioning=Hash([a], 8) ` | -| datafusion.optimizer.prefer_existing_sort | false | When true, DataFusion will opportunistically remove sorts when the data is already sorted, (i.e. setting `preserve_order` to true on `RepartitionExec` and using `SortPreservingMergeExec`) When false, DataFusion will maximize plan parallelism using `RepartitionExec` even if this requires subsequently resorting data using a `SortExec`. | -| datafusion.optimizer.skip_failed_rules | false | When set to true, the logical plan optimizer will produce warning messages if any optimization rules produce errors and then proceed to the next rule. When set to false, any rules that produce errors will cause the query to fail | -| datafusion.optimizer.max_passes | 3 | The logical optimizer applies rules in order for up to `max_passes` passes, stopping early if a pass leaves the plan unchanged or repeats an earlier plan. The physical optimizer always runs in one pass. | -| datafusion.optimizer.top_down_join_key_reordering | true | When set to true, the physical plan optimizer will run a top down process to reorder the join keys | -| datafusion.optimizer.join_reordering | true | When set to true, the physical plan optimizer may swap join inputs based on statistics. When set to false, statistics-driven join input reordering is disabled and the original join order in the query is used. | -| datafusion.optimizer.use_statistics_registry | false | (Deprecated) Ignored: the physical plan optimizer always consults the session's pluggable `StatisticsRegistry` (register providers on the `SessionState`; with none it is a no-op). | -| datafusion.optimizer.prefer_hash_join | true | When set to true, the physical plan optimizer will prefer HashJoin over SortMergeJoin. HashJoin can work more efficiently than SortMergeJoin but consumes more memory | -| datafusion.optimizer.enable_piecewise_merge_join | false | When set to true, piecewise merge join is enabled. PiecewiseMergeJoin is currently experimental. Physical planner will opt for PiecewiseMergeJoin when there is only one range filter. | -| datafusion.optimizer.hash_join_single_partition_threshold | 4194304 | The maximum estimated size in bytes for one input side of a HashJoin will be collected into a single partition | -| datafusion.optimizer.hash_join_single_partition_threshold_rows | 131072 | The maximum estimated size in rows for one input side of a HashJoin will be collected into a single partition | -| datafusion.optimizer.hash_join_inlist_pushdown_max_size | 131072 | Maximum size in bytes for the build side of a hash join to be pushed down as an InList expression for dynamic filtering. Build sides larger than this will use hash table lookups instead. Set to 0 to always use hash table lookups. InList pushdown can be more efficient for small build sides because it can result in better statistics pruning as well as use any bloom filters present on the scan side. InList expressions are also more transparent and easier to serialize over the network in distributed uses of DataFusion. On the other hand InList pushdown requires making a copy of the data and thus adds some overhead to the build side and uses more memory. This setting is per-partition, so we may end up using `hash_join_inlist_pushdown_max_size` \* `target_partitions` memory. The default is 128kB per partition. This should allow point lookup joins (e.g. joining on a unique primary key) to use InList pushdown in most cases but avoids excessive memory usage or overhead for larger joins. | -| datafusion.optimizer.hash_join_inlist_pushdown_max_distinct_values | 150 | Maximum number of distinct values (rows) in the build side of a hash join to be pushed down as an InList expression for dynamic filtering. Build sides with more rows than this will use hash table lookups instead. Set to 0 to always use hash table lookups. This provides an additional limit beyond `hash_join_inlist_pushdown_max_size` to prevent very large IN lists that might not provide much benefit over hash table lookups. This uses the deduplicated row count once the build side has been evaluated. The default is 150 values per partition. This is inspired by Trino's `max-filter-keys-per-column` setting. See: | -| datafusion.optimizer.default_filter_selectivity | 20 | The default filter selectivity used by Filter Statistics when an exact selectivity cannot be determined. Valid values are between 0 (no selectivity) and 100 (all rows are selected). | -| datafusion.optimizer.prefer_existing_union | false | When set to true, the optimizer will not attempt to convert Union to Interleave | -| datafusion.optimizer.expand_views_at_output | false | When set to true, if the returned type is a view type then the output will be coerced to a non-view. Coerces `Utf8View` to `LargeUtf8`, and `BinaryView` to `LargeBinary`. | -| datafusion.optimizer.enable_sort_pushdown | true | Enable sort pushdown optimization. When enabled, attempts to push sort requirements down to data sources that can natively handle them (e.g., by reversing file/row group read order). Returns **inexact ordering**: Sort operator is kept for correctness, but optimized input enables early termination for TopK queries (ORDER BY ... LIMIT N), providing significant speedup. Memory: No additional overhead (only changes read order). Future: Will add option to detect perfectly sorted data and eliminate Sort completely. Default: true | -| datafusion.optimizer.enable_leaf_expression_pushdown | true | When set to true, the optimizer will extract leaf expressions (such as `get_field`) from filter/sort/join nodes into projections closer to the leaf table scans, and push those projections down towards the leaf nodes. | -| datafusion.optimizer.enable_unions_to_filter | false | When set to true, the logical optimizer will rewrite `UNION DISTINCT` branches that read from the same source and differ only by filter predicates into a single branch with a combined filter. This optimization is conservative and only applies when the branches share the same source and compatible wrapper nodes such as identical projections or aliases. | -| datafusion.optimizer.single_distinct_prefer_native | false | Prefer native grouped DISTINCT aggregation when target_partitions is 1 and every aggregate explicitly supports a specialized GroupsAccumulator for its DISTINCT argument types. Defaults to false, preserving the two-phase GROUP BY rewrite. The native plan can avoid materializing distinct (group, value) pairs, but can be slower for many groups with duplicate values. It must fit each group's distinct values in memory, so queries that succeed by spilling with the rewrite may fail with this option enabled. One target partition can still have multiple input partitions and merge native states. | -| datafusion.explain.logical_plan_only | false | When set to true, the explain statement will only print logical plans | -| datafusion.explain.physical_plan_only | false | When set to true, the explain statement will only print physical plans | -| datafusion.explain.show_statistics | false | When set to true, the explain statement will print operator statistics for physical plans | -| datafusion.explain.show_sizes | true | When set to true, the explain statement will print the partition sizes | -| datafusion.explain.show_schema | false | When set to true, the explain statement will print schema information | -| datafusion.explain.format | indent | Display format of explain. Default is "indent". When set to "tree", it will print the plan in a tree-rendered format. | -| datafusion.explain.tree_maximum_render_width | 240 | (format=tree only) Maximum total width of the rendered tree. When set to 0, the tree will have no width limit. | -| datafusion.explain.analyze_level | dev | Verbosity level for "EXPLAIN ANALYZE". Default is "dev" "summary" shows common metrics for high-level insights. "dev" provides deep operator-level introspection for developers. | -| datafusion.explain.analyze_categories | all | Which metric categories to include in "EXPLAIN ANALYZE" output. Comma-separated list of: "rows", "bytes", "timing", "uncategorized". Use "none" to show plan structure only, or "all" (default) to show everything. Metrics without a declared category are treated as "uncategorized". | -| datafusion.sql_parser.parse_float_as_decimal | false | When set to true, SQL parser will parse float as decimal type | -| datafusion.sql_parser.enable_ident_normalization | true | When set to true, SQL parser will normalize ident (convert ident to lowercase when not quoted) | -| datafusion.sql_parser.enable_options_value_normalization | false | When set to true, SQL parser will normalize options value (convert value to lowercase). Note that this option is ignored and will be removed in the future. All case-insensitive values are normalized automatically. | -| datafusion.sql_parser.dialect | generic | Configure the SQL dialect used by DataFusion's parser; supported values include: Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, MsSQL, ClickHouse, BigQuery, Ansi, DuckDB, Databricks, Spark. | -| datafusion.sql_parser.support_varchar_with_length | true | If true, permit lengths for `VARCHAR` such as `VARCHAR(20)`, but ignore the length. If false, error if a `VARCHAR` with a length is specified. The Arrow type system does not have a notion of maximum string length and thus DataFusion can not enforce such limits. | -| datafusion.sql_parser.map_string_types_to_utf8view | true | If true, string types (VARCHAR, CHAR, Text, and String) are mapped to `Utf8View` during SQL planning. If false, they are mapped to `Utf8`. Default is true. | -| datafusion.sql_parser.collect_spans | false | When set to true, the source locations relative to the original SQL query (i.e. [`Span`](https://docs.rs/sqlparser/latest/sqlparser/tokenizer/struct.Span.html)) will be collected and recorded in the logical plan nodes. | -| datafusion.sql_parser.recursion_limit | 51 | Specifies the recursion depth limit when parsing complex SQL Queries | -| datafusion.sql_parser.default_null_ordering | `nulls_max` | Specifies the default null ordering for query results. There are 4 options: - `nulls_max`: Nulls appear last in ascending order. - `nulls_min`: Nulls appear first in ascending order. - `nulls_first`: Nulls always be first in any order. - `nulls_last`: Nulls always be last in any order. By default, `nulls_max` is used to follow Postgres's behavior. postgres rule: | -| datafusion.sql_parser.enable_subquery_sort_elimination | true | When set to true, DataFusion may remove `ORDER BY` clauses from subqueries or CTEs during SQL planning when their ordering cannot affect the result, such as when no `LIMIT` or other order-sensitive operator depends on them. Disable this option to preserve explicit subquery ordering in the planned query. | -| datafusion.format.safe | true | If set to `true` any formatting errors will be written to the output instead of being converted into a [`std::fmt::Error`] | -| datafusion.format.null | | Format string for nulls | -| datafusion.format.date_format | %Y-%m-%d | Date format for date arrays | -| datafusion.format.datetime_format | %Y-%m-%dT%H:%M:%S%.f | Format for DateTime arrays | -| datafusion.format.timestamp_format | %Y-%m-%dT%H:%M:%S%.f | Timestamp format for timestamp arrays | -| datafusion.format.timestamp_tz_format | NULL | Timestamp format for timestamp with timezone arrays. When `None`, ISO 8601 format is used. | -| datafusion.format.time_format | %H:%M:%S%.f | Time format for time arrays | -| datafusion.format.duration_format | pretty | Duration format. Can be either `"pretty"` or `"ISO8601"` | -| datafusion.format.types_info | false | Show types in visual representation batches | -| datafusion.spark.map_key_dedup_policy | EXCEPTION | Policy for handling duplicate keys in Spark-compatible map-construction functions (`map_from_arrays`, `map_from_entries`, `str_to_map`). Mirrors Spark's [`spark.sql.mapKeyDedupPolicy`](https://github.com/apache/spark/blob/cf3a34e19dfcf70e2d679217ff1ba21302212472/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L4961): - `EXCEPTION` (default): raise `[DUPLICATED_MAP_KEY]` at runtime on any duplicate key. - `LAST_WIN`: keep the last occurrence of each duplicate key. Values are case-insensitive. | +| key | default | description | +| ------------------------------------------------------------------------ | ----------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| datafusion.catalog.create_default_catalog_and_schema | true | Whether the default catalog and schema should be created automatically. | +| datafusion.catalog.default_catalog | datafusion | The default catalog name - this impacts what SQL queries use if not specified | +| datafusion.catalog.default_schema | public | The default schema name - this impacts what SQL queries use if not specified | +| datafusion.catalog.information_schema | false | Should DataFusion provide access to `information_schema` virtual tables for displaying schema information | +| datafusion.catalog.partition_information_schema | false | When set together with `information_schema`, the `information_schema` of each catalog only describes the objects in that catalog. When not set, the `information_schema` of each catalog describes all catalogs. | +| datafusion.catalog.location | NULL | Location scanned to load tables for `default` schema | +| datafusion.catalog.format | NULL | Type of `TableProvider` to use when loading `default` schema | +| datafusion.catalog.has_header | true | Default value for `format.has_header` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. | +| datafusion.catalog.newlines_in_values | false | Specifies whether newlines in (quoted) CSV values are supported. This is the default value for `format.newlines_in_values` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. Parsing newlines in quoted values may be affected by execution behaviour such as parallel file scanning. Setting this to `true` ensures that newlines in values are parsed successfully, which may reduce performance. | +| datafusion.execution.batch_size | 8192 | Default batch size while creating new batches, it's especially useful for buffer-in-memory batches since creating tiny batches would result in too much metadata memory consumption | +| datafusion.execution.perfect_hash_join_small_build_threshold | 1024 | A perfect hash join (see `HashJoinExec` for more details) will be considered if the range of keys (max - min) on the build side is < this threshold. This provides a fast path for joins with very small key ranges, bypassing the density check. Currently only supports cases where build_side.num_rows() < u32::MAX. Support for build_side.num_rows() >= u32::MAX will be added in the future. | +| datafusion.execution.perfect_hash_join_min_key_density | 0.15 | The minimum required density of join keys on the build side to consider a perfect hash join (see `HashJoinExec` for more details). Density is calculated as: `(number of rows) / (max_key - min_key + 1)`. A perfect hash join may be used if the actual key density > this value. Currently only supports cases where build_side.num_rows() < u32::MAX. Support for build_side.num_rows() >= u32::MAX will be added in the future. | +| datafusion.execution.coalesce_batches | true | When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting | +| datafusion.execution.collect_statistics | true | Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Defaults to true. | +| datafusion.execution.target_partitions | 0 | Number of partitions for query execution. Increasing partitions can increase concurrency. Defaults to the number of CPU cores on the system | +| datafusion.execution.time_zone | NULL | The default time zone Some functions, e.g. `now`, return timestamps in this time zone. This is also used to interpret timezone-naive timestamps in comparisons and subtraction with timezone-aware timestamps. | +| datafusion.execution.parquet.enable_page_index | true | (reading) If true, reads the Parquet data page level metadata (the Page Index), if present, to reduce the I/O and number of rows decoded. | +| datafusion.execution.parquet.pruning | true | (reading) If true, the parquet reader attempts to skip entire row groups based on the predicate in the query and the metadata (min/max values) stored in the parquet file | +| datafusion.execution.parquet.skip_metadata | true | (reading) If true, the parquet reader skip the optional embedded metadata that may be in the file Schema. This setting can help avoid schema conflicts when querying multiple parquet files with schemas containing compatible types but different metadata. Arrow extension types (the `ARROW:extension:name` and `ARROW:extension:metadata` field metadata) are kept, because they are part of the type of a column | +| datafusion.execution.parquet.metadata_size_hint | 524288 | (reading) If specified, the parquet reader will try and fetch the last `size_hint` bytes of the parquet file optimistically. If not specified, two reads are required: One read to fetch the 8-byte parquet footer and another to fetch the metadata length encoded in the footer Default setting to 512 KiB, which should be sufficient for most parquet files, it can reduce one I/O operation per parquet file. If the metadata is larger than the hint, two reads will still be performed. | +| datafusion.execution.parquet.pushdown_filters | false | (reading) If true, filter expressions are be applied during the parquet decoding operation to reduce the number of rows decoded. This optimization is sometimes called "late materialization". | +| datafusion.execution.parquet.reorder_filters | false | (reading) If true, filter expressions evaluated during the parquet decoding operation will be reordered heuristically to minimize the cost of evaluation. If false, the filters are applied in the same order as written in the query | +| datafusion.execution.parquet.force_filter_selections | false | (reading) Force the use of RowSelections for filter results, when pushdown_filters is enabled. If false, the reader will automatically choose between a RowSelection and a Bitmap based on the number and pattern of selected rows. | +| datafusion.execution.parquet.schema_force_view_types | true | (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, and `Binary/BinaryLarge` with `BinaryView`. The parquet reader is optimized for reading `Utf8View` and `BinaryView`, so such queries are significantly faster than reading `Utf8`/`Binary` and then casting to the view types. | +| datafusion.execution.parquet.binary_as_string | false | (reading) If true, parquet reader will read columns of `Binary/LargeBinary` with `Utf8`, and `BinaryView` with `Utf8View`. Parquet files generated by some legacy writers do not correctly set the UTF8 flag for strings, causing string columns to be loaded as BLOB instead. The parquet reader has special optimizations for `Utf8` validation, so reading such columns as strings is significantly faster than reading them as binary and then casting to string. | +| datafusion.execution.parquet.coerce_int96 | NULL | (reading) If set, parquet reader will read columns of physical type int96 as originating from this resolution instead of nanosecond. Valid values are: ns, us, ms, and s. This is useful for reading data from systems like Spark which store microsecond resolution timestamps in an int96, allowing them to write values with a larger date range than 64-bit timestamps with nanosecond resolution. | +| datafusion.execution.parquet.coerce_int96_tz | NULL | (reading) Optional timezone applied to INT96 columns when `coerce_int96` is set. When `Some`, INT96 columns coerce to `Timestamp(, Some())` instead of the default `Timestamp(, None)`. Spark and other systems write INT96 values as UTC-adjusted instants, so callers that need the resulting Arrow type to be timezone-aware (e.g. for Spark `TimestampType` semantics) should set this to `"UTC"`. No effect when `coerce_int96` is `None`, but the value is still validated when it is set. | +| datafusion.execution.parquet.bloom_filter_on_read | true | (reading) Use any available bloom filters when reading parquet files | +| datafusion.execution.parquet.max_predicate_cache_size | NULL | (reading) The maximum predicate cache size, in bytes. When `pushdown_filters` is enabled, sets the maximum memory used to cache the results of predicate evaluation between filter evaluation and output generation. Decreasing this value will reduce memory usage, but may increase IO and CPU usage. None means use the default parquet reader setting. 0 means no caching. | +| datafusion.execution.parquet.max_in_list_size | 20 | Maximum number of input values in an `IN (...)` list eligible for min/max pruning. Lists above this cap, or a cap of 0, skip this rewrite; other predicates and Bloom-filter pruning remain available. Within the cap, nonempty lists of at most 20 values use the existing per-value rewrite. Larger literal lists use a compact representation when the column type is string, variable-length binary, integer, decimal, date, time, timestamp, or duration. This applies to both `IN` and `NOT IN`, including lists with NULL members. `NOT IN` with NULL and all-NULL `IN` lists cannot match any rows. Compact lists containing NULL do not use the fully-matched-row-group optimization. Floating-point and other lists retain the existing per-value rewrite, so raising the cap can make those predicates expensive to build and evaluate. Defaults to 20. | +| datafusion.execution.parquet.enable_rle_to_dictionary | false | (reading) If true, top-level string and binary Parquet columns with dictionary pages are inferred and scanned as `Dictionary` / `Dictionary` instead of their plain value type. This applies only when DataFusion infers the table schema. Tables with a user-supplied schema are not promoted because the Parquet footer is not read at DDL time, so dictionary pages cannot be detected per column. See | +| datafusion.execution.parquet.row_group_range_assignment | `start_offset` | (reading) Which byte range of a split file reads each row group. `start_offset` picks the range containing the row group's start. `midpoint` picks the range containing its midpoint, as Spark does, which spreads large row groups more evenly across ranges. | +| datafusion.execution.parquet.data_pagesize_limit | 1048576 | (writing) Sets best effort maximum size of data page in bytes | +| datafusion.execution.parquet.write_batch_size | 1024 | (writing) Sets write_batch_size in rows | +| datafusion.execution.parquet.writer_version | 1.0 | (writing) Sets parquet writer version valid values are "1.0" and "2.0" | +| datafusion.execution.parquet.skip_arrow_metadata | false | (writing) Skip encoding the embedded arrow metadata in the KV_meta This is analogous to the `ArrowWriterOptions::with_skip_arrow_metadata`. Refer to | +| datafusion.execution.parquet.compression | zstd(3) | (writing) Sets default parquet compression codec. Valid values are: uncompressed, snappy, gzip(level), brotli(level), lz4, zstd(level), and lz4_raw. These values are not case sensitive. If NULL, uses default parquet writer setting Note that this default setting is not the same as the default parquet writer setting. | +| datafusion.execution.parquet.dictionary_enabled | true | (writing) Sets if dictionary encoding is enabled. If NULL, uses default parquet writer setting | +| datafusion.execution.parquet.dictionary_page_size_limit | 1048576 | (writing) Sets best effort maximum dictionary page size, in bytes | +| datafusion.execution.parquet.statistics_enabled | page | (writing) Sets if statistics are enabled for any column Valid values are: "none", "chunk", and "page" These values are not case sensitive. If NULL, uses default parquet writer setting | +| datafusion.execution.parquet.max_row_group_size | 1048576 | (writing) Target maximum number of rows in each row group (defaults to 1M rows). Writing larger row groups requires more memory to write, but can get better compression and be faster to read. When `max_row_group_bytes` is also set, the writer flushes a row group when either limit is reached, whichever comes first. | +| datafusion.execution.parquet.max_row_group_bytes | NULL | (writing) Target maximum size of each row group in bytes. When set, the writer flushes whenever either this limit or `max_row_group_size` is reached, whichever comes first. Useful for bounding writer memory on wide schemas where a row-count limit can map to very different byte sizes. Matches the behavior of `parquet.block.size` in parquet-mr. If `None` (the default), only the row-count limit applies. Currently only honored when `allow_single_file_parallelism` is `false`; by default the parallel file writer ignores this limit. | +| datafusion.execution.parquet.created_by | datafusion version 57.0.0-dev | (writing) Sets "created by" property | +| datafusion.execution.parquet.column_index_truncate_length | 64 | (writing) Sets column index truncate length | +| datafusion.execution.parquet.statistics_truncate_length | 64 | (writing) Sets statistics truncate length. If NULL, uses default parquet writer setting | +| datafusion.execution.parquet.data_page_row_count_limit | 20000 | (writing) Sets best effort maximum number of rows in data page | +| datafusion.execution.parquet.encoding | NULL | (writing) Sets default encoding for any column. Valid values are: plain, plain_dictionary, rle, bit_packed, delta_binary_packed, delta_length_byte_array, delta_byte_array, rle_dictionary, byte_stream_split, and alp. These values are not case sensitive. If NULL, uses default parquet writer setting | +| datafusion.execution.parquet.bloom_filter_on_write | false | (writing) Write bloom filters for all columns when creating parquet files | +| datafusion.execution.parquet.bloom_filter_fpp | NULL | (writing) Sets bloom filter false positive probability. If NULL, uses default parquet writer setting | +| datafusion.execution.parquet.bloom_filter_ndv | NULL | (writing) Sets bloom filter number of distinct values. If NULL, uses default parquet writer setting | +| datafusion.execution.parquet.write_row_group_number_distinct_values | false | (writing) Write the number of distinct values (NDV) for each column in row group statistics when creating parquet files. Enabling this improves NDV-based query optimizations at the cost of hashing every non-null value during write. | +| datafusion.execution.parquet.allow_single_file_parallelism | true | (writing) Controls whether DataFusion will attempt to speed up writing parquet files by serializing them in parallel. Each column in each row group in each output file are serialized in parallel leveraging a maximum possible core count of n_files\*n_row_groups\*n_columns. | +| datafusion.execution.parquet.maximum_parallel_row_group_writers | 1 | (writing) By default parallel parquet writer is tuned for minimum memory usage in a streaming execution plan. You may see a performance benefit when writing large parquet files by increasing maximum_parallel_row_group_writers and maximum_buffered_record_batches_per_stream if your system has idle cores and can tolerate additional memory usage. Boosting these values is likely worthwhile when writing out already in-memory data, such as from a cached data frame. | +| datafusion.execution.parquet.maximum_buffered_record_batches_per_stream | 2 | (writing) By default parallel parquet writer is tuned for minimum memory usage in a streaming execution plan. You may see a performance benefit when writing large parquet files by increasing maximum_parallel_row_group_writers and maximum_buffered_record_batches_per_stream if your system has idle cores and can tolerate additional memory usage. Boosting these values is likely worthwhile when writing out already in-memory data, such as from a cached data frame. | +| datafusion.execution.parquet.content_defined_chunking.enabled | false | (writing) EXPERIMENTAL: Enable content-defined chunking (CDC) when writing parquet files. When enabled, parallel writing is automatically disabled since the chunker state must persist across row groups. | +| datafusion.execution.parquet.content_defined_chunking.min_chunk_size | 262144 | Minimum chunk size in bytes. The rolling hash will not trigger a split until this many bytes have been accumulated. Default is 256 KiB. | +| datafusion.execution.parquet.content_defined_chunking.max_chunk_size | 1048576 | Maximum chunk size in bytes. A split is forced when the accumulated size exceeds this value. Default is 1 MiB. | +| datafusion.execution.parquet.content_defined_chunking.norm_level | 0 | Normalization level. Increasing this improves deduplication ratio but increases fragmentation. Recommended range is [-3, 3], default is 0. | +| datafusion.execution.planning_concurrency | 0 | Fan-out during initial physical planning. This is mostly use to plan `UNION` children in parallel. Defaults to the number of CPU cores on the system | +| datafusion.execution.skip_physical_aggregate_schema_check | false | When set to true, skips verifying that the schema produced by planning the input of `LogicalPlan::Aggregate` exactly matches the schema of the input plan. When set to false, if the schema does not match exactly (including nullability and metadata), a planning error will be raised. This is used to workaround bugs in the planner that are now caught by the new schema verification step. | +| datafusion.execution.enable_migration_aggregate | true | Whether aggregation uses the implementation from the major refactor completed in the 56.0.0 release. When set to `false`, aggregation falls back to the implementation used before 55.0.0. The fallback exists only as a workaround for bugs in the new implementation and will be removed, together with this option, after the 56.0.0 release. See for details. | +| datafusion.execution.spill_compression | uncompressed | Sets the compression codec used when spilling data to disk. Since datafusion writes spill files using the Arrow IPC Stream format, only codecs supported by the Arrow IPC Stream Writer are allowed. Valid values are: uncompressed, lz4_frame, zstd. Note: lz4_frame offers faster (de)compression, but typically results in larger spill files. In contrast, zstd achieves higher compression ratios at the cost of slower (de)compression speed. | +| datafusion.execution.sort_spill_reservation_bytes | 10485760 | Specifies the reserved memory for each spillable sort operation to facilitate an in-memory merge. When a sort operation spills to disk, the in-memory data must be sorted and merged before being written to a file. This setting reserves a specific amount of memory for that in-memory sort/merge process. Note: This setting is irrelevant if the sort operation cannot spill (i.e., if there's no `DiskManager` configured). | +| datafusion.execution.sort_in_place_threshold_bytes | 1048576 | When sorting, below what size should data be concatenated and sorted in a single RecordBatch rather than sorted in batches and merged. | +| datafusion.execution.sort_pushdown_buffer_capacity | 1073741824 | Maximum buffer capacity (in bytes) per partition for BufferExec inserted during sort pushdown optimization. When PushdownSort eliminates a SortExec under SortPreservingMergeExec, a BufferExec is inserted to replace SortExec's buffering role. This prevents I/O stalls by allowing the scan to run ahead of the merge. This uses strictly less memory than the SortExec it replaces (which buffers the entire partition). The buffer respects the global memory pool limit. Setting this to a large value is safe — actual memory usage is bounded by partition size and global memory limits. | +| datafusion.execution.max_spill_file_size_bytes | 134217728 | Maximum size in bytes for individual spill files before rotating to a new file. When operators spill data to disk (e.g., RepartitionExec), they write multiple batches to the same file until this size limit is reached, then rotate to a new file. This reduces syscall overhead compared to one-file-per-batch while preventing files from growing too large. A larger value reduces file creation overhead but may hold more disk space. A smaller value creates more files but allows finer-grained space reclamation as files can be deleted once fully consumed. Now only `RepartitionExec` supports this spill file rotation feature, other spilling operators may create spill files larger than the limit. Default: 128 MB | +| datafusion.execution.enable_nlj_coordinated_fallback | true | Enables the memory-limited fallback for `NestedLoopJoinExec` join types that emit unmatched left rows in the final output (LEFT, LEFT SEMI, LEFT ANTI, LEFT MARK, FULL) when the right side has multiple partitions. This fallback shares left-side state (the current left chunk, the visited bitmap and the probe-thread counter) across all right-side partitions, which assumes every partition runs in the same process. Distributed engines that execute each output partition as an independent task (e.g. Ballista, datafusion-distributed) give each task its own copy of this state and poll only one partition, so the cross-partition counter never reaches zero and the fallback would stall. Such engines should set this to `false`: the coordinated fallback is then disabled for left-emitting multi-partition joins, which instead fail with a resource-exhaustion error under memory pressure rather than deadlocking. Single-partition and non-left-emitting joins are unaffected and always keep the fallback. | +| datafusion.execution.meta_fetch_concurrency | 32 | Number of files to read in parallel when inferring schema and statistics | +| datafusion.execution.minimum_parallel_output_files | 4 | Guarantees a minimum level of output files running in parallel. RecordBatches will be distributed in round robin fashion to each parallel writer. Each writer is closed and a new file opened once soft_max_rows_per_output_file or soft_max_bytes_per_output_file is reached. | +| datafusion.execution.soft_max_rows_per_output_file | 50000000 | Target number of rows in output files when writing multiple. This is a soft max, so it can be exceeded slightly. There also will be one file smaller than the limit if the total number of rows written is not roughly divisible by the soft max | +| datafusion.execution.soft_max_bytes_per_output_file | 4294967295 | Target encoded size in bytes of output files when writing multiple. Writers asynchronously report the cumulative encoded size as they process RecordBatches. The final file size may exceed this limit due to batches buffered before the limit is observed, the size of a batch, and file metadata written when the file is finalized. | +| datafusion.execution.max_buffered_batches_per_output_file | 2 | This is the maximum number of RecordBatches buffered for each output file being worked. Higher values can potentially give faster write performance at the cost of higher peak memory consumption. This budget is split evenly between two independent points in the write pipeline (see the demuxer diagram in #7791): how many files can be in flight from the demuxer to a writer task, and how many RecordBatches are buffered for a single file's writer. Must be at least 2 so each half gets at least 1 unit of buffering - 0 or 1 would leave one side with a zero-capacity channel and panic at write time. | +| datafusion.execution.listing_table_ignore_subdirectory | true | Should sub directories be ignored when scanning directories for data files. Defaults to true (ignores subdirectories), consistent with Hive. Note that this setting does not affect reading partitioned tables (e.g. `/table/year=2021/month=01/data.parquet`). | +| datafusion.execution.listing_table_factory_infer_partitions | true | Should a `ListingTable` created through the `ListingTableFactory` infer table partitions from Hive compliant directories. Defaults to true (partition columns are inferred and will be represented in the table schema). | +| datafusion.execution.listing_table_hive_default_partition_name | `__HIVE_DEFAULT_PARTITION__` | The directory name Hive compliant writers use in place of a null partition value, e.g. `/table/date=__HIVE_DEFAULT_PARTITION__/data.parquet`. Partition values equal to this name are read as NULL, as Hive, Spark and Impala do, and partitioned writes reject them. Corresponds to Hive's `hive.exec.default.partition.name`. Set this to the empty string to read and write such values as the literal directory name instead. | +| datafusion.execution.enable_recursive_ctes | true | Should DataFusion support recursive CTEs | +| datafusion.execution.split_file_groups_by_statistics | false | Attempt to eliminate sorts by packing & sorting files with non-overlapping statistics into the same file groups. Currently experimental | +| datafusion.execution.keep_partition_by_columns | false | Should DataFusion keep the columns used for partition_by in the output RecordBatches | +| datafusion.execution.enable_file_stream_work_stealing | true | When `true` (the default), DataFusion's built-in file scans dynamically rebalance files across partitions at query execution time: a partition that goes idle reads files (or byte-range morsels) originally assigned to a sibling partition, which keeps all partitions busy in a single process. Executors that depend on the plan-time partition assignment — such as Ballista and datafusion-distributed, which run each partition as an isolated task and never poll the siblings — should set this to `false` so each partition reads only its own file group and no runtime reassignment occurs. | +| datafusion.execution.skip_partial_aggregation_probe_ratio_threshold | 0.8 | Aggregation ratio (number of distinct groups / number of input rows) threshold for skipping partial aggregation. If the value is greater then partial aggregation will skip aggregation for further input | +| datafusion.execution.skip_partial_aggregation_probe_rows_threshold | 100000 | Number of input rows partial aggregation partition should process, before aggregation ratio check and trying to switch to skipping aggregation mode | +| datafusion.execution.use_row_number_estimates_to_optimize_partitioning | false | Should DataFusion use row number estimates at the input to decide whether increasing parallelism is beneficial or not. By default, only exact row numbers (not estimates) are used for this decision. Setting this flag to `true` will likely produce better plans. if the source of statistics is accurate. We plan to make this the default in the future. | +| datafusion.execution.enforce_batch_size_in_joins | false | Should DataFusion enforce batch size in joins or not. By default, DataFusion will not enforce batch size in joins. Enforcing batch size in joins can reduce memory usage when joining large tables with a highly-selective join filter, but is also slightly slower. Note: this option currently only applies to the symmetric hash join. | +| datafusion.execution.objectstore_writer_buffer_size | 10485760 | Size (bytes) of data buffer DataFusion uses when writing output files. This affects the size of the data chunks that are uploaded to remote object stores (e.g. AWS S3). If very large (>= 100 GiB) output files are being written, it may be necessary to increase this size to avoid errors from the remote end point. | +| datafusion.execution.enable_ansi_mode | false | Whether to enable ANSI SQL mode. The flag is experimental and relevant only for DataFusion Spark built-in functions When `enable_ansi_mode` is set to `true`, the query engine follows ANSI SQL semantics for expressions, casting, and error handling. This means: - **Strict type coercion rules:** implicit casts between incompatible types are disallowed. - **Standard SQL arithmetic behavior:** operations such as division by zero, numeric overflow, or invalid casts raise runtime errors rather than returning `NULL` or adjusted values. - **Consistent ANSI behavior** for string concatenation, comparisons, and `NULL` handling. When `enable_ansi_mode` is `false` (the default), the engine uses a more permissive, non-ANSI mode designed for user convenience and backward compatibility. In this mode: - Implicit casts between types are allowed (e.g., string to integer when possible). - Arithmetic operations are more lenient — for example, `abs()` on the minimum representable integer value returns the input value instead of raising overflow. - Division by zero or invalid casts may return `NULL` instead of failing. # Default `false` — ANSI SQL mode is disabled by default. | +| datafusion.execution.hash_join_buffering_capacity | 0 | How many bytes to buffer in the probe side of hash joins while the build side is concurrently being built. Without this, hash joins will wait until the full materialization of the build side before polling the probe side. This is useful in scenarios where the query is not completely CPU bounded, allowing to do some early work concurrently and reducing the latency of the query. Note that when hash join buffering is enabled, the probe side will start eagerly polling data, not giving time for the producer side of dynamic filters to produce any meaningful predicate. Queries with dynamic filters might see performance degradation. Disabled by default, set to a number greater than 0 for enabling it. | +| datafusion.optimizer.enable_distinct_aggregation_soft_limit | true | When set to true, the optimizer will push a limit operation into grouped aggregations which have no aggregate expressions, as a soft limit, emitting groups once the limit is reached, before all rows in the group are read. | +| datafusion.optimizer.enable_round_robin_repartition | true | When set to true, the physical plan optimizer will try to add round robin repartitioning to increase parallelism to leverage more CPU cores | +| datafusion.optimizer.enable_topk_aggregation | true | When set to true, the optimizer will attempt to perform limit operations during aggregations, if possible | +| datafusion.optimizer.enable_window_limits | true | When set to true, the optimizer will attempt to push limit operations past window functions, if possible | +| datafusion.optimizer.enable_window_topn | false | When set to true, the optimizer will replace Filter(rn<=K) → Window(ROW_NUMBER) → Sort patterns with a PartitionedTopKExec that maintains per-partition heaps, avoiding a full sort of the input. When the window partition key has low cardinality, enabling this optimization can improve performance. However, for high cardinality keys, it may cause regressions in both memory usage and runtime. | +| datafusion.optimizer.enable_row_number_to_aggregate | false | When set to true, the optimizer will rewrite a "top-1 per group" pattern of the form `Filter(row_number() = 1)` over a `PARTITION BY` window into an `Aggregate(first_value(... ORDER BY ...) GROUP BY partition)`. This avoids buffering / fully sorting the input, which is especially beneficial on wide payloads. Disabled by default because on nested/wide value types the aggregate only becomes memory efficient once the columnar nested-type `GroupsAccumulator` fast path is available; enabling it without that support can route wide payloads onto the slow per-group accumulator path. | +| datafusion.optimizer.enable_topk_repartition | true | When set to true, the optimizer will push TopK (Sort with fetch) below hash repartition when the partition key is a prefix of the sort key, reducing data volume before the shuffle. | +| datafusion.optimizer.enable_topk_dynamic_filter_pushdown | true | When set to true, the optimizer will attempt to push down TopK dynamic filters into the file scan phase. | +| datafusion.optimizer.enable_physical_uncorrelated_scalar_subquery | true | When set to true, uncorrelated scalar subqueries are left in the logical plan and executed by `ScalarSubqueryExec` during physical execution. When set to false, all scalar subqueries (including uncorrelated ones) are rewritten to left joins by the `ScalarSubqueryToJoin` optimizer rule. Note disabling this option is not recommended. It restores pre behavior, which silently produces incorrect results for multi-row subqueries and does not support scalar subqueries in ORDER BY / JOIN ON / aggregate-function arguments. This option is intended as a temporary escape hatch for distributed execution frameworks and is planned to be removed in a future DataFusion release. | +| datafusion.optimizer.enable_join_dynamic_filter_pushdown | true | When set to true, the optimizer will attempt to push down Join dynamic filters into the file scan phase. | +| datafusion.optimizer.enable_aggregate_dynamic_filter_pushdown | true | When set to true, the optimizer will attempt to push down Aggregate dynamic filters into the file scan phase. | +| datafusion.optimizer.enable_dynamic_filter_pushdown | true | When set to true attempts to push down dynamic filters generated by operators (TopK, Join & Aggregate) into the file scan phase. For example, for a query such as `SELECT * FROM t ORDER BY timestamp DESC LIMIT 10`, the optimizer will attempt to push down the current top 10 timestamps that the TopK operator references into the file scans. This means that if we already have 10 timestamps in the year 2025 any files that only have timestamps in the year 2024 can be skipped / pruned at various stages in the scan. The config will suppress `enable_join_dynamic_filter_pushdown`, `enable_topk_dynamic_filter_pushdown` & `enable_aggregate_dynamic_filter_pushdown` So if you disable `enable_topk_dynamic_filter_pushdown`, then enable `enable_dynamic_filter_pushdown`, the `enable_topk_dynamic_filter_pushdown` will be overridden. | +| datafusion.optimizer.filter_null_join_keys | false | When set to true, the optimizer will insert filters before a join between a nullable and non-nullable column to filter out nulls on the nullable side. This filter can add additional overhead when the file format does not fully support predicate push down. | +| datafusion.optimizer.repartition_aggregations | true | Should DataFusion repartition data using the aggregate keys to execute aggregates in parallel using the provided `target_partitions` level | +| datafusion.optimizer.repartition_file_min_size | 1048576 | Minimum total file size in bytes for file-group byte-range splitting to fire. Files (or merged file groups) smaller than this stay as one partition. Lower values produce more, smaller partitions — better at filling `target_partitions` worth of cores when files are modestly sized, at the cost of slightly more per-partition open / metadata-load overhead. | +| datafusion.optimizer.repartition_joins | true | Should DataFusion repartition data using the join keys to execute joins in parallel using the provided `target_partitions` level | +| datafusion.optimizer.allow_symmetric_joins_without_pruning | true | Should DataFusion allow symmetric hash joins for unbounded data sources even when its inputs do not have any ordering or filtering If the flag is not enabled, the SymmetricHashJoin operator will be unable to prune its internal buffers, resulting in certain join types - such as Full, Left, LeftAnti, LeftSemi, Right, RightAnti, and RightSemi - being produced only at the end of the execution. This is not typical in stream processing. Additionally, without proper design for long runner execution, all types of joins may encounter out-of-memory errors. | +| datafusion.optimizer.repartition_file_scans | true | When set to `true`, datasource partitions will be repartitioned to achieve maximum parallelism. This applies to both in-memory partitions and FileSource's file groups (1 group is 1 partition). For FileSources, only Parquet and CSV formats are currently supported. If set to `true` for a FileSource, all files will be repartitioned evenly (i.e., a single large file might be partitioned into smaller chunks) for parallel scanning. If set to `false` for a FileSource, different files will be read in parallel, but repartitioning won't happen within a single file. If set to `true` for an in-memory source, all memtable's partitions will have their batches repartitioned evenly to the desired number of `target_partitions`. Repartitioning can change the total number of partitions and batches per partition, but does not slice the initial record tables provided to the MemTable on creation. | +| datafusion.optimizer.preserve_file_partitions | 0 | Minimum number of distinct partition values required to group files by their Hive partition column values (enabling output partitioning declaration). How the option is used: - preserve_file_partitions=0: Disable it. - preserve_file_partitions=1: Always enable it. - preserve_file_partitions=N, actual file partitions=M: Only enable when M >= N. This threshold preserves I/O parallelism when file partitioning is below it. Note: This may reduce parallelism, rooting from the I/O level, if the number of distinct partitions is less than the target_partitions. | +| datafusion.optimizer.repartition_windows | true | Should DataFusion repartition data using the partitions keys to execute window functions in parallel using the provided `target_partitions` level | +| datafusion.optimizer.repartition_sorts | true | Should DataFusion execute sorts in a per-partition fashion and merge afterwards instead of coalescing first and sorting globally. With this flag is enabled, plans in the form below `text "SortExec: [a@0 ASC]", " CoalescePartitionsExec", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ` would turn into the plan below which performs better in multithreaded environments `text "SortPreservingMergeExec: [a@0 ASC]", " SortExec: [a@0 ASC]", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ` | +| datafusion.optimizer.subset_repartition_threshold | 4 | Partition count threshold for subset satisfaction optimization. When the current partition count is >= this threshold, DataFusion will skip repartitioning if the required partitioning expression is a subset of the current partition expression such as Hash(a) satisfies Hash(a, b). When the current partition count is < this threshold, DataFusion will repartition to increase parallelism even when subset satisfaction applies. Set to 0 to always repartition (disable subset satisfaction optimization). Set to a high value to always use subset satisfaction. Example (subset_repartition_threshold = 4): `text Hash([a]) satisfies Hash([a, b]) because (Hash([a, b]) is subset of Hash([a]) If current partitions (3) < threshold (4), repartition: AggregateExec: mode=FinalPartitioned, gby=[a, b], aggr=[SUM(x)] RepartitionExec: partitioning=Hash([a, b], 8), input_partitions=3 AggregateExec: mode=Partial, gby=[a, b], aggr=[SUM(x)] DataSourceExec: file_groups={...}, output_partitioning=Hash([a], 3) If current partitions (8) >= threshold (4), use subset satisfaction: AggregateExec: mode=SinglePartitioned, gby=[a, b], aggr=[SUM(x)] DataSourceExec: file_groups={...}, output_partitioning=Hash([a], 8) ` | +| datafusion.optimizer.prefer_existing_sort | false | When true, DataFusion will opportunistically remove sorts when the data is already sorted, (i.e. setting `preserve_order` to true on `RepartitionExec` and using `SortPreservingMergeExec`) When false, DataFusion will maximize plan parallelism using `RepartitionExec` even if this requires subsequently resorting data using a `SortExec`. | +| datafusion.optimizer.skip_failed_rules | false | When set to true, the logical plan optimizer will produce warning messages if any optimization rules produce errors and then proceed to the next rule. When set to false, any rules that produce errors will cause the query to fail | +| datafusion.optimizer.max_passes | 3 | The logical optimizer applies rules in order for up to `max_passes` passes, stopping early if a pass leaves the plan unchanged or repeats an earlier plan. The physical optimizer always runs in one pass. | +| datafusion.optimizer.top_down_join_key_reordering | true | When set to true, the physical plan optimizer will run a top down process to reorder the join keys | +| datafusion.optimizer.join_reordering | true | When set to true, the physical plan optimizer may swap join inputs based on statistics. When set to false, statistics-driven join input reordering is disabled and the original join order in the query is used. | +| datafusion.optimizer.use_statistics_registry | false | (Deprecated) Ignored: the physical plan optimizer always consults the session's pluggable `StatisticsRegistry` (register providers on the `SessionState`; with none it is a no-op). | +| datafusion.optimizer.prefer_hash_join | true | When set to true, the physical plan optimizer will prefer HashJoin over SortMergeJoin. HashJoin can work more efficiently than SortMergeJoin but consumes more memory | +| datafusion.optimizer.enable_piecewise_merge_join | false | When set to true, piecewise merge join is enabled. PiecewiseMergeJoin is currently experimental. Physical planner will opt for PiecewiseMergeJoin when there is only one range filter. | +| datafusion.optimizer.hash_join_single_partition_threshold | 4194304 | The maximum estimated size in bytes for one input side of a HashJoin will be collected into a single partition | +| datafusion.optimizer.hash_join_single_partition_threshold_rows | 131072 | The maximum estimated size in rows for one input side of a HashJoin will be collected into a single partition | +| datafusion.optimizer.hash_join_inlist_pushdown_max_size | 131072 | Maximum size in bytes for the build side of a hash join to be pushed down as an InList expression for dynamic filtering. Build sides larger than this will use hash table lookups instead. Set to 0 to always use hash table lookups. InList pushdown can be more efficient for small build sides because it can result in better statistics pruning as well as use any bloom filters present on the scan side. InList expressions are also more transparent and easier to serialize over the network in distributed uses of DataFusion. On the other hand InList pushdown requires making a copy of the data and thus adds some overhead to the build side and uses more memory. This setting is per-partition, so we may end up using `hash_join_inlist_pushdown_max_size` \* `target_partitions` memory. The default is 128kB per partition. This should allow point lookup joins (e.g. joining on a unique primary key) to use InList pushdown in most cases but avoids excessive memory usage or overhead for larger joins. | +| datafusion.optimizer.hash_join_inlist_pushdown_max_distinct_values | 150 | Maximum number of distinct values (rows) in the build side of a hash join to be pushed down as an InList expression for dynamic filtering. Build sides with more rows than this will use hash table lookups instead. Set to 0 to always use hash table lookups. This provides an additional limit beyond `hash_join_inlist_pushdown_max_size` to prevent very large IN lists that might not provide much benefit over hash table lookups. This uses the deduplicated row count once the build side has been evaluated. The default is 150 values per partition. This is inspired by Trino's `max-filter-keys-per-column` setting. See: | +| datafusion.optimizer.hash_join_dynamic_filter_inlist_max_distinct_values | 20 | Maximum number of distinct build side keys, over all partitions of a hash join, for the global dynamic filter to test membership with one InList of all keys. Above this limit the dynamic filter uses a bloom filter (see `hash_join_bloom_filter_fpp`). Every partition must also use an InList by itself (`hash_join_inlist_pushdown_max_distinct_values` and `hash_join_inlist_pushdown_max_size`). For larger sets, a bloom filter is cheaper per probe row than an InList. | +| datafusion.optimizer.hash_join_bloom_filter_fpp | 0.01 | Target false positive probability of the bloom filter in the dynamic filter of a hash join. When the build side of a hash join has more distinct keys than `hash_join_inlist_pushdown_max_distinct_values` (over all partitions), the dynamic filter tests probe rows against one bloom filter of all build side keys. Each build partition fills a filter of its keys, sized for the estimated number of distinct keys of all partitions at this probability. Then one task ORs the partition filters and folds the result to this probability. | +| datafusion.optimizer.hash_join_bloom_filter_max_bytes | 8388608 | Maximum size, in bytes, of the bloom filter in the dynamic filter of a hash join (and of the filter of each build partition). A build side that needs a larger filter at `hash_join_bloom_filter_fpp` gets no bloom filter: such a large build side is rarely selective, so the filter is too big to be useful. The dynamic filter then tests only the min/max bounds (and keeps key range pruning). | +| datafusion.optimizer.hash_join_bloom_filter_max_fpp | 0.15 | The bloom filter of a hash join dynamic filter is not used when its estimated false positive probability is above this value (the filter is saturated, a safety net). Then the dynamic filter tests only the min/max bounds of the build keys, because a saturated filter removes few rows and costs CPU time for each probe row. | +| datafusion.optimizer.default_filter_selectivity | 20 | The default filter selectivity used by Filter Statistics when an exact selectivity cannot be determined. Valid values are between 0 (no selectivity) and 100 (all rows are selected). | +| datafusion.optimizer.prefer_existing_union | false | When set to true, the optimizer will not attempt to convert Union to Interleave | +| datafusion.optimizer.expand_views_at_output | false | When set to true, if the returned type is a view type then the output will be coerced to a non-view. Coerces `Utf8View` to `LargeUtf8`, and `BinaryView` to `LargeBinary`. | +| datafusion.optimizer.enable_sort_pushdown | true | Enable sort pushdown optimization. When enabled, attempts to push sort requirements down to data sources that can natively handle them (e.g., by reversing file/row group read order). Returns **inexact ordering**: Sort operator is kept for correctness, but optimized input enables early termination for TopK queries (ORDER BY ... LIMIT N), providing significant speedup. Memory: No additional overhead (only changes read order). Future: Will add option to detect perfectly sorted data and eliminate Sort completely. Default: true | +| datafusion.optimizer.enable_leaf_expression_pushdown | true | When set to true, the optimizer will extract leaf expressions (such as `get_field`) from filter/sort/join nodes into projections closer to the leaf table scans, and push those projections down towards the leaf nodes. | +| datafusion.optimizer.enable_unions_to_filter | false | When set to true, the logical optimizer will rewrite `UNION DISTINCT` branches that read from the same source and differ only by filter predicates into a single branch with a combined filter. This optimization is conservative and only applies when the branches share the same source and compatible wrapper nodes such as identical projections or aliases. | +| datafusion.optimizer.single_distinct_prefer_native | false | Prefer native grouped DISTINCT aggregation when target_partitions is 1 and every aggregate explicitly supports a specialized GroupsAccumulator for its DISTINCT argument types. Defaults to false, preserving the two-phase GROUP BY rewrite. The native plan can avoid materializing distinct (group, value) pairs, but can be slower for many groups with duplicate values. It must fit each group's distinct values in memory, so queries that succeed by spilling with the rewrite may fail with this option enabled. One target partition can still have multiple input partitions and merge native states. | +| datafusion.explain.logical_plan_only | false | When set to true, the explain statement will only print logical plans | +| datafusion.explain.physical_plan_only | false | When set to true, the explain statement will only print physical plans | +| datafusion.explain.show_statistics | false | When set to true, the explain statement will print operator statistics for physical plans | +| datafusion.explain.show_sizes | true | When set to true, the explain statement will print the partition sizes | +| datafusion.explain.show_schema | false | When set to true, the explain statement will print schema information | +| datafusion.explain.format | indent | Display format of explain. Default is "indent". When set to "tree", it will print the plan in a tree-rendered format. | +| datafusion.explain.tree_maximum_render_width | 240 | (format=tree only) Maximum total width of the rendered tree. When set to 0, the tree will have no width limit. | +| datafusion.explain.analyze_level | dev | Verbosity level for "EXPLAIN ANALYZE". Default is "dev" "summary" shows common metrics for high-level insights. "dev" provides deep operator-level introspection for developers. | +| datafusion.explain.analyze_categories | all | Which metric categories to include in "EXPLAIN ANALYZE" output. Comma-separated list of: "rows", "bytes", "timing", "uncategorized". Use "none" to show plan structure only, or "all" (default) to show everything. Metrics without a declared category are treated as "uncategorized". | +| datafusion.sql_parser.parse_float_as_decimal | false | When set to true, SQL parser will parse float as decimal type | +| datafusion.sql_parser.enable_ident_normalization | true | When set to true, SQL parser will normalize ident (convert ident to lowercase when not quoted) | +| datafusion.sql_parser.enable_options_value_normalization | false | When set to true, SQL parser will normalize options value (convert value to lowercase). Note that this option is ignored and will be removed in the future. All case-insensitive values are normalized automatically. | +| datafusion.sql_parser.dialect | generic | Configure the SQL dialect used by DataFusion's parser; supported values include: Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, MsSQL, ClickHouse, BigQuery, Ansi, DuckDB, Databricks, Spark. | +| datafusion.sql_parser.support_varchar_with_length | true | If true, permit lengths for `VARCHAR` such as `VARCHAR(20)`, but ignore the length. If false, error if a `VARCHAR` with a length is specified. The Arrow type system does not have a notion of maximum string length and thus DataFusion can not enforce such limits. | +| datafusion.sql_parser.map_string_types_to_utf8view | true | If true, string types (VARCHAR, CHAR, Text, and String) are mapped to `Utf8View` during SQL planning. If false, they are mapped to `Utf8`. Default is true. | +| datafusion.sql_parser.collect_spans | false | When set to true, the source locations relative to the original SQL query (i.e. [`Span`](https://docs.rs/sqlparser/latest/sqlparser/tokenizer/struct.Span.html)) will be collected and recorded in the logical plan nodes. | +| datafusion.sql_parser.recursion_limit | 51 | Specifies the recursion depth limit when parsing complex SQL Queries | +| datafusion.sql_parser.default_null_ordering | `nulls_max` | Specifies the default null ordering for query results. There are 4 options: - `nulls_max`: Nulls appear last in ascending order. - `nulls_min`: Nulls appear first in ascending order. - `nulls_first`: Nulls always be first in any order. - `nulls_last`: Nulls always be last in any order. By default, `nulls_max` is used to follow Postgres's behavior. postgres rule: | +| datafusion.sql_parser.enable_subquery_sort_elimination | true | When set to true, DataFusion may remove `ORDER BY` clauses from subqueries or CTEs during SQL planning when their ordering cannot affect the result, such as when no `LIMIT` or other order-sensitive operator depends on them. Disable this option to preserve explicit subquery ordering in the planned query. | +| datafusion.format.safe | true | If set to `true` any formatting errors will be written to the output instead of being converted into a [`std::fmt::Error`] | +| datafusion.format.null | | Format string for nulls | +| datafusion.format.date_format | %Y-%m-%d | Date format for date arrays | +| datafusion.format.datetime_format | %Y-%m-%dT%H:%M:%S%.f | Format for DateTime arrays | +| datafusion.format.timestamp_format | %Y-%m-%dT%H:%M:%S%.f | Timestamp format for timestamp arrays | +| datafusion.format.timestamp_tz_format | NULL | Timestamp format for timestamp with timezone arrays. When `None`, ISO 8601 format is used. | +| datafusion.format.time_format | %H:%M:%S%.f | Time format for time arrays | +| datafusion.format.duration_format | pretty | Duration format. Can be either `"pretty"` or `"ISO8601"` | +| datafusion.format.types_info | false | Show types in visual representation batches | +| datafusion.spark.map_key_dedup_policy | EXCEPTION | Policy for handling duplicate keys in Spark-compatible map-construction functions (`map_from_arrays`, `map_from_entries`, `str_to_map`). Mirrors Spark's [`spark.sql.mapKeyDedupPolicy`](https://github.com/apache/spark/blob/cf3a34e19dfcf70e2d679217ff1ba21302212472/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L4961): - `EXCEPTION` (default): raise `[DUPLICATED_MAP_KEY]` at runtime on any duplicate key. - `LAST_WIN`: keep the last occurrence of each duplicate key. Values are case-insensitive. | You can also reset configuration options to default settings via SQL using the `RESET` command. For example, to set and reset `datafusion.execution.batch_size`: