From bd62cfa114998bf34379d9149fbfe601d1de4132 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 21 Aug 2026 16:34:17 +0000 Subject: [PATCH 1/3] Benchmark comparison lane kernels on the CPU-feature legs Comparison bottoms out in `map_bits_into`: `vortex-array`'s `collect_zip_bits` / `collect_bits` are an allocation plus one call into that kernel, and nothing measured it directly. The existing `compare` benchmarks in `vortex-array` cover the path end to end, through expression execution, where the lane loop is not the whole cost. Add `vortex-compute/benches/compare_bits.rs`. The `zip_bits_*` and `const_bits_*` benchmarks measure the kernel alone over caller-owned words, for integer, total-ordered float, and `i128` (decimal) lanes; they carry `#[cpu_features]`, so each walltime leg reports its own series. That is the shape the attribute is for: one branch-free lane loop whose cost is how well the build auto-vectorizes it. The `collect_*` benchmarks wrap the same kernels in the allocate-and- freeze that `vortex-array` performs. They stay untagged and run in the sharded simulation job, where the wrapper is what instruction counts catch. Signed-off-by: "Joe Isaacs" --- Cargo.lock | 1 + vortex-compute/Cargo.toml | 5 + vortex-compute/benches/compare_bits.rs | 209 +++++++++++++++++++++++++ 3 files changed, 215 insertions(+) create mode 100644 vortex-compute/benches/compare_bits.rs diff --git a/Cargo.lock b/Cargo.lock index 0010a4e6cec..30f1488327e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9891,6 +9891,7 @@ dependencies = [ "codspeed-divan-compat", "num-traits", "rand 0.10.2", + "vortex-bench-support", "vortex-buffer", ] diff --git a/vortex-compute/Cargo.toml b/vortex-compute/Cargo.toml index f223d455395..8e889ff2da0 100644 --- a/vortex-compute/Cargo.toml +++ b/vortex-compute/Cargo.toml @@ -28,6 +28,7 @@ arrow-schema = { workspace = true } divan = { workspace = true } num-traits = { workspace = true } rand = { workspace = true } +vortex-bench-support = { workspace = true } [lints] workspace = true @@ -35,3 +36,7 @@ workspace = true [[bench]] name = "lane_kernels" harness = false + +[[bench]] +name = "compare_bits" +harness = false diff --git a/vortex-compute/benches/compare_bits.rs b/vortex-compute/benches/compare_bits.rs new file mode 100644 index 00000000000..1948a6537e8 --- /dev/null +++ b/vortex-compute/benches/compare_bits.rs @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Benchmarks for the lowest-level comparison kernels: bit-packing a comparison predicate +//! over lanes with [`IndexedSourceExt::map_bits_into`]. +//! +//! This is the kernel every native Vortex comparison bottoms out in. `vortex-array`'s +//! `collect_zip_bits` / `collect_bits` — used by the primitive and decimal compare paths — +//! are the allocation plus a `map_bits_into` call, so the shapes here are: +//! +//! - `zip_bits_*` / `const_bits_*`: the kernel alone, writing into caller-owned words. +//! Array-vs-array via [`LaneZip`], and array-vs-constant over a plain slice. +//! - `collect_zip_bits_*` / `collect_bits_*`: the same kernels with the `BufferMut` +//! allocation and [`BitBuffer`] freeze around them, mirroring `vortex-array` exactly. +//! +//! The `zip_bits_*` and `const_bits_*` benchmarks carry `#[cpu_features]`, so they are +//! measured on every walltime CPU-feature leg instead of in simulation. They are written once +//! and compiled differently per leg: the kernel is a branch-free lane loop whose whole cost is +//! how well it auto-vectorizes for the build, which is what comparing legs measures. The +//! `collect_*` benchmarks are untagged and stay in the sharded simulation job, where the +//! allocate-and-freeze wrapper is the part worth watching for instruction-count regressions. +//! +//! Integer lanes compare with their natural ordering; float lanes use `f64::total_cmp`, which +//! is the total ordering `vortex-array`'s `NativePType::is_lt` and friends are built on. The +//! `i128` lanes stand in for decimal comparison, which uses the same kernel. + +use divan::Bencher; +use rand::SeedableRng; +use rand::prelude::*; +use rand::rngs::StdRng; +use vortex_buffer::BitBuffer; +use vortex_buffer::Buffer; +use vortex_buffer::BufferMut; +use vortex_compute::lane_kernels::IndexedSourceExt; +use vortex_compute::lane_kernels::LaneZip; + +fn main() { + divan::main(); +} + +/// Sized to keep CodSpeed simulation under 1ms per benchmark, matching +/// `vortex-array`'s `compare` benchmarks. +const SIZES: &[usize] = &[8_192]; + +/// Two operand buffers plus a scalar operand, all drawn from one seeded RNG. +struct Fixture { + lhs: Buffer, + rhs: Buffer, + constant: T, +} + +fn fixture(n: usize, mut sample: F) -> Fixture +where + F: FnMut(&mut StdRng) -> T, +{ + let mut rng = StdRng::seed_from_u64(0xC0FFEE); + let lhs = (0..n).map(|_| sample(&mut rng)).collect::>(); + let rhs = (0..n).map(|_| sample(&mut rng)).collect::>(); + let constant = sample(&mut rng); + Fixture { lhs, rhs, constant } +} + +fn i32_fixture(n: usize) -> Fixture { + fixture(n, |rng| rng.random_range(0i32..100_000_000)) +} + +fn i64_fixture(n: usize) -> Fixture { + fixture(n, |rng| rng.random_range(0i64..100_000_000)) +} + +fn f64_fixture(n: usize) -> Fixture { + fixture(n, |rng| rng.random_range(0.0f64..1.0)) +} + +fn i128_fixture(n: usize) -> Fixture { + fixture(n, |rng| rng.random_range(0i128..100_000_000)) +} + +fn words(n: usize) -> Vec { + vec![0u64; n.div_ceil(64)] +} + +/// Bit-pack `f(lhs[i], rhs[i])` into caller-owned words — the array-vs-array kernel. +fn bench_zip( + bencher: Bencher, + n: usize, + f: Fixture, + predicate: impl Fn(T, T) -> bool + Sync, +) { + bencher.with_inputs(|| words(n)).bench_refs(|out| { + LaneZip::new(f.lhs.as_slice(), f.rhs.as_slice()) + .map_bits_into(out.as_mut_slice(), |(a, b)| predicate(a, b)); + }); +} + +/// Bit-pack `f(lhs[i], constant)` into caller-owned words — the array-vs-constant kernel. +fn bench_const( + bencher: Bencher, + n: usize, + f: Fixture, + predicate: impl Fn(T, T) -> bool + Sync, +) { + bencher.with_inputs(|| words(n)).bench_refs(|out| { + f.lhs + .as_slice() + .map_bits_into(out.as_mut_slice(), |a| predicate(a, f.constant)); + }); +} + +/// `vortex-array`'s `collect_zip_bits`: allocate the words, run the kernel, freeze the bits. +fn collect_zip_bits(lhs: &[T], rhs: &[T], predicate: impl Fn(T, T) -> bool) -> BitBuffer { + let len = lhs.len(); + let mut words = BufferMut::::zeroed(len.div_ceil(64)); + LaneZip::new(lhs, rhs).map_bits_into(words.as_mut_slice(), |(a, b)| predicate(a, b)); + bit_buffer_from_words(words, len) +} + +/// `vortex-array`'s `collect_bits`, the array-vs-constant counterpart. +fn collect_bits(values: &[T], predicate: impl Fn(T) -> bool) -> BitBuffer { + let len = values.len(); + let mut words = BufferMut::::zeroed(len.div_ceil(64)); + values.map_bits_into(words.as_mut_slice(), predicate); + bit_buffer_from_words(words, len) +} + +fn bit_buffer_from_words(words: BufferMut, len: usize) -> BitBuffer { + let mut bytes = words.into_byte_buffer(); + bytes.truncate(len.div_ceil(8)); + BitBuffer::new(bytes.freeze(), len) +} + +// ----------------------------------------------------------------------------- +// Kernel benchmarks, measured per CPU-feature leg. +// ----------------------------------------------------------------------------- + +#[vortex_bench_support::cpu_features] +#[divan::bench(args = SIZES)] +fn zip_bits_i32_gte(bencher: Bencher, n: usize) { + bench_zip(bencher, n, i32_fixture(n), |a, b| a >= b); +} + +#[vortex_bench_support::cpu_features] +#[divan::bench(args = SIZES)] +fn zip_bits_i64_gte(bencher: Bencher, n: usize) { + bench_zip(bencher, n, i64_fixture(n), |a, b| a >= b); +} + +#[vortex_bench_support::cpu_features] +#[divan::bench(args = SIZES)] +fn zip_bits_i64_eq(bencher: Bencher, n: usize) { + bench_zip(bencher, n, i64_fixture(n), |a, b| a == b); +} + +/// Float lanes under Vortex's total ordering, which is what makes this kernel harder to +/// vectorize than the integer ones. +#[vortex_bench_support::cpu_features] +#[divan::bench(args = SIZES)] +fn zip_bits_f64_lt(bencher: Bencher, n: usize) { + bench_zip(bencher, n, f64_fixture(n), |a: f64, b: f64| { + a.total_cmp(&b).is_lt() + }); +} + +/// `i128` lanes: the decimal compare path runs the same kernel over double-width lanes. +#[vortex_bench_support::cpu_features] +#[divan::bench(args = SIZES)] +fn zip_bits_i128_gte(bencher: Bencher, n: usize) { + bench_zip(bencher, n, i128_fixture(n), |a, b| a >= b); +} + +#[vortex_bench_support::cpu_features] +#[divan::bench(args = SIZES)] +fn const_bits_i64_gte(bencher: Bencher, n: usize) { + bench_const(bencher, n, i64_fixture(n), |a, b| a >= b); +} + +#[vortex_bench_support::cpu_features] +#[divan::bench(args = SIZES)] +fn const_bits_f64_lt(bencher: Bencher, n: usize) { + bench_const(bencher, n, f64_fixture(n), |a: f64, b: f64| { + a.total_cmp(&b).is_lt() + }); +} + +// ----------------------------------------------------------------------------- +// Allocate-and-freeze wrappers, measured in simulation. +// ----------------------------------------------------------------------------- + +#[divan::bench(args = SIZES)] +fn collect_zip_bits_i64_gte(bencher: Bencher, n: usize) { + let f = i64_fixture(n); + bencher.bench(|| collect_zip_bits(f.lhs.as_slice(), f.rhs.as_slice(), |a, b| a >= b)); +} + +#[divan::bench(args = SIZES)] +fn collect_zip_bits_f64_lt(bencher: Bencher, n: usize) { + let f = f64_fixture(n); + bencher.bench(|| { + collect_zip_bits(f.lhs.as_slice(), f.rhs.as_slice(), |a: f64, b: f64| { + a.total_cmp(&b).is_lt() + }) + }); +} + +#[divan::bench(args = SIZES)] +fn collect_bits_i64_gte(bencher: Bencher, n: usize) { + let f = i64_fixture(n); + bencher.bench(|| collect_bits(f.lhs.as_slice(), |a| a >= f.constant)); +} From fb1093671ff35cb121e27214b49fac85a7ed3d76 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 21 Aug 2026 16:48:16 +0000 Subject: [PATCH 2/3] Measure the collect_bool entry points on the CPU-feature legs too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `collect_bool.rs` tagged only the bare word loops, so the shipped entry points — `BitBuffer::collect_bool` under a comparison predicate, and the `&[bool]` conversion — were measured in simulation alone, never on real silicon per feature set. Predicate evaluation and allocation sit next to the pack loop in shipped code, and how the pair schedules is exactly what differs between legs. Add `*_dispatch` siblings carrying `#[cpu_features]` for both, sharing a body with the originals. The originals stay untagged and keep their simulation history: a tagged benchmark is skipped in simulation, so one name cannot serve both modes. Signed-off-by: "Joe Isaacs" --- vortex-buffer/benches/collect_bool.rs | 38 ++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/vortex-buffer/benches/collect_bool.rs b/vortex-buffer/benches/collect_bool.rs index 864179471f8..abccd84137b 100644 --- a/vortex-buffer/benches/collect_bool.rs +++ b/vortex-buffer/benches/collect_bool.rs @@ -19,6 +19,13 @@ //! through `cfg(target_feature)`, and how well the scalar loop auto-vectorizes depends on //! the build. Comparing them across legs is the point. //! +//! The public entry points are measured twice, under two names. `collect_bool_u32_gt` and +//! `from_bool_slice` stay untagged and keep their simulation history; the `*_dispatch` siblings +//! carry `#[cpu_features]` and measure the same call on the walltime legs, where predicate +//! evaluation and allocation sit alongside the pack loop as they do in shipped code. Splitting +//! them rather than tagging the originals is what keeps both modes: a tagged benchmark is +//! skipped in simulation, so one name cannot serve both. +//! //! The hand-written per-kernel benchmarks are not tagged. Each one needs an instruction set //! extension the other legs do not build for, so they stay out of CodSpeed entirely and //! remain local A/B tools, as do the historical bit-at-a-time baselines. @@ -194,12 +201,23 @@ fn from_bool_slice_old_scalar(bencher: Bencher, len: usize) { .bench_refs(|words| collect_bool_words_old(words, len, |i| bools[i])); } -#[divan::bench(args = INPUT_SIZE)] -fn from_bool_slice(bencher: Bencher, len: usize) { +/// The shipped `&[bool]` conversion: gather, multiversioned pack, and allocation together. +fn bench_from_bool_slice(bencher: Bencher, len: usize) { let bools = make_bools(len); bencher.bench(|| vortex_buffer::BitBufferMut::from(bools.as_slice())); } +#[divan::bench(args = INPUT_SIZE)] +fn from_bool_slice(bencher: Bencher, len: usize) { + bench_from_bool_slice(bencher, len); +} + +#[vortex_bench_support::cpu_features] +#[divan::bench(args = INPUT_SIZE)] +fn from_bool_slice_dispatch(bencher: Bencher, len: usize) { + bench_from_bool_slice(bencher, len); +} + #[cfg(not(codspeed))] #[divan::bench(args = INPUT_SIZE)] fn collect_bool_u32_gt_old_scalar(bencher: Bencher, len: usize) { @@ -209,8 +227,20 @@ fn collect_bool_u32_gt_old_scalar(bencher: Bencher, len: usize) { .bench_refs(|words| collect_bool_words_old(words, len, |i| values[i] > u32::MAX / 2)); } -#[divan::bench(args = INPUT_SIZE)] -fn collect_bool_u32_gt(bencher: Bencher, len: usize) { +/// The shipped `BitBuffer::collect_bool` entry point under a `u32` comparison predicate: +/// predicate evaluation, the multiversioned pack loop, and allocation together. +fn bench_collect_bool_u32_gt(bencher: Bencher, len: usize) { let values = make_u32s(len); bencher.bench(|| BitBuffer::collect_bool(len, |i| values[i] > u32::MAX / 2)); } + +#[divan::bench(args = INPUT_SIZE)] +fn collect_bool_u32_gt(bencher: Bencher, len: usize) { + bench_collect_bool_u32_gt(bencher, len); +} + +#[vortex_bench_support::cpu_features] +#[divan::bench(args = INPUT_SIZE)] +fn collect_bool_u32_gt_dispatch(bencher: Bencher, len: usize) { + bench_collect_bool_u32_gt(bencher, len); +} From f3b62bf6899d6617946b5597f87962f33f2514b6 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 21 Aug 2026 16:56:24 +0000 Subject: [PATCH 3/3] Cover every comparison operator in the primitive compare benchmarks The primitive compare kernel dispatches on the operator outside the lane loop, so each of the six is a separate instantiation that vectorizes on its own terms. The benchmarks measured `Gte` for the array, nullable, constant and float shapes and `Eq` for one of them, which said nothing about the rest. Parameterize those four shapes over all six operators and drop `compare_int_eq`, now covered by `compare_int[=]`. Boolean, string, decimal and struct comparison are unchanged. Signed-off-by: "Joe Isaacs" --- vortex-array/benches/compare.rs | 45 ++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/vortex-array/benches/compare.rs b/vortex-array/benches/compare.rs index 43969a91e39..0b196d62e91 100644 --- a/vortex-array/benches/compare.rs +++ b/vortex-array/benches/compare.rs @@ -130,44 +130,49 @@ fn compare_bool_constant(bencher: Bencher) { bench_compare(bencher, arr, constant, Operator::Eq); } -#[divan::bench] -fn compare_int(bencher: Bencher) { +/// Every comparison operator, because the primitive kernel dispatches on the operator outside +/// the lane loop: each one is a separate instantiation that vectorizes on its own terms, and +/// measuring `Gte` alone said nothing about the other five. +const COMPARE_OPERATORS: &[Operator] = &[ + Operator::Eq, + Operator::NotEq, + Operator::Gt, + Operator::Gte, + Operator::Lt, + Operator::Lte, +]; + +#[divan::bench(args = COMPARE_OPERATORS)] +fn compare_int(bencher: Bencher, op: Operator) { let mut rng = StdRng::seed_from_u64(0); let arr1 = int_array(&mut rng); let arr2 = int_array(&mut rng); - bench_compare(bencher, arr1, arr2, Operator::Gte); + bench_compare(bencher, arr1, arr2, op); } -#[divan::bench] -fn compare_int_nullable(bencher: Bencher) { +#[divan::bench(args = COMPARE_OPERATORS)] +fn compare_int_nullable(bencher: Bencher, op: Operator) { let mut rng = StdRng::seed_from_u64(0); let arr1 = int_array_nullable(&mut rng); let arr2 = int_array_nullable(&mut rng); - bench_compare(bencher, arr1, arr2, Operator::Gte); + bench_compare(bencher, arr1, arr2, op); } -#[divan::bench] -fn compare_int_constant(bencher: Bencher) { +#[divan::bench(args = COMPARE_OPERATORS)] +fn compare_int_constant(bencher: Bencher, op: Operator) { let mut rng = StdRng::seed_from_u64(0); let arr = int_array(&mut rng); let constant = ConstantArray::new(50_000_000i64, ARRAY_SIZE).into_array(); - bench_compare(bencher, arr, constant, Operator::Gte); -} - -#[divan::bench] -fn compare_int_eq(bencher: Bencher) { - let mut rng = StdRng::seed_from_u64(0); - let arr1 = int_array(&mut rng); - let arr2 = int_array(&mut rng); - bench_compare(bencher, arr1, arr2, Operator::Eq); + bench_compare(bencher, arr, constant, op); } -#[divan::bench] -fn compare_float(bencher: Bencher) { +/// Float lanes carry Vortex's total ordering, so the predicate is more than a machine compare. +#[divan::bench(args = COMPARE_OPERATORS)] +fn compare_float(bencher: Bencher, op: Operator) { let mut rng = StdRng::seed_from_u64(0); let arr1 = float_array(&mut rng); let arr2 = float_array(&mut rng); - bench_compare(bencher, arr1, arr2, Operator::Gte); + bench_compare(bencher, arr1, arr2, op); } #[divan::bench]