Exclude remaining noisy CodSpeed cases from CI

benchmarks/bench-support/src/lib.rs

@@ -106,10 +106,10 @@ pub fn cpu_features(attr: TokenStream, item: TokenStream) -> TokenStream {
     // benchmark by leaf name and arguments, so without the suffix every leg would report
     // into one `words_gather_dispatch[65536]` series instead of one series each.
     let name = function.sig.ident.to_string();
-    let separator = if existing.is_empty() {
-        quote!()
-    } else {
-        quote!(,)
+    let separator = match existing.clone().into_iter().last() {
+        None => quote!(),
+        Some(proc_macro2::TokenTree::Punct(punct)) if punct.as_char() == ',' => quote!(),
+        Some(_) => quote!(,),
     };
     let bench_path = bench.path().clone();
     bench.meta = syn::parse_quote! {

encodings/fastlanes/benches/bitpacking_take.rs

@@ -85,8 +85,14 @@ fn threshold_fixture(selected_per_chunk: usize) -> (ArrayRef, ArrayRef) {
 
 // Batch short iterations to reduce variation in the fastest walltime sample.
 // The NEON cases still vary on unchanged code, so keep those measurements local.
+// AVX2 with 80 selections still varies with batching. Retain its 64-selection case in CI.
 #[vortex_bench_support::cpu_features(exclude = ["neon"])]
-#[divan::bench(args = [64, 80], sample_size = 16)]
+#[divan::bench(
+    args = [64, 80].into_iter().filter(|&count| {
+        env!("VORTEX_BENCH_VARIANT") != "avx2" || count != 80
+    }),
+    sample_size = 16,
+)]
 fn threshold(bencher: Bencher, selected_per_chunk: usize) {
     let (packed, indices) = threshold_fixture(selected_per_chunk);
 

vortex-array/benches/binary_ops.rs

@@ -322,7 +322,8 @@ fn div_decimal_i128_nullable(bencher: Bencher) {
     bench_decimal(bencher, lhs, rhs, Operator::Div);
 }
 
-#[vortex_bench_support::cpu_features]
+// The AVX-512 minimum varies by more than 10% while the median stays stable on unchanged code.
+#[vortex_bench_support::cpu_features(exclude = ["avx512"])]
 #[divan::bench]
 fn lt_i64_nullable(bencher: Bencher) {
     let lhs = primitive_nullable(0, 7, I64_LEN).into_array();

vortex-array/benches/filter_bool.rs

@@ -296,7 +296,8 @@ fn filter_powerlaw_by_mostly_false(bencher: Bencher, n: usize) {
         });
 }
 
-#[divan::bench(args = SIZES)]
+// The small case varies in simulation on unrelated PRs. The cause is unknown.
+#[divan::bench(args = if cfg!(codspeed) { &[100_000] } else { SIZES })]
 fn filter_powerlaw_by_random(bencher: Bencher, n: usize) {
     let array = make_power_law_bool_array(n, &mut StdRng::seed_from_u64(ARRAY_SEED));
     bencher
@@ -316,7 +317,8 @@ fn filter_powerlaw_by_random(bencher: Bencher, n: usize) {
         });
 }
 
-#[divan::bench(args = SIZES)]
+// The large case repeatedly makes system calls that simulation excludes from its measurement.
+#[divan::bench(args = if cfg!(codspeed) { &[10_000] } else { SIZES })]
 fn filter_powerlaw_by_correlated_runs(bencher: Bencher, n: usize) {
     let array = make_power_law_bool_array(n, &mut StdRng::seed_from_u64(ARRAY_SEED));
     bencher

vortex-python/benchmark/conftest.py

@@ -5,11 +5,20 @@
 from pathlib import Path
 
 import pytest
+from pytest_codspeed.instruments import MeasurementMode
+from pytest_codspeed.plugin import get_plugin
 
 import vortex as vx
 from benchmark import data
 
 
+@pytest.fixture
+def codspeed_simulation(pytestconfig: pytest.Config) -> bool:
+    # Use the resolved mode, including the runner's legacy "instrumentation" alias.
+    plugin = get_plugin(pytestconfig)
+    return plugin is not None and plugin.is_codspeed_enabled and plugin.mode == MeasurementMode.Simulation
+
+
 @pytest.fixture(scope="session", params=list(data.COLUMN_SETS))
 def vxf(
     tmp_path_factory: pytest.TempPathFactory,

vortex-python/benchmark/test_filter.py

@@ -9,9 +9,17 @@
 from vortex.expr import column
 
 
+@pytest.fixture
+def filter_vxf(request: pytest.FixtureRequest, vxf: vx.VortexFile, codspeed_simulation: bool) -> vx.VortexFile:
+    assert isinstance(request.node, pytest.Function)
+    if codspeed_simulation and request.node.callspec.params["vxf"] == "int":
+        pytest.skip("System calls vary inside these filtered scans under simulation.")
+    return vxf
+
+
 @pytest.mark.benchmark(group="filter")
-def test_scan_filter(benchmark: BenchmarkFixture, vxf: vx.VortexFile) -> None:
-    benchmark(lambda: pa.concat_tables(x.to_arrow_table() for x in vxf.scan(expr=column("x") >= 50_000)))
+def test_scan_filter(benchmark: BenchmarkFixture, filter_vxf: vx.VortexFile) -> None:
+    benchmark(lambda: pa.concat_tables(x.to_arrow_table() for x in filter_vxf.scan(expr=column("x") >= 50_000)))
 
 
 @pytest.mark.benchmark(group="filter")
@@ -21,5 +29,5 @@ def test_repeated_scan_filter(benchmark: BenchmarkFixture, vxf: vx.VortexFile) -
 
 
 @pytest.mark.benchmark(group="filter")
-def test_scan_filter_to_arrow_array(benchmark: BenchmarkFixture, vxf: vx.VortexFile) -> None:
-    benchmark(lambda: vxf.scan(expr=column("x") >= 50_000).to_arrow_array())
+def test_scan_filter_to_arrow_array(benchmark: BenchmarkFixture, filter_vxf: vx.VortexFile) -> None:
+    benchmark(lambda: filter_vxf.scan(expr=column("x") >= 50_000).to_arrow_array())

vortex-python/benchmark/test_scalar_at.py

@@ -1,8 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright the Vortex contributors
 
-import os
-
 import pyarrow as pa
 import pytest
 from pytest_benchmark.fixture import BenchmarkFixture
@@ -11,13 +9,9 @@
 
 
 @pytest.fixture
-def scalar_vxf(request: pytest.FixtureRequest, vxf: vx.VortexFile) -> vx.VortexFile:
-    enabled = request.config.getoption("codspeed", default=False) or "CODSPEED_ENV" in os.environ
-    mode = request.config.getoption("codspeed_mode", default=None)
-    if mode is None and "CODSPEED_ENV" in os.environ:
-        mode = os.environ.get("CODSPEED_RUNNER_MODE", "simulation")
+def scalar_vxf(request: pytest.FixtureRequest, vxf: vx.VortexFile, codspeed_simulation: bool) -> vx.VortexFile:
     assert isinstance(request.node, pytest.Function)
-    if enabled and mode == "simulation" and request.node.callspec.params["vxf"] == "int":
+    if codspeed_simulation and request.node.callspec.params["vxf"] == "int":
         pytest.skip("Reactor polling varies inside these scalar scans under simulation.")
     return vxf