Add pco-inspired lightweight numeric encodings

Cargo.toml

@@ -62,6 +62,8 @@ members = [
     "encodings/bytebool",
     "encodings/parquet-variant",
     "encodings/onpair",
+    "encodings/block-residual",
+    "encodings/float-quant",
     # Benchmarks
     "benchmarks/bench-support",
     "benchmarks/lance-bench",
@@ -97,7 +99,7 @@ rust-version = "1.95"
 version = "0.1.0"
 
 [workspace.dependencies]
-alp = "0.0.2"
+alp = "0.0.3"
 anyhow = "1.0.100"
 arbitrary = "1.3.2"
 arc-swap = "1.9"
@@ -297,6 +299,7 @@ vortex = { version = "0.1.0", path = "./vortex" }
 vortex-alp = { version = "0.1.0", path = "./encodings/alp", default-features = false }
 vortex-array = { version = "0.1.0", path = "./vortex-array", default-features = false }
 vortex-arrow = { version = "0.1.0", path = "./vortex-arrow", default-features = false }
+vortex-block-residual = { version = "0.1.0", path = "./encodings/block-residual", default-features = false }
 vortex-btrblocks = { version = "0.1.0", path = "./vortex-btrblocks", default-features = false }
 vortex-buffer = { version = "0.1.0", path = "./vortex-buffer", default-features = false }
 vortex-bytebool = { version = "0.1.0", path = "./encodings/bytebool", default-features = false }
@@ -311,6 +314,7 @@ vortex-error = { version = "0.1.0", path = "./vortex-error", default-features =
 vortex-fastlanes = { version = "0.1.0", path = "./encodings/fastlanes", default-features = false }
 vortex-file = { version = "0.1.0", path = "./vortex-file", default-features = false }
 vortex-flatbuffers = { version = "0.1.0", path = "./vortex-flatbuffers", default-features = false }
+vortex-float-quant = { version = "0.1.0", path = "./encodings/float-quant", default-features = false }
 vortex-fsst = { version = "0.1.0", path = "./encodings/fsst", default-features = false }
 vortex-io = { version = "0.1.0", path = "./vortex-io", default-features = false }
 vortex-ipc = { version = "0.1.0", path = "./vortex-ipc", default-features = false }

benchmarks/compress-bench/Cargo.toml

@@ -35,6 +35,7 @@ tracing = { workspace = true }
 vortex = { workspace = true }
 vortex-arrow = { workspace = true }
 vortex-bench = { workspace = true }
+vortex-btrblocks = { workspace = true }
 vortex-cuda = { workspace = true, optional = true }
 
 [features]

benchmarks/compress-bench/README.md

@@ -15,6 +15,26 @@ See [`src/main.rs`](./src/main.rs) for the dataset list and CLI flags (`--format
 cargo run -p compress-bench --profile release_debug
 ```
 
+Compare the default compressor, Compact, and Parquet with Zstd:
+
+```bash
+cargo run -p compress-bench --profile release_debug -- \
+  --formats vortex,vortex-compact,parquet
+```
+
+Compare a numeric scheme bundle with the same command and one of these values:
+
+- `--vortex-numeric-bundle prior-default`
+- `--vortex-numeric-bundle block-residual`
+- `--vortex-numeric-bundle current-default`
+
+Add a local Parquet file with `--parquet-path`. Use `--datasets` to select its file stem.
+
+```bash
+cargo run -p compress-bench --profile release_debug -- \
+  --parquet-path /tmp/input.parquet --datasets '^input$'
+```
+
 ## GPU decompression
 
 `--gpu-decompress` is opt-in, requires the `cuda` feature, and restricts the suite to the

benchmarks/compress-bench/src/main.rs

@@ -24,6 +24,7 @@ use vortex_bench::Engine;
 use vortex_bench::Format;
 use vortex_bench::LogFormat;
 use vortex_bench::Target;
+use vortex_bench::VortexNumericBundle;
 use vortex_bench::compress::CompressMeasurements;
 use vortex_bench::compress::CompressOp;
 use vortex_bench::compress::Compressor;
@@ -32,6 +33,9 @@ use vortex_bench::compress::benchmark_decompress;
 use vortex_bench::compress::calculate_ratios;
 use vortex_bench::create_output_writer;
 use vortex_bench::datasets::Dataset;
+use vortex_bench::datasets::feature_vectors::GloveEmbeddingsData;
+use vortex_bench::datasets::feature_vectors::OpenAiEmbeddingsData;
+use vortex_bench::datasets::local_parquet::LocalParquetData;
 use vortex_bench::datasets::struct_list_of_ints::StructListOfInts;
 use vortex_bench::datasets::taxi_data::TaxiData;
 use vortex_bench::datasets::tpch_l_comment::TPCHLCommentCanonical;
@@ -73,6 +77,9 @@ struct Args {
     ops: Vec<CompressOp>,
     #[arg(long)]
     datasets: Option<String>,
+    /// Add a local Parquet file to the benchmark suite.
+    #[arg(long)]
+    parquet_path: Vec<PathBuf>,
     /// Print the dataset names that would run, one per line, and exit.
     /// Knowledge of datasets lies only in this binary so we need
     /// orchestrator to know whan queries to run one by one.
@@ -111,6 +118,9 @@ struct Args {
     ingest_output: Option<PathBuf>,
     #[arg(long)]
     tracing: bool,
+    /// Select the numeric scheme bundle for Vortex compression.
+    #[arg(long, value_enum, default_value_t)]
+    vortex_numeric_bundle: VortexNumericBundle,
     /// Format for the primary stderr log sink. `text` is the default human-readable format;
     /// `json` emits one JSON object per event, suitable for piping into `jq`.
     #[arg(long, value_enum, default_value_t = LogFormat::Text)]
@@ -149,13 +159,15 @@ async fn main() -> anyhow::Result<()> {
     run_compress(
         args.iterations,
         args.datasets.map(|d| Regex::new(&d)).transpose()?,
+        args.parquet_path,
         formats,
         ops,
         mode,
         args.print_datasets,
         args.display_format,
         args.output_path,
         args.ingest_output,
+        args.vortex_numeric_bundle,
     )
     .await
 }
@@ -182,13 +194,24 @@ impl BenchMode {
 }
 
 /// Get a compressor for the given format.
-fn get_compressor(format: Format, mode: BenchMode) -> Box<dyn Compressor> {
+fn get_compressor(
+    format: Format,
+    mode: BenchMode,
+    vortex_numeric_bundle: VortexNumericBundle,
+) -> Box<dyn Compressor> {
     if let BenchMode::Gpu(options) = mode {
         return gpu_compressor(format, options);
     }
 
     match format {
-        Format::OnDiskVortex => Box::new(VortexCompressor),
+        Format::OnDiskVortex => Box::new(VortexCompressor::new(
+            Format::OnDiskVortex,
+            vortex_numeric_bundle,
+        )),
+        Format::VortexCompact => Box::new(VortexCompressor::new(
+            Format::VortexCompact,
+            VortexNumericBundle::CurrentDefault,
+        )),
         Format::Parquet => Box::new(ParquetCompressor::new()),
         #[cfg(feature = "lance")]
         Format::Lance => Box::new(LanceCompressor),
@@ -209,13 +232,15 @@ const DOC_PATH: &str = "benchmarks/compress-bench/README.md";
 async fn run_compress(
     iterations: usize,
     datasets_filter: Option<Regex>,
+    parquet_paths: Vec<PathBuf>,
     formats: Vec<Format>,
     ops: Vec<CompressOp>,
     mode: BenchMode,
     print_datasets: bool,
     display_format: DisplayFormat,
     output_path: Option<PathBuf>,
     ingest_output: Option<PathBuf>,
+    vortex_numeric_bundle: VortexNumericBundle,
 ) -> anyhow::Result<()> {
     let targets = formats
         .iter()
@@ -238,6 +263,10 @@ async fn run_compress(
         //     Some(READ_PROJECTION_COLUMNS),
         // ),
     ];
+    let local_parquet = parquet_paths
+        .into_iter()
+        .map(LocalParquetData::try_new)
+        .collect::<anyhow::Result<Vec<_>>>()?;
 
     // Datasets run in GPU mode. Add one only after a `--gpu-verify` run has confirmed its CUDA
     // decode end to end; a dataset here that cannot decode fails the benchmark job. Between them
@@ -262,6 +291,8 @@ async fn run_compress(
 
     let all_datasets: Vec<&dyn Dataset> = [
         &TaxiData as &dyn Dataset,
+        &GloveEmbeddingsData,
+        &OpenAiEmbeddingsData,
         PBI_DATASETS.get(Arade),
         PBI_DATASETS.get(Bimbo),
         PBI_DATASETS.get(CMSprovider),
@@ -273,13 +304,14 @@ async fn run_compress(
         // Hatred, // panic in fsst_compress_iter
         // TableroSistemaPenal, // Unexpected type error
         // YaleLanguages, // 4th column looks like integer but also contains Y
-        &TPCHLCommentChunked,
+        &TPCHLCommentChunked as &dyn Dataset,
         &TPCHLCommentCanonical,
         &DownloadableDataset::RPlace,
         &DownloadableDataset::AirQuality,
     ]
     .into_iter()
     .chain(structlistofints.iter().map(|d| d as &dyn Dataset))
+    .chain(local_parquet.iter().map(|d| d as &dyn Dataset))
     .collect();
 
     let datasets: Vec<&dyn Dataset> = if mode.is_gpu() {
@@ -316,9 +348,16 @@ async fn run_compress(
     let survey_all = mode.is_gpu();
     let mut failures: Vec<(String, anyhow::Error)> = Vec::new();
 
-    for dataset_handle in datasets.into_iter() {
-        let run =
-            run_benchmark_for_dataset(&progress, &formats, &ops, iterations, dataset_handle, mode);
+    for dataset_handle in datasets {
+        let run = run_benchmark_for_dataset(
+            &progress,
+            &formats,
+            &ops,
+            iterations,
+            dataset_handle,
+            mode,
+            vortex_numeric_bundle,
+        );
 
         // Missing CUDA kernel support surfaces as a panic rather than an error, so the survey
         // has to catch those too or the first unsupported dataset ends the run.
@@ -403,6 +442,7 @@ async fn run_benchmark_for_dataset(
     iterations: usize,
     dataset_handle: &dyn Dataset,
     mode: BenchMode,
+    vortex_numeric_bundle: VortexNumericBundle,
 ) -> anyhow::Result<(CompressMeasurements, Vec<v3::V3Record>)> {
     let bench_name = dataset_handle.name();
     // A GPU decode and a host decode of the same dataset would otherwise publish the same
@@ -427,8 +467,7 @@ async fn run_benchmark_for_dataset(
     let mut v3_records: Vec<v3::V3Record> = Vec::new();
 
     for format in formats {
-        let compressor = get_compressor(*format, mode);
-
+        let compressor = get_compressor(*format, mode, vortex_numeric_bundle);
         for op in ops {
             let time = match op {
                 CompressOp::Compress => {