From 3e070a6c1dc3631e2fcb94b955f1a6eef8ddc630 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Tue, 18 Aug 2026 16:20:47 -0400 Subject: [PATCH 01/78] feat: add native Pco-inspired numeric encodings Signed-off-by: "Will Manning" --- Cargo.lock | 30 + Cargo.toml | 4 + docs/plans/native-pcodec.md | 394 +++ encodings/float-quant/Cargo.toml | 26 + encodings/float-quant/src/array.rs | 809 ++++++ encodings/float-quant/src/lib.rs | 17 + encodings/float-quant/src/rules.rs | 10 + encodings/float-quant/src/slice.rs | 28 + encodings/pco/src/array.rs | 36 +- encodings/pco/src/tests.rs | 19 + encodings/range-entropy/Cargo.toml | 27 + .../range-entropy/examples/encode_probe.rs | 99 + encodings/range-entropy/src/array.rs | 857 ++++++ encodings/range-entropy/src/bit_split.rs | 264 ++ .../range-entropy/src/block_residual_array.rs | 765 ++++++ encodings/range-entropy/src/codec.rs | 2337 +++++++++++++++++ encodings/range-entropy/src/lib.rs | 33 + .../range-entropy/src/ordered_float_array.rs | 399 +++ encodings/range-entropy/src/patched_for.rs | 772 ++++++ encodings/range-entropy/src/rules.rs | 10 + vortex-btrblocks/Cargo.toml | 17 + .../examples/float_quant_dataset.rs | 885 +++++++ vortex-btrblocks/examples/pco_probe.rs | 308 +++ .../examples/range_entropy_throughput.rs | 994 +++++++ vortex-btrblocks/src/builder.rs | 33 + .../src/schemes/float/float_quant.rs | 144 + vortex-btrblocks/src/schemes/float/mod.rs | 4 + .../schemes/float/ordered_block_residual.rs | 133 + .../schemes/float/scheme_selection_tests.rs | 73 + .../src/schemes/integer/bitpacking.rs | 120 +- vortex-btrblocks/src/schemes/integer/for_.rs | 49 +- vortex-btrblocks/src/schemes/mod.rs | 1 + vortex-btrblocks/src/schemes/range_entropy.rs | 62 + vortex-file/Cargo.toml | 2 + vortex-file/src/lib.rs | 2 + vortex/Cargo.toml | 2 + vortex/src/editions/core/v2026_08.rs | 7 +- vortex/src/editions/tests.rs | 86 + vortex/src/lib.rs | 10 + 39 files changed, 9782 insertions(+), 86 deletions(-) create mode 100644 docs/plans/native-pcodec.md create mode 100644 encodings/float-quant/Cargo.toml create mode 100644 encodings/float-quant/src/array.rs create mode 100644 encodings/float-quant/src/lib.rs create mode 100644 encodings/float-quant/src/rules.rs create mode 100644 encodings/float-quant/src/slice.rs create mode 100644 encodings/range-entropy/Cargo.toml create mode 100644 encodings/range-entropy/examples/encode_probe.rs create mode 100644 encodings/range-entropy/src/array.rs create mode 100644 encodings/range-entropy/src/bit_split.rs create mode 100644 encodings/range-entropy/src/block_residual_array.rs create mode 100644 encodings/range-entropy/src/codec.rs create mode 100644 encodings/range-entropy/src/lib.rs create mode 100644 encodings/range-entropy/src/ordered_float_array.rs create mode 100644 encodings/range-entropy/src/patched_for.rs create mode 100644 encodings/range-entropy/src/rules.rs create mode 100644 vortex-btrblocks/examples/float_quant_dataset.rs create mode 100644 vortex-btrblocks/examples/pco_probe.rs create mode 100644 vortex-btrblocks/examples/range_entropy_throughput.rs create mode 100644 vortex-btrblocks/src/schemes/float/float_quant.rs create mode 100644 vortex-btrblocks/src/schemes/float/ordered_block_residual.rs create mode 100644 vortex-btrblocks/src/schemes/range_entropy.rs diff --git a/Cargo.lock b/Cargo.lock index 0d55c729ae6..18535d412ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9546,6 +9546,7 @@ dependencies = [ "vortex-fastlanes", "vortex-file", "vortex-flatbuffers", + "vortex-float-quant", "vortex-fsst", "vortex-io", "vortex-ipc", @@ -9554,6 +9555,7 @@ dependencies = [ "vortex-metrics", "vortex-pco", "vortex-proto", + "vortex-range-entropy", "vortex-runend", "vortex-scan", "vortex-sequence", @@ -9752,9 +9754,12 @@ name = "vortex-btrblocks" version = "0.1.0" dependencies = [ "arrow-array 58.4.0", + "arrow-schema 58.4.0", + "arrow-select 58.4.0", "codspeed-divan-compat", "insta", "itertools 0.14.0", + "parquet 58.4.0", "pco", "rand 0.10.2", "rstest", @@ -9770,10 +9775,12 @@ dependencies = [ "vortex-decimal-byte-parts", "vortex-error", "vortex-fastlanes", + "vortex-float-quant", "vortex-fsst", "vortex-mask", "vortex-onpair", "vortex-pco", + "vortex-range-entropy", "vortex-runend", "vortex-sequence", "vortex-session", @@ -10161,6 +10168,7 @@ dependencies = [ "vortex-error", "vortex-fastlanes", "vortex-flatbuffers", + "vortex-float-quant", "vortex-fsst", "vortex-io", "vortex-layout", @@ -10168,6 +10176,7 @@ dependencies = [ "vortex-metrics", "vortex-onpair", "vortex-pco", + "vortex-range-entropy", "vortex-runend", "vortex-scan", "vortex-sequence", @@ -10188,6 +10197,16 @@ dependencies = [ "vortex-error", ] +[[package]] +name = "vortex-float-quant" +version = "0.1.0" +dependencies = [ + "vortex-array", + "vortex-buffer", + "vortex-error", + "vortex-session", +] + [[package]] name = "vortex-fsst" version = "0.1.0" @@ -10509,6 +10528,17 @@ dependencies = [ "vortex-python-abi", ] +[[package]] +name = "vortex-range-entropy" +version = "0.1.0" +dependencies = [ + "fastlanes", + "vortex-array", + "vortex-buffer", + "vortex-error", + "vortex-session", +] + [[package]] name = "vortex-row" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 65452f18300..f715d9b6d7e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,8 @@ members = [ "encodings/bytebool", "encodings/parquet-variant", "encodings/onpair", + "encodings/range-entropy", + "encodings/float-quant", # Benchmarks "benchmarks/bench-support", "benchmarks/lance-bench", @@ -321,6 +323,8 @@ vortex-metrics = { version = "0.1.0", path = "./vortex-metrics", default-feature vortex-onpair = { version = "0.1.0", path = "./encodings/onpair", default-features = false } vortex-parquet-variant = { version = "0.1.0", path = "./encodings/parquet-variant" } vortex-pco = { version = "0.1.0", path = "./encodings/pco", default-features = false } +vortex-range-entropy = { version = "0.1.0", path = "./encodings/range-entropy", default-features = false } +vortex-float-quant = { version = "0.1.0", path = "./encodings/float-quant", default-features = false } vortex-proto = { version = "0.1.0", path = "./vortex-proto", default-features = false } vortex-row = { version = "0.1.0", path = "./vortex-row", default-features = false } vortex-runend = { version = "0.1.0", path = "./encodings/runend", default-features = false } diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md new file mode 100644 index 00000000000..ef09a2ace83 --- /dev/null +++ b/docs/plans/native-pcodec.md @@ -0,0 +1,394 @@ +# Native PCodec-inspired numeric compression + +## Goal + +Improve default numeric compression where ALP and ALP-RD lose to Pco. + +Keep native Vortex arrays, bounded metadata, child arrays, and fast canonical decode. + +Pco remains a compression oracle. The new arrays do not preserve Pco byte compatibility. + +## Decision + +Add `FloatQuantArray` and `FloatQuantScheme` to default BtrBlocks compression for `f64` arrays. + +Add `OrderedFloatArray`, `BlockResidualArray`, and their BtrBlocks scheme to the default compressor. + +Keep `BitSplitArray` and `FloatMultArray` as prototypes. + +Keep `RangeEntropyArray` and `RangeEntropyScheme` experimental. Their current full-decode and compression costs fail the throughput gate. + +Keep FastLanes Delta unchanged. Its current lane-stride transform does not replace Pco adjacent Delta on the measured data. + +Do not add Delta-of-delta, Delta with lookback, convolution Delta, or Pco byte compatibility. + +## Implemented arrays + +### OrderedFloatArray + +`OrderedFloatArray` maps IEEE float bits to unsigned integers with the same order. + +The transform preserves every bit pattern. It also preserves nulls and signed zero values. + +The array stores one unsigned child. An empty metadata record identifies the transform. + +The array supports canonical decode, scalar access, slice reduction, serialization, and validation. + +### BlockResidualArray + +`BlockResidualArray` divides unsigned integers into independent blocks of 1,024 values. + +Each block stores one minimum value. Packed residuals store the difference from that minimum. + +Rare wide residuals use sorted positions and packed high bits. Scalar access uses one packed read and one binary search. + +Nine child arrays store references, widths, offsets, packed words, patch positions, and patch high bits. + +The fixed metadata record stores only lengths and slice bounds. Variable tables do not use metadata. + +The array supports canonical decode, scalar access, slice reduction, serialization, and validation. + +The `OrderedFloat(BlockResidual)` execute kernel fuses residual decode with the inverse float transform. + +### RangeEntropyArray + +`RangeEntropyArray` stores entropy-coded range-bin identifiers and fixed-width offsets. + +The logical dtype remains the source primitive dtype. The codec maps each value to an ordered unsigned latent. + +The fixed metadata record stores these fields: + +- Physical type. +- ANS table log. +- Restart block length. +- Logical slice bounds. + +Child arrays store these variable tables: + +- Bin lower bounds. +- Bin offset widths. +- Quantized ANS weights. +- Restart block byte offsets. +- Validity. + +One payload buffer stores the ANS stream and packed offsets. Independent restart blocks bound scalar access work. + +The array supports canonical decode, slice reduction, scalar access, serialization, and validation. + +### FloatQuantArray + +`FloatQuantArray` splits each float into a primary quantum and a secondary adjustment. + +Bounded metadata stores the source type and split width. Two child arrays store the integer latents. + +The BtrBlocks scheme compresses both children through the normal cascade. + +A common path uses `FloatQuant(FoR(BitPacked), Constant)`. This path targets `f32` values stored in `f64` columns. + +The array supports exact IEEE bit-pattern round trips, nulls, slices, scalar access, and serialization. + +The automatic scheme currently accepts only `f64`. Native `f32` columns keep the current ALP and dictionary choices. + +## Selection policy + +FloatQuant uses normal sample comparison against ALP, ALP-RD, dictionary, sparse, and RLE schemes. + +The scheme does not claim an unconditional win. This rule prevents poor choices on low-cardinality integer-valued floats. + +The full FloatQuant analysis runs only after the sample selects the scheme. Unselected `f64` columns pay only the sample cost. + +Callers can disable the default scheme with this builder configuration: + +```rust +let compressor = BtrBlocksCompressorBuilder::default() + .exclude_schemes([FloatQuantScheme.id()]) + .build(); +``` + +BlockResidual uses eight locality-preserving sample blocks. The ordinary BtrBlocks sample destroys the block-local float structure. + +The scheme requires a 5 percent compression ratio. It also requires a 2 percent win over the current best scheme. + +Callers can disable this scheme with this builder configuration: + +```rust +let compressor = BtrBlocksCompressorBuilder::default() + .exclude_schemes([OrderedBlockResidualScheme.id()]) + .build(); +``` + +RangeEntropy also uses sample comparison. It remains outside `ALL_SCHEMES`. + +## Performance gate + +The default candidate must meet these limits: + +- Compression throughput can regress by at most 20 percent. +- Full-decode throughput can regress by at most 20 percent. +- A selected candidate must reduce size materially. +- The result must approach Compact size on its target data. + +The release benchmark excludes source-array construction and storage I/O. + +## Synthetic target result + +The widened-f32 input contains 262,144 arbitrary `f32` values stored as `f64`. + +| Configuration | Bytes | Compression MB/s | Decode MB/s | +| --- | ---: | ---: | ---: | +| Default without FloatQuant | 1,638,436 | 762.4 | 12,932.1 | +| Default with FloatQuant | 688,130 | 635.0 | 13,636.4 | +| Compact | 658,758 | 336.8 | 5,117.1 | + +FloatQuant reduces size by 58.0 percent. It remains 4.5 percent larger than Compact. + +Compression throughput regresses by 16.7 percent. Decode throughput improves by 5.4 percent. + +The selected tree is `FloatQuant(FoR(BitPacked), Constant)`. + +### FloatQuant and RangeEntropy composition + +The stacked configuration leaves the widened-f32 tree and byte size unchanged. + +RangeEntropy loses selection for both FloatQuant children. The primary already uses FoR and bitpacking, while the secondary is constant zero. + +A repeated synthetic run reduced compression throughput by 4.9 percent because RangeEntropy still incurred sample work. + +No measured dataset selected RangeEntropy beneath FloatQuant. RangeEntropy sometimes wins as a root alternative when FloatQuant loses. + +## Pco paper data + +The benchmark reads source columns before timing and limits Parquet inputs to two million rows. + +The aggregate values below cover every supported numeric column in each downloaded input. + +### Size + +| Dataset | Without FloatQuant | Default | Default plus RangeEntropy | Compact | Pco Auto | +| --- | ---: | ---: | ---: | ---: | ---: | +| California Housing | 339,682 | 339,682 | 319,705 | 230,197 | 242,414 | +| April 2023 HVFHV Taxi | 20,894,253 | 20,894,253 | 16,443,089 | 14,533,638 | 15,403,694 | +| Air Quality | 6,135,414 | 6,135,414 | 5,149,775 | 2,311,526 | 2,303,523 | + +FloatQuant makes no selection on these inputs. Their size and decode paths remain unchanged. + +RangeEntropy reduces Taxi size by 21.3 percent and Air Quality size by 16.1 percent. + +RangeEntropy remains 13.1 percent larger than Compact on Taxi. It remains 122.8 percent larger on Air Quality. + +### Compression throughput + +| Dataset | Without FloatQuant | Default | Default plus RangeEntropy | Compact | Pco Auto | +| --- | ---: | ---: | ---: | ---: | ---: | +| California Housing | 201.9 MB/s | 204.4 MB/s | 112.9 MB/s | 70.9 MB/s | 201.6 MB/s | +| April 2023 HVFHV Taxi | 849.7 MB/s | 830.2 MB/s | 424.5 MB/s | 448.1 MB/s | 608.0 MB/s | +| Air Quality | 573.0 MB/s | 574.8 MB/s | 328.4 MB/s | 310.1 MB/s | 433.9 MB/s | + +FloatQuant changes unselected compression throughput by less than 3 percent on these inputs. + +RangeEntropy reduces compression throughput by 43 to 50 percent. This result fails the gate. + +### Decode throughput + +| Dataset | Without FloatQuant | Default | Default plus RangeEntropy | Compact | Pco Auto | +| --- | ---: | ---: | ---: | ---: | ---: | +| California Housing | 12,488.1 MB/s | 12,674.5 MB/s | 2,148.5 MB/s | 2,351.7 MB/s | 2,406.9 MB/s | +| April 2023 HVFHV Taxi | 21,718.9 MB/s | 21,918.9 MB/s | 4,509.1 MB/s | 5,063.4 MB/s | 3,517.7 MB/s | +| Air Quality | 27,375.7 MB/s | 27,799.2 MB/s | 1,557.1 MB/s | 2,199.1 MB/s | 2,170.5 MB/s | + +RangeEntropy decode is 4.9 to 17.9 times slower than the current lightweight default. + +RangeEntropy beats Pco Auto decode on Taxi. It loses to Pco Auto decode on Housing and Air Quality. + +The native implementation does not meet the default full-decode requirement. + +## Pco encode cost + +The probe separates Pco into a prepare stage and an emit stage. + +The prepare stage selects the mode and Delta transform. It also builds the final bins and ANS model. + +The emit stage finds each value's bin, creates offsets, runs reverse ANS, and writes the bit stream. + +The forced test supplies the mode and Delta transform that `Auto` selected. Both paths produce identical bytes. + +| Dataset | Auto MB/s | Forced MB/s | Auto time used for search | +| --- | ---: | ---: | ---: | +| Gaussian | 647.2 | 872.0 | 25.8 percent | +| Lognormal | 686.5 | 860.3 | 20.2 percent | +| Decimal | 604.5 | 747.3 | 19.1 percent | +| Widened f32 | 604.5 | 747.5 | 19.1 percent | +| Random walk | 572.3 | 702.6 | 18.5 percent | + +Mode and Delta search uses 18 to 26 percent of Pco's default encode time on these inputs. + +The forced path still trains the full bin model. This model and final emission use most of the time. + +The complete prepare stage uses 58 to 71 percent of automatic encode time. Final emission uses the remaining 29 to 42 percent. + +Preparation includes the selected transform, Delta application, histograms, bin optimization, weight quantization, and ANS model construction. + +Compression level 4 keeps most of level 8's size benefit with much higher throughput. + +| Dataset | Level 0 bytes | Level 0 MB/s | Level 4 bytes | Level 4 MB/s | Level 8 bytes | Level 8 MB/s | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Gaussian | 1,671,181 | 5,195.8 | 1,618,359 | 1,226.0 | 1,609,324 | 872.0 | +| Random walk | 2,097,165 | 3,322.7 | 1,611,991 | 956.8 | 1,551,142 | 702.6 | + +Level 0 proves that Pco's mode transforms do not require a slow encoder. + +The expensive part is the richer bin model. Random-walk data still needs that model or an adjacent Delta transform for good size. + +## Block residual prototype + +The block residual codec uses one or more references for each 1,024-value block. + +The single-reference form stores the block minimum. It subtracts that minimum from every ordered integer. + +FastLanes stores the low `w` residual bits for every value. A width histogram selects `w` without repeated residual scans. + +Patches store sorted 16-bit positions and the remaining high bits. Scalar access uses one packed read and one binary search. + +The multi-reference form also tests two and four references. It stores a one-bit or two-bit reference selector for each value. + +| Dataset | Form | Bytes | Encode MB/s | Decode MB/s | Scalar access ns | +| --- | --- | ---: | ---: | ---: | ---: | +| Gaussian | One reference | 1,643,520 | 4,194.7 | 15,942.8 | 3.99 | +| Gaussian | Up to four references | 1,643,520 | 1,399.9 | 15,983.3 | 4.05 | +| Random walk | One reference | 1,663,307 | 3,752.7 | 15,582.5 | 7.38 | +| Random walk | Up to four references | 1,660,912 | 1,267.9 | 14,716.9 | 7.52 | +| Four clusters | One reference | 2,102,272 | 4,667.7 | 14,492.3 | 3.01 | +| Four clusters | Up to four references | 1,978,898 | 1,151.3 | 9,348.4 | 17.21 | + +Multiple references save 0.14 percent on random-walk data and 5.9 percent on four-cluster data. + +The four-cluster result remains 17.9 percent larger than ALP-RD. It remains 44.2 percent larger than RangeEntropy. + +One local reference plus patches is the current block-residual winner. It is not the current winner for all float distributions. + +This result narrows the role of multiple references. A prefix split or range tag still handles distinct float clusters better. + +### Default candidate on random-walk floats + +| Configuration | Bytes | Encode MB/s | Decode MB/s | Scalar access ns | +| --- | ---: | ---: | ---: | ---: | +| Default without the scheme | 1,853,564 | 713.4 | 12,486.1 | 161.24 | +| Default with the scheme | 1,663,831 | 635.6 | 17,355.8 | 129.06 | +| Compact Pco | 1,551,142 | 296.2 | 3,777.2 | Not measured | + +The default scheme saves 10.2 percent against ALP-RD. It remains 7.3 percent larger than Pco. + +Decode throughput improves by 39.0 percent. Random access latency improves by 20.0 percent. + +Compression throughput regresses by 10.9 percent on the selected column. + +The direct decoder reads child buffers without copies. It also fuses the inverse ordered-float transform. + +The selector rejected the scheme on Gaussian, lognormal, decimal, widened-f32, and four-cluster inputs. + +The locality probe reduced compression throughput by 0.5 to 2.2 percent on those unselected inputs. + +## BitSplit prototype + +The `BitSplit` prototype uses one prefix dictionary per 1,024-value block. Fixed-width suffixes store the remaining bits. + +The encoder sorts each block once. Adjacent XOR widths evaluate every split point without repeated value scans. + +Scalar access uses two packed reads and one prefix lookup. No patch search is necessary. + +| Dataset | Bytes | Encode MB/s | Decode MB/s | Scalar access ns | +| --- | ---: | ---: | ---: | ---: | +| Gaussian | 1,648,800 | 1,046.7 | 5,989.0 | 5.33 | +| Random walk | 1,664,696 | 1,057.8 | 6,260.2 | 9.85 | +| Four clusters | 1,702,464 | 900.3 | 5,898.5 | 6.19 | + +The four-cluster result is 1.4 percent larger than ALP-RD. It is 14.0 percent smaller than the multiple-reference prototype. + +`BitSplit` handles separated clusters better than residual references. Full decode remains much slower than the current ALP-RD path. + +## FloatMult composition + +Pco selects FloatMult on seven Taxi columns. The selected source columns contain 112 MB of logical values. + +| Backend for FloatMult children | Aggregate bytes | +| --- | ---: | +| Normal BtrBlocks integer compression | 22,340,941 | +| Block residual packing | 19,555,263 | +| BitSplit | 24,105,632 | +| Fixed range tags | 20,860,940 | +| Two-level range tags | 19,052,586 | +| RangeEntropy | 16,730,631 | +| Current default on source floats | 20,104,147 | +| Pco Auto on source floats | 15,061,783 | + +FloatMult does not create conventional integer distributions. Normal integer compression loses 11.1 percent against the current default. + +RangeEntropy recovers most of Pco's benefit. It remains 11.1 percent larger than Pco on these columns. + +The current `RangeEntropy` decoder is too slow for the default compressor. FloatMult therefore needs a faster entropy-like child codec. + +## Why Compact still wins + +Pco does not use FloatQuant on the measured paper inputs. + +Pco selects FloatMult on five Housing columns and seven Taxi columns. + +Pco selects adjacent Delta on three Housing columns and thirteen Air Quality columns. + +These transforms explain most of the remaining Compact size gap. + +FloatMult requires a new transform array. The current prototype scope includes this array. + +Pco adjacent Delta also differs from current FastLanes lane-stride Delta. + +The FastLanes Delta diagnostic selected one Air Quality column. It reduced aggregate size by 2.6 percent. + +That diagnostic reduced compression throughput by 26 percent and decode throughput by 36 percent. + +An adjacent-Delta array requires a separate design decision. This work does not add it. + +## Dataset status + +The completed matrix includes these paper inputs: + +- Air Quality. +- California Housing. +- April 2023 high-volume for-hire Taxi. + +The matrix does not include CMS Payments, r/place, or Twitter. + +The RangeEntropy throughput failure appears on synthetic floats, Taxi floats, Housing floats, and Air Quality integers. + +More datasets cannot satisfy the gate without a faster codec implementation. + +## Default writer integration + +`FloatQuantScheme` belongs to `ALL_SCHEMES`. `BtrBlocksCompressor::default()` now evaluates it for `f64` arrays. + +The default file session registers `FloatQuantArray`. The August 2026 core edition permits the array. + +The CUDA-compatible builder excludes FloatQuant because no CUDA decode kernel exists. + +The default writer test writes, reads, and verifies an actual FloatQuant tree. + +## Current prototype scope + +Do not start RangeEntropy pushdowns before its full-decode cost improves substantially. + +The current prototype scope includes these transforms: + +- Ordered float bits. +- A high-bit and low-bit split. +- FloatMult with an arbitrary common multiplier. +- Block residual packing with optional patches. + +The current scope excludes these transforms: + +- Block-local adjacent Delta with fast random access. + +Delta-of-delta, Delta with lookback, and convolution Delta also remain excluded. + +## References + +- [PCodec paper](https://arxiv.org/html/2502.06112v2) +- [PCodec repository](https://github.com/pcodec/pcodec) diff --git a/encodings/float-quant/Cargo.toml b/encodings/float-quant/Cargo.toml new file mode 100644 index 00000000000..717ab85c13e --- /dev/null +++ b/encodings/float-quant/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "vortex-float-quant" +authors = { workspace = true } +categories = { workspace = true } +description = "Vortex float quantization array encoding" +edition = { workspace = true } +homepage = { workspace = true } +include = { workspace = true } +keywords = { workspace = true } +license = { workspace = true } +readme = { workspace = true } +repository = { workspace = true } +rust-version = { workspace = true } +version = { workspace = true } + +[dependencies] +vortex-array = { workspace = true } +vortex-buffer = { workspace = true } +vortex-error = { workspace = true } +vortex-session = { workspace = true } + +[dev-dependencies] +vortex-array = { workspace = true, features = ["_test-harness"] } + +[lints] +workspace = true diff --git a/encodings/float-quant/src/array.rs b/encodings/float-quant/src/array.rs new file mode 100644 index 00000000000..995cfb28af3 --- /dev/null +++ b/encodings/float-quant/src/array.rs @@ -0,0 +1,809 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::fmt::Display; +use std::fmt::Formatter; +use std::hash::Hash; +use std::hash::Hasher; + +use vortex_array::Array; +use vortex_array::ArrayEq; +use vortex_array::ArrayHash; +use vortex_array::ArrayId; +use vortex_array::ArrayParts; +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::EqMode; +use vortex_array::ExecutionCtx; +use vortex_array::ExecutionResult; +use vortex_array::IntoArray; +use vortex_array::TypedArrayRef; +use vortex_array::array_slots; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::Primitive; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::buffer::BufferHandle; +use vortex_array::dtype::DType; +use vortex_array::dtype::Nullability::NonNullable; +use vortex_array::dtype::PType; +use vortex_array::scalar::Scalar; +use vortex_array::serde::ArrayChildren; +use vortex_array::vtable::OperationsVTable; +use vortex_array::vtable::VTable; +use vortex_array::vtable::ValidityChild; +use vortex_array::vtable::ValidityVTableFromChild; +use vortex_buffer::Buffer; +use vortex_error::VortexExpect; +use vortex_error::VortexResult; +use vortex_error::vortex_bail; +use vortex_error::vortex_ensure; +use vortex_error::vortex_panic; +use vortex_session::VortexSession; +use vortex_session::registry::CachedId; + +use crate::rules::RULES; + +const METADATA_VERSION: u8 = 1; +const METADATA_LEN: usize = 2; + +/// A lossless float split with quantized high bits and low-bit adjustments. +pub type FloatQuantArray = Array; + +#[array_slots(FloatQuant)] +pub struct FloatQuantSlots { + /// Ordered float bits after the low `k` bits are removed. + #[slot(0)] + pub primary: ArrayRef, + /// Sign-normalized low `k` bits. + #[slot(1)] + pub secondary: ArrayRef, +} + +#[derive(Clone, Debug)] +pub struct FloatQuantData { + pub(crate) k: u8, +} + +impl Display for FloatQuantData { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "k: {}", self.k) + } +} + +impl ArrayHash for FloatQuantData { + fn array_hash(&self, state: &mut H, _accuracy: EqMode) { + self.k.hash(state); + } +} + +impl ArrayEq for FloatQuantData { + fn array_eq(&self, other: &Self, _accuracy: EqMode) -> bool { + self.k == other.k + } +} + +#[derive(Clone, Debug)] +pub struct FloatQuant; + +impl VTable for FloatQuant { + type TypedArrayData = FloatQuantData; + type OperationsVTable = Self; + type ValidityVTable = ValidityVTableFromChild; + + fn id(&self) -> ArrayId { + static ID: CachedId = CachedId::new("vortex.float_quant"); + *ID + } + + fn validate( + &self, + data: &Self::TypedArrayData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + let ptype = PType::try_from(dtype)?; + let latent_ptype = latent_ptype(ptype)?; + let precision_bits = precision_bits(ptype)?; + vortex_ensure!( + data.k > 0 && data.k <= precision_bits, + "FloatQuant k {} exceeds {ptype} precision {precision_bits}", + data.k + ); + + let slots = FloatQuantSlotsView::from_slots(slots); + let expected_primary = DType::Primitive(latent_ptype, dtype.nullability()); + let expected_secondary = DType::Primitive(latent_ptype, NonNullable); + vortex_ensure!( + slots.primary.dtype() == &expected_primary, + "expected primary dtype {expected_primary}, got {}", + slots.primary.dtype() + ); + vortex_ensure!( + slots.secondary.dtype() == &expected_secondary, + "expected secondary dtype {expected_secondary}, got {}", + slots.secondary.dtype() + ); + vortex_ensure!( + slots.primary.len() == len && slots.secondary.len() == len, + "FloatQuant child length differs from {len}" + ); + Ok(()) + } + + fn nbuffers(_array: ArrayView<'_, Self>) -> usize { + 0 + } + + fn buffer(_array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { + vortex_panic!("FloatQuantArray buffer index {idx} out of bounds") + } + + fn buffer_name(_array: ArrayView<'_, Self>, idx: usize) -> Option { + vortex_panic!("FloatQuantArray buffer_name index {idx} out of bounds") + } + + fn with_buffers( + &self, + array: ArrayView<'_, Self>, + buffers: &[BufferHandle], + ) -> VortexResult> { + vortex_array::vtable::with_empty_buffers(self, array, buffers) + } + + fn serialize( + array: ArrayView<'_, Self>, + _session: &VortexSession, + ) -> VortexResult>> { + Ok(Some(vec![METADATA_VERSION, array.data().k])) + } + + fn deserialize( + &self, + dtype: &DType, + len: usize, + metadata: &[u8], + _buffers: &[BufferHandle], + children: &dyn ArrayChildren, + _session: &VortexSession, + ) -> VortexResult> { + vortex_ensure!( + metadata.len() == METADATA_LEN, + "FloatQuant metadata requires {METADATA_LEN} bytes" + ); + vortex_ensure!( + metadata[0] == METADATA_VERSION, + "unsupported FloatQuant metadata version {}", + metadata[0] + ); + vortex_ensure!(children.len() == 2, "FloatQuant requires two children"); + + let ptype = PType::try_from(dtype)?; + let latent_ptype = latent_ptype(ptype)?; + let primary_dtype = DType::Primitive(latent_ptype, dtype.nullability()); + let secondary_dtype = DType::Primitive(latent_ptype, NonNullable); + let primary = children.get(0, &primary_dtype, len)?; + let secondary = children.get(1, &secondary_dtype, len)?; + let slots = FloatQuantSlots { primary, secondary }.into_slots(); + Ok(ArrayParts::new( + self.clone(), + dtype.clone(), + len, + FloatQuantData { k: metadata[1] }, + ) + .with_slots(slots)) + } + + fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { + FloatQuantSlots::NAMES[idx].to_string() + } + + fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionResult::done( + decode(array.as_view(), ctx)?.into_array(), + )) + } + + fn reduce_parent( + array: ArrayView<'_, Self>, + parent: &ArrayRef, + child_idx: usize, + ) -> VortexResult> { + RULES.evaluate(array, parent, child_idx) + } +} + +impl OperationsVTable for FloatQuant { + fn scalar_at( + array: ArrayView<'_, FloatQuant>, + index: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + let primary = array.primary().execute_scalar(index, ctx)?; + if primary.is_null() { + return Ok(Scalar::null(array.dtype().clone())); + } + let secondary = array.secondary().execute_scalar(index, ctx)?; + let k = array.data().k; + Ok(match PType::try_from(array.dtype())? { + PType::F32 => Scalar::primitive( + join_f32( + primary + .as_primitive() + .typed_value::() + .vortex_expect("validated primary scalar"), + secondary + .as_primitive() + .typed_value::() + .vortex_expect("validated secondary scalar"), + k, + ), + array.dtype().nullability(), + ), + PType::F64 => Scalar::primitive( + join_f64( + primary + .as_primitive() + .typed_value::() + .vortex_expect("validated primary scalar"), + secondary + .as_primitive() + .typed_value::() + .vortex_expect("validated secondary scalar"), + k, + ), + array.dtype().nullability(), + ), + ptype => vortex_panic!("unsupported FloatQuant ptype {ptype}"), + }) + } +} + +impl ValidityChild for FloatQuant { + fn validity_child(array: ArrayView<'_, FloatQuant>) -> ArrayRef { + array.primary().clone() + } +} + +pub trait FloatQuantArrayExt: TypedArrayRef + FloatQuantArraySlotsExt { + /// Return the number of split low bits. + fn k(&self) -> u8 { + self.deref().k + } +} + +impl> FloatQuantArrayExt for T {} + +impl FloatQuant { + /// Construct a float quantization array from two latent children. + pub fn try_new( + primary: ArrayRef, + secondary: ArrayRef, + float_ptype: PType, + k: u8, + ) -> VortexResult { + let dtype = DType::Primitive(float_ptype, primary.dtype().nullability()); + let len = primary.len(); + let slots = FloatQuantSlots { primary, secondary }.into_slots(); + Array::try_from_parts( + ArrayParts::new(FloatQuant, dtype, len, FloatQuantData { k }).with_slots(slots), + ) + } + + /// Split a canonical float array into two unsigned latent children. + pub fn from_primitive(array: ArrayView<'_, Primitive>, k: u8) -> VortexResult { + let validity = array.validity()?; + match array.ptype() { + PType::F32 => { + let (primary, secondary) = split_f32(array.as_slice::(), k)?; + Self::try_new( + PrimitiveArray::new(Buffer::from(primary), validity).into_array(), + PrimitiveArray::new(Buffer::from(secondary), NonNullable.into()).into_array(), + PType::F32, + k, + ) + } + PType::F64 => { + let (primary, secondary) = split_f64(array.as_slice::(), k)?; + Self::try_new( + PrimitiveArray::new(Buffer::from(primary), validity).into_array(), + PrimitiveArray::new(Buffer::from(secondary), NonNullable.into()).into_array(), + PType::F64, + k, + ) + } + ptype => vortex_bail!("FloatQuant requires f32 or f64, got {ptype}"), + } + } + + /// Split floats whose lowest `k` fraction bits are zero. + pub fn from_primitive_constant_secondary( + array: ArrayView<'_, Primitive>, + k: u8, + ) -> VortexResult { + let validity = array.validity()?; + let len = array.len(); + match array.ptype() { + PType::F32 => { + let primary = split_primary_f32(array.as_slice::(), k)?; + Self::try_new( + PrimitiveArray::new(Buffer::from(primary), validity).into_array(), + ConstantArray::new(Scalar::from(0u32), len).into_array(), + PType::F32, + k, + ) + } + PType::F64 => { + let primary = split_primary_f64(array.as_slice::(), k)?; + Self::try_new( + PrimitiveArray::new(Buffer::from(primary), validity).into_array(), + ConstantArray::new(Scalar::from(0u64), len).into_array(), + PType::F64, + k, + ) + } + ptype => vortex_bail!("FloatQuant requires f32 or f64, got {ptype}"), + } + } + + /// Split a constant-secondary float array into frame-of-reference primary values. + pub fn primary_for_primitive( + array: ArrayView<'_, Primitive>, + k: u8, + primary_min: u64, + ) -> VortexResult { + let validity = array.validity()?; + match array.ptype() { + PType::F32 => { + let primary_min = u32::try_from(primary_min)?; + let primary = split_primary_for_f32(array.as_slice::(), k, primary_min)?; + Ok(PrimitiveArray::new(Buffer::from(primary), validity)) + } + PType::F64 => { + let primary = split_primary_for_f64(array.as_slice::(), k, primary_min)?; + Ok(PrimitiveArray::new(Buffer::from(primary), validity)) + } + ptype => vortex_bail!("FloatQuant requires f32 or f64, got {ptype}"), + } + } +} + +/// Compression facts derived during FloatQuant split selection. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct FloatQuantAnalysis { + /// Selected low-bit width. + pub k: u8, + /// Bit width of the frame-of-reference primary values. + pub primary_bit_width: u8, + /// Minimum primary value before frame-of-reference subtraction. + pub primary_min: u64, + /// True when every secondary value is zero. + pub secondary_is_constant: bool, +} + +/// Estimate a useful low-bit split width for a canonical float array. +pub fn estimate_k(array: ArrayView<'_, Primitive>) -> Option { + analyze_float_quant(array).map(|analysis| analysis.k) +} + +/// Analyze a canonical float array for a FloatQuant split. +pub fn analyze_float_quant(array: ArrayView<'_, Primitive>) -> Option { + match array.ptype() { + PType::F32 => analyze_f32(array.as_slice::()), + PType::F64 => analyze_f64(array.as_slice::()), + _ => None, + } +} + +fn analyze_histogram( + mut histogram: Vec, + precision_bits: u8, + len: usize, + primary_min: u64, + primary_max: u64, +) -> Option { + if len == 0 { + return None; + } + let mut cumulative = 0usize; + for count in histogram.iter_mut().rev() { + cumulative += *count; + *count = cumulative; + } + + let mut best_k = 0u8; + let mut best_savings = 0.0; + for k in 1..=precision_bits { + let frequency = histogram[usize::from(k)] as f64 / len as f64; + if frequency == 0.0 { + continue; + } + let category_count = ((1_u64 << k) - 1) as f64; + let entropy = category_entropy(frequency) + + category_count * category_entropy((1.0 - frequency) / category_count); + let savings = f64::from(k) - entropy; + if savings > best_savings { + best_k = k; + best_savings = savings; + } else { + break; + } + } + if best_savings <= 1.5 { + return None; + } + let secondary_is_constant = histogram[usize::from(best_k)] == len; + let primary_min = primary_min >> best_k; + let primary_max = primary_max >> best_k; + let primary_bit_width = + u8::try_from(u64::BITS - (primary_max - primary_min).leading_zeros()).unwrap_or(u8::MAX); + Some(FloatQuantAnalysis { + k: best_k, + primary_bit_width, + primary_min, + secondary_is_constant, + }) +} + +fn analyze_f32(values: &[f32]) -> Option { + let mut minimum = u32::MAX; + let mut maximum = u32::MIN; + let mut histogram = vec![0usize; 24]; + for value in values { + let bits = value.to_bits(); + let ordered = ordered_u32(bits); + minimum = minimum.min(ordered); + maximum = maximum.max(ordered); + let zeros = bits.trailing_zeros().min(23); + histogram[zeros as usize] += 1; + } + analyze_histogram( + histogram, + 23, + values.len(), + u64::from(minimum), + u64::from(maximum), + ) +} + +fn analyze_f64(values: &[f64]) -> Option { + let mut minimum = u64::MAX; + let mut maximum = u64::MIN; + let mut histogram = vec![0usize; 53]; + for value in values { + let bits = value.to_bits(); + let ordered = ordered_u64(bits); + minimum = minimum.min(ordered); + maximum = maximum.max(ordered); + let zeros = bits.trailing_zeros().min(52); + histogram[zeros as usize] += 1; + } + analyze_histogram(histogram, 52, values.len(), minimum, maximum) +} + +fn category_entropy(probability: f64) -> f64 { + if probability == 0.0 || probability == 1.0 { + 0.0 + } else { + -probability * probability.log2() + } +} + +fn latent_ptype(ptype: PType) -> VortexResult { + match ptype { + PType::F32 => Ok(PType::U32), + PType::F64 => Ok(PType::U64), + _ => vortex_bail!("FloatQuant requires f32 or f64, got {ptype}"), + } +} + +fn precision_bits(ptype: PType) -> VortexResult { + match ptype { + PType::F32 => Ok(23), + PType::F64 => Ok(52), + _ => vortex_bail!("FloatQuant requires f32 or f64, got {ptype}"), + } +} + +fn ordered_u32(bits: u32) -> u32 { + if bits & (1_u32 << 31) == 0 { + bits ^ (1_u32 << 31) + } else { + !bits + } +} + +fn ordered_u64(bits: u64) -> u64 { + if bits & (1_u64 << 63) == 0 { + bits ^ (1_u64 << 63) + } else { + !bits + } +} + +fn split_f32(values: &[f32], k: u8) -> VortexResult<(Vec, Vec)> { + vortex_ensure!(k > 0 && k <= 23, "FloatQuant f32 k must be in 1..=23"); + let low_mask = (1_u32 << k) - 1; + let mut primary = Vec::with_capacity(values.len()); + let mut secondary = Vec::with_capacity(values.len()); + for &value in values { + let bits = value.to_bits(); + let ordered = ordered_u32(bits); + primary.push(ordered >> k); + let low = ordered & low_mask; + secondary.push(if bits & (1_u32 << 31) == 0 { + low + } else { + low_mask - low + }); + } + Ok((primary, secondary)) +} + +fn split_f64(values: &[f64], k: u8) -> VortexResult<(Vec, Vec)> { + vortex_ensure!(k > 0 && k <= 52, "FloatQuant f64 k must be in 1..=52"); + let low_mask = (1_u64 << k) - 1; + let mut primary = Vec::with_capacity(values.len()); + let mut secondary = Vec::with_capacity(values.len()); + for &value in values { + let bits = value.to_bits(); + let ordered = ordered_u64(bits); + primary.push(ordered >> k); + let low = ordered & low_mask; + secondary.push(if bits & (1_u64 << 63) == 0 { + low + } else { + low_mask - low + }); + } + Ok((primary, secondary)) +} + +fn split_primary_f32(values: &[f32], k: u8) -> VortexResult> { + vortex_ensure!(k > 0 && k <= 23, "FloatQuant f32 k must be in 1..=23"); + let low_mask = (1_u32 << k) - 1; + values + .iter() + .map(|value| { + let bits = value.to_bits(); + vortex_ensure!( + bits & low_mask == 0, + "FloatQuant constant secondary requires zero low bits" + ); + Ok(ordered_u32(bits) >> k) + }) + .collect() +} + +fn split_primary_f64(values: &[f64], k: u8) -> VortexResult> { + vortex_ensure!(k > 0 && k <= 52, "FloatQuant f64 k must be in 1..=52"); + let low_mask = (1_u64 << k) - 1; + values + .iter() + .map(|value| { + let bits = value.to_bits(); + vortex_ensure!( + bits & low_mask == 0, + "FloatQuant constant secondary requires zero low bits" + ); + Ok(ordered_u64(bits) >> k) + }) + .collect() +} + +fn split_primary_for_f32(values: &[f32], k: u8, primary_min: u32) -> VortexResult> { + vortex_ensure!(k > 0 && k <= 23, "FloatQuant f32 k must be in 1..=23"); + let low_mask = (1_u32 << k) - 1; + values + .iter() + .map(|value| { + let bits = value.to_bits(); + vortex_ensure!( + bits & low_mask == 0, + "FloatQuant constant secondary requires zero low bits" + ); + Ok((ordered_u32(bits) >> k) - primary_min) + }) + .collect() +} + +fn split_primary_for_f64(values: &[f64], k: u8, primary_min: u64) -> VortexResult> { + vortex_ensure!(k > 0 && k <= 52, "FloatQuant f64 k must be in 1..=52"); + let low_mask = (1_u64 << k) - 1; + values + .iter() + .map(|value| { + let bits = value.to_bits(); + vortex_ensure!( + bits & low_mask == 0, + "FloatQuant constant secondary requires zero low bits" + ); + Ok((ordered_u64(bits) >> k) - primary_min) + }) + .collect() +} + +fn join_f32(primary: u32, secondary: u32, k: u8) -> f32 { + let low_mask = (1_u32 << k) - 1; + let sign_cutoff = (1_u32 << 31) >> k; + let low = if primary >= sign_cutoff { + secondary + } else { + low_mask.wrapping_sub(secondary) + }; + let ordered = (primary << k).wrapping_add(low); + let bits = if ordered & (1_u32 << 31) == 0 { + !ordered + } else { + ordered ^ (1_u32 << 31) + }; + f32::from_bits(bits) +} + +fn join_f64(primary: u64, secondary: u64, k: u8) -> f64 { + let low_mask = (1_u64 << k) - 1; + let sign_cutoff = (1_u64 << 63) >> k; + let low = if primary >= sign_cutoff { + secondary + } else { + low_mask.wrapping_sub(secondary) + }; + let ordered = (primary << k).wrapping_add(low); + let bits = if ordered & (1_u64 << 63) == 0 { + !ordered + } else { + ordered ^ (1_u64 << 63) + }; + f64::from_bits(bits) +} + +fn decode( + array: ArrayView<'_, FloatQuant>, + ctx: &mut ExecutionCtx, +) -> VortexResult { + let primary = array.primary().clone().execute::(ctx)?; + let secondary = array.secondary().clone().execute::(ctx)?; + let validity = primary.validity()?; + let k = array.data().k; + Ok(match PType::try_from(array.dtype())? { + PType::F32 => { + let secondary_values = secondary.as_slice::(); + let mut index = 0; + let values = primary + .into_buffer::() + .map_each_in_place(|primary| { + let value = join_f32(primary, secondary_values[index], k); + index += 1; + value + }) + .freeze(); + PrimitiveArray::new(values, validity) + } + PType::F64 => { + let secondary_values = secondary.as_slice::(); + let mut index = 0; + let values = primary + .into_buffer::() + .map_each_in_place(|primary| { + let value = join_f64(primary, secondary_values[index], k); + index += 1; + value + }) + .freeze(); + PrimitiveArray::new(values, validity) + } + ptype => vortex_panic!("unsupported FloatQuant ptype {ptype}"), + }) +} + +#[cfg(test)] +mod tests { + use std::sync::LazyLock; + + use vortex_array::ArrayContext; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::array_session; + use vortex_array::assert_arrays_eq; + use vortex_array::assert_nth_scalar; + use vortex_array::serde::SerializeOptions; + use vortex_array::serde::SerializedArray; + use vortex_array::validity::Validity; + use vortex_buffer::ByteBufferMut; + use vortex_error::VortexResult; + use vortex_session::VortexSession; + use vortex_session::registry::ReadContext; + + use super::*; + + static SESSION: LazyLock = LazyLock::new(|| { + let session = array_session(); + crate::initialize(&session); + session + }); + + #[test] + fn float_bit_patterns_roundtrip() -> VortexResult<()> { + let values = [ + f64::NEG_INFINITY, + -1.5, + -0.0, + 0.0, + 1.5, + f64::INFINITY, + f64::from_bits(0x7ff8_0000_0000_1234), + f64::from_bits(0xfff8_0000_0000_5678), + ]; + let array = PrimitiveArray::from_iter(values); + let encoded = FloatQuant::from_primitive(array.as_view(), 29)?; + let decoded = encoded + .into_array() + .execute::(&mut SESSION.create_execution_ctx())?; + assert_eq!( + decoded + .as_slice::() + .iter() + .map(|value| value.to_bits()) + .collect::>(), + values + .iter() + .map(|value| value.to_bits()) + .collect::>() + ); + Ok(()) + } + + #[test] + fn nullable_slice_and_scalar_access() -> VortexResult<()> { + let array = PrimitiveArray::new( + Buffer::from(vec![1.25_f32, 0.0, -0.0, 42.5, -10.0]), + Validity::from_iter([true, false, true, true, false]), + ); + let encoded = FloatQuant::from_primitive(array.as_view(), 8)?; + let mut ctx = SESSION.create_execution_ctx(); + assert_arrays_eq!(encoded, array, &mut ctx); + assert_nth_scalar!(encoded, 3, 42.5_f32, &mut ctx); + assert!(encoded.execute_scalar(1, &mut ctx)?.is_null()); + + let sliced = encoded.into_array().slice(1..4)?; + assert!(sliced.is::()); + assert_arrays_eq!(sliced, array.into_array().slice(1..4)?, &mut ctx); + Ok(()) + } + + #[test] + fn serialization_roundtrip() -> VortexResult<()> { + let original = PrimitiveArray::from_option_iter([ + Some(f64::NEG_INFINITY), + None, + Some(-0.0), + Some(42.25), + Some(f64::from_bits(0x7ff8_0000_0000_1234)), + ]); + let encoded = FloatQuant::from_primitive(original.as_view(), 29)?; + let sliced = encoded.into_array().slice(1..5)?; + let dtype = sliced.dtype().clone(); + let len = sliced.len(); + let array_context = ArrayContext::empty(); + let serialized = + sliced.serialize(&array_context, &SESSION, &SerializeOptions::default())?; + let mut bytes = ByteBufferMut::empty(); + for buffer in serialized { + bytes.extend_from_slice(buffer.as_ref()); + } + + let decoded = SerializedArray::try_from(bytes.freeze())?.decode( + &dtype, + len, + &ReadContext::new(array_context.to_ids()), + &SESSION, + )?; + assert!(decoded.is::()); + assert_arrays_eq!( + decoded, + original.into_array().slice(1..5)?, + &mut SESSION.create_execution_ctx() + ); + Ok(()) + } +} diff --git a/encodings/float-quant/src/lib.rs b/encodings/float-quant/src/lib.rs new file mode 100644 index 00000000000..6efde148ff0 --- /dev/null +++ b/encodings/float-quant/src/lib.rs @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Lossless float quantization as two integer child arrays. + +mod array; +mod rules; +mod slice; + +pub use array::*; +use vortex_array::session::ArraySessionExt; +use vortex_session::VortexSession; + +/// Register the float quantization encoding in one session. +pub fn initialize(session: &VortexSession) { + session.arrays().register(FloatQuant); +} diff --git a/encodings/float-quant/src/rules.rs b/encodings/float-quant/src/rules.rs new file mode 100644 index 00000000000..5faedb68866 --- /dev/null +++ b/encodings/float-quant/src/rules.rs @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use vortex_array::arrays::slice::SliceReduceAdaptor; +use vortex_array::optimizer::rules::ParentRuleSet; + +use crate::FloatQuant; + +pub(crate) static RULES: ParentRuleSet = + ParentRuleSet::new(&[ParentRuleSet::lift(&SliceReduceAdaptor(FloatQuant))]); diff --git a/encodings/float-quant/src/slice.rs b/encodings/float-quant/src/slice.rs new file mode 100644 index 00000000000..3a83f48220d --- /dev/null +++ b/encodings/float-quant/src/slice.rs @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::ops::Range; + +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::IntoArray; +use vortex_array::arrays::slice::SliceReduce; +use vortex_array::dtype::PType; +use vortex_error::VortexResult; + +use crate::FloatQuant; +use crate::FloatQuantArraySlotsExt; + +impl SliceReduce for FloatQuant { + fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { + Ok(Some( + FloatQuant::try_new( + array.primary().slice(range.clone())?, + array.secondary().slice(range)?, + PType::try_from(array.dtype())?, + array.data().k, + )? + .into_array(), + )) + } +} diff --git a/encodings/pco/src/array.rs b/encodings/pco/src/array.rs index 81f5a2c94fb..bc2d1473af1 100644 --- a/encodings/pco/src/array.rs +++ b/encodings/pco/src/array.rs @@ -340,6 +340,24 @@ impl Pco { let data = PcoData::from_primitive(parray, level, values_per_page, ctx)?; Self::try_new(dtype, data, validity) } + + /// Compress a primitive array with a custom Pco configuration. + /// + /// # Errors + /// + /// Returns an error if the configuration is invalid or compression fails. + pub fn from_primitive_with_config( + parray: ArrayView<'_, Primitive>, + chunk_config: &ChunkConfig, + values_per_chunk: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + let dtype = parray.dtype().clone(); + let validity = parray.validity()?; + let data = + PcoData::from_primitive_with_config(parray, chunk_config, values_per_chunk, ctx)?; + Self::try_new(dtype, data, validity) + } } #[array_slots(Pco)] @@ -480,7 +498,6 @@ impl PcoData { values_per_page: usize, ctx: &mut ExecutionCtx, ) -> VortexResult { - let number_type = number_type_from_dtype(parray.dtype()); let values_per_page = if values_per_page == 0 { values_per_chunk } else { @@ -492,6 +509,21 @@ impl PcoData { .with_compression_level(level) .with_paging_spec(PagingSpec::EqualPagesUpTo(values_per_page)); + Self::from_primitive_with_config(parray, &chunk_config, values_per_chunk, ctx) + } + + fn from_primitive_with_config( + parray: ArrayView<'_, Primitive>, + chunk_config: &ChunkConfig, + values_per_chunk: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + vortex_ensure!( + values_per_chunk > 0, + "values per chunk must be greater than zero" + ); + let number_type = number_type_from_dtype(parray.dtype()); + let values = collect_valid(parray, ctx)?; let n_values = values.len(); @@ -510,7 +542,7 @@ impl PcoData { let values = values.to_buffer::(); let chunk = &values.as_slice()[chunk_start..chunk_end]; fc - .chunk_compressor(chunk, &chunk_config) + .chunk_compressor(chunk, chunk_config) .map_err(vortex_err_from_pco)? } ); diff --git a/encodings/pco/src/tests.rs b/encodings/pco/src/tests.rs index 8a0e1083b16..7d9613b4ea5 100644 --- a/encodings/pco/src/tests.rs +++ b/encodings/pco/src/tests.rs @@ -4,6 +4,10 @@ use std::sync::LazyLock; +use pco::ChunkConfig; +use pco::DeltaSpec; +use pco::ModeSpec; +use pco::PagingSpec; use vortex_array::ArrayContext; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; @@ -70,6 +74,21 @@ fn test_compress_decompress() { ); } +#[test] +fn test_custom_config_roundtrip() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let array = PrimitiveArray::from_iter((0..1_000).map(|value| f64::from(value).sin())); + let config = ChunkConfig::default() + .with_mode_spec(ModeSpec::Classic) + .with_delta_spec(DeltaSpec::NoOp) + .with_paging_spec(PagingSpec::EqualPagesUpTo(128)); + + let compressed = Pco::from_primitive_with_config(array.as_view(), &config, 512, &mut ctx)?; + + assert_arrays_eq!(compressed, array, &mut ctx); + Ok(()) +} + #[test] fn test_compress_decompress_small() { let mut ctx = SESSION.create_execution_ctx(); diff --git a/encodings/range-entropy/Cargo.toml b/encodings/range-entropy/Cargo.toml new file mode 100644 index 00000000000..60fb6151d86 --- /dev/null +++ b/encodings/range-entropy/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "vortex-range-entropy" +authors = { workspace = true } +categories = { workspace = true } +description = "Vortex range entropy array" +edition = { workspace = true } +homepage = { workspace = true } +include = { workspace = true } +keywords = { workspace = true } +license = { workspace = true } +readme = { workspace = true } +repository = { workspace = true } +rust-version = { workspace = true } +version = { workspace = true } + +[lints] +workspace = true + +[dependencies] +fastlanes = { workspace = true } +vortex-array = { workspace = true } +vortex-buffer = { workspace = true } +vortex-error = { workspace = true } +vortex-session = { workspace = true } + +[dev-dependencies] +vortex-array = { workspace = true, features = ["_test-harness"] } diff --git a/encodings/range-entropy/examples/encode_probe.rs b/encodings/range-entropy/examples/encode_probe.rs new file mode 100644 index 00000000000..43766f18f4a --- /dev/null +++ b/encodings/range-entropy/examples/encode_probe.rs @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::f64::consts::TAU; +use std::hint::black_box; +use std::time::Duration; +use std::time::Instant; + +use vortex_error::VortexResult; +use vortex_range_entropy::RangeEntropyCodec; + +const N: usize = 1 << 18; +const BLOCK_LEN: usize = 8192; +const ITERATIONS: usize = 200; + +struct Rng(u64); + +impl Rng { + fn next_u64(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + + fn uniform(&mut self) -> f64 { + ((self.next_u64() >> 11) as f64 + 0.5) * (1.0 / ((1_u64 << 53) as f64)) + } + + fn normal(&mut self) -> f64 { + let radius = (-2.0 * self.uniform().ln()).sqrt(); + radius * (TAU * self.uniform()).cos() + } +} + +fn ordered_f64(value: f64) -> u64 { + let bits = value.to_bits(); + if bits & (1_u64 << 63) == 0 { + bits ^ (1_u64 << 63) + } else { + !bits + } +} + +fn random_walk() -> Vec { + let mut rng = Rng(0x4d59_5df4_d0f3_3173); + let mut value = 0.0; + (0..N) + .map(|_| { + value += rng.normal() * 0.01; + ordered_f64(value) + }) + .collect() +} + +fn main() -> VortexResult<()> { + let values = random_walk(); + black_box(RangeEntropyCodec::encode(&values, BLOCK_LEN)?); + + let mut durations = Vec::with_capacity(ITERATIONS); + let mut encoded_size = 0usize; + let mut bin_count = 0usize; + for _ in 0..ITERATIONS { + let start = Instant::now(); + let encoded = RangeEntropyCodec::encode(black_box(&values), BLOCK_LEN)?; + durations.push(start.elapsed()); + encoded_size = black_box(encoded.encoded_size()); + bin_count = black_box(encoded.bin_lowers().len()); + } + + durations.sort_unstable(); + let p10 = durations[ITERATIONS / 10]; + let median = durations[ITERATIONS / 2]; + let p90 = durations[ITERATIONS * 9 / 10]; + let input_bytes = (N * size_of::()) as f64; + let throughput = input_bytes / median.as_secs_f64() / 1_000_000.0; + print_result(encoded_size, bin_count, p10, median, p90, throughput); + Ok(()) +} + +fn print_result( + encoded_size: usize, + bin_count: usize, + p10: Duration, + median: Duration, + p90: Duration, + throughput: f64, +) { + println!( + "random-walk values={N} blocks={} bins={bin_count} bytes={encoded_size}", + N.div_ceil(BLOCK_LEN), + ); + println!( + "encode p10={:.3}ms median={:.3}ms p90={:.3}ms throughput={throughput:.1}MB/s", + p10.as_secs_f64() * 1_000.0, + median.as_secs_f64() * 1_000.0, + p90.as_secs_f64() * 1_000.0, + ); +} diff --git a/encodings/range-entropy/src/array.rs b/encodings/range-entropy/src/array.rs new file mode 100644 index 00000000000..b0c0985b683 --- /dev/null +++ b/encodings/range-entropy/src/array.rs @@ -0,0 +1,857 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::fmt::Display; +use std::fmt::Formatter; +use std::hash::Hash; +use std::hash::Hasher; +use std::ops::Range; + +use vortex_array::Array; +use vortex_array::ArrayEq; +use vortex_array::ArrayHash; +use vortex_array::ArrayId; +use vortex_array::ArrayParts; +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::EqMode; +use vortex_array::ExecutionCtx; +use vortex_array::ExecutionResult; +use vortex_array::IntoArray; +use vortex_array::TypedArrayRef; +use vortex_array::array_slots; +use vortex_array::arrays::Primitive; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::slice::SliceReduce; +use vortex_array::buffer::BufferHandle; +use vortex_array::dtype::DType; +use vortex_array::dtype::Nullability; +use vortex_array::dtype::Nullability::NonNullable; +use vortex_array::dtype::PType; +use vortex_array::dtype::half::f16; +use vortex_array::scalar::Scalar; +use vortex_array::serde::ArrayChildren; +use vortex_array::validity::Validity; +use vortex_array::vtable::OperationsVTable; +use vortex_array::vtable::VTable; +use vortex_array::vtable::ValidityVTable; +use vortex_array::vtable::child_to_validity; +use vortex_array::vtable::validity_to_child; +use vortex_buffer::Buffer; +use vortex_error::VortexResult; +use vortex_error::vortex_bail; +use vortex_error::vortex_ensure; +use vortex_error::vortex_panic; +use vortex_session::VortexSession; +use vortex_session::registry::CachedId; + +use crate::RangeEntropyCodec; +use crate::RangeEntropyParts; +use crate::rules::RULES; + +const METADATA_VERSION: u8 = 1; +const METADATA_LEN: usize = 23; +const MAX_SCALE_BITS: u8 = 10; +const MAX_BINS: usize = 64; + +/// A Vortex array with range-bin identifiers and fixed-width offsets. +pub type RangeEntropyArray = Array; + +#[array_slots(RangeEntropy)] +pub struct RangeEntropySlots { + /// Lower bound for each range bin. + #[slot(0)] + pub bin_lowers: ArrayRef, + /// Fixed offset width for each range bin. + #[slot(1)] + pub offset_widths: ArrayRef, + /// Quantized tANS weight for each range bin. + #[slot(2)] + pub weights: ArrayRef, + /// Byte offset for each restart block. + #[slot(3)] + pub block_offsets: ArrayRef, + /// Validity for the unsliced logical values. + #[slot(4)] + pub validity: Option, +} + +#[derive(Clone, Debug)] +pub struct RangeEntropyData { + payload: BufferHandle, + block_len: usize, + scale_bits: u8, + unsliced_len: usize, + slice_start: usize, + slice_stop: usize, +} + +impl Display for RangeEntropyData { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "block_len: {}, scale_bits: {}, slice: {}..{}", + self.block_len, self.scale_bits, self.slice_start, self.slice_stop + ) + } +} + +impl ArrayHash for RangeEntropyData { + fn array_hash(&self, state: &mut H, accuracy: EqMode) { + self.payload.array_hash(state, accuracy); + self.block_len.hash(state); + self.scale_bits.hash(state); + self.unsliced_len.hash(state); + self.slice_start.hash(state); + self.slice_stop.hash(state); + } +} + +impl ArrayEq for RangeEntropyData { + fn array_eq(&self, other: &Self, accuracy: EqMode) -> bool { + self.payload.array_eq(&other.payload, accuracy) + && self.block_len == other.block_len + && self.scale_bits == other.scale_bits + && self.unsliced_len == other.unsliced_len + && self.slice_start == other.slice_start + && self.slice_stop == other.slice_stop + } +} + +#[derive(Clone, Debug)] +pub struct RangeEntropy; + +impl VTable for RangeEntropy { + type TypedArrayData = RangeEntropyData; + + type OperationsVTable = Self; + type ValidityVTable = Self; + + fn id(&self) -> ArrayId { + static ID: CachedId = CachedId::new("vortex.range_entropy"); + *ID + } + + fn validate( + &self, + data: &Self::TypedArrayData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + let slots = RangeEntropySlotsView::from_slots(slots); + let validity = child_to_validity(slots.validity, dtype.nullability()); + data.validate(dtype, len, slots, &validity) + } + + fn nbuffers(_array: ArrayView<'_, Self>) -> usize { + 1 + } + + fn buffer(array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { + match idx { + 0 => array.payload.clone(), + _ => vortex_panic!("RangeEntropyArray buffer index {idx} out of bounds"), + } + } + + fn buffer_name(_array: ArrayView<'_, Self>, idx: usize) -> Option { + match idx { + 0 => Some("payload".to_string()), + _ => vortex_panic!("RangeEntropyArray buffer index {idx} out of bounds"), + } + } + + fn with_buffers( + &self, + array: ArrayView<'_, Self>, + buffers: &[BufferHandle], + ) -> VortexResult> { + vortex_ensure!( + buffers.len() == 1, + "RangeEntropyArray expects one buffer, got {}", + buffers.len() + ); + let mut data = array.data().clone(); + data.payload = buffers[0].clone(); + Ok( + ArrayParts::new(self.clone(), array.dtype().clone(), array.len(), data) + .with_slots(array.slots().iter().cloned().collect()), + ) + } + + fn serialize( + array: ArrayView<'_, Self>, + _session: &VortexSession, + ) -> VortexResult>> { + let bin_count = u8::try_from(array.bin_lowers().len())?; + Ok(Some( + RangeEntropyMetadata { + scale_bits: array.scale_bits, + bin_count, + block_len: u32::try_from(array.block_len)?, + unsliced_len: u64::try_from(array.unsliced_len)?, + slice_start: u64::try_from(array.slice_start)?, + } + .encode(), + )) + } + + fn deserialize( + &self, + dtype: &DType, + len: usize, + metadata: &[u8], + buffers: &[BufferHandle], + children: &dyn ArrayChildren, + _session: &VortexSession, + ) -> VortexResult> { + vortex_ensure!( + buffers.len() == 1, + "RangeEntropyArray expects one buffer, got {}", + buffers.len() + ); + let metadata = RangeEntropyMetadata::decode(metadata)?; + let unsliced_len = usize::try_from(metadata.unsliced_len)?; + let slice_start = usize::try_from(metadata.slice_start)?; + let slice_stop = slice_start + .checked_add(len) + .ok_or_else(|| vortex_error::vortex_err!("range entropy slice length overflows"))?; + let block_len = usize::try_from(metadata.block_len)?; + vortex_ensure!(block_len > 0, "range entropy block length must be positive"); + let block_offsets_len = unsliced_len.div_ceil(block_len) + 1; + let bin_count = usize::from(metadata.bin_count); + + let bin_lowers = children.get(0, &primitive_dtype(PType::U64), bin_count)?; + let offset_widths = children.get(1, &primitive_dtype(PType::U8), bin_count)?; + let weights = children.get(2, &primitive_dtype(PType::U16), bin_count)?; + let block_offsets = children.get(3, &primitive_dtype(PType::U32), block_offsets_len)?; + let validity = match children.len() { + 4 => Validity::from(dtype.nullability()), + 5 => Validity::Array(children.get(4, &Validity::DTYPE, unsliced_len)?), + count => vortex_bail!("RangeEntropyArray expects four or five children, got {count}"), + }; + let slots = RangeEntropySlots { + bin_lowers, + offset_widths, + weights, + block_offsets, + validity: validity_to_child(&validity, unsliced_len), + } + .into_slots(); + let data = RangeEntropyData { + payload: buffers[0].clone(), + block_len, + scale_bits: metadata.scale_bits, + unsliced_len, + slice_start, + slice_stop, + }; + Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) + } + + fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { + RangeEntropySlots::NAMES[idx].to_string() + } + + fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionResult::done( + decompress_array(array.as_view(), ctx)?.into_array(), + )) + } + + fn reduce_parent( + array: ArrayView<'_, Self>, + parent: &ArrayRef, + child_idx: usize, + ) -> VortexResult> { + RULES.evaluate(array, parent, child_idx) + } +} + +impl OperationsVTable for RangeEntropy { + fn scalar_at( + array: ArrayView<'_, RangeEntropy>, + index: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + let codec = codec_from_array(array, ctx)?; + scalar_from_latent( + codec.value(array.slice_start + index)?, + array.dtype().as_ptype(), + array.dtype().nullability(), + ) + } +} + +impl ValidityVTable for RangeEntropy { + fn validity(array: ArrayView<'_, RangeEntropy>) -> VortexResult { + array + .unsliced_validity() + .slice(array.slice_start..array.slice_stop) + } +} + +impl SliceReduce for RangeEntropy { + fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { + let data = array.data().slice(range); + Ok(Some( + Array::try_from_parts( + ArrayParts::new(RangeEntropy, array.dtype().clone(), data.len(), data) + .with_slots(array.slots().iter().cloned().collect()), + )? + .into_array(), + )) + } +} + +pub trait RangeEntropyArrayExt: TypedArrayRef + RangeEntropyArraySlotsExt { + /// Return the validity for all values before a logical slice. + fn unsliced_validity(&self) -> Validity { + child_to_validity( + self.as_ref().slots()[RangeEntropySlots::VALIDITY].as_ref(), + self.as_ref().dtype().nullability(), + ) + } + + /// Decode the logical slice to a canonical primitive array. + fn decompress(&self, ctx: &mut ExecutionCtx) -> VortexResult { + decompress_array(self.to_owned().as_view(), ctx) + } +} + +impl> RangeEntropyArrayExt for T {} + +impl RangeEntropy { + /// Encode a canonical primitive array. + pub fn from_primitive( + array: ArrayView<'_, Primitive>, + block_len: usize, + ) -> VortexResult { + let dtype = array.dtype().clone(); + let validity = array.validity()?; + let codec = RangeEntropyCodec::encode(&ordered_latents(array), block_len)?; + let parts = codec.into_parts(); + let unsliced_len = parts.len; + let data = RangeEntropyData { + payload: BufferHandle::new_host(parts.payload), + block_len: parts.block_len, + scale_bits: parts.scale_bits, + unsliced_len, + slice_start: 0, + slice_stop: unsliced_len, + }; + let slots = RangeEntropySlots { + bin_lowers: PrimitiveArray::from_iter(parts.bin_lowers).into_array(), + offset_widths: PrimitiveArray::from_iter(parts.offset_widths).into_array(), + weights: PrimitiveArray::from_iter(parts.weights).into_array(), + block_offsets: PrimitiveArray::from_iter(parts.block_offsets).into_array(), + validity: validity_to_child(&validity, unsliced_len), + } + .into_slots(); + Array::try_from_parts( + ArrayParts::new(RangeEntropy, dtype, unsliced_len, data).with_slots(slots), + ) + } +} + +impl RangeEntropyData { + fn validate( + &self, + dtype: &DType, + len: usize, + slots: RangeEntropySlotsView<'_>, + validity: &Validity, + ) -> VortexResult<()> { + vortex_ensure!( + matches!(dtype, DType::Primitive(..)), + "RangeEntropyArray requires a primitive dtype" + ); + vortex_ensure!( + self.block_len > 0, + "range entropy block length must be positive" + ); + vortex_ensure!( + self.scale_bits <= MAX_SCALE_BITS, + "tANS table log {} exceeds {MAX_SCALE_BITS}", + self.scale_bits, + ); + vortex_ensure!( + self.slice_start <= self.slice_stop && self.slice_stop <= self.unsliced_len, + "range entropy slice exceeds its source length" + ); + vortex_ensure!( + len == self.len(), + "range entropy length does not match its slice" + ); + vortex_ensure!( + slots.bin_lowers.dtype() == &primitive_dtype(PType::U64), + "range entropy bin lowers require non-nullable u64 values" + ); + vortex_ensure!( + slots.offset_widths.dtype() == &primitive_dtype(PType::U8), + "range entropy offset widths require non-nullable u8 values" + ); + vortex_ensure!( + slots.weights.dtype() == &primitive_dtype(PType::U16), + "range entropy weights require non-nullable u16 values" + ); + vortex_ensure!( + slots.block_offsets.dtype() == &primitive_dtype(PType::U32), + "range entropy block offsets require non-nullable u32 values" + ); + let bin_count = slots.bin_lowers.len(); + vortex_ensure!( + bin_count <= MAX_BINS, + "range entropy bin count exceeds {MAX_BINS}" + ); + vortex_ensure!( + slots.offset_widths.len() == bin_count && slots.weights.len() == bin_count, + "range entropy bin children have different lengths" + ); + vortex_ensure!( + slots.block_offsets.len() == self.unsliced_len.div_ceil(self.block_len) + 1, + "range entropy block offset count is invalid" + ); + if let Some(validity_len) = validity.maybe_len() { + vortex_ensure!( + validity_len == self.unsliced_len, + "range entropy validity length is invalid" + ); + } + Ok(()) + } + + fn len(&self) -> usize { + self.slice_stop - self.slice_start + } + + fn slice(&self, range: Range) -> Self { + Self { + slice_start: self.slice_start + range.start, + slice_stop: self.slice_start + range.end, + ..self.clone() + } + } +} + +fn codec_from_array( + array: ArrayView<'_, RangeEntropy>, + ctx: &mut ExecutionCtx, +) -> VortexResult { + let bin_lowers = array.bin_lowers().clone().execute::(ctx)?; + let offset_widths = array + .offset_widths() + .clone() + .execute::(ctx)?; + let weights = array.weights().clone().execute::(ctx)?; + let block_offsets = array + .block_offsets() + .clone() + .execute::(ctx)?; + let payload = array.payload.clone().try_to_host_sync()?; + RangeEntropyCodec::try_from_parts(RangeEntropyParts { + len: array.unsliced_len, + block_len: array.block_len, + scale_bits: array.scale_bits, + bin_lowers: bin_lowers.as_slice::().to_vec(), + offset_widths: offset_widths.as_slice::().to_vec(), + weights: weights.as_slice::().to_vec(), + block_offsets: block_offsets.as_slice::().to_vec(), + payload, + }) +} + +fn decompress_array( + array: ArrayView<'_, RangeEntropy>, + ctx: &mut ExecutionCtx, +) -> VortexResult { + let latents = + codec_from_array(array, ctx)?.decode_range(array.slice_start..array.slice_stop)?; + primitive_from_latents(latents, array.dtype().as_ptype(), array.validity()?) +} + +#[derive(Clone, Copy)] +struct RangeEntropyMetadata { + scale_bits: u8, + bin_count: u8, + block_len: u32, + unsliced_len: u64, + slice_start: u64, +} + +impl RangeEntropyMetadata { + fn encode(self) -> Vec { + let mut bytes = Vec::with_capacity(METADATA_LEN); + bytes.push(METADATA_VERSION); + bytes.push(self.scale_bits); + bytes.push(self.bin_count); + bytes.extend_from_slice(&self.block_len.to_le_bytes()); + bytes.extend_from_slice(&self.unsliced_len.to_le_bytes()); + bytes.extend_from_slice(&self.slice_start.to_le_bytes()); + bytes + } + + fn decode(bytes: &[u8]) -> VortexResult { + vortex_ensure!( + bytes.len() == METADATA_LEN, + "RangeEntropyArray metadata requires {METADATA_LEN} bytes" + ); + vortex_ensure!( + bytes[0] == METADATA_VERSION, + "unsupported RangeEntropyArray metadata version {}", + bytes[0] + ); + Ok(Self { + scale_bits: bytes[1], + bin_count: bytes[2], + block_len: u32::from_le_bytes([bytes[3], bytes[4], bytes[5], bytes[6]]), + unsliced_len: u64::from_le_bytes([ + bytes[7], bytes[8], bytes[9], bytes[10], bytes[11], bytes[12], bytes[13], bytes[14], + ]), + slice_start: u64::from_le_bytes([ + bytes[15], bytes[16], bytes[17], bytes[18], bytes[19], bytes[20], bytes[21], + bytes[22], + ]), + }) + } +} + +fn primitive_dtype(ptype: PType) -> DType { + DType::Primitive(ptype, NonNullable) +} + +fn ordered_latents(array: ArrayView<'_, Primitive>) -> Vec { + match array.ptype() { + PType::U8 => array + .as_slice::() + .iter() + .map(|&value| u64::from(value)) + .collect(), + PType::U16 => array + .as_slice::() + .iter() + .map(|&value| u64::from(value)) + .collect(), + PType::U32 => array + .as_slice::() + .iter() + .map(|&value| u64::from(value)) + .collect(), + PType::U64 => array.as_slice::().to_vec(), + PType::I8 => array + .as_slice::() + .iter() + .map(|&value| u64::from(value.cast_unsigned() ^ (1_u8 << 7))) + .collect(), + PType::I16 => array + .as_slice::() + .iter() + .map(|&value| u64::from(value.cast_unsigned() ^ (1_u16 << 15))) + .collect(), + PType::I32 => array + .as_slice::() + .iter() + .map(|&value| u64::from(value.cast_unsigned() ^ (1_u32 << 31))) + .collect(), + PType::I64 => array + .as_slice::() + .iter() + .map(|&value| value.cast_unsigned() ^ (1_u64 << 63)) + .collect(), + PType::F16 => array + .as_slice::() + .iter() + .map(|&value| ordered_float_bits(u64::from(value.to_bits()), 16)) + .collect(), + PType::F32 => array + .as_slice::() + .iter() + .map(|&value| ordered_float_bits(u64::from(value.to_bits()), 32)) + .collect(), + PType::F64 => array + .as_slice::() + .iter() + .map(|&value| ordered_float_bits(value.to_bits(), 64)) + .collect(), + } +} + +fn ordered_float_bits(bits: u64, width: u8) -> u64 { + let sign = 1_u64 << (width - 1); + if bits & sign == 0 { + bits ^ sign + } else { + !bits & width_mask(width) + } +} + +fn float_bits_from_ordered(value: u64, width: u8) -> u64 { + let sign = 1_u64 << (width - 1); + if value & sign == 0 { + !value & width_mask(width) + } else { + value ^ sign + } +} + +fn width_mask(width: u8) -> u64 { + if width == 64 { + u64::MAX + } else { + (1_u64 << width) - 1 + } +} + +fn primitive_from_latents( + latents: Vec, + ptype: PType, + validity: Validity, +) -> VortexResult { + Ok(match ptype { + PType::U8 => PrimitiveArray::new( + Buffer::from(convert_latents(&latents, u8::try_from)?), + validity, + ), + PType::U16 => PrimitiveArray::new( + Buffer::from(convert_latents(&latents, u16::try_from)?), + validity, + ), + PType::U32 => PrimitiveArray::new( + Buffer::from(convert_latents(&latents, u32::try_from)?), + validity, + ), + PType::U64 => PrimitiveArray::new(Buffer::from(latents), validity), + PType::I8 => PrimitiveArray::new( + Buffer::from( + latents + .into_iter() + .map(|value| Ok((u8::try_from(value)? ^ (1_u8 << 7)).cast_signed())) + .collect::>>()?, + ), + validity, + ), + PType::I16 => PrimitiveArray::new( + Buffer::from( + latents + .into_iter() + .map(|value| Ok((u16::try_from(value)? ^ (1_u16 << 15)).cast_signed())) + .collect::>>()?, + ), + validity, + ), + PType::I32 => PrimitiveArray::new( + Buffer::from( + latents + .into_iter() + .map(|value| Ok((u32::try_from(value)? ^ (1_u32 << 31)).cast_signed())) + .collect::>>()?, + ), + validity, + ), + PType::I64 => PrimitiveArray::new( + Buffer::from(latents) + .map_each_in_place(|value| (value ^ (1_u64 << 63)).cast_signed()) + .freeze(), + validity, + ), + PType::F16 => PrimitiveArray::new( + Buffer::from( + latents + .into_iter() + .map(|value| { + Ok(f16::from_bits(u16::try_from(float_bits_from_ordered( + value, 16, + ))?)) + }) + .collect::>>()?, + ), + validity, + ), + PType::F32 => PrimitiveArray::new( + Buffer::from( + latents + .into_iter() + .map(|value| { + Ok(f32::from_bits(u32::try_from(float_bits_from_ordered( + value, 32, + ))?)) + }) + .collect::>>()?, + ), + validity, + ), + PType::F64 => PrimitiveArray::new( + Buffer::from(latents) + .map_each_in_place(|value| f64::from_bits(float_bits_from_ordered(value, 64))) + .freeze(), + validity, + ), + }) +} + +fn convert_latents( + latents: &[u64], + convert: impl Fn(u64) -> Result, +) -> VortexResult> { + latents + .iter() + .copied() + .map(convert) + .collect::, _>>() + .map_err(Into::into) +} + +fn scalar_from_latent(latent: u64, ptype: PType, nullability: Nullability) -> VortexResult { + Ok(match ptype { + PType::U8 => Scalar::primitive(u8::try_from(latent)?, nullability), + PType::U16 => Scalar::primitive(u16::try_from(latent)?, nullability), + PType::U32 => Scalar::primitive(u32::try_from(latent)?, nullability), + PType::U64 => Scalar::primitive(latent, nullability), + PType::I8 => Scalar::primitive( + (u8::try_from(latent)? ^ (1_u8 << 7)).cast_signed(), + nullability, + ), + PType::I16 => Scalar::primitive( + (u16::try_from(latent)? ^ (1_u16 << 15)).cast_signed(), + nullability, + ), + PType::I32 => Scalar::primitive( + (u32::try_from(latent)? ^ (1_u32 << 31)).cast_signed(), + nullability, + ), + PType::I64 => Scalar::primitive((latent ^ (1_u64 << 63)).cast_signed(), nullability), + PType::F16 => Scalar::primitive( + f16::from_bits(u16::try_from(float_bits_from_ordered(latent, 16))?), + nullability, + ), + PType::F32 => Scalar::primitive( + f32::from_bits(u32::try_from(float_bits_from_ordered(latent, 32))?), + nullability, + ), + PType::F64 => Scalar::primitive( + f64::from_bits(float_bits_from_ordered(latent, 64)), + nullability, + ), + }) +} + +#[cfg(test)] +mod tests { + use std::sync::LazyLock; + + use vortex_array::ArrayContext; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::assert_arrays_eq; + use vortex_array::assert_nth_scalar; + use vortex_array::serde::SerializeOptions; + use vortex_array::serde::SerializedArray; + use vortex_buffer::ByteBufferMut; + use vortex_error::VortexResult; + use vortex_session::VortexSession; + use vortex_session::registry::ReadContext; + + use super::*; + + static SESSION: LazyLock = LazyLock::new(|| { + let session = vortex_array::array_session(); + crate::initialize(&session); + session + }); + + fn assert_roundtrip(array: PrimitiveArray) -> VortexResult<()> { + let encoded = RangeEntropy::from_primitive(array.as_view(), 3)?; + let mut ctx = SESSION.create_execution_ctx(); + assert_arrays_eq!(encoded, array, &mut ctx); + + let slice_stop = array.len() - 1; + let slice = encoded.slice(1..slice_stop)?; + assert!(slice.is::()); + assert_arrays_eq!(slice, array.into_array().slice(1..slice_stop)?, &mut ctx); + Ok(()) + } + + #[test] + fn primitive_types_roundtrip() -> VortexResult<()> { + for array in [ + PrimitiveArray::from_iter([0_u8, 1, u8::MAX, 7]), + PrimitiveArray::from_iter([0_u16, 1, u16::MAX, 7]), + PrimitiveArray::from_iter([0_u32, 1, u32::MAX, 7]), + PrimitiveArray::from_iter([0_u64, 1, u64::MAX, 7]), + PrimitiveArray::from_iter([i8::MIN, -1, 0, i8::MAX]), + PrimitiveArray::from_iter([i16::MIN, -1, 0, i16::MAX]), + PrimitiveArray::from_iter([i32::MIN, -1, 0, i32::MAX]), + PrimitiveArray::from_iter([i64::MIN, -1, 0, i64::MAX]), + PrimitiveArray::from_iter([ + f16::NEG_INFINITY, + f16::from_f32(-0.0), + f16::from_f32(1.5), + f16::INFINITY, + ]), + PrimitiveArray::from_iter([f32::NEG_INFINITY, -0.0, 1.5, f32::INFINITY]), + PrimitiveArray::from_iter([f64::NEG_INFINITY, -0.0, 1.5, f64::INFINITY]), + ] { + assert_roundtrip(array)?; + } + Ok(()) + } + + #[test] + fn nullable_slice_and_scalar_access() -> VortexResult<()> { + let array = PrimitiveArray::from_option_iter([ + Some(-10.5_f64), + None, + Some(-0.0), + Some(42.25), + None, + Some(1_000.0), + ]); + let encoded = RangeEntropy::from_primitive(array.as_view(), 2)?; + let mut ctx = SESSION.create_execution_ctx(); + assert_arrays_eq!(encoded, array, &mut ctx); + assert_nth_scalar!(encoded, 3, 42.25_f64, &mut ctx); + assert!(encoded.execute_scalar(1, &mut ctx)?.is_null()); + + let sliced = encoded.slice(1..5)?; + assert!(sliced.is::()); + assert_arrays_eq!(sliced, array.into_array().slice(1..5)?, &mut ctx); + Ok(()) + } + + #[test] + fn serialization_roundtrip() -> VortexResult<()> { + let original = PrimitiveArray::from_option_iter([ + Some(i64::MIN), + None, + Some(-1), + Some(0), + Some(i64::MAX), + ]); + let encoded = RangeEntropy::from_primitive(original.as_view(), 2)?; + let sliced = encoded.slice(1..5)?; + let dtype = sliced.dtype().clone(); + let len = sliced.len(); + let array_context = ArrayContext::empty(); + let serialized = + sliced.serialize(&array_context, &SESSION, &SerializeOptions::default())?; + let mut bytes = ByteBufferMut::empty(); + for buffer in serialized { + bytes.extend_from_slice(buffer.as_ref()); + } + + let decoded = SerializedArray::try_from(bytes.freeze())?.decode( + &dtype, + len, + &ReadContext::new(array_context.to_ids()), + &SESSION, + )?; + assert!(decoded.is::()); + assert_arrays_eq!( + decoded, + original.into_array().slice(1..5)?, + &mut SESSION.create_execution_ctx() + ); + Ok(()) + } +} diff --git a/encodings/range-entropy/src/bit_split.rs b/encodings/range-entropy/src/bit_split.rs new file mode 100644 index 00000000000..22095099cd7 --- /dev/null +++ b/encodings/range-entropy/src/bit_split.rs @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use fastlanes::BitPacking; +use vortex_error::VortexResult; +use vortex_error::vortex_ensure; +use vortex_error::vortex_err; + +const CHUNK_LEN: usize = 1024; +const SERIALIZED_BLOCK_METADATA_BYTES: usize = 8; + +/// A prototype block-local prefix dictionary with fixed-width suffixes. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BitSplitCodec { + len: usize, + blocks: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct BitSplitBlock { + len: u16, + suffix_width: u8, + code_width: u8, + prefixes: Vec, + codes: Vec, + suffixes: Vec, +} + +impl BitSplitCodec { + /// Encode ordered unsigned latents in independent 1,024-value blocks. + pub fn encode(values: &[u64]) -> VortexResult { + let blocks = values + .chunks(CHUNK_LEN) + .map(encode_block) + .collect::>>()?; + Ok(Self { + len: values.len(), + blocks, + }) + } + + /// Decode all values. + pub fn decode(&self) -> VortexResult> { + let mut values = Vec::with_capacity(self.len); + let mut codes = [0_u64; CHUNK_LEN]; + let mut suffixes = [0_u64; CHUNK_LEN]; + for block in &self.blocks { + codes.fill(0); + suffixes.fill(0); + if block.code_width > 0 { + // SAFETY: The encoder creates one complete FastLanes chunk. + unsafe { + u64::unchecked_unpack(usize::from(block.code_width), &block.codes, &mut codes); + } + } + if block.suffix_width > 0 { + // SAFETY: The encoder creates one complete FastLanes chunk. + unsafe { + u64::unchecked_unpack( + usize::from(block.suffix_width), + &block.suffixes, + &mut suffixes, + ); + } + } + + for index in 0..usize::from(block.len) { + let prefix = block.prefixes[usize::try_from(codes[index])?]; + values.push(join(prefix, suffixes[index], block.suffix_width)); + } + } + Ok(values) + } + + /// Decode one value with two direct packed reads and one dictionary lookup. + pub fn scalar_at(&self, index: usize) -> VortexResult { + vortex_ensure!( + index < self.len, + "index {index} is out of bounds for length {}", + self.len + ); + let block = &self.blocks[index / CHUNK_LEN]; + let index_in_block = index % CHUNK_LEN; + let code = if block.code_width == 0 { + 0 + } else { + // SAFETY: The encoder creates one complete FastLanes chunk. + unsafe { + usize::try_from(u64::unchecked_unpack_single( + usize::from(block.code_width), + &block.codes, + index_in_block, + ))? + } + }; + let suffix = if block.suffix_width == 0 { + 0 + } else { + // SAFETY: The encoder creates one complete FastLanes chunk. + unsafe { + u64::unchecked_unpack_single( + usize::from(block.suffix_width), + &block.suffixes, + index_in_block, + ) + } + }; + Ok(join(block.prefixes[code], suffix, block.suffix_width)) + } + + /// Return the encoded bytes, including estimated serialized metadata. + pub fn encoded_size(&self) -> usize { + self.blocks + .iter() + .map(|block| { + SERIALIZED_BLOCK_METADATA_BYTES + + block.prefixes.len() * size_of::() + + block.codes.len() * size_of::() + + block.suffixes.len() * size_of::() + }) + .sum() + } + + /// Return the average suffix width across all blocks. + pub fn average_suffix_width(&self) -> f64 { + self.blocks + .iter() + .map(|block| f64::from(block.suffix_width)) + .sum::() + / self.blocks.len().max(1) as f64 + } + + /// Return the average prefix count across all blocks. + pub fn average_prefix_count(&self) -> f64 { + self.blocks + .iter() + .map(|block| block.prefixes.len() as f64) + .sum::() + / self.blocks.len().max(1) as f64 + } +} + +fn encode_block(values: &[u64]) -> VortexResult { + let mut sorted = values.to_vec(); + sorted.sort_unstable(); + sorted.dedup(); + + let mut divergence_counts = [0_usize; 65]; + for adjacent in sorted.windows(2) { + divergence_counts[usize::from(bit_width(adjacent[0] ^ adjacent[1]))] += 1; + } + let mut prefix_counts = [0_usize; 65]; + let mut active_divergences = 0_usize; + for suffix_width in (0..=64_usize).rev() { + prefix_counts[suffix_width] = usize::from(!sorted.is_empty()) + active_divergences; + active_divergences += divergence_counts[suffix_width]; + } + + let mut best_suffix_width = 64_u8; + let mut best_prefix_count = 1_usize; + let mut best_size = CHUNK_LEN * size_of::(); + for suffix_width in 0..=64_u8 { + let prefix_count = prefix_counts[usize::from(suffix_width)]; + let code_width = bit_width(u64::try_from(prefix_count.saturating_sub(1))?); + let size = prefix_count * size_of::() + + (CHUNK_LEN * usize::from(code_width)).div_ceil(8) + + (CHUNK_LEN * usize::from(suffix_width)).div_ceil(8); + if size < best_size { + best_size = size; + best_suffix_width = suffix_width; + best_prefix_count = prefix_count; + } + } + + let mut prefixes = Vec::with_capacity(best_prefix_count); + for value in sorted { + let prefix = prefix(value, best_suffix_width); + if prefixes.last() != Some(&prefix) { + prefixes.push(prefix); + } + } + let code_width = bit_width(u64::try_from(prefixes.len().saturating_sub(1))?); + let suffix_mask = low_mask(best_suffix_width); + let mut code_values = [0_u64; CHUNK_LEN]; + let mut suffix_values = [0_u64; CHUNK_LEN]; + for (index, &value) in values.iter().enumerate() { + let prefix = prefix(value, best_suffix_width); + let code = prefixes + .binary_search(&prefix) + .map_err(|_| vortex_err!("prefix dictionary does not contain {prefix}"))?; + code_values[index] = u64::try_from(code)?; + suffix_values[index] = value & suffix_mask; + } + + Ok(BitSplitBlock { + len: u16::try_from(values.len())?, + suffix_width: best_suffix_width, + code_width, + prefixes, + codes: fast_pack(&code_values, code_width), + suffixes: fast_pack(&suffix_values, best_suffix_width), + }) +} + +fn prefix(value: u64, suffix_width: u8) -> u64 { + if suffix_width == 64 { + 0 + } else { + value >> suffix_width + } +} + +fn join(prefix: u64, suffix: u64, suffix_width: u8) -> u64 { + if suffix_width == 64 { + suffix + } else { + (prefix << suffix_width) | suffix + } +} + +fn fast_pack(values: &[u64; CHUNK_LEN], width: u8) -> Vec { + if width == 0 { + return Vec::new(); + } + let mut packed = vec![0_u64; CHUNK_LEN * usize::from(width) / u64::BITS as usize]; + // SAFETY: Both slices have the exact lengths required for one FastLanes chunk. + unsafe { u64::unchecked_pack(usize::from(width), values, &mut packed) }; + packed +} + +fn bit_width(value: u64) -> u8 { + u8::try_from(u64::BITS - value.leading_zeros()).unwrap_or(64) +} + +fn low_mask(bits: u8) -> u64 { + match bits { + 0 => 0, + 64 => u64::MAX, + _ => (1_u64 << bits) - 1, + } +} + +#[cfg(test)] +mod tests { + use vortex_error::VortexResult; + + use super::BitSplitCodec; + + #[test] + fn roundtrip_and_scalar_access() -> VortexResult<()> { + let values = (0..2_051) + .map(|index| { + let prefix = [11_u64, 1_000, 5_000, 90_000][index % 4]; + Ok((prefix << 19) | u64::try_from(index * 17)?) + }) + .collect::>>()?; + let codec = BitSplitCodec::encode(&values)?; + assert_eq!(codec.decode()?, values); + for (index, &value) in values.iter().enumerate() { + assert_eq!(codec.scalar_at(index)?, value); + } + Ok(()) + } +} diff --git a/encodings/range-entropy/src/block_residual_array.rs b/encodings/range-entropy/src/block_residual_array.rs new file mode 100644 index 00000000000..2f247e5be19 --- /dev/null +++ b/encodings/range-entropy/src/block_residual_array.rs @@ -0,0 +1,765 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::fmt::Display; +use std::fmt::Formatter; +use std::hash::Hash; +use std::hash::Hasher; +use std::ops::Range; + +use fastlanes::BitPacking; +use vortex_array::Array; +use vortex_array::ArrayEq; +use vortex_array::ArrayHash; +use vortex_array::ArrayId; +use vortex_array::ArrayParts; +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::EqMode; +use vortex_array::ExecutionCtx; +use vortex_array::ExecutionResult; +use vortex_array::IntoArray; +use vortex_array::TypedArrayRef; +use vortex_array::array_slots; +use vortex_array::arrays::Primitive; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::slice::SliceReduce; +use vortex_array::arrays::slice::SliceReduceAdaptor; +use vortex_array::buffer::BufferHandle; +use vortex_array::dtype::DType; +use vortex_array::dtype::NativePType; +use vortex_array::dtype::Nullability::NonNullable; +use vortex_array::dtype::PType; +use vortex_array::optimizer::rules::ParentRuleSet; +use vortex_array::scalar::Scalar; +use vortex_array::serde::ArrayChildren; +use vortex_array::validity::Validity; +use vortex_array::vtable::OperationsVTable; +use vortex_array::vtable::VTable; +use vortex_array::vtable::ValidityVTable; +use vortex_array::vtable::child_to_validity; +use vortex_array::vtable::validity_to_child; +use vortex_buffer::Buffer; +use vortex_error::VortexResult; +use vortex_error::vortex_bail; +use vortex_error::vortex_ensure; +use vortex_error::vortex_panic; +use vortex_session::VortexSession; +use vortex_session::registry::CachedId; + +use crate::BlockResidualParts; +use crate::PatchedFoRCodec; +use crate::patched_for::read_wide_bits; + +const BLOCK_LEN: usize = 1024; +const METADATA_VERSION: u8 = 1; +const METADATA_LEN: usize = 41; + +/// Ordered unsigned integers with one reference and packed residuals per block. +pub type BlockResidualArray = Array; + +#[array_slots(BlockResidual)] +pub struct BlockResidualSlots { + #[slot(0)] + pub bases: ArrayRef, + #[slot(1)] + pub residual_widths: ArrayRef, + #[slot(2)] + pub high_widths: ArrayRef, + #[slot(3)] + pub residual_starts: ArrayRef, + #[slot(4)] + pub patch_starts: ArrayRef, + #[slot(5)] + pub high_starts: ArrayRef, + #[slot(6)] + pub residual_words: ArrayRef, + #[slot(7)] + pub patch_positions: ArrayRef, + #[slot(8)] + pub patch_highs: ArrayRef, + #[slot(9)] + pub validity: Option, +} + +#[derive(Clone, Debug)] +pub struct BlockResidualData { + unsliced_len: usize, + slice_start: usize, + slice_stop: usize, + residual_word_count: usize, + patch_count: usize, + patch_high_count: usize, +} + +impl Display for BlockResidualData { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "blocks: {}, slice: {}..{}", + self.unsliced_len.div_ceil(BLOCK_LEN), + self.slice_start, + self.slice_stop + ) + } +} + +impl ArrayHash for BlockResidualData { + fn array_hash(&self, state: &mut H, _accuracy: EqMode) { + self.unsliced_len.hash(state); + self.slice_start.hash(state); + self.slice_stop.hash(state); + self.residual_word_count.hash(state); + self.patch_count.hash(state); + self.patch_high_count.hash(state); + } +} + +impl ArrayEq for BlockResidualData { + fn array_eq(&self, other: &Self, _accuracy: EqMode) -> bool { + self.unsliced_len == other.unsliced_len + && self.slice_start == other.slice_start + && self.slice_stop == other.slice_stop + && self.residual_word_count == other.residual_word_count + && self.patch_count == other.patch_count + && self.patch_high_count == other.patch_high_count + } +} + +#[derive(Clone, Debug)] +pub struct BlockResidual; + +impl VTable for BlockResidual { + type TypedArrayData = BlockResidualData; + type OperationsVTable = Self; + type ValidityVTable = Self; + + fn id(&self) -> ArrayId { + static ID: CachedId = CachedId::new("vortex.block_residual"); + *ID + } + + fn validate( + &self, + data: &Self::TypedArrayData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + let slots = BlockResidualSlotsView::from_slots(slots); + let validity = child_to_validity(slots.validity, dtype.nullability()); + data.validate(dtype, len, slots, &validity) + } + + fn nbuffers(_array: ArrayView<'_, Self>) -> usize { + 0 + } + + fn buffer(_array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { + vortex_panic!("BlockResidualArray buffer index {idx} out of bounds") + } + + fn buffer_name(_array: ArrayView<'_, Self>, idx: usize) -> Option { + vortex_panic!("BlockResidualArray buffer_name {idx} out of bounds") + } + + fn with_buffers( + &self, + array: ArrayView<'_, Self>, + buffers: &[BufferHandle], + ) -> VortexResult> { + vortex_array::vtable::with_empty_buffers(self, array, buffers) + } + + fn serialize( + array: ArrayView<'_, Self>, + _session: &VortexSession, + ) -> VortexResult>> { + Ok(Some( + BlockResidualMetadata::from_data(array.data())?.encode(), + )) + } + + fn deserialize( + &self, + dtype: &DType, + len: usize, + metadata: &[u8], + _buffers: &[BufferHandle], + children: &dyn ArrayChildren, + _session: &VortexSession, + ) -> VortexResult> { + let metadata = BlockResidualMetadata::decode(metadata)?; + let unsliced_len = usize::try_from(metadata.unsliced_len)?; + let slice_start = usize::try_from(metadata.slice_start)?; + let slice_stop = slice_start + .checked_add(len) + .ok_or_else(|| vortex_error::vortex_err!("block residual slice length overflows"))?; + let residual_word_count = usize::try_from(metadata.residual_word_count)?; + let patch_count = usize::try_from(metadata.patch_count)?; + let patch_high_count = usize::try_from(metadata.patch_high_count)?; + let block_count = unsliced_len.div_ceil(BLOCK_LEN); + let bases = children.get(0, &primitive_dtype(PType::U64), block_count)?; + let residual_widths = children.get(1, &primitive_dtype(PType::U8), block_count)?; + let high_widths = children.get(2, &primitive_dtype(PType::U8), block_count)?; + let residual_starts = children.get(3, &primitive_dtype(PType::U32), block_count + 1)?; + let patch_starts = children.get(4, &primitive_dtype(PType::U32), block_count + 1)?; + let high_starts = children.get(5, &primitive_dtype(PType::U32), block_count + 1)?; + let residual_words = children.get(6, &primitive_dtype(PType::U64), residual_word_count)?; + let patch_positions = children.get(7, &primitive_dtype(PType::U16), patch_count)?; + let patch_highs = children.get(8, &primitive_dtype(PType::U8), patch_high_count)?; + let validity = match children.len() { + 9 => Validity::from(dtype.nullability()), + 10 => Validity::Array(children.get(9, &Validity::DTYPE, unsliced_len)?), + count => vortex_bail!("BlockResidualArray expects nine or ten children, got {count}"), + }; + let slots = BlockResidualSlots { + bases, + residual_widths, + high_widths, + residual_starts, + patch_starts, + high_starts, + residual_words, + patch_positions, + patch_highs, + validity: validity_to_child(&validity, unsliced_len), + } + .into_slots(); + let data = BlockResidualData { + unsliced_len, + slice_start, + slice_stop, + residual_word_count, + patch_count, + patch_high_count, + }; + Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) + } + + fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { + BlockResidualSlots::NAMES[idx].to_string() + } + + fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionResult::done( + decompress_array(array.as_view(), ctx)?.into_array(), + )) + } + + fn reduce_parent( + array: ArrayView<'_, Self>, + parent: &ArrayRef, + child_idx: usize, + ) -> VortexResult> { + RULES.evaluate(array, parent, child_idx) + } +} + +impl OperationsVTable for BlockResidual { + fn scalar_at( + array: ArrayView<'_, BlockResidual>, + index: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + let value = scalar_from_array(array, index, ctx)?; + Ok(Scalar::primitive(value, array.dtype().nullability())) + } +} + +impl ValidityVTable for BlockResidual { + fn validity(array: ArrayView<'_, BlockResidual>) -> VortexResult { + array + .unsliced_validity() + .slice(array.data().slice_start..array.data().slice_stop) + } +} + +impl SliceReduce for BlockResidual { + fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { + let data = array.data().slice(range); + Ok(Some( + Array::try_from_parts( + ArrayParts::new(BlockResidual, array.dtype().clone(), data.len(), data) + .with_slots(array.slots().iter().cloned().collect()), + )? + .into_array(), + )) + } +} + +static RULES: ParentRuleSet = + ParentRuleSet::new(&[ParentRuleSet::lift(&SliceReduceAdaptor(BlockResidual))]); + +pub trait BlockResidualArrayExt: TypedArrayRef + BlockResidualArraySlotsExt { + fn unsliced_validity(&self) -> Validity { + child_to_validity( + self.as_ref().slots()[BlockResidualSlots::VALIDITY].as_ref(), + self.as_ref().dtype().nullability(), + ) + } + + /// Decode the logical slice. + fn decompress(&self, ctx: &mut ExecutionCtx) -> VortexResult { + decompress_array(self.to_owned().as_view(), ctx) + } +} + +impl> BlockResidualArrayExt for T {} + +impl BlockResidual { + /// Encode a non-negative `u64` array in independent blocks. + pub fn from_primitive(array: ArrayView<'_, Primitive>) -> VortexResult { + vortex_ensure!( + array.ptype() == PType::U64, + "BlockResidual requires u64 values" + ); + let validity = array.validity()?; + let parts = PatchedFoRCodec::encode_single_base(array.as_slice::())? + .into_single_base_parts()?; + Self::try_new(parts, validity) + } + + fn try_new(parts: BlockResidualParts, validity: Validity) -> VortexResult { + let data = BlockResidualData { + unsliced_len: parts.len, + slice_start: 0, + slice_stop: parts.len, + residual_word_count: parts.residual_words.len(), + patch_count: parts.patch_positions.len(), + patch_high_count: parts.patch_highs.len(), + }; + let slots = BlockResidualSlots { + bases: PrimitiveArray::from_iter(parts.bases).into_array(), + residual_widths: PrimitiveArray::from_iter(parts.residual_widths).into_array(), + high_widths: PrimitiveArray::from_iter(parts.high_widths).into_array(), + residual_starts: PrimitiveArray::from_iter(parts.residual_starts).into_array(), + patch_starts: PrimitiveArray::from_iter(parts.patch_starts).into_array(), + high_starts: PrimitiveArray::from_iter(parts.high_starts).into_array(), + residual_words: PrimitiveArray::from_iter(parts.residual_words).into_array(), + patch_positions: PrimitiveArray::from_iter(parts.patch_positions).into_array(), + patch_highs: PrimitiveArray::from_iter(parts.patch_highs).into_array(), + validity: validity_to_child(&validity, data.unsliced_len), + } + .into_slots(); + Array::try_from_parts( + ArrayParts::new( + BlockResidual, + DType::Primitive(PType::U64, validity.nullability()), + data.unsliced_len, + data, + ) + .with_slots(slots), + ) + } +} + +impl BlockResidualData { + fn validate( + &self, + dtype: &DType, + len: usize, + slots: BlockResidualSlotsView<'_>, + validity: &Validity, + ) -> VortexResult<()> { + vortex_ensure!( + dtype.as_ptype() == PType::U64, + "BlockResidualArray requires a u64 dtype" + ); + vortex_ensure!( + self.slice_start <= self.slice_stop && self.slice_stop <= self.unsliced_len, + "block residual slice exceeds its source length" + ); + vortex_ensure!(len == self.len(), "block residual slice length is invalid"); + let block_count = self.unsliced_len.div_ceil(BLOCK_LEN); + for (child, ptype, child_len) in [ + (slots.bases, PType::U64, block_count), + (slots.residual_widths, PType::U8, block_count), + (slots.high_widths, PType::U8, block_count), + (slots.residual_starts, PType::U32, block_count + 1), + (slots.patch_starts, PType::U32, block_count + 1), + (slots.high_starts, PType::U32, block_count + 1), + (slots.residual_words, PType::U64, self.residual_word_count), + (slots.patch_positions, PType::U16, self.patch_count), + (slots.patch_highs, PType::U8, self.patch_high_count), + ] { + vortex_ensure!( + child.dtype() == &primitive_dtype(ptype) && child.len() == child_len, + "block residual child has an invalid dtype or length" + ); + } + if let Some(validity_len) = validity.maybe_len() { + vortex_ensure!( + validity_len == self.unsliced_len, + "block residual validity length is invalid" + ); + } + Ok(()) + } + + fn len(&self) -> usize { + self.slice_stop - self.slice_start + } + + fn slice(&self, range: Range) -> Self { + Self { + slice_start: self.slice_start + range.start, + slice_stop: self.slice_start + range.end, + ..self.clone() + } + } +} + +struct ExecutedBlockResidual { + bases: PrimitiveArray, + residual_widths: PrimitiveArray, + high_widths: PrimitiveArray, + residual_starts: PrimitiveArray, + patch_starts: PrimitiveArray, + high_starts: PrimitiveArray, + residual_words: PrimitiveArray, + patch_positions: PrimitiveArray, + patch_highs: PrimitiveArray, +} + +impl ExecutedBlockResidual { + fn new(array: ArrayView<'_, BlockResidual>, ctx: &mut ExecutionCtx) -> VortexResult { + macro_rules! execute_child { + ($accessor:ident) => { + array.$accessor().clone().execute::(ctx)? + }; + } + Ok(Self { + bases: execute_child!(bases), + residual_widths: execute_child!(residual_widths), + high_widths: execute_child!(high_widths), + residual_starts: execute_child!(residual_starts), + patch_starts: execute_child!(patch_starts), + high_starts: execute_child!(high_starts), + residual_words: execute_child!(residual_words), + patch_positions: execute_child!(patch_positions), + patch_highs: execute_child!(patch_highs), + }) + } +} + +fn decode_array_values( + array: ArrayView<'_, BlockResidual>, + ctx: &mut ExecutionCtx, + mut transform: impl FnMut(u64) -> T, +) -> VortexResult { + let children = ExecutedBlockResidual::new(array, ctx)?; + let bases = children.bases.as_slice::(); + let residual_widths = children.residual_widths.as_slice::(); + let high_widths = children.high_widths.as_slice::(); + let residual_starts = children.residual_starts.as_slice::(); + let patch_starts = children.patch_starts.as_slice::(); + let high_starts = children.high_starts.as_slice::(); + let residual_words = children.residual_words.as_slice::(); + let patch_positions = children.patch_positions.as_slice::(); + let patch_highs = children.patch_highs.as_slice::(); + let logical_range = array.data().slice_start..array.data().slice_stop; + let mut values = Vec::with_capacity(logical_range.len()); + let mut residuals = [0_u64; BLOCK_LEN]; + + for block_index in 0..bases.len() { + let block_start = block_index * BLOCK_LEN; + let block_len = (array.data().unsliced_len - block_start).min(BLOCK_LEN); + let block_stop = block_start + block_len; + if block_stop <= logical_range.start || block_start >= logical_range.end { + continue; + } + + residuals.fill(0); + let residual_width = residual_widths[block_index]; + let high_width = high_widths[block_index]; + vortex_ensure!( + residual_width <= 64 + && high_width <= 64 + && u16::from(residual_width) + u16::from(high_width) <= 64, + "block residual bit widths are invalid" + ); + let residual_payload = payload_range( + residual_starts, + block_index, + residual_words.len(), + "residual", + )?; + vortex_ensure!( + residual_payload.len() == BLOCK_LEN * usize::from(residual_width) / 64, + "block residual word count is invalid" + ); + if residual_width > 0 { + // SAFETY: The encoder writes one complete FastLanes chunk for each block. + unsafe { + u64::unchecked_unpack( + usize::from(residual_width), + &residual_words[residual_payload], + &mut residuals, + ); + } + } + + let patch_payload = + payload_range(patch_starts, block_index, patch_positions.len(), "patch")?; + let high_payload = + payload_range(high_starts, block_index, patch_highs.len(), "patch high")?; + let positions = &patch_positions[patch_payload]; + vortex_ensure!( + positions + .iter() + .all(|&position| usize::from(position) < block_len) + && positions.windows(2).all(|pair| pair[0] < pair[1]), + "block residual patch positions are invalid" + ); + let expected_high_len = if positions.is_empty() { + 0 + } else { + (positions.len() * usize::from(high_width)).div_ceil(8) + 15 + }; + vortex_ensure!( + high_payload.len() == expected_high_len, + "block residual patch high payload is invalid" + ); + let highs = &patch_highs[high_payload]; + for (patch_index, &position) in positions.iter().enumerate() { + // SAFETY: The payload includes fifteen readable padding bytes. + let high = + unsafe { read_wide_bits(highs, patch_index * usize::from(high_width), high_width) }; + residuals[usize::from(position)] |= high << residual_width; + } + + let local_start = logical_range.start.saturating_sub(block_start); + let local_stop = (logical_range.end - block_start).min(block_len); + values.extend( + residuals[local_start..local_stop] + .iter() + .map(|&residual| transform(bases[block_index].wrapping_add(residual))), + ); + } + Ok(PrimitiveArray::new(Buffer::from(values), array.validity()?)) +} + +fn decompress_array( + array: ArrayView<'_, BlockResidual>, + ctx: &mut ExecutionCtx, +) -> VortexResult { + decode_array_values(array, ctx, |value| value) +} + +pub(crate) fn decompress_ordered_f64( + array: ArrayView<'_, BlockResidual>, + ctx: &mut ExecutionCtx, +) -> VortexResult { + decode_array_values(array, ctx, |ordered| { + let bits = if ordered & (1_u64 << 63) == 0 { + !ordered + } else { + ordered ^ (1_u64 << 63) + }; + f64::from_bits(bits) + }) +} + +fn scalar_from_array( + array: ArrayView<'_, BlockResidual>, + index: usize, + ctx: &mut ExecutionCtx, +) -> VortexResult { + let source_index = array.data().slice_start + index; + let block_index = source_index / BLOCK_LEN; + let index_in_block = source_index % BLOCK_LEN; + let children = ExecutedBlockResidual::new(array, ctx)?; + let residual_width = children.residual_widths.as_slice::()[block_index]; + let high_width = children.high_widths.as_slice::()[block_index]; + vortex_ensure!( + residual_width <= 64 + && high_width <= 64 + && u16::from(residual_width) + u16::from(high_width) <= 64, + "block residual bit widths are invalid" + ); + let residual_words = children.residual_words.as_slice::(); + let residual_payload = payload_range( + children.residual_starts.as_slice::(), + block_index, + residual_words.len(), + "residual", + )?; + vortex_ensure!( + residual_payload.len() == BLOCK_LEN * usize::from(residual_width) / 64, + "block residual word count is invalid" + ); + let mut residual = if residual_width == 0 { + 0 + } else { + // SAFETY: The encoder writes one complete FastLanes chunk for each block. + unsafe { + u64::unchecked_unpack_single( + usize::from(residual_width), + &residual_words[residual_payload], + index_in_block, + ) + } + }; + + let positions = children.patch_positions.as_slice::(); + let patch_payload = payload_range( + children.patch_starts.as_slice::(), + block_index, + positions.len(), + "patch", + )?; + let block_positions = &positions[patch_payload]; + if let Ok(patch_index) = block_positions.binary_search(&u16::try_from(index_in_block)?) { + let highs = children.patch_highs.as_slice::(); + let high_payload = payload_range( + children.high_starts.as_slice::(), + block_index, + highs.len(), + "patch high", + )?; + let high_payload = &highs[high_payload]; + vortex_ensure!( + high_payload.len() + >= (block_positions.len() * usize::from(high_width)).div_ceil(8) + 15, + "block residual patch high payload is invalid" + ); + // SAFETY: The payload includes fifteen readable padding bytes. + let high = unsafe { + read_wide_bits( + high_payload, + patch_index * usize::from(high_width), + high_width, + ) + }; + residual |= high << residual_width; + } + Ok(children.bases.as_slice::()[block_index].wrapping_add(residual)) +} + +fn payload_range( + starts: &[u32], + block_index: usize, + payload_len: usize, + name: &str, +) -> VortexResult> { + let start = usize::try_from( + *starts + .get(block_index) + .ok_or_else(|| vortex_error::vortex_err!("block residual {name} start is missing"))?, + )?; + let stop = usize::try_from( + *starts + .get(block_index + 1) + .ok_or_else(|| vortex_error::vortex_err!("block residual {name} stop is missing"))?, + )?; + vortex_ensure!( + start <= stop && stop <= payload_len, + "block residual {name} offsets are invalid" + ); + Ok(start..stop) +} + +#[derive(Clone, Copy)] +struct BlockResidualMetadata { + unsliced_len: u64, + slice_start: u64, + residual_word_count: u64, + patch_count: u64, + patch_high_count: u64, +} + +impl BlockResidualMetadata { + fn from_data(data: &BlockResidualData) -> VortexResult { + Ok(Self { + unsliced_len: u64::try_from(data.unsliced_len)?, + slice_start: u64::try_from(data.slice_start)?, + residual_word_count: u64::try_from(data.residual_word_count)?, + patch_count: u64::try_from(data.patch_count)?, + patch_high_count: u64::try_from(data.patch_high_count)?, + }) + } + + fn encode(self) -> Vec { + let mut bytes = Vec::with_capacity(METADATA_LEN); + bytes.push(METADATA_VERSION); + for value in [ + self.unsliced_len, + self.slice_start, + self.residual_word_count, + self.patch_count, + self.patch_high_count, + ] { + bytes.extend_from_slice(&value.to_le_bytes()); + } + bytes + } + + fn decode(bytes: &[u8]) -> VortexResult { + vortex_ensure!( + bytes.len() == METADATA_LEN, + "BlockResidualArray metadata requires {METADATA_LEN} bytes" + ); + vortex_ensure!( + bytes[0] == METADATA_VERSION, + "unsupported BlockResidualArray metadata version {}", + bytes[0] + ); + let read = |offset: usize| { + u64::from_le_bytes([ + bytes[offset], + bytes[offset + 1], + bytes[offset + 2], + bytes[offset + 3], + bytes[offset + 4], + bytes[offset + 5], + bytes[offset + 6], + bytes[offset + 7], + ]) + }; + Ok(Self { + unsliced_len: read(1), + slice_start: read(9), + residual_word_count: read(17), + patch_count: read(25), + patch_high_count: read(33), + }) + } +} + +fn primitive_dtype(ptype: PType) -> DType { + DType::Primitive(ptype, NonNullable) +} + +#[cfg(test)] +mod tests { + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::array_session; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::assert_arrays_eq; + use vortex_error::VortexResult; + + use super::BlockResidual; + + #[test] + fn roundtrip_and_scalar_access() -> VortexResult<()> { + let values = (0..4_099) + .map(|index| Ok(1_000_000_u64 + u64::try_from(index * index)?)) + .collect::>>()?; + let primitive = PrimitiveArray::from_iter(values.clone()); + let encoded = BlockResidual::from_primitive(primitive.as_view())?; + let session = array_session(); + crate::initialize(&session); + let mut ctx = session.create_execution_ctx(); + assert_arrays_eq!(encoded.clone(), primitive.into_array(), &mut ctx); + for index in [0, 1, 1_023, 1_024, 4_098] { + let scalar = encoded.execute_scalar(index, &mut ctx)?; + assert_eq!( + scalar.as_primitive().typed_value::(), + Some(values[index]) + ); + } + Ok(()) + } +} diff --git a/encodings/range-entropy/src/codec.rs b/encodings/range-entropy/src/codec.rs new file mode 100644 index 00000000000..05ec8f09531 --- /dev/null +++ b/encodings/range-entropy/src/codec.rs @@ -0,0 +1,2337 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::cmp::Ordering; +use std::cmp::Reverse; +use std::ops::Deref; +use std::ops::DerefMut; +use std::ops::Range; + +use vortex_buffer::ByteBuffer; +use vortex_error::VortexResult; +use vortex_error::vortex_ensure; + +const MAX_SCALE_BITS: u8 = 10; +const ANS_INTERLEAVING: usize = 4; +const ANS_STATE_BYTES: usize = ANS_INTERLEAVING * size_of::(); +const ANS_PADDING: usize = size_of::(); +const DECODE_BATCH_SIZE: usize = 256; +const OFFSET_PADDING: usize = 15; +const MAX_BINS: usize = 64; +const SPLIT_CANDIDATES: usize = 64; +const TRAINING_SAMPLE_SIZE: usize = 8192; +const BIN_TABLE_COST_BITS: f64 = 128.0; + +#[repr(align(64))] +struct DecodeScratch([T; DECODE_BATCH_SIZE]); + +impl Deref for DecodeScratch { + type Target = [T; DECODE_BATCH_SIZE]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for DecodeScratch { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +/// A restart-block range entropy stream for ordered unsigned latents. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RangeEntropyCodec { + len: usize, + block_len: usize, + scale_bits: u8, + bin_lowers: Vec, + offset_widths: Vec, + weights: Vec, + block_offsets: Vec, + payload: ByteBuffer, +} + +/// Owned components of a range entropy stream. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RangeEntropyParts { + /// Logical value count. + pub len: usize, + /// Logical values per restart block. + pub block_len: usize, + /// Base-two logarithm of the ANS table size. + pub scale_bits: u8, + /// Lower bound for each range bin. + pub bin_lowers: Vec, + /// Fixed offset width for each range bin. + pub offset_widths: Vec, + /// Quantized ANS weight for each range bin. + pub weights: Vec, + /// Byte offsets for restart blocks. + pub block_offsets: Vec, + /// Encoded ANS and offset streams. + pub payload: ByteBuffer, +} + +/// A restart-block range codec with fixed-width bin identifiers. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RangePackedCodec { + len: usize, + block_len: usize, + symbol_width: u8, + bin_lowers: Vec, + offset_widths: Vec, + block_offsets: Vec, + payload: ByteBuffer, +} + +/// A restart-block range codec with hot-bin tags and a cold-bin escape stream. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RangeTwoLevelCodec { + len: usize, + block_len: usize, + tag_width: u8, + cold_width: u8, + bin_lowers: Vec, + offset_widths: Vec, + block_offsets: Vec, + payload: ByteBuffer, +} + +/// A restart-block range codec with one fixed-width residual stream per bin. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RangeGroupedCodec { + len: usize, + block_len: usize, + symbol_width: u8, + bin_lowers: Vec, + offset_widths: Vec, + block_offsets: Vec, + payload: ByteBuffer, +} + +impl RangeEntropyCodec { + /// Encode ordered unsigned latents into independent restart blocks. + pub fn encode(values: &[u64], block_len: usize) -> VortexResult { + vortex_ensure!(block_len > 0, "block length must be greater than zero"); + + if values.is_empty() { + return Ok(Self { + len: 0, + block_len, + scale_bits: 0, + bin_lowers: Vec::new(), + offset_widths: Vec::new(), + weights: Vec::new(), + block_offsets: vec![0], + payload: ByteBuffer::empty(), + }); + } + + let (training_sample, minimum, maximum) = training_sample(values); + let bins = cover_domain(optimize_bins(&training_sample), minimum, maximum); + let (bins, symbols, counts) = assign_bins(values, bins)?; + let bin_lowers: Vec<_> = bins.iter().map(|bin| bin.lower).collect(); + let offset_widths: Vec<_> = bins.iter().map(|bin| bin.offset_bits).collect(); + + let scale_bits = choose_scale_bits(values.len(), bins.len()); + let weights = quantize_weights(&counts, scale_bits)?; + let table = AnsEncoderTable::new(&weights, scale_bits)?; + + let n_blocks = values.len().div_ceil(block_len); + let mut block_offsets = Vec::with_capacity(n_blocks + 1); + let mut payload = Vec::new(); + block_offsets.push(0); + + for block_idx in 0..n_blocks { + let start = block_idx * block_len; + let stop = (start + block_len).min(values.len()); + encode_block( + &values[start..stop], + &symbols[start..stop], + &bins, + &table, + &mut payload, + )?; + block_offsets.push(u32::try_from(payload.len())?); + } + + let codec = Self { + len: values.len(), + block_len, + scale_bits, + bin_lowers, + offset_widths, + weights, + block_offsets, + payload: ByteBuffer::from(payload), + }; + codec.validate()?; + Ok(codec) + } + + /// Construct a codec from stored components. + pub fn try_from_parts(parts: RangeEntropyParts) -> VortexResult { + let codec = Self { + len: parts.len, + block_len: parts.block_len, + scale_bits: parts.scale_bits, + bin_lowers: parts.bin_lowers, + offset_widths: parts.offset_widths, + weights: parts.weights, + block_offsets: parts.block_offsets, + payload: parts.payload, + }; + codec.validate()?; + Ok(codec) + } + + /// Consume the codec and return its stored components. + pub fn into_parts(self) -> RangeEntropyParts { + RangeEntropyParts { + len: self.len, + block_len: self.block_len, + scale_bits: self.scale_bits, + bin_lowers: self.bin_lowers, + offset_widths: self.offset_widths, + weights: self.weights, + block_offsets: self.block_offsets, + payload: self.payload, + } + } + + /// Decode all values. + pub fn decode(&self) -> VortexResult> { + self.decode_range(0..self.len) + } + + /// Decode one logical range through the intersecting restart blocks. + pub fn decode_range(&self, range: Range) -> VortexResult> { + self.validate()?; + vortex_ensure!( + range.start <= range.end && range.end <= self.len, + "decode range {:?} exceeds length {}", + range, + self.len + ); + if range.is_empty() { + return Ok(Vec::new()); + } + + let first_block = range.start / self.block_len; + let last_block = (range.end - 1) / self.block_len; + let bins = self.bins()?; + let table = AnsDecoderTable::new(&self.weights, self.scale_bits, &bins)?; + let mut values = Vec::with_capacity(range.len()); + for block_idx in first_block..=last_block { + let block_start = block_idx * self.block_len; + let start = range.start.saturating_sub(block_start); + let stop = (range.end - block_start).min(self.block_value_count(block_idx)); + self.decode_block_into(block_idx, start..stop, &table, &mut values)?; + } + Ok(values) + } + + /// Decode one restart block. + pub fn decode_block(&self, block_idx: usize) -> VortexResult> { + self.validate()?; + vortex_ensure!( + block_idx < self.n_blocks(), + "block index {block_idx} is out of bounds for {} blocks", + self.n_blocks() + ); + + let bins = self.bins()?; + let table = AnsDecoderTable::new(&self.weights, self.scale_bits, &bins)?; + let n_values = self.block_value_count(block_idx); + let mut values = Vec::with_capacity(n_values); + self.decode_block_into(block_idx, 0..n_values, &table, &mut values)?; + Ok(values) + } + + fn decode_block_into( + &self, + block_idx: usize, + output_range: Range, + table: &AnsDecoderTable, + values: &mut Vec, + ) -> VortexResult<()> { + let start = usize::try_from(self.block_offsets[block_idx])?; + let stop = usize::try_from(self.block_offsets[block_idx + 1])?; + let n_values = self.block_value_count(block_idx); + decode_block_into( + &self.payload.as_slice()[start..stop], + n_values, + output_range, + table, + values, + ) + } + + /// Return one value after one restart-block decode. + pub fn value(&self, index: usize) -> VortexResult { + vortex_ensure!( + index < self.len, + "value index {index} is out of bounds for length {}", + self.len + ); + let block_idx = index / self.block_len; + let within_block = index % self.block_len; + Ok(self.decode_block(block_idx)?[within_block]) + } + + /// Return the logical value count. + pub fn len(&self) -> usize { + self.len + } + + /// Return true when the stream has no values. + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + /// Return the restart block length. + pub fn block_len(&self) -> usize { + self.block_len + } + + /// Return the ANS table size log. + pub fn scale_bits(&self) -> u8 { + self.scale_bits + } + + /// Return the bin lower bounds. + pub fn bin_lowers(&self) -> &[u64] { + &self.bin_lowers + } + + /// Return the fixed offset width for each bin. + pub fn offset_widths(&self) -> &[u8] { + &self.offset_widths + } + + /// Return the quantized ANS weight for each bin. + pub fn weights(&self) -> &[u16] { + &self.weights + } + + /// Return the byte offsets for restart blocks. + pub fn block_offsets(&self) -> &[u32] { + &self.block_offsets + } + + /// Return the encoded payload. + pub fn payload(&self) -> &[u8] { + self.payload.as_slice() + } + + /// Return the total bytes in the variable tables and payload. + pub fn encoded_size(&self) -> usize { + self.bin_lowers.len() * size_of::() + + self.offset_widths.len() * size_of::() + + self.weights.len() * size_of::() + + self.block_offsets.len() * size_of::() + + self.payload.len() + } + + fn validate(&self) -> VortexResult<()> { + vortex_ensure!(self.block_len > 0, "block length must be greater than zero"); + vortex_ensure!( + self.scale_bits <= MAX_SCALE_BITS, + "ANS table log {} exceeds {MAX_SCALE_BITS}", + self.scale_bits + ); + vortex_ensure!( + self.bin_lowers.len() == self.offset_widths.len() + && self.bin_lowers.len() == self.weights.len(), + "range entropy bin tables have different lengths" + ); + + if self.len == 0 { + vortex_ensure!(self.bin_lowers.is_empty(), "empty streams cannot have bins"); + vortex_ensure!( + self.payload.is_empty(), + "empty streams cannot have payload bytes" + ); + vortex_ensure!( + self.block_offsets == [0], + "empty streams require one zero block offset" + ); + return Ok(()); + } + + vortex_ensure!( + !self.bin_lowers.is_empty(), + "non-empty streams require bins" + ); + vortex_ensure!( + self.bin_lowers.len() <= MAX_BINS, + "bin count {} exceeds maximum {MAX_BINS}", + self.bin_lowers.len() + ); + vortex_ensure!( + self.offset_widths.iter().all(|&width| width <= 64), + "offset width exceeds 64 bits" + ); + vortex_ensure!( + self.weights.iter().all(|&weight| weight > 0), + "ANS weights must be positive" + ); + vortex_ensure!( + self.weights + .iter() + .map(|&weight| u32::from(weight)) + .sum::() + == 1_u32 << self.scale_bits, + "ANS weights do not sum to the table size" + ); + vortex_ensure!( + self.block_offsets.len() == self.n_blocks() + 1, + "expected {} block offsets, got {}", + self.n_blocks() + 1, + self.block_offsets.len() + ); + vortex_ensure!( + self.block_offsets[0] == 0, + "first block offset must be zero" + ); + vortex_ensure!( + self.block_offsets.windows(2).all(|pair| pair[0] <= pair[1]), + "block offsets must be monotonic" + ); + vortex_ensure!( + usize::try_from(*self.block_offsets.last().unwrap_or(&0))? == self.payload.len(), + "last block offset does not match payload length" + ); + Ok(()) + } + + fn bins(&self) -> VortexResult> { + self.bin_lowers + .iter() + .copied() + .zip(self.offset_widths.iter().copied()) + .map(|(lower, offset_bits)| Bin::try_new(lower, offset_bits)) + .collect() + } + + fn n_blocks(&self) -> usize { + self.len.div_ceil(self.block_len) + } + + fn block_value_count(&self, block_idx: usize) -> usize { + let start = block_idx * self.block_len; + (self.len - start).min(self.block_len) + } +} + +impl RangePackedCodec { + /// Encode ordered unsigned latents with fixed-width bin identifiers. + pub fn encode(values: &[u64], block_len: usize) -> VortexResult { + vortex_ensure!(block_len > 0, "block length must be greater than zero"); + + if values.is_empty() { + return Ok(Self { + len: 0, + block_len, + symbol_width: 0, + bin_lowers: Vec::new(), + offset_widths: Vec::new(), + block_offsets: vec![0], + payload: ByteBuffer::empty(), + }); + } + + let (training_sample, minimum, maximum) = training_sample(values); + let bins = cover_domain(optimize_bins(&training_sample), minimum, maximum); + let (bins, symbols, _) = assign_bins(values, bins)?; + let symbol_width = bit_width(u64::try_from(bins.len() - 1)?); + let bin_lowers = bins.iter().map(|bin| bin.lower).collect(); + let offset_widths = bins.iter().map(|bin| bin.offset_bits).collect(); + + let n_blocks = values.len().div_ceil(block_len); + let mut block_offsets = Vec::with_capacity(n_blocks + 1); + let mut payload = Vec::new(); + block_offsets.push(0); + for block_idx in 0..n_blocks { + let start = block_idx * block_len; + let stop = (start + block_len).min(values.len()); + encode_packed_block( + &values[start..stop], + &symbols[start..stop], + &bins, + symbol_width, + &mut payload, + ); + block_offsets.push(u32::try_from(payload.len())?); + } + + let codec = Self { + len: values.len(), + block_len, + symbol_width, + bin_lowers, + offset_widths, + block_offsets, + payload: ByteBuffer::from(payload), + }; + codec.validate()?; + Ok(codec) + } + + /// Decode all values. + pub fn decode(&self) -> VortexResult> { + self.validate()?; + let bins = self.bins()?; + let mut values = Vec::with_capacity(self.len); + for block_idx in 0..self.n_blocks() { + let start = usize::try_from(self.block_offsets[block_idx])?; + let stop = usize::try_from(self.block_offsets[block_idx + 1])?; + decode_packed_block_into( + &self.payload.as_slice()[start..stop], + self.block_value_count(block_idx), + self.symbol_width, + &bins, + &mut values, + )?; + } + Ok(values) + } + + /// Return the logical value count. + pub fn len(&self) -> usize { + self.len + } + + /// Return true when the stream has no values. + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + /// Return the fixed bin identifier width. + pub fn symbol_width(&self) -> u8 { + self.symbol_width + } + + /// Return the bin lower bounds. + pub fn bin_lowers(&self) -> &[u64] { + &self.bin_lowers + } + + /// Return the total bytes in the variable tables and payload. + pub fn encoded_size(&self) -> usize { + self.bin_lowers.len() * size_of::() + + self.offset_widths.len() * size_of::() + + self.block_offsets.len() * size_of::() + + self.payload.len() + } + + fn validate(&self) -> VortexResult<()> { + vortex_ensure!(self.block_len > 0, "block length must be greater than zero"); + vortex_ensure!( + self.bin_lowers.len() == self.offset_widths.len(), + "range packed bin tables have different lengths" + ); + + if self.len == 0 { + vortex_ensure!(self.bin_lowers.is_empty(), "empty streams cannot have bins"); + vortex_ensure!( + self.payload.is_empty(), + "empty streams cannot have payload bytes" + ); + vortex_ensure!( + self.block_offsets == [0], + "empty streams require one zero block offset" + ); + return Ok(()); + } + + vortex_ensure!( + !self.bin_lowers.is_empty() && self.bin_lowers.len() <= MAX_BINS, + "range packed bin count is invalid" + ); + vortex_ensure!( + self.symbol_width == bit_width(u64::try_from(self.bin_lowers.len() - 1)?), + "range packed symbol width does not match its bin count" + ); + vortex_ensure!( + self.offset_widths.iter().all(|&width| width <= 64), + "offset width exceeds 64 bits" + ); + vortex_ensure!( + self.block_offsets.len() == self.n_blocks() + 1, + "range packed block offset count is invalid" + ); + vortex_ensure!( + self.block_offsets.first() == Some(&0) + && self.block_offsets.windows(2).all(|pair| pair[0] <= pair[1]), + "range packed block offsets are invalid" + ); + vortex_ensure!( + usize::try_from(*self.block_offsets.last().unwrap_or(&0))? == self.payload.len(), + "last block offset does not match payload length" + ); + Ok(()) + } + + fn bins(&self) -> VortexResult> { + self.bin_lowers + .iter() + .copied() + .zip(self.offset_widths.iter().copied()) + .map(|(lower, offset_bits)| Bin::try_new(lower, offset_bits)) + .collect() + } + + fn n_blocks(&self) -> usize { + self.len.div_ceil(self.block_len) + } + + fn block_value_count(&self, block_idx: usize) -> usize { + let start = block_idx * self.block_len; + (self.len - start).min(self.block_len) + } +} + +impl RangeTwoLevelCodec { + /// Encode ordered unsigned latents with hot-bin tags and cold-bin escapes. + pub fn encode(values: &[u64], block_len: usize) -> VortexResult { + vortex_ensure!(block_len > 0, "block length must be greater than zero"); + if values.is_empty() { + return Ok(Self { + len: 0, + block_len, + tag_width: 0, + cold_width: 0, + bin_lowers: Vec::new(), + offset_widths: Vec::new(), + block_offsets: vec![0], + payload: ByteBuffer::empty(), + }); + } + + let (training_sample, minimum, maximum) = training_sample(values); + let bins = cover_domain(optimize_bins(&training_sample), minimum, maximum); + let (bins, symbols, counts) = assign_bins(values, bins)?; + let (tag_width, direct_count, order) = choose_two_level_layout(&counts); + let cold_count = bins.len() - direct_count; + let cold_width = bit_width(u64::try_from(cold_count.saturating_sub(1))?); + let mut rank_by_symbol = vec![0usize; bins.len()]; + for (rank, &symbol) in order.iter().enumerate() { + rank_by_symbol[symbol] = rank; + } + let reordered_bins = order.iter().map(|&symbol| bins[symbol]).collect::>(); + let ranks = symbols + .into_iter() + .map(|symbol| rank_by_symbol[symbol]) + .collect::>(); + + let n_blocks = values.len().div_ceil(block_len); + let mut block_offsets = Vec::with_capacity(n_blocks + 1); + let mut payload = Vec::new(); + block_offsets.push(0); + for block_idx in 0..n_blocks { + let start = block_idx * block_len; + let stop = (start + block_len).min(values.len()); + encode_two_level_block( + &values[start..stop], + &ranks[start..stop], + &reordered_bins, + tag_width, + direct_count, + cold_width, + &mut payload, + )?; + block_offsets.push(u32::try_from(payload.len())?); + } + + Ok(Self { + len: values.len(), + block_len, + tag_width, + cold_width, + bin_lowers: reordered_bins.iter().map(|bin| bin.lower).collect(), + offset_widths: reordered_bins.iter().map(|bin| bin.offset_bits).collect(), + block_offsets, + payload: ByteBuffer::from(payload), + }) + } + + /// Decode all values. + pub fn decode(&self) -> VortexResult> { + if self.len == 0 { + return Ok(Vec::new()); + } + let bins = self + .bin_lowers + .iter() + .copied() + .zip(self.offset_widths.iter().copied()) + .map(|(lower, offset_bits)| Bin::try_new(lower, offset_bits)) + .collect::>>()?; + let direct_count = ((1usize << self.tag_width) - 1).min(bins.len()); + let mut values = Vec::with_capacity(self.len); + for block_idx in 0..self.len.div_ceil(self.block_len) { + let start = usize::try_from(self.block_offsets[block_idx])?; + let stop = usize::try_from(self.block_offsets[block_idx + 1])?; + let block_start = block_idx * self.block_len; + let block_value_count = (self.len - block_start).min(self.block_len); + decode_two_level_block_into( + &self.payload.as_slice()[start..stop], + block_value_count, + self.tag_width, + direct_count, + self.cold_width, + &bins, + &mut values, + )?; + } + Ok(values) + } + + /// Return the fixed hot-bin tag width. + pub fn tag_width(&self) -> u8 { + self.tag_width + } + + /// Return the fixed cold-bin identifier width. + pub fn cold_width(&self) -> u8 { + self.cold_width + } + + /// Return the total bytes in the variable tables and payload. + pub fn encoded_size(&self) -> usize { + self.bin_lowers.len() * size_of::() + + self.offset_widths.len() * size_of::() + + self.block_offsets.len() * size_of::() + + self.payload.len() + } +} + +impl RangeGroupedCodec { + /// Encode ordered unsigned latents with one residual stream per bin. + pub fn encode(values: &[u64], block_len: usize) -> VortexResult { + vortex_ensure!(block_len > 0, "block length must be greater than zero"); + if values.is_empty() { + return Ok(Self { + len: 0, + block_len, + symbol_width: 0, + bin_lowers: Vec::new(), + offset_widths: Vec::new(), + block_offsets: vec![0], + payload: ByteBuffer::empty(), + }); + } + + let (training_sample, minimum, maximum) = training_sample(values); + let bins = cover_domain(optimize_bins(&training_sample), minimum, maximum); + let (bins, symbols, _) = assign_bins(values, bins)?; + let symbol_width = bit_width(u64::try_from(bins.len() - 1)?); + let n_blocks = values.len().div_ceil(block_len); + let mut block_offsets = Vec::with_capacity(n_blocks + 1); + let mut payload = Vec::new(); + block_offsets.push(0); + for block_idx in 0..n_blocks { + let start = block_idx * block_len; + let stop = (start + block_len).min(values.len()); + encode_grouped_block( + &values[start..stop], + &symbols[start..stop], + &bins, + symbol_width, + &mut payload, + )?; + block_offsets.push(u32::try_from(payload.len())?); + } + + Ok(Self { + len: values.len(), + block_len, + symbol_width, + bin_lowers: bins.iter().map(|bin| bin.lower).collect(), + offset_widths: bins.iter().map(|bin| bin.offset_bits).collect(), + block_offsets, + payload: ByteBuffer::from(payload), + }) + } + + /// Decode all values. + pub fn decode(&self) -> VortexResult> { + if self.len == 0 { + return Ok(Vec::new()); + } + let bins = self + .bin_lowers + .iter() + .copied() + .zip(self.offset_widths.iter().copied()) + .map(|(lower, offset_bits)| Bin::try_new(lower, offset_bits)) + .collect::>>()?; + let mut values = Vec::with_capacity(self.len); + for block_idx in 0..self.len.div_ceil(self.block_len) { + let start = usize::try_from(self.block_offsets[block_idx])?; + let stop = usize::try_from(self.block_offsets[block_idx + 1])?; + let block_start = block_idx * self.block_len; + let block_value_count = (self.len - block_start).min(self.block_len); + decode_grouped_block_into( + &self.payload.as_slice()[start..stop], + block_value_count, + self.symbol_width, + &bins, + &mut values, + )?; + } + Ok(values) + } + + /// Return the total bytes in the variable tables and payload. + pub fn encoded_size(&self) -> usize { + self.bin_lowers.len() * size_of::() + + self.offset_widths.len() * size_of::() + + self.block_offsets.len() * size_of::() + + self.payload.len() + } +} + +#[derive(Clone, Copy, Debug)] +struct Bin { + lower: u64, + offset_bits: u8, +} + +impl Bin { + fn try_new(lower: u64, offset_bits: u8) -> VortexResult { + vortex_ensure!(offset_bits <= 64, "offset width exceeds 64 bits"); + Ok(Self { lower, offset_bits }) + } + + fn from_values(lower: u64, upper: u64) -> Self { + let offset_bits = bit_width(upper - lower); + Self { lower, offset_bits } + } + + fn contains(&self, value: u64) -> bool { + if value < self.lower { + return false; + } + self.offset_bits == 64 || value - self.lower <= low_mask(self.offset_bits) + } +} + +fn bit_width(value: u64) -> u8 { + u8::try_from(u64::BITS - value.leading_zeros()).unwrap_or(64) +} + +fn low_mask(bits: u8) -> u64 { + if bits == 64 { + u64::MAX + } else if bits == 0 { + 0 + } else { + (1_u64 << bits) - 1 + } +} + +fn find_bin(value: u64, bins: &[Bin], search_order: &[usize]) -> VortexResult { + search_order + .iter() + .copied() + .find(|&index| bins[index].contains(value)) + .ok_or_else(|| vortex_error::vortex_err!("no range entropy bin contains value {value}")) +} + +fn assign_bins( + values: &[u64], + mut bins: Vec, +) -> VortexResult<(Vec, Vec, Vec)> { + loop { + let mut search_order: Vec<_> = (0..bins.len()).collect(); + search_order.sort_by_key(|&index| bins[index].offset_bits); + let mut counts = vec![0usize; bins.len()]; + let mut symbols = Vec::with_capacity(values.len()); + for &value in values { + let symbol = find_bin(value, &bins, &search_order)?; + counts[symbol] += 1; + symbols.push(symbol); + } + if counts.iter().all(|&count| count > 0) { + return Ok((bins, symbols, counts)); + } + bins = bins + .into_iter() + .zip(counts) + .filter_map(|(bin, count)| (count > 0).then_some(bin)) + .collect(); + } +} + +fn choose_two_level_layout(counts: &[usize]) -> (u8, usize, Vec) { + let mut order = (0..counts.len()).collect::>(); + order.sort_unstable_by_key(|&symbol| Reverse(counts[symbol])); + if counts.len() <= 1 { + return (0, counts.len(), order); + } + + let flat_width = bit_width((counts.len() - 1) as u64); + let mut best = (usize::MAX, flat_width, counts.len()); + let minimum_tag_width = flat_width.min(2); + for tag_width in minimum_tag_width..=flat_width { + let direct_count = ((1usize << tag_width) - 1).min(counts.len()); + let cold_count = counts.len() - direct_count; + let cold_width = bit_width(cold_count.saturating_sub(1) as u64); + let cold_values = order[direct_count..] + .iter() + .map(|&symbol| counts[symbol]) + .sum::(); + let cost = counts.iter().sum::() * usize::from(tag_width) + + cold_values * usize::from(cold_width); + if cost < best.0 { + best = (cost, tag_width, direct_count); + } + } + (best.1, best.2, order) +} + +fn optimize_bins(sorted: &[u64]) -> Vec { + debug_assert!(!sorted.is_empty()); + let mut segments = Vec::with_capacity(MAX_BINS); + segments.push(0..sorted.len()); + + while segments.len() < MAX_BINS { + let best = segments + .iter() + .enumerate() + .filter_map(|(segment_idx, segment)| { + best_split(sorted, segment).map(|split| (segment_idx, split)) + }) + .max_by(|(_, left), (_, right)| { + left.gain + .partial_cmp(&right.gain) + .unwrap_or(Ordering::Equal) + }); + + let Some((segment_idx, split)) = best else { + break; + }; + if split.gain <= BIN_TABLE_COST_BITS { + break; + } + + let segment = segments.remove(segment_idx); + segments.push(segment.start..split.at); + segments.push(split.at..segment.end); + segments.sort_unstable_by_key(|range| range.start); + } + + segments + .into_iter() + .map(|range| Bin::from_values(sorted[range.start], sorted[range.end - 1])) + .collect() +} + +fn training_sample(values: &[u64]) -> (Vec, u64, u64) { + let mut minimum = u64::MAX; + let mut maximum = u64::MIN; + for &value in values { + minimum = minimum.min(value); + maximum = maximum.max(value); + } + + let mut sample = if values.len() <= TRAINING_SAMPLE_SIZE { + values.to_vec() + } else { + (0..TRAINING_SAMPLE_SIZE) + .map(|index| values[index * values.len() / TRAINING_SAMPLE_SIZE]) + .collect() + }; + sample.push(minimum); + sample.push(maximum); + sample.sort_unstable(); + (sample, minimum, maximum) +} + +fn cover_domain(mut bins: Vec, minimum: u64, maximum: u64) -> Vec { + if let Some(first) = bins.first_mut() { + first.lower = minimum; + } + for index in 0..bins.len().saturating_sub(1) { + let upper = bins[index + 1].lower - 1; + bins[index].offset_bits = bit_width(upper - bins[index].lower); + } + if let Some(last) = bins.last_mut() { + last.offset_bits = bit_width(maximum - last.lower); + } + bins +} + +#[derive(Clone, Copy)] +struct Split { + at: usize, + gain: f64, +} + +fn best_split(sorted: &[u64], segment: &Range) -> Option { + let len = segment.len(); + if len < 2 || sorted[segment.start] == sorted[segment.end - 1] { + return None; + } + + let whole_width = f64::from(bit_width(sorted[segment.end - 1] - sorted[segment.start])); + let whole_cost = len as f64 * whole_width; + let mut best: Option = None; + + for candidate in 1..SPLIT_CANDIDATES { + let mut at = segment.start + len * candidate / SPLIT_CANDIDATES; + if at <= segment.start || at >= segment.end { + continue; + } + while at < segment.end && sorted[at - 1] == sorted[at] { + at += 1; + } + if at >= segment.end { + continue; + } + + let left_len = at - segment.start; + let right_len = segment.end - at; + let left_width = f64::from(bit_width(sorted[at - 1] - sorted[segment.start])); + let right_width = f64::from(bit_width(sorted[segment.end - 1] - sorted[at])); + let symbol_cost = entropy_split_cost(left_len, right_len); + let split_cost = + left_len as f64 * left_width + right_len as f64 * right_width + symbol_cost; + let gain = whole_cost - split_cost; + + if best.is_none_or(|current| gain > current.gain) { + best = Some(Split { at, gain }); + } + } + best +} + +fn entropy_split_cost(left: usize, right: usize) -> f64 { + let total = (left + right) as f64; + let left = left as f64; + let right = right as f64; + left * (total / left).log2() + right * (total / right).log2() +} + +fn choose_scale_bits(value_count: usize, bin_count: usize) -> u8 { + if bin_count <= 1 { + return 0; + } + + let required_bits = usize::BITS - (bin_count - 1).leading_zeros(); + let useful_bits = usize::BITS - (value_count - 1).leading_zeros(); + u8::try_from(required_bits.max(useful_bits.min(u32::from(MAX_SCALE_BITS)))) + .unwrap_or(MAX_SCALE_BITS) +} + +fn quantize_weights(counts: &[usize], scale_bits: u8) -> VortexResult> { + let total_weight = 1_u32 << scale_bits; + let used: Vec<_> = counts + .iter() + .enumerate() + .filter(|(_, count)| **count > 0) + .map(|(index, &count)| (index, count)) + .collect(); + vortex_ensure!(!used.is_empty(), "cannot quantize an empty histogram"); + vortex_ensure!( + used.len() <= total_weight as usize, + "symbol count exceeds the ANS table size" + ); + + let total_count: usize = used.iter().map(|(_, count)| count).sum(); + let distributable = usize::try_from(total_weight)? - used.len(); + let mut weights = vec![0u16; counts.len()]; + let mut remainders = Vec::with_capacity(used.len()); + let mut allocated = 0usize; + + for &(index, count) in &used { + let scaled = count * distributable; + let extra = scaled / total_count; + let remainder = scaled % total_count; + weights[index] = u16::try_from(1 + extra)?; + allocated += 1 + extra; + remainders.push((index, remainder)); + } + + remainders.sort_unstable_by_key(|entry| Reverse(entry.1)); + for &(index, _) in remainders + .iter() + .take(usize::try_from(total_weight)? - allocated) + { + weights[index] += 1; + } + Ok(weights) +} + +struct AnsEncoderSymbol { + renorm_bit_cutoff: u32, + min_renorm_bits: u8, + next_states: Vec, +} + +#[derive(Clone, Copy)] +struct AnsDecoderNode { + next_state_idx_base: u16, + offset_bits: u8, + bits_to_read: u8, +} + +struct AnsEncoderTable { + table_size: u32, + encoder_symbols: Vec, +} + +struct AnsDecoderTable { + table_size: u32, + decoder_nodes: Vec, + lowers: Vec, +} + +fn spread_symbols(weights: &[u16], scale_bits: u8) -> VortexResult> { + let table_size = 1_u32 << scale_bits; + let mut state_symbols = vec![usize::MAX; usize::try_from(table_size)?]; + let mut stride = 3 * table_size / 5; + if stride.is_multiple_of(2) { + stride += 1; + } + let mask = table_size - 1; + let mut step = 0u32; + for (symbol, &weight) in weights.iter().enumerate() { + for _ in 0..weight { + let state_idx = (stride * step) & mask; + state_symbols[usize::try_from(state_idx)?] = symbol; + step += 1; + } + } + vortex_ensure!( + state_symbols.iter().all(|&symbol| symbol != usize::MAX), + "ANS table has unassigned states" + ); + Ok(state_symbols) +} + +fn validate_weights(weights: &[u16], scale_bits: u8) -> VortexResult { + let table_size = 1_u32 << scale_bits; + vortex_ensure!( + weights.iter().map(|&weight| u32::from(weight)).sum::() == table_size, + "ANS weights do not sum to the table size" + ); + vortex_ensure!( + weights.iter().all(|&weight| weight > 0), + "ANS weights must be positive" + ); + Ok(table_size) +} + +impl AnsEncoderTable { + fn new(weights: &[u16], scale_bits: u8) -> VortexResult { + let table_size = 1_u32 << scale_bits; + validate_weights(weights, scale_bits)?; + let state_symbols = spread_symbols(weights, scale_bits)?; + + let mut encoder_symbols = Vec::with_capacity(weights.len()); + for &weight in weights { + let frequency = u32::from(weight); + let max_x_s = 2 * frequency - 1; + let min_renorm_bits = scale_bits - u8::try_from(max_x_s.ilog2())?; + encoder_symbols.push(AnsEncoderSymbol { + renorm_bit_cutoff: 2 * frequency * (1_u32 << min_renorm_bits), + min_renorm_bits, + next_states: Vec::with_capacity(usize::from(weight)), + }); + } + for (state_idx, &symbol) in state_symbols.iter().enumerate() { + encoder_symbols[symbol] + .next_states + .push(table_size + u32::try_from(state_idx)?); + } + + Ok(Self { + table_size, + encoder_symbols, + }) + } +} + +impl AnsDecoderTable { + fn new(weights: &[u16], scale_bits: u8, bins: &[Bin]) -> VortexResult { + let table_size = validate_weights(weights, scale_bits)?; + vortex_ensure!( + weights.len() == bins.len(), + "ANS weights and range bins have different lengths" + ); + let state_symbols = spread_symbols(weights, scale_bits)?; + let mut symbol_x_s: Vec<_> = weights.iter().map(|&weight| u32::from(weight)).collect(); + let mut decoder_nodes = Vec::with_capacity(usize::try_from(table_size)?); + let mut lowers = Vec::with_capacity(usize::try_from(table_size)?); + for symbol in state_symbols { + let next_state_base = symbol_x_s[symbol]; + let bits_to_read = + u8::try_from(next_state_base.leading_zeros() - table_size.leading_zeros())?; + let next_state_idx_base = (next_state_base << bits_to_read) - table_size; + let bin = bins[symbol]; + decoder_nodes.push(AnsDecoderNode { + next_state_idx_base: u16::try_from(next_state_idx_base)?, + offset_bits: bin.offset_bits, + bits_to_read, + }); + lowers.push(bin.lower); + symbol_x_s[symbol] += 1; + } + + Ok(Self { + table_size, + decoder_nodes, + lowers, + }) + } +} + +fn encode_block( + values: &[u64], + symbols: &[usize], + bins: &[Bin], + table: &AnsEncoderTable, + payload: &mut Vec, +) -> VortexResult<()> { + let ans = encode_symbols(symbols, table)?; + payload.extend_from_slice(&u32::try_from(ans.len())?.to_le_bytes()); + payload.extend_from_slice(&ans); + + let mut offsets = BitWriter::with_capacity(size_of_val(values)); + for (&value, &symbol) in values.iter().zip(symbols) { + let bin = &bins[symbol]; + offsets.write(value - bin.lower, bin.offset_bits); + } + payload.extend_from_slice(&offsets.finish()); + payload.extend_from_slice(&[0; OFFSET_PADDING]); + Ok(()) +} + +fn encode_packed_block( + values: &[u64], + symbols: &[usize], + bins: &[Bin], + symbol_width: u8, + payload: &mut Vec, +) { + let mut encoded_symbols = BitWriter::with_capacity(symbols.len()); + for &symbol in symbols { + encoded_symbols.write(symbol as u64, symbol_width); + } + payload.extend_from_slice(&encoded_symbols.finish()); + payload.extend_from_slice(&[0; ANS_PADDING]); + + let mut offsets = BitWriter::with_capacity(size_of_val(values)); + for (&value, &symbol) in values.iter().zip(symbols) { + let bin = &bins[symbol]; + offsets.write(value - bin.lower, bin.offset_bits); + } + payload.extend_from_slice(&offsets.finish()); + payload.extend_from_slice(&[0; OFFSET_PADDING]); +} + +fn encode_two_level_block( + values: &[u64], + ranks: &[usize], + bins: &[Bin], + tag_width: u8, + direct_count: usize, + cold_width: u8, + payload: &mut Vec, +) -> VortexResult<()> { + let escape_tag = direct_count; + let cold_value_count = ranks.iter().filter(|&&rank| rank >= direct_count).count(); + payload.extend_from_slice(&u32::try_from(cold_value_count)?.to_le_bytes()); + + let mut tags = BitWriter::with_capacity(ranks.len()); + let mut cold = BitWriter::with_capacity(cold_value_count); + for &rank in ranks { + if rank < direct_count { + tags.write(rank as u64, tag_width); + } else { + tags.write(escape_tag as u64, tag_width); + cold.write((rank - direct_count) as u64, cold_width); + } + } + payload.extend_from_slice(&tags.finish()); + payload.extend_from_slice(&[0; ANS_PADDING]); + payload.extend_from_slice(&cold.finish()); + payload.extend_from_slice(&[0; ANS_PADDING]); + + let mut offsets = BitWriter::with_capacity(size_of_val(values)); + for (&value, &rank) in values.iter().zip(ranks) { + let bin = &bins[rank]; + offsets.write(value - bin.lower, bin.offset_bits); + } + payload.extend_from_slice(&offsets.finish()); + payload.extend_from_slice(&[0; OFFSET_PADDING]); + Ok(()) +} + +fn encode_grouped_block( + values: &[u64], + symbols: &[usize], + bins: &[Bin], + symbol_width: u8, + payload: &mut Vec, +) -> VortexResult<()> { + let mut encoded_symbols = BitWriter::with_capacity(symbols.len()); + let mut streams = (0..bins.len()) + .map(|_| BitWriter::with_capacity(values.len() / bins.len().max(1))) + .collect::>(); + for (&value, &symbol) in values.iter().zip(symbols) { + let bin = &bins[symbol]; + encoded_symbols.write(symbol as u64, symbol_width); + streams[symbol].write(value - bin.lower, bin.offset_bits); + } + payload.extend_from_slice(&encoded_symbols.finish()); + payload.extend_from_slice(&[0; ANS_PADDING]); + + let mut stream_offsets = Vec::with_capacity(bins.len() + 1); + let mut encoded_streams = Vec::new(); + stream_offsets.push(0u32); + for stream in streams { + encoded_streams.extend_from_slice(&stream.finish()); + encoded_streams.extend_from_slice(&[0; OFFSET_PADDING]); + stream_offsets.push(u32::try_from(encoded_streams.len())?); + } + for offset in stream_offsets { + payload.extend_from_slice(&offset.to_le_bytes()); + } + payload.extend_from_slice(&encoded_streams); + Ok(()) +} + +#[expect( + clippy::cast_possible_truncation, + reason = "bin identifiers and stream positions fit their destination types" +)] +fn decode_grouped_block_into( + payload: &[u8], + n_values: usize, + symbol_width: u8, + bins: &[Bin], + values: &mut Vec, +) -> VortexResult<()> { + let symbol_bytes = (n_values * usize::from(symbol_width)).div_ceil(8); + let offset_table_start = symbol_bytes + ANS_PADDING; + let offset_table_bytes = (bins.len() + 1) * size_of::(); + let streams_start = offset_table_start + offset_table_bytes; + vortex_ensure!( + payload.len() >= streams_start, + "grouped restart block is too short" + ); + let mut stream_starts = [0usize; MAX_BINS]; + let mut stream_stops = [0usize; MAX_BINS]; + for index in 0..bins.len() { + let read_offset = |entry: usize| { + let start = offset_table_start + entry * size_of::(); + u32::from_le_bytes([ + payload[start], + payload[start + 1], + payload[start + 2], + payload[start + 3], + ]) + }; + stream_starts[index] = streams_start + usize::try_from(read_offset(index))?; + stream_stops[index] = streams_start + usize::try_from(read_offset(index + 1))?; + vortex_ensure!( + stream_starts[index] <= stream_stops[index] + && stream_stops[index] <= payload.len() + && stream_stops[index] - stream_starts[index] >= OFFSET_PADDING, + "grouped residual stream is invalid" + ); + } + + let mut symbols = BitReader::new(payload, symbol_bytes)?; + let mut table = [PackedDecodeNode::default(); MAX_BINS]; + for (index, bin) in bins.iter().enumerate() { + table[index] = PackedDecodeNode { + lower: bin.lower, + offset_bits: u32::from(bin.offset_bits), + }; + } + let mut stream_positions = [0u32; MAX_BINS]; + let symbol_mask = low_mask(symbol_width); + let mut seen_symbols = 0u64; + + for _ in (0..n_values / 8 * 8).step_by(8) { + // SAFETY: The padded symbol stream provides eight readable bytes. + let packed = unsafe { symbols.read_unchecked(symbol_width * 8) }; + for lane in 0..8 { + let symbol = ((packed >> (lane * usize::from(symbol_width))) & symbol_mask) as usize; + seen_symbols |= 1_u64 << symbol; + let node = table[symbol]; + let bit_position = stream_positions[symbol]; + let byte_position = stream_starts[symbol] + bit_position as usize / 8; + let bits_past_byte = bit_position % 8; + // SAFETY: Each stream contains fifteen padding bytes. + let first = unsafe { read_u64_unaligned(payload.as_ptr().add(byte_position)) }; + let offset = if node.offset_bits <= 57 { + (first >> bits_past_byte) & ((1_u64 << node.offset_bits) - 1) + } else { + // SAFETY: Each stream contains fifteen padding bytes. + let second = unsafe { read_u64_unaligned(payload.as_ptr().add(byte_position + 7)) }; + let processed = 56 - bits_past_byte; + ((first >> bits_past_byte) | (second << processed)) + & low_mask(node.offset_bits as u8) + }; + stream_positions[symbol] += node.offset_bits; + values.push(node.lower.wrapping_add(offset)); + } + } + for _ in n_values / 8 * 8..n_values { + let symbol = symbols.read(symbol_width)? as usize; + seen_symbols |= 1_u64 << symbol; + let node = table[symbol]; + let bit_position = stream_positions[symbol]; + let byte_position = stream_starts[symbol] + bit_position as usize / 8; + let bits_past_byte = bit_position % 8; + // SAFETY: Each stream contains fifteen padding bytes. + let first = unsafe { read_u64_unaligned(payload.as_ptr().add(byte_position)) }; + let offset = if node.offset_bits <= 57 { + (first >> bits_past_byte) & ((1_u64 << node.offset_bits) - 1) + } else { + // SAFETY: Each stream contains fifteen padding bytes. + let second = unsafe { read_u64_unaligned(payload.as_ptr().add(byte_position + 7)) }; + let processed = 56 - bits_past_byte; + ((first >> bits_past_byte) | (second << processed)) & low_mask(node.offset_bits as u8) + }; + stream_positions[symbol] += node.offset_bits; + values.push(node.lower.wrapping_add(offset)); + } + + let valid_symbols = low_mask(u8::try_from(bins.len())?); + vortex_ensure!( + seen_symbols & !valid_symbols == 0, + "grouped symbol exceeds bin table" + ); + symbols.finish()?; + for index in 0..bins.len() { + let used_bytes = usize::try_from(stream_positions[index])?.div_ceil(8); + vortex_ensure!( + stream_starts[index] + used_bytes + OFFSET_PADDING == stream_stops[index], + "grouped residual stream has unused bytes" + ); + } + Ok(()) +} + +#[derive(Clone, Copy, Default)] +struct PackedDecodeNode { + lower: u64, + offset_bits: u32, +} + +#[expect( + clippy::cast_possible_truncation, + reason = "bin identifiers and block offset positions fit their destination types" +)] +fn decode_two_level_block_into( + payload: &[u8], + n_values: usize, + tag_width: u8, + direct_count: usize, + cold_width: u8, + bins: &[Bin], + values: &mut Vec, +) -> VortexResult<()> { + vortex_ensure!(payload.len() >= 4, "two-level restart block is too short"); + let cold_value_count = usize::try_from(u32::from_le_bytes([ + payload[0], payload[1], payload[2], payload[3], + ]))?; + let tag_bytes = (n_values * usize::from(tag_width)).div_ceil(8); + let cold_bytes = (cold_value_count * usize::from(cold_width)).div_ceil(8); + let tags_start = 4; + let cold_start = tags_start + tag_bytes + ANS_PADDING; + let offsets_start = cold_start + cold_bytes + ANS_PADDING; + vortex_ensure!( + payload.len() >= offsets_start + OFFSET_PADDING, + "two-level restart block is too short" + ); + let mut tags = BitReader::new(&payload[tags_start..], tag_bytes)?; + let mut cold = BitReader::new(&payload[cold_start..], cold_bytes)?; + let offsets = &payload[offsets_start..]; + let mut table = [PackedDecodeNode::default(); MAX_BINS]; + for (index, bin) in bins.iter().enumerate() { + table[index] = PackedDecodeNode { + lower: bin.lower, + offset_bits: u32::from(bin.offset_bits), + }; + } + + let escape_tag = direct_count; + let tag_mask = low_mask(tag_width); + let mut offset_bit_position = 0u32; + let full_len = n_values / 8 * 8; + for _ in (0..full_len).step_by(8) { + // SAFETY: The encoded codec invariant provides eight tag padding bytes. + let packed = unsafe { tags.read_unchecked(tag_width * 8) }; + for lane in 0..8 { + let tag = ((packed >> (lane * usize::from(tag_width))) & tag_mask) as usize; + let rank = if tag == escape_tag { + // SAFETY: The encoded codec invariant provides eight cold padding bytes. + direct_count + unsafe { cold.read_unchecked(cold_width) } as usize + } else { + tag + }; + let node = table[rank]; + // SAFETY: The private codec fields come from the validated encoder. + let offset = unsafe { read_packed_offset(offsets, offset_bit_position, node) }; + offset_bit_position += node.offset_bits; + values.push(node.lower.wrapping_add(offset)); + } + } + for _ in full_len..n_values { + let tag = tags.read(tag_width)? as usize; + let rank = if tag == escape_tag { + direct_count + cold.read(cold_width)? as usize + } else { + tag + }; + let node = table[rank]; + // SAFETY: The private codec fields come from the validated encoder. + let offset = unsafe { read_packed_offset(offsets, offset_bit_position, node) }; + offset_bit_position += node.offset_bits; + values.push(node.lower.wrapping_add(offset)); + } + + tags.finish()?; + cold.finish()?; + let offset_bytes = offset_bit_position.div_ceil(8) as usize; + vortex_ensure!( + offsets.len() == offset_bytes + OFFSET_PADDING, + "offset stream has unused bytes" + ); + Ok(()) +} + +#[inline(always)] +#[expect( + clippy::cast_possible_truncation, + reason = "validated offset widths do not exceed 64 bits" +)] +unsafe fn read_packed_offset(offsets: &[u8], bit_position: u32, node: PackedDecodeNode) -> u64 { + let byte_position = bit_position as usize / 8; + let bits_past_byte = bit_position % 8; + // SAFETY: The encoded codec invariant provides fifteen readable padding bytes. + let first = unsafe { read_u64_unaligned(offsets.as_ptr().add(byte_position)) }; + if node.offset_bits <= 57 { + (first >> bits_past_byte) & ((1_u64 << node.offset_bits) - 1) + } else { + // SAFETY: The encoded codec invariant provides fifteen readable padding bytes. + let second = unsafe { read_u64_unaligned(offsets.as_ptr().add(byte_position + 7)) }; + let processed = 56 - bits_past_byte; + ((first >> bits_past_byte) | (second << processed)) & low_mask(node.offset_bits as u8) + } +} + +#[expect( + clippy::cast_possible_truncation, + reason = "bin identifiers and block offset positions fit their destination types" +)] +fn decode_packed_block_into( + payload: &[u8], + n_values: usize, + symbol_width: u8, + bins: &[Bin], + values: &mut Vec, +) -> VortexResult<()> { + let symbol_bits = n_values * usize::from(symbol_width); + let symbol_bytes = symbol_bits.div_ceil(8); + vortex_ensure!( + payload.len() >= symbol_bytes + ANS_PADDING + OFFSET_PADDING, + "range packed restart block is too short" + ); + let offsets = &payload[symbol_bytes + ANS_PADDING..]; + let mut symbols = BitReader::new(payload, symbol_bytes)?; + let mut table = [PackedDecodeNode::default(); MAX_BINS]; + for (index, bin) in bins.iter().enumerate() { + table[index] = PackedDecodeNode { + lower: bin.lower, + offset_bits: u32::from(bin.offset_bits), + }; + } + + let symbol_mask = low_mask(symbol_width); + let mut offset_bit_position = 0u32; + let full_len = n_values / 8 * 8; + for _ in (0..full_len).step_by(8) { + // SAFETY: The encoded codec invariant provides eight symbol padding bytes. + let packed = unsafe { symbols.read_unchecked(symbol_width * 8) }; + for lane in 0..8 { + let symbol = ((packed >> (lane * usize::from(symbol_width))) & symbol_mask) as usize; + let node = table[symbol]; + // SAFETY: The private codec fields come from the validated encoder. + let offset = unsafe { read_packed_offset(offsets, offset_bit_position, node) }; + offset_bit_position += node.offset_bits; + values.push(node.lower.wrapping_add(offset)); + } + } + for _ in full_len..n_values { + let symbol = symbols.read(symbol_width)? as usize; + let node = table[symbol]; + // SAFETY: The private codec fields come from the validated encoder. + let offset = unsafe { read_packed_offset(offsets, offset_bit_position, node) }; + offset_bit_position += node.offset_bits; + values.push(node.lower.wrapping_add(offset)); + } + + symbols.finish()?; + let offset_bytes = offset_bit_position.div_ceil(8) as usize; + vortex_ensure!( + offsets.len() == offset_bytes + OFFSET_PADDING, + "offset stream has unused bytes" + ); + vortex_ensure!( + offsets[offset_bytes..].iter().all(|&byte| byte == 0), + "offset stream has nonzero padding" + ); + Ok(()) +} + +#[inline(never)] +#[expect( + clippy::cast_possible_truncation, + reason = "ANS transitions read at most MAX_SCALE_BITS bits" +)] +unsafe fn read_full_ans_batch( + symbols: &mut BitReader<'_>, + table: &AnsDecoderTable, + states: &mut [u32; ANS_INTERLEAVING], + lowers: &mut DecodeScratch, + widths: &mut DecodeScratch, + bit_positions: &mut DecodeScratch, +) -> u32 { + let [mut state_0, mut state_1, mut state_2, mut state_3] = *states; + let mut offset_bit_position = 0u32; + + for base_index in (0..DECODE_BATCH_SIZE).step_by(ANS_INTERLEAVING) { + // SAFETY: Initial states are validated, and each ANS transition stays in the table. + let node_0 = unsafe { table.decoder_nodes.get_unchecked(state_0 as usize) }; + // SAFETY: Initial states are validated, and each ANS transition stays in the table. + let node_1 = unsafe { table.decoder_nodes.get_unchecked(state_1 as usize) }; + // SAFETY: Initial states are validated, and each ANS transition stays in the table. + let node_2 = unsafe { table.decoder_nodes.get_unchecked(state_2 as usize) }; + // SAFETY: Initial states are validated, and each ANS transition stays in the table. + let node_3 = unsafe { table.decoder_nodes.get_unchecked(state_3 as usize) }; + let width_0 = node_0.bits_to_read; + let width_1 = node_1.bits_to_read; + let width_2 = node_2.bits_to_read; + let width_3 = node_3.bits_to_read; + let packed_width = width_0 + width_1 + width_2 + width_3; + // SAFETY: The caller verifies enough readable bytes for the worst-case batch. + let packed = unsafe { symbols.read_unchecked(packed_width) }; + let shift_1 = width_0; + let shift_2 = shift_1 + width_1; + let shift_3 = shift_2 + width_2; + + macro_rules! write_symbol { + ($index:expr, $state:ident, $node:ident, $shift:expr, $width:ident) => {{ + // SAFETY: The decoder node uses the same validated state index. + let lower = unsafe { *table.lowers.get_unchecked($state as usize) }; + // SAFETY: Every scratch index is within this full batch. + unsafe { *lowers.get_unchecked_mut($index) = lower }; + // SAFETY: Every scratch index is within this full batch. + unsafe { *bit_positions.get_unchecked_mut($index) = offset_bit_position }; + // SAFETY: Every scratch index is within this full batch. + unsafe { *widths.get_unchecked_mut($index) = u32::from($node.offset_bits) }; + offset_bit_position += u32::from($node.offset_bits); + $state = u32::from($node.next_state_idx_base) + + ((packed >> $shift) & ((1_u64 << $width) - 1)) as u32; + }}; + } + + write_symbol!(base_index, state_0, node_0, 0, width_0); + write_symbol!(base_index + 1, state_1, node_1, shift_1, width_1); + write_symbol!(base_index + 2, state_2, node_2, shift_2, width_2); + write_symbol!(base_index + 3, state_3, node_3, shift_3, width_3); + } + + *states = [state_0, state_1, state_2, state_3]; + offset_bit_position +} + +#[expect( + clippy::cast_possible_truncation, + reason = "validated ANS and offset widths fit their destination types" +)] +fn decode_block_into( + payload: &[u8], + n_values: usize, + output_range: Range, + table: &AnsDecoderTable, + values: &mut Vec, +) -> VortexResult<()> { + vortex_ensure!( + output_range.start <= output_range.end && output_range.end <= n_values, + "output range exceeds restart block" + ); + vortex_ensure!( + payload.len() >= 4 + ANS_STATE_BYTES, + "restart block is too short" + ); + let ans_len = usize::try_from(u32::from_le_bytes([ + payload[0], payload[1], payload[2], payload[3], + ]))?; + vortex_ensure!( + ans_len >= ANS_STATE_BYTES + ANS_PADDING, + "ANS stream is too short" + ); + vortex_ensure!( + 4 + ans_len <= payload.len(), + "ANS stream exceeds restart block" + ); + + let encoded_symbols = &payload[4..4 + ans_len]; + let mut state_idxs = [0u32; ANS_INTERLEAVING]; + for (index, state_idx) in state_idxs.iter_mut().enumerate() { + let byte_idx = index * size_of::(); + *state_idx = u32::from(u16::from_le_bytes([ + encoded_symbols[byte_idx], + encoded_symbols[byte_idx + 1], + ])); + } + vortex_ensure!( + state_idxs + .iter() + .all(|&state_idx| state_idx < table.table_size), + "ANS state index exceeds the table" + ); + let offsets = &payload[4 + ans_len..]; + let symbol_bytes = &payload[4 + ANS_STATE_BYTES..]; + let symbol_unpadded_len = ans_len - ANS_STATE_BYTES - ANS_PADDING; + let mut symbols = BitReader::new(symbol_bytes, symbol_unpadded_len)?; + let [mut state_0, mut state_1, mut state_2, mut state_3] = state_idxs; + let mut offset_bit_position = 0u32; + let mut lowers = DecodeScratch([0u64; DECODE_BATCH_SIZE]); + let mut widths = DecodeScratch([0u32; DECODE_BATCH_SIZE]); + let mut bit_positions = DecodeScratch([0u32; DECODE_BATCH_SIZE]); + let mut decoded = DecodeScratch([0u64; DECODE_BATCH_SIZE]); + + for batch_start in (0..n_values).step_by(DECODE_BATCH_SIZE) { + let batch_len = (n_values - batch_start).min(DECODE_BATCH_SIZE); + let full_batch_len = batch_len / ANS_INTERLEAVING * ANS_INTERLEAVING; + let batch_offset_start = offset_bit_position; + let mut batch_offset_bits = 0u32; + let fast_full_batch = batch_len == DECODE_BATCH_SIZE + && symbols.can_read_unchecked(DECODE_BATCH_SIZE * usize::from(MAX_SCALE_BITS)); + if fast_full_batch { + let mut states = [state_0, state_1, state_2, state_3]; + // SAFETY: The size check provides readable bytes for the worst-case batch. + batch_offset_bits = unsafe { + read_full_ans_batch( + &mut symbols, + table, + &mut states, + &mut lowers, + &mut widths, + &mut bit_positions, + ) + }; + [state_0, state_1, state_2, state_3] = states; + symbols.check_in_bounds()?; + } else { + for base_index in (0..full_batch_len).step_by(ANS_INTERLEAVING) { + // SAFETY: Initial states are validated, and each ANS transition stays in the table. + let node_0 = unsafe { table.decoder_nodes.get_unchecked(state_0 as usize) }; + // SAFETY: Initial states are validated, and each ANS transition stays in the table. + let node_1 = unsafe { table.decoder_nodes.get_unchecked(state_1 as usize) }; + // SAFETY: Initial states are validated, and each ANS transition stays in the table. + let node_2 = unsafe { table.decoder_nodes.get_unchecked(state_2 as usize) }; + // SAFETY: Initial states are validated, and each ANS transition stays in the table. + let node_3 = unsafe { table.decoder_nodes.get_unchecked(state_3 as usize) }; + let width_0 = node_0.bits_to_read; + let width_1 = node_1.bits_to_read; + let width_2 = node_2.bits_to_read; + let width_3 = node_3.bits_to_read; + let packed_width = width_0 + width_1 + width_2 + width_3; + let packed = symbols.read(packed_width)?; + let shift_1 = width_0; + let shift_2 = shift_1 + width_1; + let shift_3 = shift_2 + width_2; + // SAFETY: The corresponding decoder nodes use the same validated state indexes. + lowers[base_index] = unsafe { *table.lowers.get_unchecked(state_0 as usize) }; + // SAFETY: The corresponding decoder nodes use the same validated state indexes. + lowers[base_index + 1] = unsafe { *table.lowers.get_unchecked(state_1 as usize) }; + // SAFETY: The corresponding decoder nodes use the same validated state indexes. + lowers[base_index + 2] = unsafe { *table.lowers.get_unchecked(state_2 as usize) }; + // SAFETY: The corresponding decoder nodes use the same validated state indexes. + lowers[base_index + 3] = unsafe { *table.lowers.get_unchecked(state_3 as usize) }; + for (index, width) in [ + node_0.offset_bits, + node_1.offset_bits, + node_2.offset_bits, + node_3.offset_bits, + ] + .into_iter() + .enumerate() + { + bit_positions[base_index + index] = batch_offset_bits; + widths[base_index + index] = u32::from(width); + batch_offset_bits += u32::from(width); + } + state_0 = u32::from(node_0.next_state_idx_base) + + (packed & ((1_u64 << width_0) - 1)) as u32; + state_1 = u32::from(node_1.next_state_idx_base) + + ((packed >> shift_1) & ((1_u64 << width_1) - 1)) as u32; + state_2 = u32::from(node_2.next_state_idx_base) + + ((packed >> shift_2) & ((1_u64 << width_2) - 1)) as u32; + state_3 = u32::from(node_3.next_state_idx_base) + + ((packed >> shift_3) & ((1_u64 << width_3) - 1)) as u32; + } + } + + state_idxs = [state_0, state_1, state_2, state_3]; + for index in full_batch_len..batch_len { + let lane = index - full_batch_len; + let state_idx = &mut state_idxs[lane]; + let current_state = *state_idx; + // SAFETY: Initial states are validated, and each ANS transition stays in the table. + let node = unsafe { table.decoder_nodes.get_unchecked(current_state as usize) }; + *state_idx = + u32::from(node.next_state_idx_base) + symbols.read(node.bits_to_read)? as u32; + // SAFETY: The corresponding decoder node was read from the same state index. + lowers[index] = unsafe { *table.lowers.get_unchecked(current_state as usize) }; + bit_positions[index] = batch_offset_bits; + widths[index] = u32::from(node.offset_bits); + batch_offset_bits += u32::from(node.offset_bits); + } + [state_0, state_1, state_2, state_3] = state_idxs; + + let last_bit_position = batch_offset_start + bit_positions[batch_len - 1]; + let last_byte_position = last_bit_position as usize / 8; + vortex_ensure!( + last_byte_position + OFFSET_PADDING <= offsets.len(), + "offset stream is truncated" + ); + decoded[..batch_len].copy_from_slice(&lowers[..batch_len]); + if widths[..batch_len].iter().all(|&width| width <= 57) { + // SAFETY: The caller verified eight readable bytes after each start position. + unsafe { + read_offsets::<8>( + offsets, + batch_offset_start, + &bit_positions.0, + &widths.0, + &mut decoded.0, + batch_len, + ) + } + } else { + // SAFETY: The caller verified fifteen readable bytes after each start position. + unsafe { + read_offsets::<15>( + offsets, + batch_offset_start, + &bit_positions.0, + &widths.0, + &mut decoded.0, + batch_len, + ) + } + } + offset_bit_position += batch_offset_bits; + + let output_start = output_range.start.max(batch_start); + let output_stop = output_range.end.min(batch_start + batch_len); + if output_start < output_stop { + values + .extend_from_slice(&decoded[output_start - batch_start..output_stop - batch_start]); + } + } + symbols.finish()?; + let offset_bytes = offset_bit_position.div_ceil(8) as usize; + vortex_ensure!( + offsets.len() == offset_bytes + OFFSET_PADDING, + "offset stream has unused bytes" + ); + vortex_ensure!( + offsets[offset_bytes..].iter().all(|&byte| byte == 0), + "offset stream has nonzero padding" + ); + if !offset_bit_position.is_multiple_of(8) { + let used_bits = offset_bit_position % 8; + let trailing_mask = !low_mask(u8::try_from(used_bits)?).to_le_bytes()[0]; + vortex_ensure!( + offsets[offset_bytes - 1] & trailing_mask == 0, + "offset stream has nonzero trailing bits" + ); + } + state_idxs = [state_0, state_1, state_2, state_3]; + vortex_ensure!( + state_idxs.iter().all(|&state_idx| state_idx == 0), + "ANS stream ended in an invalid state" + ); + Ok(()) +} + +#[inline(never)] +#[expect( + clippy::cast_possible_truncation, + reason = "validated offset widths do not exceed 64 bits" +)] +unsafe fn read_offsets( + bytes: &[u8], + base_bit_position: u32, + bit_positions: &[u32], + widths: &[u32], + values: &mut [u64], + len: usize, +) { + for index in 0..len { + let bit_position = base_bit_position + bit_positions[index]; + let byte_position = bit_position as usize / 8; + let bits_past_byte = bit_position % 8; + let first = unsafe { read_u64_unaligned(bytes.as_ptr().add(byte_position)) }; + let offset = match READ_BYTES { + 8 => { + debug_assert!(widths[index] <= 57); + (first >> bits_past_byte) & ((1_u64 << widths[index]) - 1) + } + 15 => { + let second = unsafe { read_u64_unaligned(bytes.as_ptr().add(byte_position + 7)) }; + let processed = 56 - bits_past_byte; + ((first >> bits_past_byte) | (second << processed)) & low_mask(widths[index] as u8) + } + _ => unreachable!("invalid offset read width {READ_BYTES}"), + }; + values[index] = values[index].wrapping_add(offset); + } +} + +type ReadOffsetsFn = unsafe fn(&[u8], u32, &[u32], &[u32], &mut [u64], usize); + +#[used] +static FORCE_EXPORT_READ_OFFSETS_8: ReadOffsetsFn = read_offsets::<8>; + +#[used] +static FORCE_EXPORT_READ_OFFSETS_15: ReadOffsetsFn = read_offsets::<15>; + +unsafe fn read_u64_unaligned(pointer: *const u8) -> u64 { + // SAFETY: The caller provides eight readable bytes at the pointer. + u64::from_le(unsafe { pointer.cast::().read_unaligned() }) +} + +fn encode_symbols(symbols: &[usize], table: &AnsEncoderTable) -> VortexResult> { + let mut states = [table.table_size; ANS_INTERLEAVING]; + let mut writes = vec![(0u32, 0u8); symbols.len()]; + let full_len = symbols.len() / ANS_INTERLEAVING * ANS_INTERLEAVING; + + let mut encode_one = |index: usize, lane: usize| -> VortexResult<()> { + let symbol = symbols[index]; + let state = states[lane]; + let symbol_info = table + .encoder_symbols + .get(symbol) + .ok_or_else(|| vortex_error::vortex_err!("symbol {symbol} exceeds ANS table"))?; + let renorm_bits = if state >= symbol_info.renorm_bit_cutoff { + symbol_info.min_renorm_bits + 1 + } else { + symbol_info.min_renorm_bits + }; + let frequency = u32::try_from(symbol_info.next_states.len())?; + let x_s = state >> renorm_bits; + let next_state_idx = usize::try_from(x_s - frequency)?; + let next_state = *symbol_info + .next_states + .get(next_state_idx) + .ok_or_else(|| vortex_error::vortex_err!("ANS state index exceeds the symbol table"))?; + writes[index] = (state, renorm_bits); + states[lane] = next_state; + Ok(()) + }; + + for lane in (0..symbols.len() - full_len).rev() { + encode_one(full_len + lane, lane)?; + } + for base_index in (0..full_len).step_by(ANS_INTERLEAVING).rev() { + for lane in (0..ANS_INTERLEAVING).rev() { + encode_one(base_index + lane, lane)?; + } + } + + let mut encoded = Vec::with_capacity(ANS_STATE_BYTES + symbols.len() / 2); + for state in states { + encoded.extend_from_slice(&u16::try_from(state - table.table_size)?.to_le_bytes()); + } + let mut writer = BitWriter::with_capacity(symbols.len() * 2); + for (value, width) in writes { + writer.write(u64::from(value) & low_mask(width), width); + } + encoded.extend_from_slice(&writer.finish()); + encoded.extend_from_slice(&[0; ANS_PADDING]); + Ok(encoded) +} + +struct BitWriter { + bytes: Vec, + pending: u64, + pending_bits: u8, +} + +impl BitWriter { + fn with_capacity(capacity: usize) -> Self { + Self { + bytes: Vec::with_capacity(capacity), + pending: 0, + pending_bits: 0, + } + } + + fn write(&mut self, value: u64, width: u8) { + debug_assert!(width <= 64); + debug_assert!(width == 64 || value <= low_mask(width)); + if width == 0 { + return; + } + + let available = 64 - self.pending_bits; + self.pending |= value << self.pending_bits; + if width < available { + self.pending_bits += width; + return; + } + + self.bytes.extend_from_slice(&self.pending.to_le_bytes()); + let remaining = width - available; + self.pending = if remaining == 0 { + 0 + } else { + value >> available + }; + self.pending_bits = remaining; + } + + fn finish(mut self) -> Vec { + if self.pending_bits > 0 { + let byte_count = usize::from(self.pending_bits).div_ceil(8); + self.bytes + .extend_from_slice(&self.pending.to_le_bytes()[..byte_count]); + } + self.bytes + } +} + +struct BitReader<'a> { + bytes: &'a [u8], + unpadded_len: usize, + bit_position: usize, +} + +impl<'a> BitReader<'a> { + fn new(bytes: &'a [u8], unpadded_len: usize) -> VortexResult { + vortex_ensure!( + bytes.len() >= unpadded_len + ANS_PADDING, + "ANS stream is too short" + ); + Ok(Self { + bytes, + unpadded_len, + bit_position: 0, + }) + } + + fn can_read_unchecked(&self, max_bits: usize) -> bool { + let byte_position = self.bit_position / 8; + let bits_past_byte = self.bit_position % 8; + let max_bytes = (bits_past_byte + max_bits).div_ceil(8) + size_of::(); + byte_position + max_bytes <= self.bytes.len() + } + + fn check_in_bounds(&self) -> VortexResult<()> { + vortex_ensure!( + self.bit_position <= self.unpadded_len * 8, + "ANS stream is truncated" + ); + Ok(()) + } + + #[inline(always)] + unsafe fn read_unchecked(&mut self, width: u8) -> u64 { + let byte_position = self.bit_position / 8; + let bits_past_byte = self.bit_position % 8; + // SAFETY: The caller verifies eight readable bytes after the current position. + let packed = unsafe { read_u64_unaligned(self.bytes.as_ptr().add(byte_position)) }; + self.bit_position += usize::from(width); + (packed >> bits_past_byte) & ((1_u64 << width) - 1) + } + + #[inline(always)] + fn read(&mut self, width: u8) -> VortexResult { + vortex_ensure!(width <= 57, "ANS read width exceeds 57 bits"); + let required = self.bit_position + usize::from(width); + vortex_ensure!(required <= self.unpadded_len * 8, "ANS stream is truncated"); + // SAFETY: The bounds check and stream padding provide eight readable bytes. + Ok(unsafe { self.read_unchecked(width) }) + } + + fn finish(&self) -> VortexResult<()> { + let used_bytes = self.bit_position.div_ceil(8); + vortex_ensure!( + used_bytes == self.unpadded_len, + "ANS stream has unused bytes" + ); + vortex_ensure!( + self.bytes[self.unpadded_len..self.unpadded_len + ANS_PADDING] + .iter() + .all(|&byte| byte == 0), + "ANS stream has nonzero padding" + ); + if !self.bit_position.is_multiple_of(8) { + let used_bits = self.bit_position % 8; + let trailing_mask = !low_mask(u8::try_from(used_bits)?).to_le_bytes()[0]; + vortex_ensure!( + self.bytes[used_bytes - 1] & trailing_mask == 0, + "ANS stream has nonzero trailing bits" + ); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use vortex_buffer::ByteBuffer; + use vortex_error::VortexResult; + + use super::RangeEntropyCodec; + use super::RangeEntropyParts; + use super::RangeGroupedCodec; + use super::RangePackedCodec; + use super::RangeTwoLevelCodec; + + #[test] + fn empty_roundtrip() -> VortexResult<()> { + let encoded = RangeEntropyCodec::encode(&[], 256)?; + assert!(encoded.is_empty()); + assert_eq!(encoded.decode()?, Vec::::new()); + Ok(()) + } + + #[test] + fn constant_roundtrip() -> VortexResult<()> { + let values = vec![42; 10_000]; + let encoded = RangeEntropyCodec::encode(&values, 256)?; + assert_eq!(encoded.bin_lowers(), [42]); + assert_eq!(encoded.offset_widths(), [0]); + assert_eq!(encoded.decode()?, values); + Ok(()) + } + + #[test] + fn clustered_roundtrip_across_blocks() -> VortexResult<()> { + let values: Vec<_> = (0..20_000) + .map(|index| match index % 4 { + 0 => 1_000 + index % 17, + 1 => 1_000_000 + index % 31, + 2 => u64::MAX - index % 13, + _ => index % 7, + }) + .collect(); + let encoded = RangeEntropyCodec::encode(&values, 1_024)?; + assert!(encoded.bin_lowers().len() > 1); + assert_eq!( + encoded.block_offsets().len(), + values.len().div_ceil(1_024) + 1 + ); + assert_eq!(encoded.decode()?, values); + assert_eq!(encoded.value(10_777)?, values[10_777]); + Ok(()) + } + + #[test] + fn full_width_roundtrip() -> VortexResult<()> { + let values = [0, u64::MAX, 1, u64::MAX - 1]; + let encoded = RangeEntropyCodec::encode(&values, 3)?; + assert_eq!(encoded.decode()?, values); + Ok(()) + } + + #[test] + fn rejects_zero_block_length() { + assert!(RangeEntropyCodec::encode(&[1, 2, 3], 0).is_err()); + } + + #[test] + fn restart_blocks_are_independent() -> VortexResult<()> { + let values: Vec<_> = (0_u64..1_000).map(|value| value * value).collect(); + let encoded = RangeEntropyCodec::encode(&values, 100)?; + for block_idx in 0..10 { + assert_eq!( + encoded.decode_block(block_idx)?, + values[block_idx * 100..(block_idx + 1) * 100] + ); + } + Ok(()) + } + + #[test] + fn range_decode_touches_boundary_blocks() -> VortexResult<()> { + let values: Vec<_> = (0_u64..1_000).map(|value| value * value).collect(); + let encoded = RangeEntropyCodec::encode(&values, 100)?; + assert_eq!(encoded.decode_range(95..205)?, values[95..205]); + assert_eq!(encoded.decode_range(500..500)?, Vec::::new()); + assert!(encoded.decode_range(999..1_001).is_err()); + Ok(()) + } + + #[test] + fn stored_parts_roundtrip() -> VortexResult<()> { + let values: Vec<_> = (0_u64..4_000).map(|value| value % 97).collect(); + let parts = RangeEntropyCodec::encode(&values, 256)?.into_parts(); + let restored = RangeEntropyCodec::try_from_parts(parts)?; + assert_eq!(restored.decode()?, values); + Ok(()) + } + + #[test] + fn corrupt_block_offset_is_rejected() -> VortexResult<()> { + let values: Vec<_> = (0_u64..1_000).collect(); + let mut parts = RangeEntropyCodec::encode(&values, 100)?.into_parts(); + parts.block_offsets.pop(); + assert!(RangeEntropyCodec::try_from_parts(parts).is_err()); + Ok(()) + } + + #[test] + fn corrupt_payload_is_rejected_during_decode() -> VortexResult<()> { + let values: Vec<_> = (0_u64..1_000).map(|value| value % 31).collect(); + let mut parts = RangeEntropyCodec::encode(&values, 100)?.into_parts(); + let mut payload = parts.payload.to_vec(); + payload.pop(); + parts.payload = ByteBuffer::from(payload); + if let Some(last) = parts.block_offsets.last_mut() { + *last = u32::try_from(parts.payload.len())?; + } + let restored = RangeEntropyCodec::try_from_parts(parts)?; + assert!(restored.decode().is_err()); + Ok(()) + } + + #[test] + fn pseudo_random_roundtrip() -> VortexResult<()> { + let mut state = 0x4d59_5df4_d0f3_3173_u64; + let values: Vec<_> = (0..8_192) + .map(|index| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + match index % 3 { + 0 => state, + 1 => state % 1_000, + _ => u64::MAX - state % 257, + } + }) + .collect(); + for block_len in [1, 7, 256, 1_024, 8_192] { + assert_eq!( + RangeEntropyCodec::encode(&values, block_len)?.decode()?, + values + ); + } + Ok(()) + } + + #[test] + fn range_packed_roundtrip() -> VortexResult<()> { + let mut state = 0x4d59_5df4_d0f3_3173_u64; + let values: Vec<_> = (0..20_000) + .map(|index| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + match index % 4 { + 0 => state, + 1 => state % 1_000, + 2 => u64::MAX - state % 257, + _ => 42, + } + }) + .collect(); + for block_len in [1, 7, 256, 1_024, 8_192] { + assert_eq!( + RangePackedCodec::encode(&values, block_len)?.decode()?, + values + ); + } + Ok(()) + } + + #[test] + fn range_packed_empty_and_constant_roundtrip() -> VortexResult<()> { + let empty = RangePackedCodec::encode(&[], 256)?; + assert!(empty.is_empty()); + assert_eq!(empty.decode()?, Vec::::new()); + + let values = vec![42; 10_000]; + let constant = RangePackedCodec::encode(&values, 256)?; + assert_eq!(constant.symbol_width(), 0); + assert_eq!(constant.decode()?, values); + Ok(()) + } + + #[test] + fn range_two_level_roundtrip() -> VortexResult<()> { + let mut state = 0x4d59_5df4_d0f3_3173_u64; + let values: Vec<_> = (0..20_000) + .map(|index| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + match index % 4 { + 0 => state, + 1 => state % 1_000, + 2 => u64::MAX - state % 257, + _ => 42, + } + }) + .collect(); + for block_len in [1, 7, 256, 1_024, 8_192] { + assert_eq!( + RangeTwoLevelCodec::encode(&values, block_len)?.decode()?, + values + ); + } + Ok(()) + } + + #[test] + fn range_grouped_roundtrip() -> VortexResult<()> { + let values = (0_u64..20_000) + .map(|index| match index % 4 { + 0 => index, + 1 => 1_000_000 + index % 31, + 2 => u64::MAX - index % 13, + _ => 42, + }) + .collect::>(); + for block_len in [1, 7, 256, 1_024, 8_192] { + assert_eq!( + RangeGroupedCodec::encode(&values, block_len)?.decode()?, + values + ); + } + Ok(()) + } + + #[test] + fn invalid_weight_sum_is_rejected() -> VortexResult<()> { + let parts = RangeEntropyParts { + len: 1, + block_len: 1, + scale_bits: 12, + bin_lowers: vec![0], + offset_widths: vec![0], + weights: vec![1], + block_offsets: vec![0, 8], + payload: ByteBuffer::from(vec![0; 8]), + }; + assert!(RangeEntropyCodec::try_from_parts(parts).is_err()); + Ok(()) + } +} diff --git a/encodings/range-entropy/src/lib.rs b/encodings/range-entropy/src/lib.rs new file mode 100644 index 00000000000..d3c8ffe6371 --- /dev/null +++ b/encodings/range-entropy/src/lib.rs @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Native range-bin and ANS entropy codec for Vortex numeric arrays. + +mod array; +mod bit_split; +mod block_residual_array; +mod codec; +mod ordered_float_array; +mod patched_for; +mod rules; + +pub use array::*; +pub use bit_split::BitSplitCodec; +pub use block_residual_array::*; +pub use codec::RangeEntropyCodec; +pub use codec::RangeEntropyParts; +pub use codec::RangeGroupedCodec; +pub use codec::RangePackedCodec; +pub use codec::RangeTwoLevelCodec; +pub use ordered_float_array::*; +pub use patched_for::BlockResidualParts; +pub use patched_for::PatchedFoRCodec; +use vortex_array::session::ArraySessionExt; +use vortex_session::VortexSession; + +/// Register the range entropy encoding in one session. +pub fn initialize(session: &VortexSession) { + session.arrays().register(RangeEntropy); + session.arrays().register(BlockResidual); + session.arrays().register(OrderedFloat); +} diff --git a/encodings/range-entropy/src/ordered_float_array.rs b/encodings/range-entropy/src/ordered_float_array.rs new file mode 100644 index 00000000000..690ed357175 --- /dev/null +++ b/encodings/range-entropy/src/ordered_float_array.rs @@ -0,0 +1,399 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::fmt::Display; +use std::fmt::Formatter; +use std::hash::Hasher; +use std::ops::Range; + +use vortex_array::Array; +use vortex_array::ArrayEq; +use vortex_array::ArrayHash; +use vortex_array::ArrayId; +use vortex_array::ArrayParts; +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::EqMode; +use vortex_array::ExecutionCtx; +use vortex_array::ExecutionResult; +use vortex_array::IntoArray; +use vortex_array::TypedArrayRef; +use vortex_array::array_slots; +use vortex_array::arrays::Primitive; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::slice::SliceReduce; +use vortex_array::arrays::slice::SliceReduceAdaptor; +use vortex_array::buffer::BufferHandle; +use vortex_array::dtype::DType; +use vortex_array::dtype::PType; +use vortex_array::optimizer::rules::ParentRuleSet; +use vortex_array::scalar::Scalar; +use vortex_array::serde::ArrayChildren; +use vortex_array::vtable::OperationsVTable; +use vortex_array::vtable::VTable; +use vortex_array::vtable::ValidityChild; +use vortex_array::vtable::ValidityVTableFromChild; +use vortex_buffer::Buffer; +use vortex_error::VortexExpect; +use vortex_error::VortexResult; +use vortex_error::vortex_bail; +use vortex_error::vortex_ensure; +use vortex_error::vortex_panic; +use vortex_session::VortexSession; +use vortex_session::registry::CachedId; + +use crate::BlockResidual; +use crate::block_residual_array::decompress_ordered_f64; + +/// IEEE floats mapped to unsigned integers that preserve numeric order. +pub type OrderedFloatArray = Array; + +#[array_slots(OrderedFloat)] +pub struct OrderedFloatSlots { + /// Ordered unsigned float bits. + #[slot(0)] + pub encoded: ArrayRef, +} + +#[derive(Clone, Debug, Default)] +pub struct OrderedFloatData; + +impl Display for OrderedFloatData { + fn fmt(&self, _f: &mut Formatter<'_>) -> std::fmt::Result { + Ok(()) + } +} + +impl ArrayHash for OrderedFloatData { + fn array_hash(&self, _state: &mut H, _accuracy: EqMode) {} +} + +impl ArrayEq for OrderedFloatData { + fn array_eq(&self, _other: &Self, _accuracy: EqMode) -> bool { + true + } +} + +#[derive(Clone, Debug)] +pub struct OrderedFloat; + +impl VTable for OrderedFloat { + type TypedArrayData = OrderedFloatData; + type OperationsVTable = Self; + type ValidityVTable = ValidityVTableFromChild; + + fn id(&self) -> ArrayId { + static ID: CachedId = CachedId::new("vortex.ordered_float"); + *ID + } + + fn validate( + &self, + _data: &Self::TypedArrayData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + let ptype = PType::try_from(dtype)?; + vortex_ensure!( + matches!(ptype, PType::F32 | PType::F64), + "OrderedFloatArray requires f32 or f64" + ); + let encoded = OrderedFloatSlotsView::from_slots(slots).encoded; + let expected = DType::Primitive(ordered_ptype(ptype)?, dtype.nullability()); + vortex_ensure!( + encoded.dtype() == &expected, + "OrderedFloatArray expected child dtype {expected}, got {}", + encoded.dtype() + ); + vortex_ensure!( + encoded.len() == len, + "OrderedFloatArray child length differs" + ); + Ok(()) + } + + fn nbuffers(_array: ArrayView<'_, Self>) -> usize { + 0 + } + + fn buffer(_array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { + vortex_panic!("OrderedFloatArray buffer index {idx} out of bounds") + } + + fn buffer_name(_array: ArrayView<'_, Self>, idx: usize) -> Option { + vortex_panic!("OrderedFloatArray buffer_name {idx} out of bounds") + } + + fn with_buffers( + &self, + array: ArrayView<'_, Self>, + buffers: &[BufferHandle], + ) -> VortexResult> { + vortex_array::vtable::with_empty_buffers(self, array, buffers) + } + + fn serialize( + _array: ArrayView<'_, Self>, + _session: &VortexSession, + ) -> VortexResult>> { + Ok(Some(Vec::new())) + } + + fn deserialize( + &self, + dtype: &DType, + len: usize, + metadata: &[u8], + _buffers: &[BufferHandle], + children: &dyn ArrayChildren, + _session: &VortexSession, + ) -> VortexResult> { + vortex_ensure!( + metadata.is_empty(), + "OrderedFloatArray metadata must be empty" + ); + vortex_ensure!(children.len() == 1, "OrderedFloatArray requires one child"); + let ptype = PType::try_from(dtype)?; + let child_dtype = DType::Primitive(ordered_ptype(ptype)?, dtype.nullability()); + let encoded = children.get(0, &child_dtype, len)?; + Ok( + ArrayParts::new(self.clone(), dtype.clone(), len, OrderedFloatData) + .with_slots(OrderedFloatSlots { encoded }.into_slots()), + ) + } + + fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { + OrderedFloatSlots::NAMES[idx].to_string() + } + + fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { + let decoded = if let Some(block_residual) = array.encoded().as_typed::() { + vortex_ensure!( + array.dtype().as_ptype() == PType::F64, + "fused BlockResidual decode requires f64" + ); + decompress_ordered_f64(block_residual, ctx)? + } else { + decode_primitive(array.as_view(), ctx)? + }; + Ok(ExecutionResult::done(decoded.into_array())) + } + + fn reduce_parent( + array: ArrayView<'_, Self>, + parent: &ArrayRef, + child_idx: usize, + ) -> VortexResult> { + RULES.evaluate(array, parent, child_idx) + } +} + +impl OperationsVTable for OrderedFloat { + fn scalar_at( + array: ArrayView<'_, OrderedFloat>, + index: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + let scalar = array.encoded().execute_scalar(index, ctx)?; + if scalar.is_null() { + return Ok(Scalar::null(array.dtype().clone())); + } + Ok(match array.dtype().as_ptype() { + PType::F32 => Scalar::primitive( + f32::from_bits(unordered_u32( + scalar + .as_primitive() + .typed_value::() + .vortex_expect("validated ordered float scalar"), + )), + array.dtype().nullability(), + ), + PType::F64 => Scalar::primitive( + f64::from_bits(unordered_u64( + scalar + .as_primitive() + .typed_value::() + .vortex_expect("validated ordered float scalar"), + )), + array.dtype().nullability(), + ), + ptype => vortex_panic!("unsupported OrderedFloat ptype {ptype}"), + }) + } +} + +impl ValidityChild for OrderedFloat { + fn validity_child(array: ArrayView<'_, OrderedFloat>) -> ArrayRef { + array.encoded().clone() + } +} + +impl SliceReduce for OrderedFloat { + fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { + Ok(Some( + OrderedFloat::try_new(array.encoded().slice(range)?, array.dtype().as_ptype())? + .into_array(), + )) + } +} + +static RULES: ParentRuleSet = + ParentRuleSet::new(&[ParentRuleSet::lift(&SliceReduceAdaptor(OrderedFloat))]); + +pub trait OrderedFloatArrayExt: TypedArrayRef + OrderedFloatArraySlotsExt {} + +impl> OrderedFloatArrayExt for T {} + +impl OrderedFloat { + /// Construct an ordered float array from an unsigned child. + pub fn try_new(encoded: ArrayRef, float_ptype: PType) -> VortexResult { + vortex_ensure!( + matches!(float_ptype, PType::F32 | PType::F64), + "OrderedFloat requires f32 or f64" + ); + let dtype = DType::Primitive(float_ptype, encoded.dtype().nullability()); + let len = encoded.len(); + Array::try_from_parts( + ArrayParts::new(OrderedFloat, dtype, len, OrderedFloatData) + .with_slots(OrderedFloatSlots { encoded }.into_slots()), + ) + } + + /// Map canonical floats to ordered unsigned integer bits. + pub fn from_primitive(array: ArrayView<'_, Primitive>) -> VortexResult { + let validity = array.validity()?; + match array.ptype() { + PType::F32 => Self::try_new( + PrimitiveArray::new( + Buffer::from( + array + .as_slice::() + .iter() + .map(|value| ordered_u32(value.to_bits())) + .collect::>(), + ), + validity, + ) + .into_array(), + PType::F32, + ), + PType::F64 => Self::try_new( + PrimitiveArray::new( + Buffer::from( + array + .as_slice::() + .iter() + .map(|value| ordered_u64(value.to_bits())) + .collect::>(), + ), + validity, + ) + .into_array(), + PType::F64, + ), + ptype => vortex_bail!("OrderedFloat requires f32 or f64, got {ptype}"), + } + } +} + +fn decode_primitive( + array: ArrayView<'_, OrderedFloat>, + ctx: &mut ExecutionCtx, +) -> VortexResult { + let encoded = array.encoded().clone().execute::(ctx)?; + Ok(match array.dtype().as_ptype() { + PType::F32 => PrimitiveArray::new( + Buffer::from( + encoded + .as_slice::() + .iter() + .map(|&value| f32::from_bits(unordered_u32(value))) + .collect::>(), + ), + encoded.validity()?, + ), + PType::F64 => PrimitiveArray::new( + Buffer::from( + encoded + .as_slice::() + .iter() + .map(|&value| f64::from_bits(unordered_u64(value))) + .collect::>(), + ), + encoded.validity()?, + ), + ptype => vortex_panic!("unsupported OrderedFloat ptype {ptype}"), + }) +} + +fn ordered_ptype(ptype: PType) -> VortexResult { + match ptype { + PType::F32 => Ok(PType::U32), + PType::F64 => Ok(PType::U64), + _ => vortex_bail!("OrderedFloat requires f32 or f64, got {ptype}"), + } +} + +fn ordered_u32(bits: u32) -> u32 { + if bits & (1_u32 << 31) == 0 { + bits ^ (1_u32 << 31) + } else { + !bits + } +} + +fn unordered_u32(value: u32) -> u32 { + if value & (1_u32 << 31) == 0 { + !value + } else { + value ^ (1_u32 << 31) + } +} + +fn ordered_u64(bits: u64) -> u64 { + if bits & (1_u64 << 63) == 0 { + bits ^ (1_u64 << 63) + } else { + !bits + } +} + +fn unordered_u64(value: u64) -> u64 { + if value & (1_u64 << 63) == 0 { + !value + } else { + value ^ (1_u64 << 63) + } +} + +#[cfg(test)] +mod tests { + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::array_session; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::assert_arrays_eq; + use vortex_error::VortexResult; + + use super::OrderedFloat; + + #[test] + fn roundtrip_special_values() -> VortexResult<()> { + let primitive = PrimitiveArray::from_iter([ + f64::NEG_INFINITY, + -1.0, + -0.0, + 0.0, + 1.0, + f64::INFINITY, + f64::from_bits(0x7ff8_0000_0000_0042), + ]); + let encoded = OrderedFloat::from_primitive(primitive.as_view())?; + let session = array_session(); + crate::initialize(&session); + let mut ctx = session.create_execution_ctx(); + assert_arrays_eq!(encoded, primitive.into_array(), &mut ctx); + Ok(()) + } +} diff --git a/encodings/range-entropy/src/patched_for.rs b/encodings/range-entropy/src/patched_for.rs new file mode 100644 index 00000000000..deecaac75e2 --- /dev/null +++ b/encodings/range-entropy/src/patched_for.rs @@ -0,0 +1,772 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use fastlanes::BitPacking; +use vortex_error::VortexResult; + +const CHUNK_LEN: usize = 1024; +const BASE_SAMPLE_LEN: usize = 64; +const HIGH_PADDING: usize = 15; +const SERIALIZED_BLOCK_METADATA_BYTES: usize = 12; + +/// A prototype patched frame-of-reference codec for ordered unsigned latents. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PatchedFoRCodec { + len: usize, + blocks: Vec, +} + +/// Serialized children for the one-reference block residual codec. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BlockResidualParts { + pub len: usize, + pub bases: Vec, + pub residual_widths: Vec, + pub high_widths: Vec, + pub residual_starts: Vec, + pub patch_starts: Vec, + pub high_starts: Vec, + pub residual_words: Vec, + pub patch_positions: Vec, + pub patch_highs: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct PatchedFoRBlock { + len: u16, + bases: Vec, + cluster_width: u8, + residual_width: u8, + high_width: u8, + clusters: Vec, + residuals: Vec, + patch_positions: Vec, + patch_highs: Vec, +} + +impl PatchedFoRCodec { + /// Encode ordered unsigned latents in independent 1024-value blocks. + pub fn encode(values: &[u64]) -> VortexResult { + let blocks = values + .chunks(CHUNK_LEN) + .map(encode_block) + .collect::>>()?; + Ok(Self { + len: values.len(), + blocks, + }) + } + + /// Encode with one base and a residual-width histogram per block. + pub fn encode_single_base(values: &[u64]) -> VortexResult { + let blocks = values + .chunks(CHUNK_LEN) + .map(encode_single_base_block) + .collect::>>()?; + Ok(Self { + len: values.len(), + blocks, + }) + } + + /// Convert a one-reference codec into serialized array children. + pub fn into_single_base_parts(self) -> VortexResult { + let mut parts = BlockResidualParts { + len: self.len, + bases: Vec::with_capacity(self.blocks.len()), + residual_widths: Vec::with_capacity(self.blocks.len()), + high_widths: Vec::with_capacity(self.blocks.len()), + residual_starts: Vec::with_capacity(self.blocks.len() + 1), + patch_starts: Vec::with_capacity(self.blocks.len() + 1), + high_starts: Vec::with_capacity(self.blocks.len() + 1), + residual_words: Vec::new(), + patch_positions: Vec::new(), + patch_highs: Vec::new(), + }; + parts.residual_starts.push(0); + parts.patch_starts.push(0); + parts.high_starts.push(0); + for block in self.blocks { + vortex_error::vortex_ensure!( + block.bases.len() == 1 && block.cluster_width == 0, + "block residual parts require one reference per block" + ); + parts.bases.push(block.bases[0]); + parts.residual_widths.push(block.residual_width); + parts.high_widths.push(block.high_width); + parts.residual_words.extend(block.residuals); + parts.patch_positions.extend(block.patch_positions); + parts.patch_highs.extend(block.patch_highs); + parts + .residual_starts + .push(u32::try_from(parts.residual_words.len())?); + parts + .patch_starts + .push(u32::try_from(parts.patch_positions.len())?); + parts + .high_starts + .push(u32::try_from(parts.patch_highs.len())?); + } + Ok(parts) + } + + /// Reconstruct a one-reference codec from serialized array children. + pub fn try_from_single_base_parts(parts: BlockResidualParts) -> VortexResult { + let block_count = parts.len.div_ceil(CHUNK_LEN); + vortex_error::vortex_ensure!( + parts.bases.len() == block_count + && parts.residual_widths.len() == block_count + && parts.high_widths.len() == block_count, + "block residual metadata child lengths are invalid" + ); + validate_starts( + &parts.residual_starts, + block_count, + parts.residual_words.len(), + "residual", + )?; + validate_starts( + &parts.patch_starts, + block_count, + parts.patch_positions.len(), + "patch", + )?; + validate_starts( + &parts.high_starts, + block_count, + parts.patch_highs.len(), + "patch high", + )?; + + let mut blocks = Vec::with_capacity(block_count); + for block_index in 0..block_count { + let residual_start = usize::try_from(parts.residual_starts[block_index])?; + let residual_stop = usize::try_from(parts.residual_starts[block_index + 1])?; + let patch_start = usize::try_from(parts.patch_starts[block_index])?; + let patch_stop = usize::try_from(parts.patch_starts[block_index + 1])?; + let high_start = usize::try_from(parts.high_starts[block_index])?; + let high_stop = usize::try_from(parts.high_starts[block_index + 1])?; + let block_start = block_index * CHUNK_LEN; + let block_len = (parts.len - block_start).min(CHUNK_LEN); + let residual_width = parts.residual_widths[block_index]; + let high_width = parts.high_widths[block_index]; + vortex_error::vortex_ensure!( + residual_width <= 64 && high_width <= 64, + "block residual bit width exceeds 64" + ); + vortex_error::vortex_ensure!( + residual_stop - residual_start + == CHUNK_LEN * usize::from(residual_width) / u64::BITS as usize, + "block residual packed word count is invalid" + ); + let patch_positions = &parts.patch_positions[patch_start..patch_stop]; + vortex_error::vortex_ensure!( + patch_positions + .iter() + .all(|&position| usize::from(position) < block_len) + && patch_positions + .windows(2) + .all(|window| window[0] < window[1]), + "block residual patch positions are invalid" + ); + let expected_high_len = if patch_positions.is_empty() { + 0 + } else { + (patch_positions.len() * usize::from(high_width)).div_ceil(8) + HIGH_PADDING + }; + vortex_error::vortex_ensure!( + high_stop - high_start == expected_high_len, + "block residual patch high payload length is invalid" + ); + blocks.push(PatchedFoRBlock { + len: u16::try_from(block_len)?, + bases: vec![parts.bases[block_index]], + cluster_width: 0, + residual_width, + high_width, + clusters: Vec::new(), + residuals: parts.residual_words[residual_start..residual_stop].to_vec(), + patch_positions: parts.patch_positions[patch_start..patch_stop].to_vec(), + patch_highs: parts.patch_highs[high_start..high_stop].to_vec(), + }); + } + Ok(Self { + len: parts.len, + blocks, + }) + } + + /// Decode all values. + pub fn decode(&self) -> VortexResult> { + let mut values = Vec::with_capacity(self.len); + let mut clusters = [0u64; CHUNK_LEN]; + let mut residuals = [0u64; CHUNK_LEN]; + for block in &self.blocks { + clusters.fill(0); + residuals.fill(0); + if block.cluster_width > 0 { + // SAFETY: The encoder creates one complete FastLanes chunk. + unsafe { + u64::unchecked_unpack( + usize::from(block.cluster_width), + &block.clusters, + &mut clusters, + ); + } + } + if block.residual_width > 0 { + // SAFETY: The encoder creates one complete FastLanes chunk. + unsafe { + u64::unchecked_unpack( + usize::from(block.residual_width), + &block.residuals, + &mut residuals, + ); + } + } + + let mut high_bit_position = 0usize; + for &position in &block.patch_positions { + // SAFETY: The encoder appends fifteen readable padding bytes. + let high = unsafe { + read_wide_bits(&block.patch_highs, high_bit_position, block.high_width) + }; + residuals[usize::from(position)] |= high << block.residual_width; + high_bit_position += usize::from(block.high_width); + } + + let block_len = usize::from(block.len); + match block.bases.as_slice() { + [base] => { + for residual in &mut residuals[..block_len] { + *residual = residual.wrapping_add(*base); + } + } + bases => { + for (index, residual) in residuals[..block_len].iter_mut().enumerate() { + let cluster = usize::try_from(clusters[index])?; + *residual = residual.wrapping_add(bases[cluster]); + } + } + } + values.extend_from_slice(&residuals[..block_len]); + } + Ok(values) + } + + /// Decode ordered `f64` latents with a fused inverse transform. + pub fn decode_ordered_f64(&self) -> VortexResult> { + let mut values = Vec::with_capacity(self.len); + let mut clusters = [0_u64; CHUNK_LEN]; + let mut residuals = [0_u64; CHUNK_LEN]; + for block in &self.blocks { + clusters.fill(0); + residuals.fill(0); + if block.cluster_width > 0 { + // SAFETY: The encoder creates one complete FastLanes chunk. + unsafe { + u64::unchecked_unpack( + usize::from(block.cluster_width), + &block.clusters, + &mut clusters, + ); + } + } + if block.residual_width > 0 { + // SAFETY: The encoder creates one complete FastLanes chunk. + unsafe { + u64::unchecked_unpack( + usize::from(block.residual_width), + &block.residuals, + &mut residuals, + ); + } + } + let mut high_bit_position = 0_usize; + for &position in &block.patch_positions { + // SAFETY: The encoder appends fifteen readable padding bytes. + let high = unsafe { + read_wide_bits(&block.patch_highs, high_bit_position, block.high_width) + }; + residuals[usize::from(position)] |= high << block.residual_width; + high_bit_position += usize::from(block.high_width); + } + let block_len = usize::from(block.len); + match block.bases.as_slice() { + [base] => { + for residual in &mut residuals[..block_len] { + *residual = residual.wrapping_add(*base); + } + } + bases => { + for (index, residual) in residuals[..block_len].iter_mut().enumerate() { + let cluster = usize::try_from(clusters[index])?; + *residual = residual.wrapping_add(bases[cluster]); + } + } + } + values.extend(residuals[..block_len].iter().map(|&ordered| { + let bits = if ordered & (1_u64 << 63) == 0 { + !ordered + } else { + ordered ^ (1_u64 << 63) + }; + f64::from_bits(bits) + })); + } + Ok(values) + } + + /// Decode one value with direct packed access and a binary patch search. + pub fn scalar_at(&self, index: usize) -> VortexResult { + vortex_error::vortex_ensure!( + index < self.len, + "index {index} is out of bounds for length {}", + self.len + ); + let block = &self.blocks[index / CHUNK_LEN]; + let index_in_block = index % CHUNK_LEN; + let cluster = if block.cluster_width == 0 { + 0 + } else { + // SAFETY: The encoder creates one complete FastLanes chunk. + unsafe { + usize::try_from(u64::unchecked_unpack_single( + usize::from(block.cluster_width), + &block.clusters, + index_in_block, + ))? + } + }; + let mut residual = if block.residual_width == 0 { + 0 + } else { + // SAFETY: The encoder creates one complete FastLanes chunk. + unsafe { + u64::unchecked_unpack_single( + usize::from(block.residual_width), + &block.residuals, + index_in_block, + ) + } + }; + if let Ok(patch_index) = block + .patch_positions + .binary_search(&u16::try_from(index_in_block)?) + { + // SAFETY: The encoder appends fifteen readable padding bytes. + let high = unsafe { + read_wide_bits( + &block.patch_highs, + patch_index * usize::from(block.high_width), + block.high_width, + ) + }; + residual |= high << block.residual_width; + } + Ok(block.bases[cluster].wrapping_add(residual)) + } + + /// Return the logical value count. + pub fn len(&self) -> usize { + self.len + } + + /// Return true when the codec contains no values. + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + /// Return the encoded bytes, including estimated serialized block metadata. + pub fn encoded_size(&self) -> usize { + self.blocks + .iter() + .map(|block| { + SERIALIZED_BLOCK_METADATA_BYTES + + block.bases.len() * size_of::() + + block.clusters.len() * size_of::() + + block.residuals.len() * size_of::() + + block.patch_positions.len() * size_of::() + + block.patch_highs.len() + }) + .sum() + } + + /// Return the logical block count. + pub fn block_count(&self) -> usize { + self.blocks.len() + } + + /// Return the total patch count. + pub fn patch_count(&self) -> usize { + self.blocks + .iter() + .map(|block| block.patch_positions.len()) + .sum() + } + + /// Return the sum of base counts across all blocks. + pub fn total_base_count(&self) -> usize { + self.blocks.iter().map(|block| block.bases.len()).sum() + } + + /// Return the sum of main residual widths across all blocks. + pub fn total_residual_width(&self) -> usize { + self.blocks + .iter() + .map(|block| usize::from(block.residual_width)) + .sum() + } +} + +fn encode_single_base_block(values: &[u64]) -> VortexResult { + let base = values.iter().copied().min().unwrap_or(0); + let mut residuals = Vec::with_capacity(CHUNK_LEN); + let mut width_counts = [0usize; 65]; + let mut maximum_width = 0u8; + for &value in values { + let residual = value - base; + let width = bit_width(residual); + residuals.push(residual); + width_counts[usize::from(width)] += 1; + maximum_width = maximum_width.max(width); + } + residuals.resize(CHUNK_LEN, 0); + + let mut patch_count = values.len(); + let mut best = (usize::MAX, maximum_width, 0u8, 0usize); + for residual_width in 0..=maximum_width { + patch_count -= width_counts[usize::from(residual_width)]; + let high_width = if patch_count == 0 { + 0 + } else { + maximum_width - residual_width + }; + let cost_bits = usize::from(residual_width) * CHUNK_LEN + + patch_count * (u16::BITS as usize + usize::from(high_width)) + + u64::BITS as usize + + SERIALIZED_BLOCK_METADATA_BYTES * 8 + + usize::from(patch_count > 0) * HIGH_PADDING * 8; + if cost_bits < best.0 { + best = (cost_bits, residual_width, high_width, patch_count); + } + } + + materialize_block( + values, + BlockPlan { + bases: vec![base], + cluster_width: 0, + residual_width: best.1, + high_width: best.2, + clusters: vec![0; CHUNK_LEN], + residuals, + patch_count: best.3, + cost_bits: best.0, + }, + ) +} + +fn validate_starts( + starts: &[u32], + block_count: usize, + payload_len: usize, + name: &str, +) -> VortexResult<()> { + vortex_error::vortex_ensure!( + starts.len() == block_count + 1, + "block residual {name} offsets have an invalid length" + ); + vortex_error::vortex_ensure!( + starts.first() == Some(&0) && usize::try_from(*starts.last().unwrap_or(&0))? == payload_len, + "block residual {name} offsets do not cover the payload" + ); + vortex_error::vortex_ensure!( + starts.windows(2).all(|window| window[0] <= window[1]), + "block residual {name} offsets are not ordered" + ); + Ok(()) +} + +fn encode_block(values: &[u64]) -> VortexResult { + let sampled = sampled_values(values); + let mut best: Option = None; + for requested_base_count in [1, 2, 4] { + let bases = quantile_bases(&sampled, requested_base_count); + let plan = plan_block(values, bases)?; + if best + .as_ref() + .is_none_or(|current| plan.cost_bits < current.cost_bits) + { + best = Some(plan); + } + } + materialize_block( + values, + best.ok_or_else(|| vortex_error::vortex_err!("patched FoR block has no plan"))?, + ) +} + +fn sampled_values(values: &[u64]) -> Vec { + if values.len() <= BASE_SAMPLE_LEN { + let mut sampled = values.to_vec(); + sampled.sort_unstable(); + return sampled; + } + + let mut sampled = (0..BASE_SAMPLE_LEN) + .map(|index| values[index * values.len() / BASE_SAMPLE_LEN]) + .collect::>(); + sampled.push(values.iter().copied().min().unwrap_or(0)); + sampled.sort_unstable(); + sampled +} + +struct BlockPlan { + bases: Vec, + cluster_width: u8, + residual_width: u8, + high_width: u8, + clusters: Vec, + residuals: Vec, + patch_count: usize, + cost_bits: usize, +} + +fn quantile_bases(sorted: &[u64], requested_count: usize) -> Vec { + let mut bases = (0..requested_count) + .map(|index| sorted[index * sorted.len() / requested_count]) + .collect::>(); + bases.dedup(); + bases +} + +fn plan_block(values: &[u64], bases: Vec) -> VortexResult { + let cluster_width = bit_width(u64::try_from(bases.len() - 1)?); + let mut clusters = Vec::with_capacity(CHUNK_LEN); + let mut residuals = Vec::with_capacity(CHUNK_LEN); + let mut width_counts = [0usize; 65]; + let mut maximum_width = 0u8; + for &value in values { + let cluster = bases + .partition_point(|&base| base <= value) + .saturating_sub(1); + let residual = value - bases[cluster]; + clusters.push(u64::try_from(cluster)?); + residuals.push(residual); + let width = bit_width(residual); + width_counts[usize::from(width)] += 1; + maximum_width = maximum_width.max(width); + } + clusters.resize(CHUNK_LEN, 0); + residuals.resize(CHUNK_LEN, 0); + + let mut patch_count = values.len(); + let mut best = (usize::MAX, maximum_width, 0u8, patch_count); + for residual_width in 0..=maximum_width { + patch_count -= width_counts[usize::from(residual_width)]; + let high_width = if patch_count == 0 { + 0 + } else { + maximum_width - residual_width + }; + let cost_bits = usize::from(cluster_width) * CHUNK_LEN + + usize::from(residual_width) * CHUNK_LEN + + patch_count * (u16::BITS as usize + usize::from(high_width)) + + bases.len() * u64::BITS as usize + + SERIALIZED_BLOCK_METADATA_BYTES * 8 + + usize::from(patch_count > 0) * HIGH_PADDING * 8; + if cost_bits < best.0 { + best = (cost_bits, residual_width, high_width, patch_count); + } + } + + Ok(BlockPlan { + bases, + cluster_width, + residual_width: best.1, + high_width: best.2, + clusters, + residuals, + patch_count: best.3, + cost_bits: best.0, + }) +} + +fn materialize_block(values: &[u64], plan: BlockPlan) -> VortexResult { + let clusters = fast_pack(&plan.clusters, plan.cluster_width); + let residual_mask = low_mask(plan.residual_width); + let low_residuals = plan + .residuals + .iter() + .map(|&residual| residual & residual_mask) + .collect::>(); + let residuals = fast_pack(&low_residuals, plan.residual_width); + let mut patch_positions = Vec::with_capacity(plan.patch_count); + let mut patch_highs = BitWriter::with_capacity(plan.patch_count * 8); + if plan.high_width > 0 { + for (position, &residual) in plan.residuals[..values.len()].iter().enumerate() { + let high = residual >> plan.residual_width; + if high != 0 { + patch_positions.push(u16::try_from(position)?); + patch_highs.write(high, plan.high_width); + } + } + } + let patch_highs = if patch_positions.is_empty() { + Vec::new() + } else { + let mut encoded = patch_highs.finish(); + encoded.extend_from_slice(&[0; HIGH_PADDING]); + encoded + }; + + Ok(PatchedFoRBlock { + len: u16::try_from(values.len())?, + bases: plan.bases, + cluster_width: plan.cluster_width, + residual_width: plan.residual_width, + high_width: plan.high_width, + clusters, + residuals, + patch_positions, + patch_highs, + }) +} + +fn fast_pack(values: &[u64], width: u8) -> Vec { + if width == 0 { + return Vec::new(); + } + let mut packed = vec![0u64; CHUNK_LEN * usize::from(width) / u64::BITS as usize]; + // SAFETY: Both slices have the exact lengths required for one FastLanes chunk. + unsafe { u64::unchecked_pack(usize::from(width), values, &mut packed) }; + packed +} + +fn bit_width(value: u64) -> u8 { + u8::try_from(u64::BITS - value.leading_zeros()).unwrap_or(64) +} + +fn low_mask(bits: u8) -> u64 { + match bits { + 0 => 0, + 64 => u64::MAX, + _ => (1_u64 << bits) - 1, + } +} + +pub(crate) unsafe fn read_wide_bits(bytes: &[u8], bit_position: usize, width: u8) -> u64 { + let byte_position = bit_position / 8; + let bits_past_byte = bit_position % 8; + // SAFETY: The caller provides fifteen readable padding bytes. + let first = unsafe { read_u64_unaligned(bytes.as_ptr().add(byte_position)) }; + if width <= 57 { + (first >> bits_past_byte) & low_mask(width) + } else { + // SAFETY: The caller provides fifteen readable padding bytes. + let second = unsafe { read_u64_unaligned(bytes.as_ptr().add(byte_position + 7)) }; + let processed = 56 - bits_past_byte; + ((first >> bits_past_byte) | (second << processed)) & low_mask(width) + } +} + +unsafe fn read_u64_unaligned(pointer: *const u8) -> u64 { + // SAFETY: The caller provides eight readable bytes at the pointer. + u64::from_le(unsafe { pointer.cast::().read_unaligned() }) +} + +struct BitWriter { + bytes: Vec, + pending: u64, + pending_bits: u8, +} + +impl BitWriter { + fn with_capacity(capacity: usize) -> Self { + Self { + bytes: Vec::with_capacity(capacity), + pending: 0, + pending_bits: 0, + } + } + + fn write(&mut self, value: u64, width: u8) { + if width == 0 { + return; + } + let available = 64 - self.pending_bits; + self.pending |= value << self.pending_bits; + if width < available { + self.pending_bits += width; + return; + } + self.bytes.extend_from_slice(&self.pending.to_le_bytes()); + let remaining = width - available; + self.pending = if remaining == 0 { + 0 + } else { + value >> available + }; + self.pending_bits = remaining; + } + + fn finish(mut self) -> Vec { + if self.pending_bits > 0 { + let byte_count = usize::from(self.pending_bits).div_ceil(8); + self.bytes + .extend_from_slice(&self.pending.to_le_bytes()[..byte_count]); + } + self.bytes + } +} + +#[cfg(test)] +mod tests { + use vortex_error::VortexResult; + + use super::PatchedFoRCodec; + + #[test] + fn roundtrip_lengths_and_domains() -> VortexResult<()> { + for len in [0, 1, 1_023, 1_024, 1_025, 20_000] { + let values = (0..len) + .map(|index| match index % 4 { + 0 => index as u64, + 1 => 1_000_000 + index as u64 % 31, + 2 => u64::MAX - index as u64 % 13, + _ => 42, + }) + .collect::>(); + let codec = PatchedFoRCodec::encode(&values)?; + assert_eq!(codec.decode()?, values); + for (index, &value) in values.iter().enumerate() { + assert_eq!(codec.scalar_at(index)?, value); + } + } + Ok(()) + } + + #[test] + fn constant_roundtrip() -> VortexResult<()> { + let values = vec![42; 10_000]; + let codec = PatchedFoRCodec::encode(&values)?; + assert_eq!(codec.decode()?, values); + assert_eq!(codec.scalar_at(4_321)?, 42); + Ok(()) + } + + #[test] + fn single_base_parts_roundtrip() -> VortexResult<()> { + let values = (0..4_099) + .map(|index| { + let value = u64::try_from(index)?; + Ok(1_000_000_u64.wrapping_add(value * value)) + }) + .collect::>>()?; + let parts = PatchedFoRCodec::encode_single_base(&values)?.into_single_base_parts()?; + let codec = PatchedFoRCodec::try_from_single_base_parts(parts)?; + assert_eq!(codec.decode()?, values); + Ok(()) + } +} diff --git a/encodings/range-entropy/src/rules.rs b/encodings/range-entropy/src/rules.rs new file mode 100644 index 00000000000..060e7943666 --- /dev/null +++ b/encodings/range-entropy/src/rules.rs @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use vortex_array::arrays::slice::SliceReduceAdaptor; +use vortex_array::optimizer::rules::ParentRuleSet; + +use crate::RangeEntropy; + +pub(crate) static RULES: ParentRuleSet = + ParentRuleSet::new(&[ParentRuleSet::lift(&SliceReduceAdaptor(RangeEntropy))]); diff --git a/vortex-btrblocks/Cargo.toml b/vortex-btrblocks/Cargo.toml index 4e22f042adf..6312637c6e8 100644 --- a/vortex-btrblocks/Cargo.toml +++ b/vortex-btrblocks/Cargo.toml @@ -28,9 +28,11 @@ vortex-datetime-parts = { workspace = true } vortex-decimal-byte-parts = { workspace = true } vortex-error = { workspace = true } vortex-fastlanes = { workspace = true } +vortex-float-quant = { workspace = true } vortex-fsst = { workspace = true } vortex-onpair = { workspace = true, optional = true } vortex-pco = { workspace = true, optional = true } +vortex-range-entropy = { workspace = true } vortex-runend = { workspace = true } vortex-sequence = { workspace = true } vortex-sparse = { workspace = true } @@ -40,8 +42,11 @@ vortex-zstd = { workspace = true, optional = true } [dev-dependencies] arrow-array = { workspace = true } +arrow-schema = { workspace = true } +arrow-select = { workspace = true } divan = { workspace = true } insta = { workspace = true } +parquet = { workspace = true } rstest = { workspace = true } test-with = { workspace = true } tpchgen = { workspace = true } @@ -69,3 +74,15 @@ test = false name = "compress_listview" harness = false test = false + +[[example]] +name = "pco_probe" +required-features = ["pco"] + +[[example]] +name = "range_entropy_throughput" +required-features = ["pco", "zstd"] + +[[example]] +name = "float_quant_dataset" +required-features = ["pco", "zstd"] diff --git a/vortex-btrblocks/examples/float_quant_dataset.rs b/vortex-btrblocks/examples/float_quant_dataset.rs new file mode 100644 index 00000000000..b9fb8e955e8 --- /dev/null +++ b/vortex-btrblocks/examples/float_quant_dataset.rs @@ -0,0 +1,885 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::fs::File; +use std::hint::black_box; +use std::io::BufRead; +use std::io::BufReader; +use std::path::Path; +use std::sync::Arc; +use std::time::Duration; +use std::time::Instant; + +use arrow_array::ArrayRef as ArrowArrayRef; +use arrow_schema::DataType; +use arrow_select::concat::concat; +use parquet::arrow::ProjectionMask; +use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; +use pco::ChunkConfig; +use pco::DeltaSpec; +use pco::ModeSpec; +use pco::data_types::Number; +use pco::metadata::DeltaEncoding; +use pco::metadata::Mode; +use pco::wrapped::FileCompressor; +use vortex_array::ArrayRef; +use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::array_session; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::assert_arrays_eq; +use vortex_array::dtype::PType; +use vortex_arrow::ArrowSessionExt; +use vortex_btrblocks::BtrBlocksCompressor; +use vortex_btrblocks::BtrBlocksCompressorBuilder; +use vortex_btrblocks::SchemeExt; +use vortex_btrblocks::schemes::float::FloatQuantScheme; +use vortex_btrblocks::schemes::range_entropy::RangeEntropyScheme; +use vortex_error::VortexResult; +use vortex_error::vortex_ensure; +use vortex_error::vortex_err; +use vortex_pco::Pco; +use vortex_range_entropy::BitSplitCodec; +use vortex_range_entropy::PatchedFoRCodec; +use vortex_range_entropy::RangeEntropyCodec; +use vortex_range_entropy::RangePackedCodec; +use vortex_range_entropy::RangeTwoLevelCodec; +use vortex_session::VortexSession; + +const COLUMN_NAMES: [&str; 9] = [ + "longitude", + "latitude", + "housingMedianAge", + "totalRooms", + "totalBedrooms", + "population", + "households", + "medianIncome", + "medianHouseValue", +]; +const DEFAULT_ROW_LIMIT: usize = 2_000_000; + +struct Column { + name: String, + primitive: PrimitiveArray, + array: ArrayRef, +} + +#[derive(Clone, Copy)] +struct FloatMultBackendSizes { + bit_split: usize, + block_residual: usize, + range_entropy: usize, + range_packed: usize, + range_two_level: usize, +} + +enum FloatMultPrototype { + F32 { + base: f32, + primary: ArrayRef, + secondary: ArrayRef, + backend_sizes: FloatMultBackendSizes, + }, + F64 { + base: f64, + primary: ArrayRef, + secondary: ArrayRef, + backend_sizes: FloatMultBackendSizes, + }, +} + +impl FloatMultPrototype { + fn encoded_size(&self) -> u64 { + let (primary, secondary) = match self { + Self::F32 { + primary, secondary, .. + } + | Self::F64 { + primary, secondary, .. + } => (primary, secondary), + }; + primary.nbytes() + secondary.nbytes() + 16 + } + + fn structure(&self) -> String { + let (primary, secondary) = match self { + Self::F32 { + primary, secondary, .. + } + | Self::F64 { + primary, secondary, .. + } => (primary, secondary), + }; + format!( + "float-mult({},{})", + encoding_tree(primary), + encoding_tree(secondary) + ) + } + + fn backend_sizes(&self) -> FloatMultBackendSizes { + match self { + Self::F32 { backend_sizes, .. } | Self::F64 { backend_sizes, .. } => *backend_sizes, + } + } + + fn decode(&self, session: &VortexSession) -> VortexResult { + match self { + Self::F32 { + base, + primary, + secondary, + .. + } => { + let primary = primary + .clone() + .execute::(&mut session.create_execution_ctx())?; + let secondary = secondary + .clone() + .execute::(&mut session.create_execution_ctx())?; + let mask = primary + .as_view() + .validity()? + .execute_mask(primary.len(), &mut session.create_execution_ctx())?; + Ok(PrimitiveArray::from_option_iter( + primary + .as_slice::() + .iter() + .copied() + .zip(secondary.as_slice::().iter().copied()) + .zip(mask.iter()) + .map(|((primary, secondary), valid)| { + valid.then_some(join_float_mult_f32(primary, secondary, *base)) + }), + ) + .into_array()) + } + Self::F64 { + base, + primary, + secondary, + .. + } => { + let primary = primary + .clone() + .execute::(&mut session.create_execution_ctx())?; + let secondary = secondary + .clone() + .execute::(&mut session.create_execution_ctx())?; + let mask = primary + .as_view() + .validity()? + .execute_mask(primary.len(), &mut session.create_execution_ctx())?; + Ok(PrimitiveArray::from_option_iter( + primary + .as_slice::() + .iter() + .copied() + .zip(secondary.as_slice::().iter().copied()) + .zip(mask.iter()) + .map(|((primary, secondary), valid)| { + valid.then_some(join_float_mult_f64(primary, secondary, *base)) + }), + ) + .into_array()) + } + } + } +} + +enum Encoder<'a> { + BtrBlocks(&'a BtrBlocksCompressor), + Pco(&'a ChunkConfig), +} + +impl Encoder<'_> { + fn encode(&self, columns: &[Column], session: &VortexSession) -> VortexResult> { + match self { + Self::BtrBlocks(compressor) => columns + .iter() + .map(|column| { + compressor.compress(&column.array, &mut session.create_execution_ctx()) + }) + .collect(), + Self::Pco(config) => columns + .iter() + .map(|column| { + Ok(Pco::from_primitive_with_config( + column.primitive.as_view(), + config, + pco::DEFAULT_MAX_PAGE_N, + &mut session.create_execution_ctx(), + )? + .into_array()) + }) + .collect(), + } + } +} + +fn read_california_housing(path: &Path) -> VortexResult> { + let reader = BufReader::new(File::open(path)?); + let mut values = std::array::from_fn::<_, 9, _>(|_| Vec::::new()); + for (line_index, line) in reader.lines().enumerate() { + let line = line?; + let fields = line.split(',').collect::>(); + vortex_ensure!( + fields.len() == COLUMN_NAMES.len(), + "line {} contains {} fields instead of {}", + line_index + 1, + fields.len(), + COLUMN_NAMES.len() + ); + for (column_index, field) in fields.into_iter().enumerate() { + values[column_index].push(field.parse::().map_err(|error| { + vortex_err!( + "cannot parse line {} column {} as f32: {}", + line_index + 1, + column_index + 1, + error + ) + })?); + } + } + + Ok(COLUMN_NAMES + .into_iter() + .zip(values) + .map(|(name, values)| { + let primitive = PrimitiveArray::from_iter(values); + let array = primitive.clone().into_array(); + Column { + name: name.to_string(), + primitive, + array, + } + }) + .collect::>()) +} + +fn read_parquet_numeric( + path: &Path, + row_limit: usize, + session: &VortexSession, +) -> VortexResult> { + let file = File::open(path)?; + let builder = ParquetRecordBatchReaderBuilder::try_new(file) + .map_err(|error| vortex_err!("cannot read Parquet metadata: {error}"))?; + let schema = Arc::clone(builder.schema()); + let selected = schema + .fields() + .iter() + .enumerate() + .filter_map(|(index, field)| { + matches!( + field.data_type(), + DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::Float32 + | DataType::Float64 + ) + .then_some(index) + }) + .collect::>(); + vortex_ensure!( + !selected.is_empty(), + "Parquet file contains no numeric columns" + ); + let mask = ProjectionMask::roots(builder.parquet_schema(), selected.iter().copied()); + let reader = builder + .with_projection(mask) + .with_batch_size(65_536) + .build() + .map_err(|error| vortex_err!("cannot build Parquet reader: {error}"))?; + let mut chunks = (0..selected.len()) + .map(|_| Vec::::new()) + .collect::>(); + let mut row_count = 0usize; + for batch in reader { + let batch = batch.map_err(|error| vortex_err!("cannot read Parquet batch: {error}"))?; + let batch_len = batch.num_rows().min(row_limit - row_count); + for (column_chunks, array) in chunks.iter_mut().zip(batch.columns()) { + column_chunks.push(array.slice(0, batch_len)); + } + row_count += batch_len; + if row_count == row_limit { + break; + } + } + vortex_ensure!(row_count > 0, "Parquet file contains no rows"); + + selected + .into_iter() + .zip(chunks) + .map(|(field_index, chunks)| { + let field = &schema.fields()[field_index]; + let chunk_refs = chunks + .iter() + .map(|chunk| chunk.as_ref()) + .collect::>(); + let combined = concat(&chunk_refs) + .map_err(|error| vortex_err!("cannot concatenate {}: {error}", field.name()))?; + let array = session.arrow().from_arrow_array(combined, field.as_ref())?; + let primitive = array.execute::(&mut session.create_execution_ctx())?; + let array = primitive.clone().into_array(); + Ok(Column { + name: field.name().clone(), + primitive, + array, + }) + }) + .collect() +} + +fn percentile(durations: &mut [Duration], numerator: usize, denominator: usize) -> Duration { + durations.sort_unstable(); + durations[durations.len() * numerator / denominator] +} + +fn encoding_tree(array: &ArrayRef) -> String { + let children = array.children(); + if children.is_empty() { + return array.encoding_id().to_string(); + } + let children = children + .iter() + .map(encoding_tree) + .collect::>() + .join(","); + format!("{}({children})", array.encoding_id()) +} + +fn describe_pco_values(values: &[T], config: &ChunkConfig) -> VortexResult { + let compressor = FileCompressor::default(); + let chunk = compressor + .chunk_compressor(values, config) + .map_err(|error| vortex_err!("cannot inspect Pco selection: {error}"))?; + let mode = match &chunk.meta().mode { + Mode::Classic => "classic".to_string(), + Mode::IntMult(_) => "int-mult".to_string(), + Mode::FloatMult(_) => "float-mult".to_string(), + Mode::FloatQuant(bits) => format!("float-quant-{bits}"), + Mode::Dict(_) => "dict".to_string(), + _ => "unknown".to_string(), + }; + let delta = match &chunk.meta().delta_encoding { + DeltaEncoding::NoOp => "none".to_string(), + DeltaEncoding::Consecutive { order, .. } => format!("consecutive-{order}"), + DeltaEncoding::Lookback { .. } => "lookback".to_string(), + DeltaEncoding::Conv1(_) => "conv1".to_string(), + _ => "unknown".to_string(), + }; + Ok(format!("{mode}\t{delta}")) +} + +fn describe_pco( + column: &Column, + config: &ChunkConfig, + session: &VortexSession, +) -> VortexResult { + let mask = column + .primitive + .as_view() + .validity()? + .execute_mask(column.primitive.len(), &mut session.create_execution_ctx())?; + macro_rules! describe_typed { + ($T:ty) => {{ + let values = column + .primitive + .as_slice::<$T>() + .iter() + .copied() + .zip(mask.iter()) + .filter_map(|(value, valid)| valid.then_some(value)) + .collect::>(); + describe_pco_values(&values, config) + }}; + } + match column.primitive.ptype() { + PType::I16 => describe_typed!(i16), + PType::I32 => describe_typed!(i32), + PType::I64 => describe_typed!(i64), + PType::U16 => describe_typed!(u16), + PType::U32 => describe_typed!(u32), + PType::U64 => describe_typed!(u64), + PType::F32 => describe_typed!(f32), + PType::F64 => describe_typed!(f64), + ptype => Err(vortex_err!("unsupported numeric type {ptype}")), + } +} + +fn ordered_f32(value: f32) -> u32 { + let bits = value.to_bits(); + if bits & (1_u32 << 31) == 0 { + bits ^ (1_u32 << 31) + } else { + !bits + } +} + +fn from_ordered_f32(value: u32) -> f32 { + if value & (1_u32 << 31) == 0 { + f32::from_bits(!value) + } else { + f32::from_bits(value ^ (1_u32 << 31)) + } +} + +fn ordered_f64(value: f64) -> u64 { + let bits = value.to_bits(); + if bits & (1_u64 << 63) == 0 { + bits ^ (1_u64 << 63) + } else { + !bits + } +} + +fn from_ordered_f64(value: u64) -> f64 { + if value & (1_u64 << 63) == 0 { + f64::from_bits(!value) + } else { + f64::from_bits(value ^ (1_u64 << 63)) + } +} + +#[expect( + clippy::cast_possible_truncation, + reason = "the branch limits the float to the exact u32 integer range" +)] +fn int_float_to_u32(value: f32) -> u32 { + let absolute = value.abs(); + let greatest_precise_integer = 1_u32 << f32::MANTISSA_DIGITS; + let greatest_precise_float = greatest_precise_integer as f32; + let absolute_integer = if absolute < greatest_precise_float { + absolute as u32 + } else { + greatest_precise_integer + (absolute.to_bits() - greatest_precise_float.to_bits()) + }; + if value.is_sign_positive() { + (1_u32 << 31) + absolute_integer + } else { + (1_u32 << 31) - 1 - absolute_integer + } +} + +fn int_float_from_u32(value: u32) -> f32 { + let middle = 1_u32 << 31; + let (negative, absolute_integer) = if value >= middle { + (false, value - middle) + } else { + (true, middle - 1 - value) + }; + let greatest_precise_integer = 1_u32 << f32::MANTISSA_DIGITS; + let absolute = if absolute_integer < greatest_precise_integer { + absolute_integer as f32 + } else { + f32::from_bits( + (greatest_precise_integer as f32).to_bits() + + (absolute_integer - greatest_precise_integer), + ) + }; + if negative { -absolute } else { absolute } +} + +#[expect( + clippy::cast_possible_truncation, + reason = "the branch limits the float to the exact u64 integer range" +)] +fn int_float_to_u64(value: f64) -> u64 { + let absolute = value.abs(); + let greatest_precise_integer = 1_u64 << f64::MANTISSA_DIGITS; + let greatest_precise_float = greatest_precise_integer as f64; + let absolute_integer = if absolute < greatest_precise_float { + absolute as u64 + } else { + greatest_precise_integer + (absolute.to_bits() - greatest_precise_float.to_bits()) + }; + if value.is_sign_positive() { + (1_u64 << 63) + absolute_integer + } else { + (1_u64 << 63) - 1 - absolute_integer + } +} + +fn int_float_from_u64(value: u64) -> f64 { + let middle = 1_u64 << 63; + let (negative, absolute_integer) = if value >= middle { + (false, value - middle) + } else { + (true, middle - 1 - value) + }; + let greatest_precise_integer = 1_u64 << f64::MANTISSA_DIGITS; + let absolute = if absolute_integer < greatest_precise_integer { + absolute_integer as f64 + } else { + f64::from_bits( + (greatest_precise_integer as f64).to_bits() + + (absolute_integer - greatest_precise_integer), + ) + }; + if negative { -absolute } else { absolute } +} + +fn split_float_mult_f32(value: f32, base: f32) -> (u32, u32) { + let multiple = (value / base).round(); + let primary = int_float_to_u32(multiple); + let secondary = ordered_f32(value) + .wrapping_sub(ordered_f32(multiple * base)) + .wrapping_add(1_u32 << 31); + (primary, secondary) +} + +fn split_float_mult_f64(value: f64, base: f64) -> (u64, u64) { + let multiple = (value / base).round(); + let primary = int_float_to_u64(multiple); + let secondary = ordered_f64(value) + .wrapping_sub(ordered_f64(multiple * base)) + .wrapping_add(1_u64 << 63); + (primary, secondary) +} + +fn join_float_mult_f32(primary: u32, secondary: u32, base: f32) -> f32 { + let approximate = int_float_from_u32(primary) * base; + from_ordered_f32(ordered_f32(approximate).wrapping_add(secondary.wrapping_add(1_u32 << 31))) +} + +fn join_float_mult_f64(primary: u64, secondary: u64, base: f64) -> f64 { + let approximate = int_float_from_u64(primary) * base; + from_ordered_f64(ordered_f64(approximate).wrapping_add(secondary.wrapping_add(1_u64 << 63))) +} + +fn float_mult_prototype( + column: &Column, + pco_config: &ChunkConfig, + compressor: &BtrBlocksCompressor, + session: &VortexSession, +) -> VortexResult> { + let mask = column + .primitive + .as_view() + .validity()? + .execute_mask(column.primitive.len(), &mut session.create_execution_ctx())?; + macro_rules! build { + ($T:ty, $L:ty, $variant:ident, $ordered:ident, $split:ident) => {{ + let values = column.primitive.as_slice::<$T>(); + let valid_values = values + .iter() + .copied() + .zip(mask.iter()) + .filter_map(|(value, valid)| valid.then_some(value)) + .collect::>(); + let chunk = FileCompressor::default() + .chunk_compressor(&valid_values, pco_config) + .map_err(|error| vortex_err!("cannot inspect Pco selection: {error}"))?; + let Mode::FloatMult(base) = &chunk.meta().mode else { + return Ok(None); + }; + let base = $ordered( + *base + .downcast_ref::<$L>() + .ok_or_else(|| vortex_err!("Pco FloatMult base has the wrong latent type"))?, + ); + let (primary, secondary): (Vec<$L>, Vec<$L>) = values + .iter() + .copied() + .map(|value| $split(value, base)) + .unzip(); + let primary_u64 = primary.iter().copied().map(u64::from).collect::>(); + let secondary_u64 = secondary.iter().copied().map(u64::from).collect::>(); + let backend_sizes = FloatMultBackendSizes { + bit_split: BitSplitCodec::encode(&primary_u64)?.encoded_size() + + BitSplitCodec::encode(&secondary_u64)?.encoded_size() + + 16, + block_residual: PatchedFoRCodec::encode_single_base(&primary_u64)?.encoded_size() + + PatchedFoRCodec::encode_single_base(&secondary_u64)?.encoded_size() + + 16, + range_entropy: RangeEntropyCodec::encode(&primary_u64, 8_192)?.encoded_size() + + RangeEntropyCodec::encode(&secondary_u64, 8_192)?.encoded_size() + + 16, + range_packed: RangePackedCodec::encode(&primary_u64, 8_192)?.encoded_size() + + RangePackedCodec::encode(&secondary_u64, 8_192)?.encoded_size() + + 16, + range_two_level: RangeTwoLevelCodec::encode(&primary_u64, 8_192)?.encoded_size() + + RangeTwoLevelCodec::encode(&secondary_u64, 8_192)?.encoded_size() + + 16, + }; + let primary = PrimitiveArray::from_option_iter( + primary + .into_iter() + .zip(mask.iter()) + .map(|(value, valid)| valid.then_some(value)), + ) + .into_array(); + let secondary = PrimitiveArray::from_iter(secondary).into_array(); + let primary = compressor.compress(&primary, &mut session.create_execution_ctx())?; + let secondary = compressor.compress(&secondary, &mut session.create_execution_ctx())?; + Ok(Some(FloatMultPrototype::$variant { + base, + primary, + secondary, + backend_sizes, + })) + }}; + } + match column.primitive.ptype() { + PType::F32 => build!(f32, u32, F32, from_ordered_f32, split_float_mult_f32), + PType::F64 => build!(f64, u64, F64, from_ordered_f64, split_float_mult_f64), + _ => Ok(None), + } +} + +fn decode_all(arrays: &[ArrayRef], session: &VortexSession) -> VortexResult<()> { + for array in arrays { + let decoded = array + .clone() + .execute::(&mut session.create_execution_ctx())?; + black_box(decoded.nbytes()); + } + Ok(()) +} + +fn measure_compressors( + dataset: &str, + configs: &[(&str, Encoder<'_>)], + columns: &[Column], + input_bytes: u64, + session: &VortexSession, +) -> VortexResult<()> { + let iterations = (200_000_000u64 / input_bytes).clamp(3, 100) as usize; + for (_, encoder) in configs { + black_box(encoder.encode(columns, session)?); + } + + let mut durations = (0..configs.len()) + .map(|_| Vec::with_capacity(iterations)) + .collect::>(); + for iteration in 0..iterations { + for offset in 0..configs.len() { + let index = (iteration + offset) % configs.len(); + let start = Instant::now(); + black_box(configs[index].1.encode(columns, session)?); + durations[index].push(start.elapsed()); + } + } + + for (index, (name, _)) in configs.iter().enumerate() { + let p10 = percentile(&mut durations[index].clone(), 1, 10); + let p90 = percentile(&mut durations[index].clone(), 9, 10); + let median = percentile(&mut durations[index], 1, 2); + let throughput = input_bytes as f64 / median.as_secs_f64() / 1_000_000.0; + println!( + "compress\t{dataset}\t{name}\t{input_bytes}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", + p10.as_secs_f64() * 1_000.0, + median.as_secs_f64() * 1_000.0, + p90.as_secs_f64() * 1_000.0, + ); + } + Ok(()) +} + +fn measure_decoders( + dataset: &str, + configs: &[(&str, Vec)], + input_bytes: u64, + session: &VortexSession, +) -> VortexResult<()> { + let iterations = (1_000_000_000u64 / input_bytes).clamp(5, 200) as usize; + for (_, arrays) in configs { + black_box(decode_all(arrays, session)?); + } + + let mut durations = (0..configs.len()) + .map(|_| Vec::with_capacity(iterations)) + .collect::>(); + for iteration in 0..iterations { + for offset in 0..configs.len() { + let index = (iteration + offset) % configs.len(); + let start = Instant::now(); + black_box(decode_all(&configs[index].1, session)?); + durations[index].push(start.elapsed()); + } + } + + for (index, (name, arrays)) in configs.iter().enumerate() { + let output_bytes = arrays.iter().map(ArrayRef::nbytes).sum::(); + let p10 = percentile(&mut durations[index].clone(), 1, 10); + let p90 = percentile(&mut durations[index].clone(), 9, 10); + let median = percentile(&mut durations[index], 1, 2); + let throughput = input_bytes as f64 / median.as_secs_f64() / 1_000_000.0; + println!( + "decode\t{dataset}\t{name}\t{output_bytes}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", + p10.as_secs_f64() * 1_000.0, + median.as_secs_f64() * 1_000.0, + p90.as_secs_f64() * 1_000.0, + ); + } + Ok(()) +} + +fn measure_float_mult_decoder( + dataset: &str, + prototypes: &[(&Column, FloatMultPrototype)], + session: &VortexSession, +) -> VortexResult<()> { + if prototypes.is_empty() { + return Ok(()); + } + let input_bytes = prototypes + .iter() + .map(|(column, _)| column.primitive.nbytes()) + .sum::(); + let encoded_bytes = prototypes + .iter() + .map(|(_, prototype)| prototype.encoded_size()) + .sum::(); + let iterations = (1_000_000_000_u64 / input_bytes).clamp(5, 200) as usize; + for (_, prototype) in prototypes { + black_box(prototype.decode(session)?); + } + let mut durations = Vec::with_capacity(iterations); + for _ in 0..iterations { + let start = Instant::now(); + for (_, prototype) in prototypes { + black_box(prototype.decode(session)?); + } + durations.push(start.elapsed()); + } + let p10 = percentile(&mut durations.clone(), 1, 10); + let p90 = percentile(&mut durations.clone(), 9, 10); + let median = percentile(&mut durations, 1, 2); + let throughput = input_bytes as f64 / median.as_secs_f64() / 1_000_000.0; + println!( + "decode\t{dataset}\tfloat-mult-prototype\t{encoded_bytes}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", + p10.as_secs_f64() * 1_000.0, + median.as_secs_f64() * 1_000.0, + p90.as_secs_f64() * 1_000.0, + ); + Ok(()) +} + +fn main() -> VortexResult<()> { + let path = std::env::args() + .nth(1) + .ok_or_else(|| vortex_err!("usage: float_quant_dataset [row limit]"))?; + let row_limit = std::env::args() + .nth(2) + .map(|value| { + value + .parse::() + .map_err(|error| vortex_err!("invalid row limit: {error}")) + }) + .transpose()? + .unwrap_or(DEFAULT_ROW_LIMIT); + let path = Path::new(&path); + let dataset = path + .file_stem() + .and_then(|name| name.to_str()) + .ok_or_else(|| vortex_err!("data path has no valid file name"))?; + let session = array_session(); + let columns = if path + .extension() + .is_some_and(|extension| extension == "parquet") + { + read_parquet_numeric(path, row_limit, &session)? + } else { + read_california_housing(path)? + }; + let input_bytes = columns + .iter() + .map(|column| column.primitive.nbytes()) + .sum::(); + let baseline = BtrBlocksCompressorBuilder::default() + .exclude_schemes([FloatQuantScheme.id()]) + .build(); + let range_candidate = BtrBlocksCompressorBuilder::default() + .exclude_schemes([FloatQuantScheme.id()]) + .with_new_scheme(&RangeEntropyScheme) + .build(); + let float_candidate = BtrBlocksCompressor::default(); + let stacked_candidate = BtrBlocksCompressorBuilder::default() + .with_new_scheme(&RangeEntropyScheme) + .build(); + let compact = BtrBlocksCompressorBuilder::default().with_compact().build(); + let pco_config = ChunkConfig::default() + .with_mode_spec(ModeSpec::Auto) + .with_delta_spec(DeltaSpec::Auto); + let configs = [ + ("default-without-float-quant", Encoder::BtrBlocks(&baseline)), + ( + "default-without-float-quant+range-entropy", + Encoder::BtrBlocks(&range_candidate), + ), + ("default", Encoder::BtrBlocks(&float_candidate)), + ( + "default+range-entropy", + Encoder::BtrBlocks(&stacked_candidate), + ), + ("compact", Encoder::BtrBlocks(&compact)), + ("pco-auto", Encoder::Pco(&pco_config)), + ]; + let encoded = configs + .iter() + .map(|(name, encoder)| Ok((*name, encoder.encode(&columns, &session)?))) + .collect::>>()?; + let float_mult_prototypes = columns + .iter() + .filter_map(|column| { + float_mult_prototype(column, &pco_config, &baseline, &session) + .transpose() + .map(|result| result.map(|prototype| (column, prototype))) + }) + .collect::>>()?; + + println!("pco-selection\tdataset\tcolumn\tmode\tdelta"); + for column in &columns { + println!( + "pco-selection\t{dataset}\t{}\t{}", + column.name, + describe_pco(column, &pco_config, &session)? + ); + } + println!("structure\tdataset\tcolumn\tconfig\tencoding\tbytes"); + for (config_name, arrays) in &encoded { + for (column, array) in columns.iter().zip(arrays) { + let mut verify_ctx = session.create_execution_ctx(); + assert_arrays_eq!(array, column.array, &mut verify_ctx); + println!( + "structure\t{dataset}\t{}\t{config_name}\t{}\t{}", + column.name, + encoding_tree(array), + array.nbytes() + ); + } + } + for (column, prototype) in &float_mult_prototypes { + let decoded = prototype.decode(&session)?; + let mut verify_ctx = session.create_execution_ctx(); + assert_arrays_eq!(decoded, column.array, &mut verify_ctx); + let backend_sizes = prototype.backend_sizes(); + println!( + "structure\t{dataset}\t{}\tfloat-mult-prototype\t{}\t{}", + column.name, + prototype.structure(), + prototype.encoded_size(), + ); + println!( + "float-mult-backends\t{dataset}\t{}\tbit-split={}\tblock-residual={}\trange-entropy={}\trange-packed={}\trange-two-level={}", + column.name, + backend_sizes.bit_split, + backend_sizes.block_residual, + backend_sizes.range_entropy, + backend_sizes.range_packed, + backend_sizes.range_two_level, + ); + } + println!("operation\tdataset\tconfig\tbytes\tp10-ms\tmedian-ms\tp90-ms\tMB/s"); + measure_decoders(dataset, &encoded, input_bytes, &session)?; + measure_float_mult_decoder(dataset, &float_mult_prototypes, &session)?; + measure_compressors(dataset, &configs, &columns, input_bytes, &session)?; + Ok(()) +} diff --git a/vortex-btrblocks/examples/pco_probe.rs b/vortex-btrblocks/examples/pco_probe.rs new file mode 100644 index 00000000000..acd6c96e8c3 --- /dev/null +++ b/vortex-btrblocks/examples/pco_probe.rs @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::f64::consts::TAU; +use std::time::Duration; +use std::time::Instant; + +use pco::ChunkConfig; +use pco::DeltaSpec; +use pco::ModeSpec; +use pco::PagingSpec; +use pco::metadata::DeltaEncoding; +use pco::metadata::Mode; +use pco::wrapped::FileCompressor; +use vortex_array::ArrayRef; +use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::array_session; +use vortex_array::arrays::PrimitiveArray; +use vortex_btrblocks::BtrBlocksCompressor; +use vortex_btrblocks::BtrBlocksCompressorBuilder; +use vortex_btrblocks::schemes::float::PcoScheme; +use vortex_btrblocks::schemes::range_entropy::RangeEntropyScheme; +use vortex_error::VortexResult; +use vortex_error::vortex_err; +use vortex_pco::Pco; +use vortex_range_entropy::RangeEntropyCodec; + +const N: usize = 1 << 18; +const VALUES_PER_PAGE: usize = 8192; +const STAGE_ITERATIONS: usize = 15; + +struct Rng(u64); + +impl Rng { + fn next_u64(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + + fn uniform(&mut self) -> f64 { + ((self.next_u64() >> 11) as f64 + 0.5) * (1.0 / ((1_u64 << 53) as f64)) + } + + fn normal(&mut self) -> f64 { + let radius = (-2.0 * self.uniform().ln()).sqrt(); + radius * (TAU * self.uniform()).cos() + } +} + +#[expect( + clippy::cast_possible_truncation, + reason = "the probe models f32 values stored as f64" +)] +fn widen_f32(value: f64) -> f64 { + value as f32 as f64 +} + +fn datasets() -> Vec<(&'static str, Vec)> { + let mut rng = Rng(0x4d59_5df4_d0f3_3173); + let gaussian = (0..N).map(|_| 1_000.0 + 17.0 * rng.normal()).collect(); + let lognormal = (0..N).map(|_| rng.normal().exp()).collect(); + let decimal = (0..N) + .map(|_| (rng.next_u64() % 1_000_000) as f64 / 100.0) + .collect(); + let widened_f32 = (0..N) + .map(|_| widen_f32(1_000.0 + 17.0 * rng.normal())) + .collect(); + let mut value = 0.0; + let random_walk = (0..N) + .map(|_| { + value += rng.normal() * 0.01; + value + }) + .collect(); + + vec![ + ("gaussian", gaussian), + ("lognormal", lognormal), + ("decimal", decimal), + ("widened-f32", widened_f32), + ("random-walk", random_walk), + ] +} + +fn pco_config(mode: ModeSpec, delta: DeltaSpec) -> ChunkConfig { + ChunkConfig::default() + .with_mode_spec(mode) + .with_delta_spec(delta) + .with_paging_spec(PagingSpec::EqualPagesUpTo(VALUES_PER_PAGE)) +} + +fn percentile(durations: &mut [Duration], numerator: usize, denominator: usize) -> Duration { + durations.sort_unstable(); + durations[durations.len() * numerator / denominator] +} + +fn selected_config(dataset: &str) -> ChunkConfig { + match dataset { + "gaussian" | "lognormal" => pco_config(ModeSpec::Classic, DeltaSpec::NoOp), + "decimal" => pco_config(ModeSpec::TryFloatMult(0.01), DeltaSpec::NoOp), + "widened-f32" => pco_config(ModeSpec::TryFloatQuant(29), DeltaSpec::NoOp), + "random-walk" => pco_config(ModeSpec::Classic, DeltaSpec::TryConsecutive(1)), + _ => unreachable!("unknown synthetic dataset"), + } +} + +fn pco_stage_once( + values: &[f64], + config: &ChunkConfig, +) -> VortexResult<(Duration, Duration, usize)> { + let compressor = FileCompressor::default(); + let prepare_start = Instant::now(); + let mut chunk = compressor + .chunk_compressor(values, config) + .map_err(|error| vortex_err!("{}", error))?; + let prepare = prepare_start.elapsed(); + + let emit_start = Instant::now(); + let mut encoded_bytes = 0usize; + let mut metadata = Vec::with_capacity(chunk.meta_size_hint()); + chunk + .write_meta(&mut metadata) + .map_err(|error| vortex_err!("{}", error))?; + encoded_bytes += metadata.len(); + for page_idx in 0..chunk.n_per_page().len() { + let mut page = Vec::with_capacity(chunk.page_size_hint(page_idx)); + chunk + .write_page(page_idx, &mut page) + .map_err(|error| vortex_err!("{}", error))?; + encoded_bytes += page.len(); + } + let emit = emit_start.elapsed(); + Ok((prepare, emit, encoded_bytes)) +} + +fn report_pco_stages( + dataset: &str, + config_name: &str, + values: &[f64], + config: &ChunkConfig, +) -> VortexResult<()> { + for _ in 0..3 { + std::hint::black_box(pco_stage_once(std::hint::black_box(values), config)?); + } + + let mut prepare_durations = Vec::with_capacity(STAGE_ITERATIONS); + let mut emit_durations = Vec::with_capacity(STAGE_ITERATIONS); + let mut total_durations = Vec::with_capacity(STAGE_ITERATIONS); + let mut encoded_bytes = 0usize; + for _ in 0..STAGE_ITERATIONS { + let (prepare, emit, bytes) = pco_stage_once(std::hint::black_box(values), config)?; + prepare_durations.push(prepare); + emit_durations.push(emit); + total_durations.push(prepare + emit); + encoded_bytes = std::hint::black_box(bytes); + } + + let prepare = percentile(&mut prepare_durations, 1, 2); + let emit = percentile(&mut emit_durations, 1, 2); + let total = percentile(&mut total_durations, 1, 2); + let input_bytes = size_of_val(values); + let throughput = input_bytes as f64 / total.as_secs_f64() / 1_000_000.0; + println!( + "pco-stage\t{dataset}\t{config_name}\t{encoded_bytes}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", + prepare.as_secs_f64() * 1_000.0, + emit.as_secs_f64() * 1_000.0, + total.as_secs_f64() * 1_000.0, + ); + Ok(()) +} + +fn report_auto_selection(dataset: &str, array: &PrimitiveArray) -> VortexResult<()> { + let config = pco_config(ModeSpec::Auto, DeltaSpec::Auto); + let compressor = FileCompressor::default(); + let chunk = compressor + .chunk_compressor(array.as_slice::(), &config) + .map_err(|error| vortex_err!("{}", error))?; + let mode = match &chunk.meta().mode { + Mode::Classic => "classic".to_owned(), + Mode::IntMult(_) => "int-mult".to_owned(), + Mode::FloatMult(_) => "float-mult".to_owned(), + Mode::FloatQuant(bits) => format!("float-quant-{bits}"), + Mode::Dict(_) => "dict".to_owned(), + _ => "unknown".to_owned(), + }; + let delta = match &chunk.meta().delta_encoding { + DeltaEncoding::NoOp => "none".to_owned(), + DeltaEncoding::Consecutive { order, .. } => format!("consecutive-{order}"), + DeltaEncoding::Lookback { .. } => "lookback".to_owned(), + DeltaEncoding::Conv1(_) => "conv1".to_owned(), + _ => "unknown".to_owned(), + }; + eprintln!("{dataset}: pco-auto selected {mode} with {delta}"); + Ok(()) +} + +fn report( + dataset: &str, + encoder: &str, + input_bytes: u64, + encode: impl FnOnce() -> VortexResult, +) -> VortexResult<()> { + let start = Instant::now(); + let compressed = encode()?; + let elapsed = start.elapsed(); + let encoded_bytes = compressed.nbytes(); + let bits_per_value = 8.0 * encoded_bytes as f64 / N as f64; + let throughput = input_bytes as f64 / elapsed.as_secs_f64() / 1_000_000.0; + println!("{dataset}\t{encoder}\t{encoded_bytes}\t{bits_per_value:.3}\t{throughput:.1}"); + Ok(()) +} + +fn ordered_f64(value: f64) -> u64 { + let bits = value.to_bits(); + if bits & (1_u64 << 63) == 0 { + bits ^ (1_u64 << 63) + } else { + !bits + } +} + +fn report_native(dataset: &str, values: &[f64], input_bytes: u64) -> VortexResult<()> { + let latents: Vec<_> = values.iter().copied().map(ordered_f64).collect(); + let start = Instant::now(); + let compressed = RangeEntropyCodec::encode(&latents, VALUES_PER_PAGE)?; + let elapsed = start.elapsed(); + let encoded_bytes = u64::try_from(compressed.encoded_size())?; + let bits_per_value = 8.0 * encoded_bytes as f64 / N as f64; + let throughput = input_bytes as f64 / elapsed.as_secs_f64() / 1_000_000.0; + eprintln!( + "{dataset}: native range entropy used {} bins", + compressed.bin_lowers().len() + ); + println!("{dataset}\trange-entropy\t{encoded_bytes}\t{bits_per_value:.3}\t{throughput:.1}"); + Ok(()) +} + +fn main() -> VortexResult<()> { + let session = array_session(); + let default = BtrBlocksCompressor::default(); + let with_auto = BtrBlocksCompressorBuilder::default() + .with_new_scheme(&PcoScheme) + .build(); + let with_range_entropy = BtrBlocksCompressorBuilder::default() + .with_new_scheme(&RangeEntropyScheme) + .build(); + + println!("dataset\tencoder\tbytes\tbits/value\tMB/s"); + println!("pco-stage columns: dataset, config, bytes, prepare-ms, emit-ms, total-ms, MB/s"); + for (name, values) in datasets() { + let array = PrimitiveArray::from_iter(values.clone()); + let array_ref = array.clone().into_array(); + let input_bytes = array_ref.nbytes(); + report_auto_selection(name, &array)?; + report_pco_stages( + name, + "auto", + &values, + &pco_config(ModeSpec::Auto, DeltaSpec::Auto), + )?; + for level in [0, 4, 8, 12] { + let config = selected_config(name).with_compression_level(level); + report_pco_stages(name, &format!("selected-level-{level}"), &values, &config)?; + } + report_native(name, &values, input_bytes)?; + + report(name, "btrblocks", input_bytes, || { + default.compress(&array_ref, &mut session.create_execution_ctx()) + })?; + report(name, "btrblocks+pco-auto", input_bytes, || { + with_auto.compress(&array_ref, &mut session.create_execution_ctx()) + })?; + report(name, "btrblocks+range-entropy", input_bytes, || { + with_range_entropy.compress(&array_ref, &mut session.create_execution_ctx()) + })?; + + for (encoder, config) in [ + ( + "pco-classic", + pco_config(ModeSpec::Classic, DeltaSpec::NoOp), + ), + ( + "pco-classic-delta1", + pco_config(ModeSpec::Classic, DeltaSpec::TryConsecutive(1)), + ), + ( + "pco-classic-delta2", + pco_config(ModeSpec::Classic, DeltaSpec::TryConsecutive(2)), + ), + ("pco-auto", pco_config(ModeSpec::Auto, DeltaSpec::Auto)), + ] { + report(name, encoder, input_bytes, || { + Ok(Pco::from_primitive_with_config( + array.as_view(), + &config, + pco::DEFAULT_MAX_PAGE_N, + &mut session.create_execution_ctx(), + )? + .into_array()) + })?; + } + } + + Ok(()) +} diff --git a/vortex-btrblocks/examples/range_entropy_throughput.rs b/vortex-btrblocks/examples/range_entropy_throughput.rs new file mode 100644 index 00000000000..c4886fc63fe --- /dev/null +++ b/vortex-btrblocks/examples/range_entropy_throughput.rs @@ -0,0 +1,994 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::f64::consts::TAU; +use std::hint::black_box; +use std::time::Duration; +use std::time::Instant; + +use pco::ChunkConfig; +use pco::DeltaSpec; +use pco::ModeSpec; +use pco::PagingSpec; +use vortex_array::ArrayRef; +use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::array_session; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::assert_arrays_eq; +use vortex_btrblocks::BtrBlocksCompressor; +use vortex_btrblocks::BtrBlocksCompressorBuilder; +use vortex_btrblocks::SchemeExt; +use vortex_btrblocks::schemes::float::FloatQuantScheme; +use vortex_btrblocks::schemes::float::OrderedBlockResidualScheme; +use vortex_btrblocks::schemes::range_entropy::RangeEntropyScheme; +use vortex_error::VortexResult; +use vortex_float_quant::FloatQuant; +use vortex_float_quant::FloatQuantArraySlotsExt; +use vortex_float_quant::estimate_k; +use vortex_pco::Pco; +use vortex_range_entropy::BitSplitCodec; +use vortex_range_entropy::PatchedFoRCodec; +use vortex_range_entropy::RangeEntropy; +use vortex_range_entropy::RangeEntropyCodec; +use vortex_range_entropy::RangeGroupedCodec; +use vortex_range_entropy::RangePackedCodec; +use vortex_range_entropy::RangeTwoLevelCodec; +use vortex_session::VortexSession; + +const N: usize = 1 << 18; +const VALUES_PER_PAGE: usize = 8192; +const DECODE_ITERATIONS: usize = 100; +const COMPRESS_ITERATIONS: usize = 40; +const SCALAR_ACCESS_ITERATIONS: usize = N * 8; +const ARRAY_SCALAR_ACCESS_ITERATIONS: usize = N; + +struct Rng(u64); + +impl Rng { + fn next_u64(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + + fn uniform(&mut self) -> f64 { + ((self.next_u64() >> 11) as f64 + 0.5) * (1.0 / ((1_u64 << 53) as f64)) + } + + fn normal(&mut self) -> f64 { + let radius = (-2.0 * self.uniform().ln()).sqrt(); + radius * (TAU * self.uniform()).cos() + } +} + +#[expect( + clippy::cast_possible_truncation, + reason = "the benchmark models f32 values stored as f64" +)] +fn widen_f32(value: f64) -> f64 { + value as f32 as f64 +} + +fn datasets() -> Vec<(&'static str, Vec)> { + let mut rng = Rng(0x4d59_5df4_d0f3_3173); + let gaussian = (0..N).map(|_| 1_000.0 + 17.0 * rng.normal()).collect(); + let lognormal = (0..N).map(|_| rng.normal().exp()).collect(); + let decimal = (0..N) + .map(|_| (rng.next_u64() % 1_000_000) as f64 / 100.0) + .collect(); + let widened_f32 = (0..N) + .map(|_| widen_f32(1_000.0 + 17.0 * rng.normal())) + .collect(); + let mut value = 0.0; + let random_walk = (0..N) + .map(|_| { + value += rng.normal() * 0.01; + value + }) + .collect(); + let four_clusters = (0..N) + .map(|_| { + let center = match rng.next_u64() & 3 { + 0 => -1_000_000_000.0, + 1 => -1_000.0, + 2 => 1_000.0, + _ => 1_000_000_000.0, + }; + center + 17.0 * rng.normal() + }) + .collect(); + + vec![ + ("gaussian", gaussian), + ("lognormal", lognormal), + ("decimal", decimal), + ("widened-f32", widened_f32), + ("random-walk", random_walk), + ("four-clusters", four_clusters), + ] +} + +fn pco_config(mode: ModeSpec, delta: DeltaSpec) -> ChunkConfig { + ChunkConfig::default() + .with_mode_spec(mode) + .with_delta_spec(delta) + .with_paging_spec(PagingSpec::EqualPagesUpTo(VALUES_PER_PAGE)) +} + +fn percentile(durations: &mut [Duration], numerator: usize, denominator: usize) -> Duration { + durations.sort_unstable(); + durations[durations.len() * numerator / denominator] +} + +fn encoding_tree(array: &ArrayRef) -> String { + let children = array.children(); + if children.is_empty() { + return array.encoding_id().to_string(); + } + let children = children + .iter() + .map(encoding_tree) + .collect::>() + .join(","); + format!("{}({children})", array.encoding_id()) +} + +fn decode_once(array: &ArrayRef, session: &VortexSession) -> VortexResult { + let start = Instant::now(); + let decoded = array + .clone() + .execute::(&mut session.create_execution_ctx())?; + black_box(decoded.nbytes()); + Ok(start.elapsed()) +} + +fn report_decode( + dataset: &str, + encoder: &str, + array: &ArrayRef, + durations: &mut [Duration], +) -> VortexResult<()> { + let mut for_p10 = durations.to_vec(); + let mut for_median = durations.to_vec(); + let p10 = percentile(&mut for_p10, 1, 10); + let median = percentile(&mut for_median, 1, 2); + let p90 = percentile(durations, 9, 10); + let throughput = (N * size_of::()) as f64 / median.as_secs_f64() / 1_000_000.0; + println!( + "decode\t{dataset}\t{encoder}\t{}\t{}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", + array.encoding_id(), + array.nbytes(), + p10.as_secs_f64() * 1_000.0, + median.as_secs_f64() * 1_000.0, + p90.as_secs_f64() * 1_000.0, + ); + Ok(()) +} + +fn measure_decoders( + dataset: &str, + configs: &[(&str, &ArrayRef)], + session: &VortexSession, +) -> VortexResult<()> { + for _ in 0..3 { + for (_, array) in configs { + black_box(decode_once(array, session)?); + } + } + + let mut durations = vec![Vec::with_capacity(DECODE_ITERATIONS); configs.len()]; + for iteration in 0..DECODE_ITERATIONS { + for offset in 0..configs.len() { + let config_index = (iteration + offset) % configs.len(); + durations[config_index].push(decode_once(configs[config_index].1, session)?); + } + } + + for (config_index, (config, array)) in configs.iter().enumerate() { + report_decode(dataset, config, array, &mut durations[config_index])?; + } + Ok(()) +} + +fn measure_range_entropy_codec( + dataset: &str, + values: &[f64], + codec: &RangeEntropyCodec, +) -> VortexResult<()> { + for _ in 0..3 { + black_box(codec.decode()?); + } + let mut durations = Vec::with_capacity(DECODE_ITERATIONS); + for _ in 0..DECODE_ITERATIONS { + let start = Instant::now(); + black_box(codec.decode()?); + durations.push(start.elapsed()); + } + let p10 = percentile(&mut durations.to_vec(), 1, 10); + let p90 = percentile(&mut durations.to_vec(), 9, 10); + let median = percentile(&mut durations, 1, 2); + let throughput = + values.len() as f64 * size_of::() as f64 / median.as_secs_f64() / 1_000_000.0; + println!( + "decode\t{dataset}\trange-entropy-codec\tvortex.range_entropy_codec\t{}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", + codec.encoded_size(), + p10.as_secs_f64() * 1_000.0, + median.as_secs_f64() * 1_000.0, + p90.as_secs_f64() * 1_000.0, + ); + Ok(()) +} + +fn measure_range_packed_codec( + dataset: &str, + values: &[f64], + codec: &RangePackedCodec, +) -> VortexResult<()> { + for _ in 0..3 { + black_box(codec.decode()?); + } + let mut durations = Vec::with_capacity(DECODE_ITERATIONS); + for _ in 0..DECODE_ITERATIONS { + let start = Instant::now(); + black_box(codec.decode()?); + durations.push(start.elapsed()); + } + let p10 = percentile(&mut durations.to_vec(), 1, 10); + let p90 = percentile(&mut durations.to_vec(), 9, 10); + let median = percentile(&mut durations, 1, 2); + let throughput = + values.len() as f64 * size_of::() as f64 / median.as_secs_f64() / 1_000_000.0; + println!( + "decode\t{dataset}\trange-packed-codec\tvortex.range_packed_codec\t{}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", + codec.encoded_size(), + p10.as_secs_f64() * 1_000.0, + median.as_secs_f64() * 1_000.0, + p90.as_secs_f64() * 1_000.0, + ); + Ok(()) +} + +fn measure_range_two_level_codec( + dataset: &str, + values: &[f64], + codec: &RangeTwoLevelCodec, +) -> VortexResult<()> { + for _ in 0..3 { + black_box(codec.decode()?); + } + let mut durations = Vec::with_capacity(DECODE_ITERATIONS); + for _ in 0..DECODE_ITERATIONS { + let start = Instant::now(); + black_box(codec.decode()?); + durations.push(start.elapsed()); + } + let p10 = percentile(&mut durations.to_vec(), 1, 10); + let p90 = percentile(&mut durations.to_vec(), 9, 10); + let median = percentile(&mut durations, 1, 2); + let throughput = + values.len() as f64 * size_of::() as f64 / median.as_secs_f64() / 1_000_000.0; + println!( + "decode\t{dataset}\trange-two-level-codec\tvortex.range_two_level_codec\t{}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", + codec.encoded_size(), + p10.as_secs_f64() * 1_000.0, + median.as_secs_f64() * 1_000.0, + p90.as_secs_f64() * 1_000.0, + ); + Ok(()) +} + +fn measure_range_grouped_codec( + dataset: &str, + values: &[f64], + codec: &RangeGroupedCodec, +) -> VortexResult<()> { + for _ in 0..3 { + black_box(codec.decode()?); + } + let mut durations = Vec::with_capacity(DECODE_ITERATIONS); + for _ in 0..DECODE_ITERATIONS { + let start = Instant::now(); + black_box(codec.decode()?); + durations.push(start.elapsed()); + } + let p10 = percentile(&mut durations.to_vec(), 1, 10); + let p90 = percentile(&mut durations.to_vec(), 9, 10); + let median = percentile(&mut durations, 1, 2); + let throughput = + values.len() as f64 * size_of::() as f64 / median.as_secs_f64() / 1_000_000.0; + println!( + "decode\t{dataset}\trange-grouped-codec\tvortex.range_grouped_codec\t{}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", + codec.encoded_size(), + p10.as_secs_f64() * 1_000.0, + median.as_secs_f64() * 1_000.0, + p90.as_secs_f64() * 1_000.0, + ); + Ok(()) +} + +fn measure_patched_for_codec( + dataset: &str, + config: &str, + values: &[f64], + codec: &PatchedFoRCodec, +) -> VortexResult<()> { + for _ in 0..3 { + black_box(codec.decode()?); + } + let mut durations = Vec::with_capacity(DECODE_ITERATIONS); + for _ in 0..DECODE_ITERATIONS { + let start = Instant::now(); + black_box(codec.decode()?); + durations.push(start.elapsed()); + } + let p10 = percentile(&mut durations.to_vec(), 1, 10); + let p90 = percentile(&mut durations.to_vec(), 9, 10); + let median = percentile(&mut durations, 1, 2); + let throughput = + values.len() as f64 * size_of::() as f64 / median.as_secs_f64() / 1_000_000.0; + println!( + "decode\t{dataset}\t{config}\tvortex.patched_for_codec\t{}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", + codec.encoded_size(), + p10.as_secs_f64() * 1_000.0, + median.as_secs_f64() * 1_000.0, + p90.as_secs_f64() * 1_000.0, + ); + Ok(()) +} + +fn measure_ordered_float_patched_for_codec( + dataset: &str, + values: &[f64], + codec: &PatchedFoRCodec, +) -> VortexResult<()> { + let decode = || -> VortexResult> { + Ok(codec + .decode()? + .into_iter() + .map(|ordered| { + let bits = if ordered & (1_u64 << 63) == 0 { + !ordered + } else { + ordered ^ (1_u64 << 63) + }; + f64::from_bits(bits) + }) + .collect::>()) + }; + for _ in 0..3 { + black_box(decode()?); + } + let mut durations = Vec::with_capacity(DECODE_ITERATIONS); + for _ in 0..DECODE_ITERATIONS { + let start = Instant::now(); + black_box(decode()?); + durations.push(start.elapsed()); + } + let p10 = percentile(&mut durations.to_vec(), 1, 10); + let p90 = percentile(&mut durations.to_vec(), 9, 10); + let median = percentile(&mut durations, 1, 2); + let throughput = + values.len() as f64 * size_of::() as f64 / median.as_secs_f64() / 1_000_000.0; + println!( + "decode\t{dataset}\tordered-float+patched-for-single-base\tvortex.prototype_stack\t{}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", + codec.encoded_size(), + p10.as_secs_f64() * 1_000.0, + median.as_secs_f64() * 1_000.0, + p90.as_secs_f64() * 1_000.0, + ); + + for _ in 0..3 { + black_box(codec.decode_ordered_f64()?); + } + let mut durations = Vec::with_capacity(DECODE_ITERATIONS); + for _ in 0..DECODE_ITERATIONS { + let start = Instant::now(); + black_box(codec.decode_ordered_f64()?); + durations.push(start.elapsed()); + } + let p10 = percentile(&mut durations.to_vec(), 1, 10); + let p90 = percentile(&mut durations.to_vec(), 9, 10); + let median = percentile(&mut durations, 1, 2); + let throughput = + values.len() as f64 * size_of::() as f64 / median.as_secs_f64() / 1_000_000.0; + println!( + "decode\t{dataset}\tordered-float+patched-for-single-base-fused\tvortex.prototype_fused\t{}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", + codec.encoded_size(), + p10.as_secs_f64() * 1_000.0, + median.as_secs_f64() * 1_000.0, + p90.as_secs_f64() * 1_000.0, + ); + Ok(()) +} + +fn measure_patched_for_scalar_at( + dataset: &str, + config: &str, + codec: &PatchedFoRCodec, +) -> VortexResult<()> { + let mut checksum = 0_u64; + for access in 0..N { + let index = access.wrapping_mul(0x9e37_79b1) & (N - 1); + checksum ^= black_box(codec.scalar_at(index)?); + } + black_box(checksum); + + let mut durations = Vec::with_capacity(9); + for _ in 0..9 { + let start = Instant::now(); + let mut checksum = 0_u64; + for access in 0..SCALAR_ACCESS_ITERATIONS { + let index = access.wrapping_mul(0x9e37_79b1) & (N - 1); + checksum ^= black_box(codec.scalar_at(index)?); + } + durations.push(start.elapsed()); + black_box(checksum); + } + let median = percentile(&mut durations, 1, 2); + let nanoseconds = median.as_secs_f64() * 1_000_000_000.0 / SCALAR_ACCESS_ITERATIONS as f64; + let accesses_per_second = SCALAR_ACCESS_ITERATIONS as f64 / median.as_secs_f64() / 1_000_000.0; + println!( + "scalar-at\t{dataset}\t{config}\tvortex.patched_for_codec\t{}\t{nanoseconds:.2}\t{accesses_per_second:.1}", + codec.encoded_size(), + ); + Ok(()) +} + +fn measure_array_scalar_at( + dataset: &str, + config: &str, + array: &ArrayRef, + session: &VortexSession, +) -> VortexResult<()> { + let mut durations = Vec::with_capacity(5); + for _ in 0..5 { + let mut ctx = session.create_execution_ctx(); + let start = Instant::now(); + let mut checksum = 0_u64; + for access in 0..ARRAY_SCALAR_ACCESS_ITERATIONS { + let index = access.wrapping_mul(0x9e37_79b1) & (N - 1); + let value = array + .execute_scalar(index, &mut ctx)? + .as_primitive() + .typed_value::() + .ok_or_else(|| vortex_error::vortex_err!("benchmark value is null"))?; + checksum ^= black_box(value.to_bits()); + } + durations.push(start.elapsed()); + black_box(checksum); + } + let median = percentile(&mut durations, 1, 2); + let nanoseconds = + median.as_secs_f64() * 1_000_000_000.0 / ARRAY_SCALAR_ACCESS_ITERATIONS as f64; + let accesses_per_second = + ARRAY_SCALAR_ACCESS_ITERATIONS as f64 / median.as_secs_f64() / 1_000_000.0; + println!( + "scalar-at\t{dataset}\t{config}\t{}\t{}\t{nanoseconds:.2}\t{accesses_per_second:.1}", + array.encoding_id(), + array.nbytes(), + ); + Ok(()) +} + +fn measure_bit_split_codec( + dataset: &str, + values: &[f64], + codec: &BitSplitCodec, +) -> VortexResult<()> { + for _ in 0..3 { + black_box(codec.decode()?); + } + let mut durations = Vec::with_capacity(DECODE_ITERATIONS); + for _ in 0..DECODE_ITERATIONS { + let start = Instant::now(); + black_box(codec.decode()?); + durations.push(start.elapsed()); + } + let p10 = percentile(&mut durations.to_vec(), 1, 10); + let p90 = percentile(&mut durations.to_vec(), 9, 10); + let median = percentile(&mut durations, 1, 2); + let throughput = + values.len() as f64 * size_of::() as f64 / median.as_secs_f64() / 1_000_000.0; + println!( + "decode\t{dataset}\tbit-split-codec\tvortex.bit_split_codec\t{}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", + codec.encoded_size(), + p10.as_secs_f64() * 1_000.0, + median.as_secs_f64() * 1_000.0, + p90.as_secs_f64() * 1_000.0, + ); + Ok(()) +} + +fn measure_bit_split_scalar_at(dataset: &str, codec: &BitSplitCodec) -> VortexResult<()> { + let mut durations = Vec::with_capacity(9); + for _ in 0..9 { + let start = Instant::now(); + let mut checksum = 0_u64; + for access in 0..SCALAR_ACCESS_ITERATIONS { + let index = access.wrapping_mul(0x9e37_79b1) & (N - 1); + checksum ^= black_box(codec.scalar_at(index)?); + } + durations.push(start.elapsed()); + black_box(checksum); + } + let median = percentile(&mut durations, 1, 2); + let nanoseconds = median.as_secs_f64() * 1_000_000_000.0 / SCALAR_ACCESS_ITERATIONS as f64; + let accesses_per_second = SCALAR_ACCESS_ITERATIONS as f64 / median.as_secs_f64() / 1_000_000.0; + println!( + "scalar-at\t{dataset}\tbit-split-codec\tvortex.bit_split_codec\t{}\t{nanoseconds:.2}\t{accesses_per_second:.1}", + codec.encoded_size(), + ); + Ok(()) +} + +fn measure_native_codec_encoders(dataset: &str, values: &[u64]) -> VortexResult<()> { + for (name, encode) in [( + "range-entropy-codec", + RangeEntropyCodec::encode as fn(&[u64], usize) -> VortexResult, + )] { + let mut durations = Vec::with_capacity(COMPRESS_ITERATIONS); + let mut size = 0; + for _ in 0..COMPRESS_ITERATIONS { + let start = Instant::now(); + let codec = encode(black_box(values), VALUES_PER_PAGE)?; + durations.push(start.elapsed()); + size = black_box(codec.encoded_size()); + } + report_native_codec_encoder(dataset, name, size, &mut durations); + } + + let mut durations = Vec::with_capacity(COMPRESS_ITERATIONS); + let mut size = 0; + for _ in 0..COMPRESS_ITERATIONS { + let start = Instant::now(); + let codec = RangePackedCodec::encode(black_box(values), VALUES_PER_PAGE)?; + durations.push(start.elapsed()); + size = black_box(codec.encoded_size()); + } + report_native_codec_encoder(dataset, "range-packed-codec", size, &mut durations); + + let mut durations = Vec::with_capacity(COMPRESS_ITERATIONS); + let mut size = 0; + for _ in 0..COMPRESS_ITERATIONS { + let start = Instant::now(); + let codec = RangeTwoLevelCodec::encode(black_box(values), VALUES_PER_PAGE)?; + durations.push(start.elapsed()); + size = black_box(codec.encoded_size()); + } + report_native_codec_encoder(dataset, "range-two-level-codec", size, &mut durations); + + let mut durations = Vec::with_capacity(COMPRESS_ITERATIONS); + let mut size = 0; + for _ in 0..COMPRESS_ITERATIONS { + let start = Instant::now(); + let codec = RangeGroupedCodec::encode(black_box(values), VALUES_PER_PAGE)?; + durations.push(start.elapsed()); + size = black_box(codec.encoded_size()); + } + report_native_codec_encoder(dataset, "range-grouped-codec", size, &mut durations); + + let mut durations = Vec::with_capacity(COMPRESS_ITERATIONS); + let mut size = 0; + for _ in 0..COMPRESS_ITERATIONS { + let start = Instant::now(); + let codec = PatchedFoRCodec::encode(black_box(values))?; + durations.push(start.elapsed()); + size = black_box(codec.encoded_size()); + } + report_native_codec_encoder(dataset, "patched-for-codec", size, &mut durations); + + let mut durations = Vec::with_capacity(COMPRESS_ITERATIONS); + let mut size = 0; + for _ in 0..COMPRESS_ITERATIONS { + let start = Instant::now(); + let codec = PatchedFoRCodec::encode_single_base(black_box(values))?; + durations.push(start.elapsed()); + size = black_box(codec.encoded_size()); + } + report_native_codec_encoder(dataset, "patched-for-single-base", size, &mut durations); + + let mut durations = Vec::with_capacity(COMPRESS_ITERATIONS); + let mut size = 0; + for _ in 0..COMPRESS_ITERATIONS { + let start = Instant::now(); + let codec = BitSplitCodec::encode(black_box(values))?; + durations.push(start.elapsed()); + size = black_box(codec.encoded_size()); + } + report_native_codec_encoder(dataset, "bit-split-codec", size, &mut durations); + Ok(()) +} + +fn measure_ordered_float_patched_for_encoder(dataset: &str, values: &[f64]) -> VortexResult<()> { + let mut durations = Vec::with_capacity(COMPRESS_ITERATIONS); + let mut size = 0; + for _ in 0..COMPRESS_ITERATIONS { + let start = Instant::now(); + let ordered = values + .iter() + .map(|value| { + let bits = value.to_bits(); + if bits & (1_u64 << 63) == 0 { + bits ^ (1_u64 << 63) + } else { + !bits + } + }) + .collect::>(); + let codec = PatchedFoRCodec::encode_single_base(black_box(&ordered))?; + durations.push(start.elapsed()); + size = black_box(codec.encoded_size()); + } + report_native_codec_encoder( + dataset, + "ordered-float+patched-for-single-base", + size, + &mut durations, + ); + Ok(()) +} + +fn report_native_codec_encoder(dataset: &str, name: &str, size: usize, durations: &mut [Duration]) { + let p10 = percentile(&mut durations.to_vec(), 1, 10); + let p90 = percentile(&mut durations.to_vec(), 9, 10); + let median = percentile(durations, 1, 2); + let throughput = (N * size_of::()) as f64 / median.as_secs_f64() / 1_000_000.0; + println!( + "codec-encode\t{dataset}\t{name}\tnative-codec\t{size}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", + p10.as_secs_f64() * 1_000.0, + median.as_secs_f64() * 1_000.0, + p90.as_secs_f64() * 1_000.0, + ); +} + +fn report_symbol_models(dataset: &str, codec: &RangeEntropyCodec) { + let total_weight = (1_u64 << codec.scale_bits()) as f64; + let mut probabilities = codec + .weights() + .iter() + .map(|&weight| f64::from(weight) / total_weight) + .collect::>(); + let entropy = probabilities + .iter() + .map(|&probability| -probability * probability.log2()) + .sum::(); + probabilities.sort_by(|left, right| right.total_cmp(left)); + let bin_count = probabilities.len(); + let flat_width = usize::from(u8::try_from(codec.bin_lowers().len().ilog2()).unwrap_or(u8::MAX)) + + usize::from(!codec.bin_lowers().len().is_power_of_two()); + let mut best = (f64::INFINITY, 0usize, 0usize, 0.0); + let minimum_tag_width = flat_width.min(2); + for tag_width in minimum_tag_width..=flat_width { + let direct_count = ((1usize << tag_width) - 1).min(bin_count); + let escape_probability = 1.0 - probabilities[..direct_count].iter().sum::(); + let cold_count = bin_count - direct_count; + let cold_width = if cold_count <= 1 { + 0 + } else { + usize::from(u8::try_from(cold_count.ilog2()).unwrap_or(u8::MAX)) + + usize::from(!cold_count.is_power_of_two()) + }; + let cost = tag_width as f64 + escape_probability * cold_width as f64; + if cost < best.0 { + best = (cost, tag_width, cold_width, escape_probability); + } + } + println!( + "symbol-model\t{dataset}\tbins={bin_count}\tentropy={entropy:.3}\tflat={flat_width}\ttwo-level={:.3}\ttag={}\tcold={}\tescape={:.3}", + best.0, best.1, best.2, best.3 + ); +} + +fn measure_float_quant_stages( + dataset: &str, + primitive: &PrimitiveArray, + compressor: &BtrBlocksCompressor, + session: &VortexSession, +) -> VortexResult<()> { + let Some(k) = estimate_k(primitive.as_view()) else { + return Ok(()); + }; + let encoded = FloatQuant::from_primitive(primitive.as_view(), k)?; + let primary = encoded.primary().clone(); + let secondary = encoded.secondary().clone(); + let mut estimate_times = Vec::with_capacity(COMPRESS_ITERATIONS); + let mut split_times = Vec::with_capacity(COMPRESS_ITERATIONS); + let mut primary_times = Vec::with_capacity(COMPRESS_ITERATIONS); + let mut secondary_times = Vec::with_capacity(COMPRESS_ITERATIONS); + for _ in 0..COMPRESS_ITERATIONS { + let start = Instant::now(); + black_box(estimate_k(primitive.as_view())); + estimate_times.push(start.elapsed()); + + let start = Instant::now(); + black_box(FloatQuant::from_primitive(primitive.as_view(), k)?); + split_times.push(start.elapsed()); + + let start = Instant::now(); + black_box(compressor.compress(&primary, &mut session.create_execution_ctx())?); + primary_times.push(start.elapsed()); + + let start = Instant::now(); + black_box(compressor.compress(&secondary, &mut session.create_execution_ctx())?); + secondary_times.push(start.elapsed()); + } + + for (stage, durations) in [ + ("estimate-k", &mut estimate_times), + ("split", &mut split_times), + ("primary-child", &mut primary_times), + ("secondary-child", &mut secondary_times), + ] { + let median = percentile(durations, 1, 2); + println!( + "stage\t{dataset}\t{stage}\tk={k}\t{:.3}", + median.as_secs_f64() * 1_000.0 + ); + } + Ok(()) +} + +fn compress_once( + compressor: &BtrBlocksCompressor, + values: &[f64], + session: &VortexSession, +) -> VortexResult<(Duration, ArrayRef)> { + let array = PrimitiveArray::from_iter(values.iter().copied()).into_array(); + let start = Instant::now(); + let compressed = compressor.compress(&array, &mut session.create_execution_ctx())?; + Ok((start.elapsed(), compressed)) +} + +fn measure_compressors( + dataset: &str, + values: &[f64], + configs: &[(&str, &BtrBlocksCompressor)], + session: &VortexSession, +) -> VortexResult<()> { + for _ in 0..2 { + for (_, compressor) in configs { + black_box(compress_once(compressor, values, session)?); + } + } + + let mut durations = (0..configs.len()) + .map(|_| Vec::with_capacity(COMPRESS_ITERATIONS)) + .collect::>(); + let mut outputs = vec![None; configs.len()]; + for iteration in 0..COMPRESS_ITERATIONS { + for offset in 0..configs.len() { + let config_index = (iteration + offset) % configs.len(); + record_compression( + configs[config_index].1, + values, + session, + &mut durations[config_index], + &mut outputs[config_index], + )?; + } + } + + for (index, (name, _)) in configs.iter().enumerate() { + report_compressor(dataset, name, outputs[index].take(), &mut durations[index])?; + } + Ok(()) +} + +fn record_compression( + compressor: &BtrBlocksCompressor, + values: &[f64], + session: &VortexSession, + durations: &mut Vec, + last_output: &mut Option, +) -> VortexResult<()> { + let (elapsed, output) = compress_once(compressor, values, session)?; + durations.push(elapsed); + *last_output = Some(output); + Ok(()) +} + +fn report_compressor( + dataset: &str, + config: &str, + output: Option, + durations: &mut [Duration], +) -> VortexResult<()> { + let output = + output.ok_or_else(|| vortex_error::vortex_err!("compressor produced no output"))?; + let p10 = percentile(&mut durations.to_vec(), 1, 10); + let p90 = percentile(&mut durations.to_vec(), 9, 10); + let median = percentile(durations, 1, 2); + let throughput = (N * size_of::()) as f64 / median.as_secs_f64() / 1_000_000.0; + println!( + "compress\t{dataset}\t{config}\t{}\t{}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", + output.encoding_id(), + output.nbytes(), + p10.as_secs_f64() * 1_000.0, + median.as_secs_f64() * 1_000.0, + p90.as_secs_f64() * 1_000.0, + ); + Ok(()) +} + +#[expect( + clippy::cognitive_complexity, + reason = "the benchmark driver constructs and measures one fixed configuration matrix" +)] +fn main() -> VortexResult<()> { + let dataset_filter = std::env::args().nth(1); + let session = array_session(); + let baseline = BtrBlocksCompressorBuilder::default() + .exclude_schemes([FloatQuantScheme.id(), OrderedBlockResidualScheme.id()]) + .build(); + let range_candidate = BtrBlocksCompressorBuilder::default() + .exclude_schemes([FloatQuantScheme.id(), OrderedBlockResidualScheme.id()]) + .with_new_scheme(&RangeEntropyScheme) + .build(); + let default_without_block_residual = BtrBlocksCompressorBuilder::default() + .exclude_schemes([OrderedBlockResidualScheme.id()]) + .build(); + let float_candidate = BtrBlocksCompressor::default(); + let stacked_candidate = BtrBlocksCompressorBuilder::default() + .with_new_scheme(&RangeEntropyScheme) + .build(); + let compact = BtrBlocksCompressorBuilder::default().with_compact().build(); + + println!("operation\tdataset\tconfig\tencoding\tbytes\tp10-ms\tmedian-ms\tp90-ms\tMB/s"); + for (name, values) in datasets() { + if dataset_filter + .as_deref() + .is_some_and(|filter| filter != name) + { + continue; + } + let primitive = PrimitiveArray::from_iter(values.iter().copied()); + let expected = primitive.clone().into_array(); + let ordered_values = values + .iter() + .map(|value| { + let bits = value.to_bits(); + if bits & (1_u64 << 63) == 0 { + bits ^ (1_u64 << 63) + } else { + !bits + } + }) + .collect::>(); + let native_codec = RangeEntropyCodec::encode(&ordered_values, VALUES_PER_PAGE)?; + let range_packed_codec = RangePackedCodec::encode(&ordered_values, VALUES_PER_PAGE)?; + let range_two_level_codec = RangeTwoLevelCodec::encode(&ordered_values, VALUES_PER_PAGE)?; + let range_grouped_codec = RangeGroupedCodec::encode(&ordered_values, VALUES_PER_PAGE)?; + let patched_for_codec = PatchedFoRCodec::encode(&ordered_values)?; + let patched_for_single_base = PatchedFoRCodec::encode_single_base(&ordered_values)?; + let bit_split_codec = BitSplitCodec::encode(&ordered_values)?; + println!( + "patched-model\t{name}\tblocks={}\tavg-bases={:.2}\tavg-width={:.2}\tpatch-rate={:.4}", + patched_for_codec.block_count(), + patched_for_codec.total_base_count() as f64 + / patched_for_codec.block_count().max(1) as f64, + patched_for_codec.total_residual_width() as f64 + / patched_for_codec.block_count().max(1) as f64, + patched_for_codec.patch_count() as f64 / ordered_values.len().max(1) as f64, + ); + println!( + "bit-split-model\t{name}\tavg-prefixes={:.2}\tavg-suffix-width={:.2}", + bit_split_codec.average_prefix_count(), + bit_split_codec.average_suffix_width(), + ); + report_symbol_models(name, &native_codec); + let native = + RangeEntropy::from_primitive(primitive.as_view(), VALUES_PER_PAGE)?.into_array(); + let pco_classic = Pco::from_primitive_with_config( + primitive.as_view(), + &pco_config(ModeSpec::Classic, DeltaSpec::NoOp), + pco::DEFAULT_MAX_PAGE_N, + &mut session.create_execution_ctx(), + )? + .into_array(); + let pco_auto = Pco::from_primitive_with_config( + primitive.as_view(), + &pco_config(ModeSpec::Auto, DeltaSpec::Auto), + pco::DEFAULT_MAX_PAGE_N, + &mut session.create_execution_ctx(), + )? + .into_array(); + let default_encoded = baseline.compress(&expected, &mut session.create_execution_ctx())?; + let range_encoded = + range_candidate.compress(&expected, &mut session.create_execution_ctx())?; + let float_encoded = + float_candidate.compress(&expected, &mut session.create_execution_ctx())?; + let default_without_block_residual_encoded = default_without_block_residual + .compress(&expected, &mut session.create_execution_ctx())?; + let stacked_encoded = + stacked_candidate.compress(&expected, &mut session.create_execution_ctx())?; + let compact_encoded = compact.compress(&expected, &mut session.create_execution_ctx())?; + println!( + "structure\t{name}\twithout-new-float={}\twithout-new-float+range={}\tdefault-off={}\tdefault-on={}\tdefault+range={}\tcompact={}", + encoding_tree(&default_encoded), + encoding_tree(&range_encoded), + encoding_tree(&default_without_block_residual_encoded), + encoding_tree(&float_encoded), + encoding_tree(&stacked_encoded), + encoding_tree(&compact_encoded) + ); + + let mut verify_ctx = session.create_execution_ctx(); + assert_eq!(range_packed_codec.decode()?, ordered_values); + assert_eq!(range_two_level_codec.decode()?, ordered_values); + assert_eq!(range_grouped_codec.decode()?, ordered_values); + assert_eq!(patched_for_codec.decode()?, ordered_values); + assert_eq!(patched_for_single_base.decode()?, ordered_values); + assert_eq!(bit_split_codec.decode()?, ordered_values); + assert_arrays_eq!(native, expected, &mut verify_ctx); + assert_arrays_eq!(pco_classic, expected, &mut verify_ctx); + assert_arrays_eq!(pco_auto, expected, &mut verify_ctx); + assert_arrays_eq!(default_encoded, expected, &mut verify_ctx); + assert_arrays_eq!(range_encoded, expected, &mut verify_ctx); + assert_arrays_eq!( + default_without_block_residual_encoded, + expected, + &mut verify_ctx + ); + assert_arrays_eq!(float_encoded, expected, &mut verify_ctx); + assert_arrays_eq!(stacked_encoded, expected, &mut verify_ctx); + assert_arrays_eq!(compact_encoded, expected, &mut verify_ctx); + + measure_decoders( + name, + &[ + ("default-without-new-float", &default_encoded), + ("default-without-new-float+range-entropy", &range_encoded), + ("default-off", &default_without_block_residual_encoded), + ("default-on", &float_encoded), + ("default+range-entropy", &stacked_encoded), + ("compact", &compact_encoded), + ("range-entropy", &native), + ("pco-classic", &pco_classic), + ("pco-auto", &pco_auto), + ], + &session, + )?; + measure_array_scalar_at( + name, + "default-off", + &default_without_block_residual_encoded, + &session, + )?; + measure_array_scalar_at(name, "default-on", &float_encoded, &session)?; + measure_range_entropy_codec(name, &values, &native_codec)?; + measure_range_packed_codec(name, &values, &range_packed_codec)?; + measure_range_two_level_codec(name, &values, &range_two_level_codec)?; + measure_range_grouped_codec(name, &values, &range_grouped_codec)?; + measure_patched_for_codec(name, "patched-for-codec", &values, &patched_for_codec)?; + measure_patched_for_codec( + name, + "patched-for-single-base", + &values, + &patched_for_single_base, + )?; + measure_ordered_float_patched_for_codec(name, &values, &patched_for_single_base)?; + measure_patched_for_scalar_at(name, "patched-for-codec", &patched_for_codec)?; + measure_patched_for_scalar_at(name, "patched-for-single-base", &patched_for_single_base)?; + measure_bit_split_codec(name, &values, &bit_split_codec)?; + measure_bit_split_scalar_at(name, &bit_split_codec)?; + measure_native_codec_encoders(name, &ordered_values)?; + measure_ordered_float_patched_for_encoder(name, &values)?; + measure_float_quant_stages(name, &primitive, &baseline, &session)?; + measure_compressors( + name, + &values, + &[ + ("default-without-new-float", &baseline), + ("default-without-new-float+range-entropy", &range_candidate), + ("default-off", &default_without_block_residual), + ("default-on", &float_candidate), + ("default+range-entropy", &stacked_candidate), + ("compact", &compact), + ], + &session, + )?; + } + Ok(()) +} diff --git a/vortex-btrblocks/src/builder.rs b/vortex-btrblocks/src/builder.rs index 53cfc1d2be4..3020d9ae348 100644 --- a/vortex-btrblocks/src/builder.rs +++ b/vortex-btrblocks/src/builder.rs @@ -44,6 +44,8 @@ pub const ALL_SCHEMES: &[&dyn Scheme] = &[ //////////////////////////////////////////////////////////////////////////////////////////////// &float::ALPScheme, &float::ALPRDScheme, + &float::FloatQuantScheme, + &float::OrderedBlockResidualScheme, &float::FloatDictScheme, &float::NullDominatedSparseScheme, &float::FloatRLEScheme, @@ -173,6 +175,8 @@ impl BtrBlocksCompressorBuilder { integer::SparseScheme.id(), integer::IntRLEScheme.id(), float::ALPRDScheme.id(), + float::FloatQuantScheme.id(), + float::OrderedBlockResidualScheme.id(), float::FloatRLEScheme.id(), float::NullDominatedSparseScheme.id(), string::StringDictScheme.id(), @@ -236,6 +240,24 @@ mod tests { fn default_includes_all_schemes() { let builder = BtrBlocksCompressorBuilder::default(); assert_eq!(builder.schemes.len(), ALL_SCHEMES.len()); + assert!( + builder + .schemes + .iter() + .any(|scheme| scheme.id() == float::FloatQuantScheme.id()) + ); + } + + #[test] + fn float_quant_can_be_excluded() { + let builder = + BtrBlocksCompressorBuilder::default().exclude_schemes([float::FloatQuantScheme.id()]); + assert!( + !builder + .schemes + .iter() + .any(|scheme| scheme.id() == float::FloatQuantScheme.id()) + ); } #[test] @@ -270,6 +292,17 @@ mod tests { ); } + #[test] + fn cuda_compatible_excludes_float_quant() { + let builder = BtrBlocksCompressorBuilder::default().only_cuda_compatible(); + assert!( + !builder + .schemes + .iter() + .any(|scheme| scheme.id() == float::FloatQuantScheme.id()) + ); + } + #[test] fn cuda_compatible_uses_fsst_for_strings() { let builder = BtrBlocksCompressorBuilder::default().only_cuda_compatible(); diff --git a/vortex-btrblocks/src/schemes/float/float_quant.rs b/vortex-btrblocks/src/schemes/float/float_quant.rs new file mode 100644 index 00000000000..44009a09dcb --- /dev/null +++ b/vortex-btrblocks/src/schemes/float/float_quant.rs @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Lossless float quantization with recursively compressed integer children. + +use vortex_array::ArrayId; +use vortex_array::ArrayRef; +use vortex_array::Canonical; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::VTable; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::dtype::PType; +use vortex_array::scalar::Scalar; +use vortex_compressor::scheme::CompressionEstimate; +use vortex_compressor::scheme::DeferredEstimate; +use vortex_compressor::scheme::EstimateVerdict; +use vortex_error::VortexResult; +use vortex_fastlanes::FoR; +use vortex_fastlanes::bitpack_compress::bitpack_encode_unchecked; +use vortex_float_quant::FloatQuant; +use vortex_float_quant::FloatQuantAnalysis; +use vortex_float_quant::FloatQuantArraySlotsExt; +use vortex_float_quant::analyze_float_quant; + +use crate::ArrayAndStats; +use crate::CascadingCompressor; +use crate::CompressorContext; +use crate::Scheme; +use crate::SchemeExt; + +const ALWAYS_USE_MIN_RATIO: f64 = 2.0; + +fn is_strong_constant_split(analysis: FloatQuantAnalysis, ptype: PType) -> bool { + analysis.secondary_is_constant + && analysis.primary_bit_width > 0 + && ptype.bit_width() as f64 / f64::from(analysis.primary_bit_width) >= ALWAYS_USE_MIN_RATIO +} + +/// FloatQuant split with normal BtrBlocks compression for both latent children. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct FloatQuantScheme; + +impl Scheme for FloatQuantScheme { + fn scheme_name(&self) -> &'static str { + "vortex.float.float_quant" + } + + fn matches(&self, canonical: &Canonical) -> bool { + canonical.dtype().is_float() + } + + fn produced_encodings(&self) -> Vec { + vec![FloatQuant.id()] + } + + fn num_children(&self) -> usize { + 2 + } + + fn expected_compression_ratio( + &self, + data: &ArrayAndStats, + compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> CompressionEstimate { + let primitive = data.array_as_primitive(); + if compress_ctx.finished_cascading() || primitive.ptype() != PType::F64 { + return CompressionEstimate::Verdict(EstimateVerdict::Skip); + } + CompressionEstimate::Deferred(DeferredEstimate::Sample) + } + + fn compress( + &self, + compressor: &CascadingCompressor, + data: &ArrayAndStats, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, + ) -> VortexResult { + let primitive = data.array_as_primitive(); + let Some(analysis) = analyze_float_quant(primitive) else { + return Ok(primitive.array().clone()); + }; + if is_strong_constant_split(analysis, primitive.ptype()) { + let biased = + FloatQuant::primary_for_primitive(primitive, analysis.k, analysis.primary_min)?; + // SAFETY: The analysis computes this width from the exact primary minimum and maximum. + let compressed_primary = + unsafe { bitpack_encode_unchecked(biased, analysis.primary_bit_width)? } + .into_array(); + let reference = match primitive.ptype() { + PType::F32 => Scalar::from(u32::try_from(analysis.primary_min)?), + PType::F64 => Scalar::from(analysis.primary_min), + _ => unreachable!(), + }; + let compressed_primary = FoR::try_new(compressed_primary, reference)?.into_array(); + let compressed_secondary = match primitive.ptype() { + PType::F32 => ConstantArray::new(Scalar::from(0u32), primitive.len()).into_array(), + PType::F64 => ConstantArray::new(Scalar::from(0u64), primitive.len()).into_array(), + _ => unreachable!(), + }; + return Ok(FloatQuant::try_new( + compressed_primary, + compressed_secondary, + primitive.ptype(), + analysis.k, + )? + .into_array()); + } + + let encoded = FloatQuant::from_primitive(primitive, analysis.k)?; + let primary = encoded + .primary() + .clone() + .execute::(exec_ctx)?; + let secondary = encoded + .secondary() + .clone() + .execute::(exec_ctx)?; + let compressed_primary = compressor.compress_child( + &primary.into_array(), + &compress_ctx, + self.id(), + 0, + exec_ctx, + )?; + let compressed_secondary = compressor.compress_child( + &secondary.into_array(), + &compress_ctx, + self.id(), + 1, + exec_ctx, + )?; + Ok(FloatQuant::try_new( + compressed_primary, + compressed_secondary, + primitive.ptype(), + analysis.k, + )? + .into_array()) + } +} diff --git a/vortex-btrblocks/src/schemes/float/mod.rs b/vortex-btrblocks/src/schemes/float/mod.rs index 1301184ac0c..e2f8aecb398 100644 --- a/vortex-btrblocks/src/schemes/float/mod.rs +++ b/vortex-btrblocks/src/schemes/float/mod.rs @@ -5,6 +5,8 @@ mod alp; mod alprd; +mod float_quant; +mod ordered_block_residual; mod rle; mod sparse; @@ -13,6 +15,8 @@ mod pco; pub use alp::ALPScheme; pub use alprd::ALPRDScheme; +pub use float_quant::FloatQuantScheme; +pub use ordered_block_residual::OrderedBlockResidualScheme; #[cfg(feature = "pco")] pub use pco::PcoScheme; pub use rle::FloatRLEScheme; diff --git a/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs b/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs new file mode 100644 index 00000000000..7afb49174e7 --- /dev/null +++ b/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Ordered float bits with one block-local reference and packed residuals. + +use vortex_array::ArrayId; +use vortex_array::ArrayRef; +use vortex_array::Canonical; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::VTable; +use vortex_array::arrays::Primitive; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::dtype::PType; +use vortex_compressor::scheme::CompressionEstimate; +use vortex_compressor::scheme::DeferredEstimate; +use vortex_compressor::scheme::EstimateScore; +use vortex_compressor::scheme::EstimateVerdict; +use vortex_error::VortexResult; +use vortex_range_entropy::BlockResidual; +use vortex_range_entropy::OrderedFloat; +use vortex_range_entropy::OrderedFloatArraySlotsExt; + +use crate::ArrayAndStats; +use crate::CascadingCompressor; +use crate::CompressorContext; +use crate::Scheme; + +const BLOCK_LEN: usize = 1024; +const ESTIMATE_BLOCKS: usize = 8; +const MIN_COMPRESSION_RATIO: f64 = 1.05; +const MIN_WIN_OVER_INCUMBENT: f64 = 1.02; + +/// Compress `f64` values as block-local residuals of ordered IEEE bits. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct OrderedBlockResidualScheme; + +impl Scheme for OrderedBlockResidualScheme { + fn scheme_name(&self) -> &'static str { + "vortex.float.ordered_block_residual" + } + + fn matches(&self, canonical: &Canonical) -> bool { + canonical.dtype().is_float() && canonical.dtype().as_ptype() == PType::F64 + } + + fn produced_encodings(&self) -> Vec { + vec![OrderedFloat.id()] + } + + fn num_children(&self) -> usize { + 1 + } + + fn expected_compression_ratio( + &self, + data: &ArrayAndStats, + compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> CompressionEstimate { + if compress_ctx.finished_cascading() || data.array_as_primitive().ptype() != PType::F64 { + return CompressionEstimate::Verdict(EstimateVerdict::Skip); + } + CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( + |_compressor, data, best_so_far, _compress_ctx, exec_ctx| { + let sample = locality_sample(data.array_as_primitive(), exec_ctx)?; + let before_nbytes = sample.nbytes(); + let ordered = OrderedFloat::from_primitive(sample.as_view())?; + let residuals = + BlockResidual::from_primitive(ordered.encoded().as_::())?; + let after_nbytes = residuals.nbytes(); + if after_nbytes == 0 { + return Ok(EstimateVerdict::Skip); + } + + let ratio = before_nbytes as f64 / after_nbytes as f64; + if ratio < MIN_COMPRESSION_RATIO { + return Ok(EstimateVerdict::Skip); + } + let incumbent = best_so_far.and_then(EstimateScore::finite_ratio); + if incumbent.is_some_and(|best| ratio < best * MIN_WIN_OVER_INCUMBENT) { + return Ok(EstimateVerdict::Skip); + } + Ok(EstimateVerdict::Ratio(ratio)) + }, + ))) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> VortexResult { + let ordered = OrderedFloat::from_primitive(data.array_as_primitive())?; + let ordered_values = ordered.encoded().as_::(); + let residuals = BlockResidual::from_primitive(ordered_values)?; + Ok(OrderedFloat::try_new(residuals.into_array(), PType::F64)?.into_array()) + } +} + +fn locality_sample( + primitive: vortex_array::ArrayView<'_, Primitive>, + exec_ctx: &mut ExecutionCtx, +) -> VortexResult { + let values = primitive.as_slice::(); + let validity = primitive + .validity()? + .execute_mask(primitive.len(), exec_ctx)?; + let full_blocks = primitive.len() / BLOCK_LEN; + + if full_blocks <= ESTIMATE_BLOCKS { + return Ok(PrimitiveArray::from_option_iter( + values + .iter() + .copied() + .enumerate() + .map(|(index, value)| validity.value(index).then_some(value)), + )); + } + + let sample_blocks = ESTIMATE_BLOCKS.min(full_blocks); + let mut sample = Vec::with_capacity(sample_blocks * BLOCK_LEN); + for sample_index in 0..sample_blocks { + let block_index = sample_index * full_blocks / sample_blocks; + let start = block_index * BLOCK_LEN; + sample.extend( + (start..start + BLOCK_LEN).map(|index| validity.value(index).then_some(values[index])), + ); + } + Ok(PrimitiveArray::from_option_iter(sample)) +} diff --git a/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs index 4c7a5f85fa6..582a48b4dfa 100644 --- a/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs @@ -3,6 +3,7 @@ //! Tests to verify that each float compression scheme produces the expected encoding. +use std::f64::consts::TAU; use std::sync::LazyLock; use vortex_alp::ALP; @@ -17,6 +18,9 @@ use vortex_array::dtype::Nullability; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_error::VortexResult; +use vortex_float_quant::FloatQuant; +use vortex_range_entropy::BlockResidual; +use vortex_range_entropy::OrderedFloat; use vortex_session::VortexSession; use crate::BtrBlocksCompressor; @@ -71,3 +75,72 @@ fn test_null_dominated_compressed() -> VortexResult<()> { assert_eq!(compressed.len(), 100); Ok(()) } + +#[test] +fn test_widened_f32_uses_float_quant() -> VortexResult<()> { + let values = (0u32..16_384) + .map(|index| { + let mantissa = index.wrapping_mul(7_919) & 0x007f_ffff; + f64::from(f32::from_bits(0x3f80_0000 | mantissa)) + }) + .collect::>(); + let array = PrimitiveArray::from_iter(values).into_array(); + let compressed = + BtrBlocksCompressor::default().compress(&array, &mut SESSION.create_execution_ctx())?; + assert!(compressed.is::()); + Ok(()) +} + +#[test] +fn test_f32_does_not_use_float_quant() -> VortexResult<()> { + let values = (0u32..16_384) + .map(|index| { + let mantissa = index.wrapping_mul(7_919) & 0x007f_ffff; + f32::from_bits(0x3f80_0000 | mantissa) + }) + .collect::>(); + let array = PrimitiveArray::from_iter(values).into_array(); + let compressed = + BtrBlocksCompressor::default().compress(&array, &mut SESSION.create_execution_ctx())?; + assert!(!compressed.is::()); + Ok(()) +} + +#[test] +fn test_repeated_f64_prefers_existing_scheme() -> VortexResult<()> { + let values = (0u32..16_384) + .map(|index| f64::from(index % 8)) + .collect::>(); + let array = PrimitiveArray::from_iter(values).into_array(); + let compressed = + BtrBlocksCompressor::default().compress(&array, &mut SESSION.create_execution_ctx())?; + assert!(!compressed.is::()); + Ok(()) +} + +#[test] +fn test_random_walk_uses_ordered_block_residual() -> VortexResult<()> { + fn uniform(state: &mut u64) -> f64 { + *state ^= *state << 13; + *state ^= *state >> 7; + *state ^= *state << 17; + ((*state >> 11) as f64 + 0.5) / (1_u64 << 53) as f64 + } + + let mut state = 0x4d59_5df4_d0f3_3173_u64; + let mut value = 0.0_f64; + let values = (0..65_536) + .map(|_| { + let radius = (-2.0 * uniform(&mut state).ln()).sqrt(); + let normal = radius * (TAU * uniform(&mut state)).cos(); + value += normal * 0.01; + value + }) + .collect::>(); + let array = PrimitiveArray::from_iter(values).into_array(); + let compressed = + BtrBlocksCompressor::default().compress(&array, &mut SESSION.create_execution_ctx())?; + assert!(compressed.is::()); + assert!(compressed.children()[0].is::()); + Ok(()) +} diff --git a/vortex-btrblocks/src/schemes/integer/bitpacking.rs b/vortex-btrblocks/src/schemes/integer/bitpacking.rs index 5ac7d0e4078..b11e54299b9 100644 --- a/vortex-btrblocks/src/schemes/integer/bitpacking.rs +++ b/vortex-btrblocks/src/schemes/integer/bitpacking.rs @@ -10,6 +10,7 @@ use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::VTable; use vortex_array::arrays::Patched; +use vortex_array::arrays::Primitive; use vortex_array::arrays::patched::use_experimental_patches; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_compressor::scheme::CompressionEstimate; @@ -31,6 +32,65 @@ use crate::compress_patches; #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct BitPackingScheme; +pub(crate) fn compress_bitpacked( + primitive_array: vortex_array::ArrayView<'_, Primitive>, + exec_ctx: &mut ExecutionCtx, +) -> VortexResult { + let histogram = bit_width_histogram(primitive_array, exec_ctx)?; + let bw = find_best_bit_width(primitive_array.ptype(), &histogram)?; + + if bw as usize == primitive_array.ptype().bit_width() { + return Ok(primitive_array.array().clone()); + } + + let primitive_array = primitive_array.into_owned(); + let packed = bitpack_encode(&primitive_array, bw, Some(&histogram), exec_ctx)?; + let packed_stats = packed.statistics().to_owned(); + let ptype = packed.dtype().as_ptype(); + let mut parts = BitPacked::into_parts(packed); + + let array = if use_experimental_patches() { + let patches = parts.patches.take(); + let array = BitPacked::try_new( + parts.packed, + ptype, + parts.validity, + None, + parts.bit_width, + parts.len, + parts.offset, + )? + .into_array(); + + match patches { + None => array, + Some(patches) => Patched::from_array_and_patches(array, &patches, exec_ctx)? + .with_stats_set(packed_stats) + .into_array(), + } + } else { + let patches = parts + .patches + .take() + .map(|patches| compress_patches(patches, exec_ctx)) + .transpose()?; + parts.patches = patches; + BitPacked::try_new( + parts.packed, + ptype, + parts.validity, + parts.patches, + parts.bit_width, + parts.len, + parts.offset, + )? + .with_stats_set(packed_stats) + .into_array() + }; + + Ok(array) +} + impl Scheme for BitPackingScheme { fn scheme_name(&self) -> &'static str { "vortex.int.bitpacking" @@ -72,64 +132,6 @@ impl Scheme for BitPackingScheme { exec_ctx: &mut ExecutionCtx, ) -> VortexResult { let primitive_array = data.array_as_primitive(); - - let histogram = bit_width_histogram(primitive_array, exec_ctx)?; - let bw = find_best_bit_width(primitive_array.ptype(), &histogram)?; - - // If best bw is determined to be the current bit-width, return the original array. - if bw as usize == primitive_array.ptype().bit_width() { - return Ok(primitive_array.array().clone()); - } - - // Otherwise we can bitpack the array. - let primitive_array = primitive_array.into_owned(); - let packed = bitpack_encode(&primitive_array, bw, Some(&histogram), exec_ctx)?; - - let packed_stats = packed.statistics().to_owned(); - let ptype = packed.dtype().as_ptype(); - let mut parts = BitPacked::into_parts(packed); - - let array = if use_experimental_patches() { - let patches = parts.patches.take(); - // Transpose patches into G-ALP style PatchedArray, wrapping an inner BitPackedArray. - let array = BitPacked::try_new( - parts.packed, - ptype, - parts.validity, - None, - parts.bit_width, - parts.len, - parts.offset, - )? - .into_array(); - - match patches { - None => array, - Some(p) => Patched::from_array_and_patches(array, &p, exec_ctx)? - .with_stats_set(packed_stats) - .into_array(), - } - } else { - // Compress patches and place back into BitPackedArray. - let patches = parts - .patches - .take() - .map(|p| compress_patches(p, exec_ctx)) - .transpose()?; - parts.patches = patches; - BitPacked::try_new( - parts.packed, - ptype, - parts.validity, - parts.patches, - parts.bit_width, - parts.len, - parts.offset, - )? - .with_stats_set(packed_stats) - .into_array() - }; - - Ok(array) + compress_bitpacked(primitive_array, exec_ctx) } } diff --git a/vortex-btrblocks/src/schemes/integer/for_.rs b/vortex-btrblocks/src/schemes/integer/for_.rs index 476a0dec282..3c43a4a5c0b 100644 --- a/vortex-btrblocks/src/schemes/integer/for_.rs +++ b/vortex-btrblocks/src/schemes/integer/for_.rs @@ -31,6 +31,31 @@ use crate::CompressorContext; use crate::Scheme; use crate::SchemeExt; +pub(crate) fn compress_for_primitive( + compressor: &CascadingCompressor, + primitive: PrimitiveArray, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, +) -> VortexResult { + let for_array = FoR::encode(primitive, exec_ctx)?; + let biased = for_array + .encoded() + .clone() + .execute::(exec_ctx)?; + + let leaf_ctx = compress_ctx.clone().as_leaf(); + let biased_data = ArrayAndStats::new(biased.into_array(), compress_ctx.merged_stats_options()); + let compressed = BitPackingScheme.compress(compressor, &biased_data, leaf_ctx, exec_ctx)?; + + let for_compressed = FoR::try_new(compressed, for_array.reference_scalar().clone())?; + for_compressed + .as_ref() + .statistics() + .inherit_from(for_array.as_ref().statistics()); + + Ok(for_compressed.into_array()) +} + /// Frame of Reference encoding. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct FoRScheme; @@ -130,28 +155,6 @@ impl Scheme for FoRScheme { exec_ctx: &mut ExecutionCtx, ) -> VortexResult { let primitive = data.array().clone().execute::(exec_ctx)?; - let for_array = FoR::encode(primitive, exec_ctx)?; - let biased = for_array - .encoded() - .clone() - .execute::(exec_ctx)?; - - // Immediately bitpack. If any other scheme was preferable, it would be chosen instead - // of bitpacking. - // NOTE: we could delegate in the future if we had another downstream codec that performs - // as well. - let leaf_ctx = compress_ctx.clone().as_leaf(); - let biased_data = - ArrayAndStats::new(biased.into_array(), compress_ctx.merged_stats_options()); - let compressed = BitPackingScheme.compress(compressor, &biased_data, leaf_ctx, exec_ctx)?; - - // TODO(connor): This should really be `new_unchecked`. - let for_compressed = FoR::try_new(compressed, for_array.reference_scalar().clone())?; - for_compressed - .as_ref() - .statistics() - .inherit_from(for_array.as_ref().statistics()); - - Ok(for_compressed.into_array()) + compress_for_primitive(compressor, primitive, compress_ctx, exec_ctx) } } diff --git a/vortex-btrblocks/src/schemes/mod.rs b/vortex-btrblocks/src/schemes/mod.rs index a0e9b042a66..9bf88d5f732 100644 --- a/vortex-btrblocks/src/schemes/mod.rs +++ b/vortex-btrblocks/src/schemes/mod.rs @@ -6,6 +6,7 @@ pub mod binary; pub mod float; pub mod integer; +pub mod range_entropy; pub mod string; pub mod decimal; diff --git a/vortex-btrblocks/src/schemes/range_entropy.rs b/vortex-btrblocks/src/schemes/range_entropy.rs new file mode 100644 index 00000000000..c070107e1fb --- /dev/null +++ b/vortex-btrblocks/src/schemes/range_entropy.rs @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Native range entropy compression for primitive numeric arrays. + +use vortex_array::ArrayId; +use vortex_array::ArrayRef; +use vortex_array::Canonical; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::VTable; +use vortex_compressor::scheme::CompressionEstimate; +use vortex_compressor::scheme::DeferredEstimate; +use vortex_error::VortexResult; +use vortex_range_entropy::RangeEntropy; + +use crate::ArrayAndStats; +use crate::CascadingCompressor; +use crate::CompressorContext; +use crate::Scheme; + +const RESTART_BLOCK_LEN: usize = 8192; + +/// Native range bins plus tANS for primitive numeric arrays. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct RangeEntropyScheme; + +impl Scheme for RangeEntropyScheme { + fn scheme_name(&self) -> &'static str { + "vortex.numeric.range_entropy" + } + + fn matches(&self, canonical: &Canonical) -> bool { + canonical.dtype().is_int() || canonical.dtype().is_float() + } + + fn produced_encodings(&self) -> Vec { + vec![RangeEntropy.id()] + } + + fn expected_compression_ratio( + &self, + _data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> CompressionEstimate { + CompressionEstimate::Deferred(DeferredEstimate::Sample) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> VortexResult { + Ok( + RangeEntropy::from_primitive(data.array_as_primitive(), RESTART_BLOCK_LEN)? + .into_array(), + ) + } +} diff --git a/vortex-file/Cargo.toml b/vortex-file/Cargo.toml index 72a47d88f50..eb3663c7846 100644 --- a/vortex-file/Cargo.toml +++ b/vortex-file/Cargo.toml @@ -44,6 +44,7 @@ vortex-edition = { workspace = true } vortex-error = { workspace = true } vortex-fastlanes = { workspace = true } vortex-flatbuffers = { workspace = true, features = ["file"] } +vortex-float-quant = { workspace = true } vortex-fsst = { workspace = true } vortex-io = { workspace = true } vortex-layout = { workspace = true } @@ -51,6 +52,7 @@ vortex-mask = { workspace = true } vortex-metrics = { workspace = true } vortex-onpair = { workspace = true } vortex-pco = { workspace = true } +vortex-range-entropy = { workspace = true } vortex-runend = { workspace = true } vortex-scan = { workspace = true } vortex-sequence = { workspace = true } diff --git a/vortex-file/src/lib.rs b/vortex-file/src/lib.rs index 25760f0890e..020792b6998 100644 --- a/vortex-file/src/lib.rs +++ b/vortex-file/src/lib.rs @@ -190,6 +190,8 @@ pub fn register_default_encodings(session: &VortexSession) { vortex_datetime_parts::initialize(session); vortex_decimal_byte_parts::initialize(session); vortex_fastlanes::initialize(session); + vortex_float_quant::initialize(session); + vortex_range_entropy::initialize(session); vortex_runend::initialize(session); vortex_sequence::initialize(session); vortex_sparse::initialize(session); diff --git a/vortex/Cargo.toml b/vortex/Cargo.toml index 57392ed627d..2ce314106ab 100644 --- a/vortex/Cargo.toml +++ b/vortex/Cargo.toml @@ -37,6 +37,7 @@ vortex-error = { workspace = true } vortex-fastlanes = { workspace = true } vortex-file = { workspace = true, optional = true, default-features = true } vortex-flatbuffers = { workspace = true } +vortex-float-quant = { workspace = true } vortex-fsst = { workspace = true } vortex-io = { workspace = true } vortex-ipc = { workspace = true } @@ -44,6 +45,7 @@ vortex-layout = { workspace = true } vortex-mask = { workspace = true } vortex-metrics = { workspace = true } vortex-pco = { workspace = true } +vortex-range-entropy = { workspace = true } vortex-proto = { workspace = true, default-features = true } vortex-runend = { workspace = true } vortex-scan = { workspace = true } diff --git a/vortex/src/editions/core/v2026_08.rs b/vortex/src/editions/core/v2026_08.rs index 108402c76df..b64faf56025 100644 --- a/vortex/src/editions/core/v2026_08.rs +++ b/vortex/src/editions/core/v2026_08.rs @@ -1,14 +1,14 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! The August 2026 core edition adding the canonical Map encoding and the zone map aggregates. +//! The August 2026 core edition with new arrays and zone map aggregates. use vortex_edition::Edition; use vortex_edition::EditionDeclaration; use vortex_edition::EditionId; use vortex_edition::EditionMember; -/// The August 2026 core edition containing canonical Map arrays. +/// The August 2026 core edition with new numeric and map arrays. pub const CORE_2026_08: EditionId = EditionId::new("core", 2026, 8, 0); /// The declaration of [`CORE_2026_08`] and the components that join the family at it. @@ -26,7 +26,10 @@ pub static DECLARATION: EditionDeclaration = EditionDeclaration { min_vortex_version: Some("0.84.0"), }, added: &[ + EditionMember::array(&"vortex.block_residual"), + EditionMember::array(&"vortex.float_quant"), EditionMember::array(&"vortex.map"), + EditionMember::array(&"vortex.ordered_float"), EditionMember::aggregate(&"vortex.bounded_max"), EditionMember::aggregate(&"vortex.bounded_min"), EditionMember::aggregate(&"vortex.max"), diff --git a/vortex/src/editions/tests.rs b/vortex/src/editions/tests.rs index d418a8aa412..a833854d71a 100644 --- a/vortex/src/editions/tests.rs +++ b/vortex/src/editions/tests.rs @@ -1,11 +1,13 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::f64::consts::TAU; use std::sync::Arc; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; use vortex_array::array_session; use vortex_array::arrays::ChunkedArray; use vortex_array::arrays::PrimitiveArray; @@ -33,11 +35,14 @@ use vortex_error::vortex_err; use vortex_file::OpenOptionsSessionExt; use vortex_file::WriteOptionsSessionExt; use vortex_file::WriteStrategyBuilder; +use vortex_float_quant::FloatQuant; use vortex_io::session::RuntimeSession; use vortex_layout::LayoutStrategy; use vortex_layout::layouts::compressed::CompressingStrategy; use vortex_layout::layouts::flat::writer::FlatLayoutStrategy; use vortex_layout::session::LayoutSession; +use vortex_range_entropy::BlockResidual; +use vortex_range_entropy::OrderedFloat; use vortex_sequence::Sequence; use vortex_session::VortexSession; use vortex_session::registry::Id; @@ -490,6 +495,87 @@ async fn default_strategy_round_trip_uses_only_enabled_encodings() -> VortexResu .await } +#[tokio::test] +async fn default_writer_round_trips_float_quant() -> VortexResult<()> { + use crate::VortexSessionDefault; + + let session = VortexSession::default(); + let values = (0u32..65_536) + .map(|index| { + let mantissa = index.wrapping_mul(7_919) & 0x007f_ffff; + f64::from(f32::from_bits(0x3f80_0000 | mantissa)) + }) + .collect::>(); + let expected = PrimitiveArray::from_iter(values.clone()).into_array(); + let buffer = write_with(&session, expected.clone()).await?; + let actual = session + .open_options() + .open_buffer(buffer)? + .scan()? + .into_array_stream()? + .read_all() + .await?; + assert!( + actual + .depth_first_traversal() + .any(|array| array.is::()) + ); + let decoded = actual.execute::(&mut session.create_execution_ctx())?; + assert_eq!(decoded.as_slice::(), values); + Ok(()) +} + +#[tokio::test] +async fn default_writer_round_trips_ordered_block_residual() -> VortexResult<()> { + use crate::VortexSessionDefault; + + fn uniform(state: &mut u64) -> f64 { + *state ^= *state << 13; + *state ^= *state >> 7; + *state ^= *state << 17; + ((*state >> 11) as f64 + 0.5) / (1_u64 << 53) as f64 + } + + let session = VortexSession::default(); + let mut state = 0x4d59_5df4_d0f3_3173_u64; + let mut value = 0.0_f64; + let values = (0..65_536) + .map(|_| { + let radius = (-2.0 * uniform(&mut state).ln()).sqrt(); + let normal = radius * (TAU * uniform(&mut state)).cos(); + value += normal * 0.01; + value + }) + .collect::>(); + let expected = PrimitiveArray::from_iter(values.clone()).into_array(); + let buffer = write_with(&session, expected).await?; + let actual = session + .open_options() + .open_buffer(buffer)? + .scan()? + .into_array_stream()? + .read_all() + .await?; + let encoding_ids = actual + .depth_first_traversal() + .map(|array| array.encoding_id().to_string()) + .collect::>(); + assert!( + actual + .depth_first_traversal() + .any(|array| array.is::()), + "written tree does not contain OrderedFloat: {encoding_ids:?}" + ); + assert!( + actual + .depth_first_traversal() + .any(|array| array.is::()) + ); + let decoded = actual.execute::(&mut session.create_execution_ctx())?; + assert_eq!(decoded.as_slice::(), values); + Ok(()) +} + #[tokio::test] async fn replacement_default_builder_round_trip_uses_only_enabled_encodings() -> VortexResult<()> { let session = writer_test_session()?; diff --git a/vortex/src/lib.rs b/vortex/src/lib.rs index 289500e2543..43a5188816c 100644 --- a/vortex/src/lib.rs +++ b/vortex/src/lib.rs @@ -260,6 +260,11 @@ pub mod encodings { pub use vortex_fastlanes::*; } + /// Lossless float quantization with integer child arrays. + pub mod float_quant { + pub use vortex_float_quant::*; + } + /// Fast Static Symbol Table string encoding. pub mod fsst { pub use vortex_fsst::*; @@ -270,6 +275,11 @@ pub mod encodings { pub use vortex_pco::*; } + /// Ordered-float, block-residual, and range-entropy numeric encodings. + pub mod range_entropy { + pub use vortex_range_entropy::*; + } + /// Arrow-compatible run-end encoding. pub mod runend { pub use vortex_runend::*; From 5786fd763faeb66fb32a7620d57a1809b6588c57 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Tue, 18 Aug 2026 16:39:01 -0400 Subject: [PATCH 02/78] bench: compare FloatMult child codecs Signed-off-by: "Will Manning" --- docs/plans/native-pcodec.md | 76 +++-- .../examples/float_quant_dataset.rs | 314 +++++++++++++++--- 2 files changed, 320 insertions(+), 70 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index ef09a2ace83..c901dfdb029 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -164,39 +164,39 @@ The aggregate values below cover every supported numeric column in each download ### Size -| Dataset | Without FloatQuant | Default | Default plus RangeEntropy | Compact | Pco Auto | +| Dataset | Without new float schemes | Default | Default plus RangeEntropy | Compact | Pco Auto | | --- | ---: | ---: | ---: | ---: | ---: | | California Housing | 339,682 | 339,682 | 319,705 | 230,197 | 242,414 | -| April 2023 HVFHV Taxi | 20,894,253 | 20,894,253 | 16,443,089 | 14,533,638 | 15,403,694 | +| April 2023 HVFHV Taxi | 28,540,793 | 28,540,793 | 23,331,922 | 21,127,765 | 21,994,290 | | Air Quality | 6,135,414 | 6,135,414 | 5,149,775 | 2,311,526 | 2,303,523 | -FloatQuant makes no selection on these inputs. Their size and decode paths remain unchanged. +FloatQuant and OrderedFloat with BlockResidual make no selection on these inputs. Their size and decode paths remain unchanged. -RangeEntropy reduces Taxi size by 21.3 percent and Air Quality size by 16.1 percent. +RangeEntropy reduces Taxi size by 18.3 percent and Air Quality size by 16.1 percent. -RangeEntropy remains 13.1 percent larger than Compact on Taxi. It remains 122.8 percent larger on Air Quality. +RangeEntropy remains 10.4 percent larger than Compact on Taxi. It remains 122.8 percent larger on Air Quality. ### Compression throughput -| Dataset | Without FloatQuant | Default | Default plus RangeEntropy | Compact | Pco Auto | -| --- | ---: | ---: | ---: | ---: | ---: | -| California Housing | 201.9 MB/s | 204.4 MB/s | 112.9 MB/s | 70.9 MB/s | 201.6 MB/s | -| April 2023 HVFHV Taxi | 849.7 MB/s | 830.2 MB/s | 424.5 MB/s | 448.1 MB/s | 608.0 MB/s | -| Air Quality | 573.0 MB/s | 574.8 MB/s | 328.4 MB/s | 310.1 MB/s | 433.9 MB/s | +| Dataset | Without new float schemes | Without BlockResidual | Default | Default plus RangeEntropy | Compact | Pco Auto | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| California Housing | 205.8 MB/s | 208.5 MB/s | 210.0 MB/s | 114.8 MB/s | 72.1 MB/s | 206.3 MB/s | +| April 2023 HVFHV Taxi | 886.2 MB/s | 838.4 MB/s | 828.8 MB/s | 421.7 MB/s | 432.3 MB/s | 574.0 MB/s | +| Air Quality | 548.2 MB/s | 549.1 MB/s | 550.5 MB/s | 318.3 MB/s | 303.7 MB/s | 415.2 MB/s | -FloatQuant changes unselected compression throughput by less than 3 percent on these inputs. +The BlockResidual locality probe changes compression throughput by 1.2 percent or less on these inputs. -RangeEntropy reduces compression throughput by 43 to 50 percent. This result fails the gate. +RangeEntropy reduces compression throughput by 42 to 49 percent. This result fails the gate. ### Decode throughput -| Dataset | Without FloatQuant | Default | Default plus RangeEntropy | Compact | Pco Auto | -| --- | ---: | ---: | ---: | ---: | ---: | -| California Housing | 12,488.1 MB/s | 12,674.5 MB/s | 2,148.5 MB/s | 2,351.7 MB/s | 2,406.9 MB/s | -| April 2023 HVFHV Taxi | 21,718.9 MB/s | 21,918.9 MB/s | 4,509.1 MB/s | 5,063.4 MB/s | 3,517.7 MB/s | -| Air Quality | 27,375.7 MB/s | 27,799.2 MB/s | 1,557.1 MB/s | 2,199.1 MB/s | 2,170.5 MB/s | +| Dataset | Without new float schemes | Without BlockResidual | Default | Default plus RangeEntropy | Compact | Pco Auto | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| California Housing | 12,392.7 MB/s | 12,531.9 MB/s | 12,558.4 MB/s | 2,221.6 MB/s | 2,336.9 MB/s | 2,413.8 MB/s | +| April 2023 HVFHV Taxi | 23,963.4 MB/s | 23,879.0 MB/s | 24,200.2 MB/s | 3,809.3 MB/s | 4,825.0 MB/s | 3,644.8 MB/s | +| Air Quality | 25,013.7 MB/s | 24,807.0 MB/s | 25,371.2 MB/s | 1,471.8 MB/s | 2,088.0 MB/s | 2,092.8 MB/s | -RangeEntropy decode is 4.9 to 17.9 times slower than the current lightweight default. +RangeEntropy decode is 5.7 to 17.2 times slower than the current lightweight default. RangeEntropy beats Pco Auto decode on Taxi. It loses to Pco Auto decode on Housing and Air Quality. @@ -310,23 +310,41 @@ The four-cluster result is 1.4 percent larger than ALP-RD. It is 14.0 percent sm Pco selects FloatMult on seven Taxi columns. The selected source columns contain 112 MB of logical values. -| Backend for FloatMult children | Aggregate bytes | -| --- | ---: | -| Normal BtrBlocks integer compression | 22,340,941 | -| Block residual packing | 19,555,263 | -| BitSplit | 24,105,632 | -| Fixed range tags | 20,860,940 | -| Two-level range tags | 19,052,586 | -| RangeEntropy | 16,730,631 | -| Current default on source floats | 20,104,147 | -| Pco Auto on source floats | 15,061,783 | +| Backend for FloatMult children | Aggregate bytes | Encode MB/s | Decode MB/s | +| --- | ---: | ---: | ---: | +| Normal BtrBlocks integer compression | 22,340,941 | Not isolated | 5,079.1 | +| Block residual packing | 19,555,263 | 1,826.0 | 3,628.8 | +| BitSplit | 24,105,632 | 814.7 | 2,487.2 | +| Fixed range tags | 20,860,940 | 703.5 | 3,057.3 | +| Two-level range tags | 19,052,586 | 477.3 | 2,088.3 | +| RangeEntropy | 16,730,631 | 461.9 | 1,884.1 | +| Current default on source floats | 20,104,147 | Not isolated | Not isolated | +| Pco Auto on source floats | 15,061,783 | Not isolated | Not isolated | FloatMult does not create conventional integer distributions. Normal integer compression loses 11.1 percent against the current default. -RangeEntropy recovers most of Pco's benefit. It remains 11.1 percent larger than Pco on these columns. +Block residual packing saves only 2.7 percent against the current default on the Taxi source columns. + +RangeEntropy saves 16.8 percent against the current default. It remains 11.1 percent larger than Pco on these columns. + +The direct decode test includes child decode, FloatMult reconstruction, and output allocation. + +The isolated encode test excludes FloatMult base search and the split from floats to latent integers. + +California Housing gives a different result. Block residual packing uses 170,645 bytes for five FloatMult columns. + +The current default uses 209,265 bytes for those source columns. Block residual packing saves 18.5 percent. + +The full block residual path decodes at 2,941.6 MB/s and encodes its existing latent integers at 937.6 MB/s. + +Normal BtrBlocks children use 172,046 bytes and decode at 6,991.6 MB/s on those columns. The current `RangeEntropy` decoder is too slow for the default compressor. FloatMult therefore needs a faster entropy-like child codec. +Block residual packing is the fastest specialized encoder. Its decode path is still too slow for the default compressor. + +The next FloatMult prototype will use normal integer children first. Its selector must reject Taxi-like inputs with larger encoded children. + ## Why Compact still wins Pco does not use FloatQuant on the measured paper inputs. diff --git a/vortex-btrblocks/examples/float_quant_dataset.rs b/vortex-btrblocks/examples/float_quant_dataset.rs index b9fb8e955e8..544691e90b8 100644 --- a/vortex-btrblocks/examples/float_quant_dataset.rs +++ b/vortex-btrblocks/examples/float_quant_dataset.rs @@ -34,7 +34,9 @@ use vortex_btrblocks::BtrBlocksCompressor; use vortex_btrblocks::BtrBlocksCompressorBuilder; use vortex_btrblocks::SchemeExt; use vortex_btrblocks::schemes::float::FloatQuantScheme; +use vortex_btrblocks::schemes::float::OrderedBlockResidualScheme; use vortex_btrblocks::schemes::range_entropy::RangeEntropyScheme; +use vortex_buffer::Buffer; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_err; @@ -74,18 +76,25 @@ struct FloatMultBackendSizes { range_two_level: usize, } +struct FloatMultLatents { + primary: Vec, + secondary: Vec, +} + enum FloatMultPrototype { F32 { base: f32, primary: ArrayRef, secondary: ArrayRef, backend_sizes: FloatMultBackendSizes, + latents: FloatMultLatents, }, F64 { base: f64, primary: ArrayRef, secondary: ArrayRef, backend_sizes: FloatMultBackendSizes, + latents: FloatMultLatents, }, } @@ -124,6 +133,43 @@ impl FloatMultPrototype { } } + fn latents(&self) -> &FloatMultLatents { + match self { + Self::F32 { latents, .. } | Self::F64 { latents, .. } => latents, + } + } + + #[allow(clippy::cast_possible_truncation)] + fn reconstruct(&self, primary: &[u64], secondary: &[u64]) -> VortexResult<()> { + vortex_ensure!( + primary.len() == secondary.len(), + "FloatMult latent lengths differ" + ); + match self { + Self::F32 { base, .. } => { + let values = primary + .iter() + .copied() + .zip(secondary.iter().copied()) + .map(|(primary, secondary)| { + join_float_mult_f32(primary as u32, secondary as u32, *base) + }) + .collect::>(); + black_box(values); + } + Self::F64 { base, .. } => { + let values = primary + .iter() + .copied() + .zip(secondary.iter().copied()) + .map(|(primary, secondary)| join_float_mult_f64(primary, secondary, *base)) + .collect::>(); + black_box(values); + } + } + Ok(()) + } + fn decode(&self, session: &VortexSession) -> VortexResult { match self { Self::F32 { @@ -138,20 +184,20 @@ impl FloatMultPrototype { let secondary = secondary .clone() .execute::(&mut session.create_execution_ctx())?; - let mask = primary - .as_view() - .validity()? - .execute_mask(primary.len(), &mut session.create_execution_ctx())?; - Ok(PrimitiveArray::from_option_iter( - primary - .as_slice::() - .iter() - .copied() - .zip(secondary.as_slice::().iter().copied()) - .zip(mask.iter()) - .map(|((primary, secondary), valid)| { - valid.then_some(join_float_mult_f32(primary, secondary, *base)) - }), + let validity = primary.as_view().validity()?; + Ok(PrimitiveArray::new( + Buffer::from( + primary + .as_slice::() + .iter() + .copied() + .zip(secondary.as_slice::().iter().copied()) + .map(|(primary, secondary)| { + join_float_mult_f32(primary, secondary, *base) + }) + .collect::>(), + ), + validity, ) .into_array()) } @@ -167,20 +213,20 @@ impl FloatMultPrototype { let secondary = secondary .clone() .execute::(&mut session.create_execution_ctx())?; - let mask = primary - .as_view() - .validity()? - .execute_mask(primary.len(), &mut session.create_execution_ctx())?; - Ok(PrimitiveArray::from_option_iter( - primary - .as_slice::() - .iter() - .copied() - .zip(secondary.as_slice::().iter().copied()) - .zip(mask.iter()) - .map(|((primary, secondary), valid)| { - valid.then_some(join_float_mult_f64(primary, secondary, *base)) - }), + let validity = primary.as_view().validity()?; + Ok(PrimitiveArray::new( + Buffer::from( + primary + .as_slice::() + .iter() + .copied() + .zip(secondary.as_slice::().iter().copied()) + .map(|(primary, secondary)| { + join_float_mult_f64(primary, secondary, *base) + }) + .collect::>(), + ), + validity, ) .into_array()) } @@ -188,6 +234,100 @@ impl FloatMultPrototype { } } +#[derive(Clone, Copy)] +enum FloatMultLatentBackend { + BitSplit, + BlockResidual, + RangeEntropy, + RangePacked, + RangeTwoLevel, +} + +impl FloatMultLatentBackend { + const ALL: [Self; 5] = [ + Self::BitSplit, + Self::BlockResidual, + Self::RangeEntropy, + Self::RangePacked, + Self::RangeTwoLevel, + ]; + + fn name(self) -> &'static str { + match self { + Self::BitSplit => "bit-split", + Self::BlockResidual => "block-residual", + Self::RangeEntropy => "range-entropy", + Self::RangePacked => "range-packed", + Self::RangeTwoLevel => "range-two-level", + } + } + + fn encode(self, latents: &FloatMultLatents) -> VortexResult { + Ok(match self { + Self::BitSplit => FloatMultCodecPair::BitSplit( + BitSplitCodec::encode(&latents.primary)?, + BitSplitCodec::encode(&latents.secondary)?, + ), + Self::BlockResidual => FloatMultCodecPair::BlockResidual( + PatchedFoRCodec::encode_single_base(&latents.primary)?, + PatchedFoRCodec::encode_single_base(&latents.secondary)?, + ), + Self::RangeEntropy => FloatMultCodecPair::RangeEntropy( + RangeEntropyCodec::encode(&latents.primary, 8_192)?, + RangeEntropyCodec::encode(&latents.secondary, 8_192)?, + ), + Self::RangePacked => FloatMultCodecPair::RangePacked( + RangePackedCodec::encode(&latents.primary, 8_192)?, + RangePackedCodec::encode(&latents.secondary, 8_192)?, + ), + Self::RangeTwoLevel => FloatMultCodecPair::RangeTwoLevel( + RangeTwoLevelCodec::encode(&latents.primary, 8_192)?, + RangeTwoLevelCodec::encode(&latents.secondary, 8_192)?, + ), + }) + } +} + +enum FloatMultCodecPair { + BitSplit(BitSplitCodec, BitSplitCodec), + BlockResidual(PatchedFoRCodec, PatchedFoRCodec), + RangeEntropy(RangeEntropyCodec, RangeEntropyCodec), + RangePacked(RangePackedCodec, RangePackedCodec), + RangeTwoLevel(RangeTwoLevelCodec, RangeTwoLevelCodec), +} + +impl FloatMultCodecPair { + fn encoded_size(&self) -> usize { + match self { + Self::BitSplit(primary, secondary) => { + primary.encoded_size() + secondary.encoded_size() + 16 + } + Self::BlockResidual(primary, secondary) => { + primary.encoded_size() + secondary.encoded_size() + 16 + } + Self::RangeEntropy(primary, secondary) => { + primary.encoded_size() + secondary.encoded_size() + 16 + } + Self::RangePacked(primary, secondary) => { + primary.encoded_size() + secondary.encoded_size() + 16 + } + Self::RangeTwoLevel(primary, secondary) => { + primary.encoded_size() + secondary.encoded_size() + 16 + } + } + } + + fn decode(&self) -> VortexResult<(Vec, Vec)> { + match self { + Self::BitSplit(primary, secondary) => Ok((primary.decode()?, secondary.decode()?)), + Self::BlockResidual(primary, secondary) => Ok((primary.decode()?, secondary.decode()?)), + Self::RangeEntropy(primary, secondary) => Ok((primary.decode()?, secondary.decode()?)), + Self::RangePacked(primary, secondary) => Ok((primary.decode()?, secondary.decode()?)), + Self::RangeTwoLevel(primary, secondary) => Ok((primary.decode()?, secondary.decode()?)), + } + } +} + enum Encoder<'a> { BtrBlocks(&'a BtrBlocksCompressor), Pco(&'a ChunkConfig), @@ -608,14 +748,13 @@ fn float_mult_prototype( + RangeTwoLevelCodec::encode(&secondary_u64, 8_192)?.encoded_size() + 16, }; - let primary = PrimitiveArray::from_option_iter( - primary - .into_iter() - .zip(mask.iter()) - .map(|(value, valid)| valid.then_some(value)), - ) - .into_array(); - let secondary = PrimitiveArray::from_iter(secondary).into_array(); + let latents = FloatMultLatents { + primary: primary_u64, + secondary: secondary_u64, + }; + let validity = column.primitive.as_view().validity()?; + let primary = PrimitiveArray::new(Buffer::from(primary), validity.clone()).into_array(); + let secondary = PrimitiveArray::new(Buffer::from(secondary), validity).into_array(); let primary = compressor.compress(&primary, &mut session.create_execution_ctx())?; let secondary = compressor.compress(&secondary, &mut session.create_execution_ctx())?; Ok(Some(FloatMultPrototype::$variant { @@ -623,6 +762,7 @@ fn float_mult_prototype( primary, secondary, backend_sizes, + latents, })) }}; } @@ -762,6 +902,90 @@ fn measure_float_mult_decoder( Ok(()) } +fn measure_float_mult_backends( + dataset: &str, + prototypes: &[(&Column, FloatMultPrototype)], +) -> VortexResult<()> { + if prototypes.is_empty() { + return Ok(()); + } + let input_bytes = prototypes + .iter() + .map(|(column, _)| column.primitive.nbytes()) + .sum::(); + let encoded = FloatMultLatentBackend::ALL + .into_iter() + .map(|backend| { + let codecs = prototypes + .iter() + .map(|(_, prototype)| backend.encode(prototype.latents())) + .collect::>>()?; + Ok((backend, codecs)) + }) + .collect::>>()?; + + let decode_iterations = (1_000_000_000_u64 / input_bytes).clamp(5, 50) as usize; + for (backend, codecs) in &encoded { + for ((_, prototype), codec) in prototypes.iter().zip(codecs) { + let (primary, secondary) = codec.decode()?; + prototype.reconstruct(&primary, &secondary)?; + } + let mut durations = Vec::with_capacity(decode_iterations); + for _ in 0..decode_iterations { + let start = Instant::now(); + for ((_, prototype), codec) in prototypes.iter().zip(codecs) { + let (primary, secondary) = codec.decode()?; + prototype.reconstruct(&primary, &secondary)?; + } + durations.push(start.elapsed()); + } + let encoded_bytes = codecs + .iter() + .map(FloatMultCodecPair::encoded_size) + .sum::(); + let p10 = percentile(&mut durations.clone(), 1, 10); + let p90 = percentile(&mut durations.clone(), 9, 10); + let median = percentile(&mut durations, 1, 2); + let throughput = input_bytes as f64 / median.as_secs_f64() / 1_000_000.0; + println!( + "float-mult-backend-decode\t{dataset}\t{}\t{encoded_bytes}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", + backend.name(), + p10.as_secs_f64() * 1_000.0, + median.as_secs_f64() * 1_000.0, + p90.as_secs_f64() * 1_000.0, + ); + } + + let encode_iterations = (200_000_000_u64 / input_bytes).clamp(3, 20) as usize; + for backend in FloatMultLatentBackend::ALL { + let mut durations = Vec::with_capacity(encode_iterations); + for _ in 0..encode_iterations { + let start = Instant::now(); + let encoded_bytes = prototypes + .iter() + .try_fold(0_usize, |size, (_, prototype)| { + Ok::<_, vortex_error::VortexError>( + size + backend.encode(prototype.latents())?.encoded_size(), + ) + })?; + durations.push(start.elapsed()); + black_box(encoded_bytes); + } + let p10 = percentile(&mut durations.clone(), 1, 10); + let p90 = percentile(&mut durations.clone(), 9, 10); + let median = percentile(&mut durations, 1, 2); + let throughput = input_bytes as f64 / median.as_secs_f64() / 1_000_000.0; + println!( + "float-mult-backend-encode\t{dataset}\t{}\t{input_bytes}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", + backend.name(), + p10.as_secs_f64() * 1_000.0, + median.as_secs_f64() * 1_000.0, + p90.as_secs_f64() * 1_000.0, + ); + } + Ok(()) +} + fn main() -> VortexResult<()> { let path = std::env::args() .nth(1) @@ -794,12 +1018,15 @@ fn main() -> VortexResult<()> { .map(|column| column.primitive.nbytes()) .sum::(); let baseline = BtrBlocksCompressorBuilder::default() - .exclude_schemes([FloatQuantScheme.id()]) + .exclude_schemes([FloatQuantScheme.id(), OrderedBlockResidualScheme.id()]) .build(); let range_candidate = BtrBlocksCompressorBuilder::default() - .exclude_schemes([FloatQuantScheme.id()]) + .exclude_schemes([FloatQuantScheme.id(), OrderedBlockResidualScheme.id()]) .with_new_scheme(&RangeEntropyScheme) .build(); + let default_without_block_residual = BtrBlocksCompressorBuilder::default() + .exclude_schemes([OrderedBlockResidualScheme.id()]) + .build(); let float_candidate = BtrBlocksCompressor::default(); let stacked_candidate = BtrBlocksCompressorBuilder::default() .with_new_scheme(&RangeEntropyScheme) @@ -809,12 +1036,16 @@ fn main() -> VortexResult<()> { .with_mode_spec(ModeSpec::Auto) .with_delta_spec(DeltaSpec::Auto); let configs = [ - ("default-without-float-quant", Encoder::BtrBlocks(&baseline)), + ("default-without-new-float", Encoder::BtrBlocks(&baseline)), ( - "default-without-float-quant+range-entropy", + "default-without-new-float+range-entropy", Encoder::BtrBlocks(&range_candidate), ), - ("default", Encoder::BtrBlocks(&float_candidate)), + ( + "default-off", + Encoder::BtrBlocks(&default_without_block_residual), + ), + ("default-on", Encoder::BtrBlocks(&float_candidate)), ( "default+range-entropy", Encoder::BtrBlocks(&stacked_candidate), @@ -880,6 +1111,7 @@ fn main() -> VortexResult<()> { println!("operation\tdataset\tconfig\tbytes\tp10-ms\tmedian-ms\tp90-ms\tMB/s"); measure_decoders(dataset, &encoded, input_bytes, &session)?; measure_float_mult_decoder(dataset, &float_mult_prototypes, &session)?; + measure_float_mult_backends(dataset, &float_mult_prototypes)?; measure_compressors(dataset, &configs, &columns, input_bytes, &session)?; Ok(()) } From a20651aca603687ee236acc0c3d90e8c12f55da1 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Tue, 18 Aug 2026 17:08:20 -0400 Subject: [PATCH 03/78] feat: add native FloatMult encoding Signed-off-by: "Will Manning" --- Cargo.lock | 1 + docs/plans/native-pcodec.md | 81 +- encodings/float-quant/Cargo.toml | 3 +- encodings/float-quant/src/float_mult.rs | 1108 +++++++++++++++++ encodings/float-quant/src/lib.rs | 3 + encodings/float-quant/src/rules.rs | 4 + encodings/float-quant/src/slice.rs | 17 + .../examples/float_quant_dataset.rs | 52 +- vortex-btrblocks/src/builder.rs | 2 + .../src/schemes/float/float_mult.rs | 251 ++++ vortex-btrblocks/src/schemes/float/mod.rs | 2 + .../schemes/float/scheme_selection_tests.rs | 13 + vortex-btrblocks/src/schemes/float/tests.rs | 24 +- vortex/src/editions/core/v2026_08.rs | 1 + vortex/src/editions/tests.rs | 28 + 15 files changed, 1568 insertions(+), 22 deletions(-) create mode 100644 encodings/float-quant/src/float_mult.rs create mode 100644 vortex-btrblocks/src/schemes/float/float_mult.rs diff --git a/Cargo.lock b/Cargo.lock index 18535d412ad..8126733f397 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10205,6 +10205,7 @@ dependencies = [ "vortex-buffer", "vortex-error", "vortex-session", + "vortex-utils", ] [[package]] diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index c901dfdb029..711b282a9c4 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -12,9 +12,11 @@ Pco remains a compression oracle. The new arrays do not preserve Pco byte compat Add `FloatQuantArray` and `FloatQuantScheme` to default BtrBlocks compression for `f64` arrays. +Add `FloatMultArray` and a restricted `FloatMultScheme` to default BtrBlocks compression for `f32` arrays. + Add `OrderedFloatArray`, `BlockResidualArray`, and their BtrBlocks scheme to the default compressor. -Keep `BitSplitArray` and `FloatMultArray` as prototypes. +Keep `BitSplitArray` as a prototype. Keep `RangeEntropyArray` and `RangeEntropyScheme` experimental. Their current full-decode and compression costs fail the throughput gate. @@ -87,13 +89,29 @@ A common path uses `FloatQuant(FoR(BitPacked), Constant)`. This path targets `f3 The array supports exact IEEE bit-pattern round trips, nulls, slices, scalar access, and serialization. -The automatic scheme currently accepts only `f64`. Native `f32` columns keep the current ALP and dictionary choices. +The automatic FloatQuant scheme accepts only `f64`. Native `f32` columns remain eligible for the separate FloatMult scheme. + +### FloatMultArray + +`FloatMultArray` represents each float as an integer multiple of one base plus a signed ULP adjustment. + +The array stores the base in at most nine metadata bytes. Two unsigned child arrays store the multiples and adjustments. + +The default scheme accepts only `f32` inputs whose sampled adjustments all represent zero. + +This restriction lets the encoder omit the adjustment vector. It also lets the decoder skip adjustment materialization. + +The array retains general `f32` and `f64` support for experiments with nonconstant adjustments. + +The array supports exact IEEE bit-pattern round trips, nulls, slices, scalar access, and serialization. ## Selection policy FloatQuant uses normal sample comparison against ALP, ALP-RD, dictionary, sparse, and RLE schemes. -The scheme does not claim an unconditional win. This rule prevents poor choices on low-cardinality integer-valued floats. +The selected encoding can displace ALP, ALP-RD, dictionary, sparse, or RLE. + +The selector compares estimated size after an encoding-specific speed penalty. It does not prefer an encoding identity. The full FloatQuant analysis runs only after the sample selects the scheme. Unselected `f64` columns pay only the sample cost. @@ -119,6 +137,34 @@ let compressor = BtrBlocksCompressorBuilder::default() RangeEntropy also uses sample comparison. It remains outside `ALL_SCHEMES`. +FloatMult tests decimal bases, one exact binary divisor, and one approximate common divisor. + +The default scheme requires these sample properties: + +- The source type is `f32`. +- Every adjustment represents zero. +- The primary span uses at most 17 bits. +- The estimated compression ratio is at least 1.5. +- The estimate beats the incumbent by at least 5 percent. + +The selector applies a 15 percent penalty to the FloatMult ratio before comparison. + +This penalty represents its measured encode cost. A sufficient size win can still displace ALP or another incumbent. + +The first implementation compresses the primary child through the normal integer cascade. + +A fused scheme can select a fixed integer layout for both latent children. Its internal layout does not need to consume the generic cascade depth. + +This fused design remains a benchmark candidate. It can trade more internal work for a better size and speed point. + +Callers can disable this scheme with this builder configuration: + +```rust +let compressor = BtrBlocksCompressorBuilder::default() + .exclude_schemes([FloatMultScheme.id()]) + .build(); +``` + ## Performance gate The default candidate must meet these limits: @@ -343,7 +389,22 @@ The current `RangeEntropy` decoder is too slow for the default compressor. Float Block residual packing is the fastest specialized encoder. Its decode path is still too slow for the default compressor. -The next FloatMult prototype will use normal integer children first. Its selector must reject Taxi-like inputs with larger encoded children. +The native exact-multiple scheme uses normal integer compression for its primary child. + +It selects five California Housing columns. It rejects all Taxi and Air Quality columns. + +| Dataset | Configuration | Bytes | Encode MB/s | Decode MB/s | +| --- | --- | ---: | ---: | ---: | +| California Housing | Without FloatMult | 339,682 | 200.8 | 12,298.5 | +| California Housing | Default | 309,499 | 164.5 | 12,728.7 | +| April 2023 HVFHV Taxi | Without FloatMult | 28,540,793 | 873.4 | 21,651.0 | +| April 2023 HVFHV Taxi | Default | 28,540,793 | 872.0 | 21,514.7 | +| Air Quality | Without FloatMult | 6,135,414 | 536.4 | 23,568.2 | +| Air Quality | Default | 6,135,414 | 530.4 | 23,548.0 | + +FloatMult reduces total Housing size by 8.9 percent. Compression throughput regresses by 16.5 percent. + +Housing decode changes by less than measurement noise. Unselected Taxi and Air Quality results remain effectively unchanged. ## Why Compact still wins @@ -355,7 +416,7 @@ Pco selects adjacent Delta on three Housing columns and thirteen Air Quality col These transforms explain most of the remaining Compact size gap. -FloatMult requires a new transform array. The current prototype scope includes this array. +The restricted FloatMult scheme does not capture Pco's Taxi gains. Those gains require useful nonconstant adjustments and a stronger child codec. Pco adjacent Delta also differs from current FastLanes lane-stride Delta. @@ -381,13 +442,15 @@ More datasets cannot satisfy the gate without a faster codec implementation. ## Default writer integration -`FloatQuantScheme` belongs to `ALL_SCHEMES`. `BtrBlocksCompressor::default()` now evaluates it for `f64` arrays. +`FloatQuantScheme` and `FloatMultScheme` belong to `ALL_SCHEMES`. + +`BtrBlocksCompressor::default()` evaluates FloatQuant for `f64` and FloatMult for `f32`. -The default file session registers `FloatQuantArray`. The August 2026 core edition permits the array. +The default file session registers both arrays. The August 2026 core edition permits both arrays. -The CUDA-compatible builder excludes FloatQuant because no CUDA decode kernel exists. +The CUDA-compatible builder excludes both schemes because no CUDA decode kernels exist. -The default writer test writes, reads, and verifies an actual FloatQuant tree. +The default writer tests write, read, and verify actual FloatQuant and FloatMult trees. ## Current prototype scope diff --git a/encodings/float-quant/Cargo.toml b/encodings/float-quant/Cargo.toml index 717ab85c13e..88cb361b555 100644 --- a/encodings/float-quant/Cargo.toml +++ b/encodings/float-quant/Cargo.toml @@ -2,7 +2,7 @@ name = "vortex-float-quant" authors = { workspace = true } categories = { workspace = true } -description = "Vortex float quantization array encoding" +description = "Vortex lossless float transform array encodings" edition = { workspace = true } homepage = { workspace = true } include = { workspace = true } @@ -18,6 +18,7 @@ vortex-array = { workspace = true } vortex-buffer = { workspace = true } vortex-error = { workspace = true } vortex-session = { workspace = true } +vortex-utils = { workspace = true } [dev-dependencies] vortex-array = { workspace = true, features = ["_test-harness"] } diff --git a/encodings/float-quant/src/float_mult.rs b/encodings/float-quant/src/float_mult.rs new file mode 100644 index 00000000000..112c743b661 --- /dev/null +++ b/encodings/float-quant/src/float_mult.rs @@ -0,0 +1,1108 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::fmt::Display; +use std::fmt::Formatter; +use std::hash::Hash; +use std::hash::Hasher; + +use vortex_array::Array; +use vortex_array::ArrayEq; +use vortex_array::ArrayHash; +use vortex_array::ArrayId; +use vortex_array::ArrayParts; +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::EqMode; +use vortex_array::ExecutionCtx; +use vortex_array::ExecutionResult; +use vortex_array::IntoArray; +use vortex_array::TypedArrayRef; +use vortex_array::array_slots; +use vortex_array::arrays::Constant; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::Primitive; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::buffer::BufferHandle; +use vortex_array::dtype::DType; +use vortex_array::dtype::Nullability::NonNullable; +use vortex_array::dtype::PType; +use vortex_array::scalar::Scalar; +use vortex_array::serde::ArrayChildren; +use vortex_array::vtable::OperationsVTable; +use vortex_array::vtable::VTable; +use vortex_array::vtable::ValidityChild; +use vortex_array::vtable::ValidityVTableFromChild; +use vortex_buffer::Buffer; +use vortex_error::VortexExpect; +use vortex_error::VortexResult; +use vortex_error::vortex_bail; +use vortex_error::vortex_ensure; +use vortex_error::vortex_panic; +use vortex_session::VortexSession; +use vortex_session::registry::CachedId; +use vortex_utils::aliases::hash_set::HashSet; + +use crate::rules::FLOAT_MULT_RULES; + +const METADATA_VERSION: u8 = 1; +const F32_METADATA_LEN: usize = 5; +const F64_METADATA_LEN: usize = 9; +const MIN_ANALYSIS_VALUES: usize = 32; + +/// A lossless float split into approximate multiples and ULP adjustments. +pub type FloatMultArray = Array; + +#[array_slots(FloatMult)] +pub struct FloatMultSlots { + /// Signed integer multiples encoded in float order. + #[slot(0)] + pub primary: ArrayRef, + /// Signed ULP adjustments with zero moved to the unsigned midpoint. + #[slot(1)] + pub secondary: ArrayRef, +} + +#[derive(Clone, Debug)] +pub struct FloatMultData { + base_bits: u64, +} + +impl Display for FloatMultData { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "base_bits: {}", self.base_bits) + } +} + +impl ArrayHash for FloatMultData { + fn array_hash(&self, state: &mut H, _accuracy: EqMode) { + self.base_bits.hash(state); + } +} + +impl ArrayEq for FloatMultData { + fn array_eq(&self, other: &Self, _accuracy: EqMode) -> bool { + self.base_bits == other.base_bits + } +} + +#[derive(Clone, Debug)] +pub struct FloatMult; + +impl VTable for FloatMult { + type TypedArrayData = FloatMultData; + type OperationsVTable = Self; + type ValidityVTable = ValidityVTableFromChild; + + fn id(&self) -> ArrayId { + static ID: CachedId = CachedId::new("vortex.float_mult"); + *ID + } + + fn validate( + &self, + data: &Self::TypedArrayData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + let ptype = PType::try_from(dtype)?; + let latent_ptype = latent_ptype(ptype)?; + let base = data.base(ptype)?; + vortex_ensure!( + base.is_finite() && base.abs() > 0.0, + "FloatMult base must be finite and nonzero" + ); + + let slots = FloatMultSlotsView::from_slots(slots); + let expected_primary = DType::Primitive(latent_ptype, dtype.nullability()); + let expected_secondary = DType::Primitive(latent_ptype, NonNullable); + vortex_ensure!( + slots.primary.dtype() == &expected_primary, + "expected primary dtype {expected_primary}, got {}", + slots.primary.dtype() + ); + vortex_ensure!( + slots.secondary.dtype() == &expected_secondary, + "expected secondary dtype {expected_secondary}, got {}", + slots.secondary.dtype() + ); + vortex_ensure!( + slots.primary.len() == len && slots.secondary.len() == len, + "FloatMult child length differs from {len}" + ); + Ok(()) + } + + fn nbuffers(_array: ArrayView<'_, Self>) -> usize { + 0 + } + + fn buffer(_array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { + vortex_panic!("FloatMultArray buffer index {idx} out of bounds") + } + + fn buffer_name(_array: ArrayView<'_, Self>, idx: usize) -> Option { + vortex_panic!("FloatMultArray buffer_name index {idx} out of bounds") + } + + fn with_buffers( + &self, + array: ArrayView<'_, Self>, + buffers: &[BufferHandle], + ) -> VortexResult> { + vortex_array::vtable::with_empty_buffers(self, array, buffers) + } + + fn serialize( + array: ArrayView<'_, Self>, + _session: &VortexSession, + ) -> VortexResult>> { + let mut metadata = vec![METADATA_VERSION]; + match PType::try_from(array.dtype())? { + PType::F32 => metadata.extend_from_slice( + &u32::try_from(array.data().base_bits) + .vortex_expect("validated f32 FloatMult base bits") + .to_le_bytes(), + ), + PType::F64 => metadata.extend_from_slice(&array.data().base_bits.to_le_bytes()), + ptype => vortex_bail!("FloatMult requires f32 or f64, got {ptype}"), + } + Ok(Some(metadata)) + } + + fn deserialize( + &self, + dtype: &DType, + len: usize, + metadata: &[u8], + _buffers: &[BufferHandle], + children: &dyn ArrayChildren, + _session: &VortexSession, + ) -> VortexResult> { + let ptype = PType::try_from(dtype)?; + let expected_metadata_len = match ptype { + PType::F32 => F32_METADATA_LEN, + PType::F64 => F64_METADATA_LEN, + _ => vortex_bail!("FloatMult requires f32 or f64, got {ptype}"), + }; + vortex_ensure!( + metadata.len() == expected_metadata_len, + "FloatMult metadata requires {expected_metadata_len} bytes" + ); + vortex_ensure!( + metadata[0] == METADATA_VERSION, + "unsupported FloatMult metadata version {}", + metadata[0] + ); + vortex_ensure!(children.len() == 2, "FloatMult requires two children"); + + let base_bits = match ptype { + PType::F32 => { + let mut bytes = [0; 4]; + bytes.copy_from_slice(&metadata[1..5]); + u64::from(u32::from_le_bytes(bytes)) + } + PType::F64 => { + let mut bytes = [0; 8]; + bytes.copy_from_slice(&metadata[1..9]); + u64::from_le_bytes(bytes) + } + _ => unreachable!(), + }; + let latent_ptype = latent_ptype(ptype)?; + let primary_dtype = DType::Primitive(latent_ptype, dtype.nullability()); + let secondary_dtype = DType::Primitive(latent_ptype, NonNullable); + let primary = children.get(0, &primary_dtype, len)?; + let secondary = children.get(1, &secondary_dtype, len)?; + let slots = FloatMultSlots { primary, secondary }.into_slots(); + Ok(ArrayParts::new( + self.clone(), + dtype.clone(), + len, + FloatMultData { base_bits }, + ) + .with_slots(slots)) + } + + fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { + FloatMultSlots::NAMES[idx].to_string() + } + + fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionResult::done( + decode(array.as_view(), ctx)?.into_array(), + )) + } + + fn reduce_parent( + array: ArrayView<'_, Self>, + parent: &ArrayRef, + child_idx: usize, + ) -> VortexResult> { + FLOAT_MULT_RULES.evaluate(array, parent, child_idx) + } +} + +impl OperationsVTable for FloatMult { + fn scalar_at( + array: ArrayView<'_, FloatMult>, + index: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + let primary = array.primary().execute_scalar(index, ctx)?; + if primary.is_null() { + return Ok(Scalar::null(array.dtype().clone())); + } + let secondary = array.secondary().execute_scalar(index, ctx)?; + Ok(match PType::try_from(array.dtype())? { + PType::F32 => Scalar::primitive( + join_f32( + primary + .as_primitive() + .typed_value::() + .vortex_expect("validated primary scalar"), + secondary + .as_primitive() + .typed_value::() + .vortex_expect("validated secondary scalar"), + array.data().base_f32()?, + ), + array.dtype().nullability(), + ), + PType::F64 => Scalar::primitive( + join_f64( + primary + .as_primitive() + .typed_value::() + .vortex_expect("validated primary scalar"), + secondary + .as_primitive() + .typed_value::() + .vortex_expect("validated secondary scalar"), + array.data().base_f64()?, + ), + array.dtype().nullability(), + ), + ptype => vortex_panic!("unsupported FloatMult ptype {ptype}"), + }) + } +} + +impl ValidityChild for FloatMult { + fn validity_child(array: ArrayView<'_, FloatMult>) -> ArrayRef { + array.primary().clone() + } +} + +impl FloatMultData { + fn base(&self, ptype: PType) -> VortexResult { + match ptype { + PType::F32 => Ok(f64::from(self.base_f32()?)), + PType::F64 => self.base_f64(), + _ => vortex_bail!("FloatMult requires f32 or f64, got {ptype}"), + } + } + + fn base_f32(&self) -> VortexResult { + Ok(f32::from_bits(u32::try_from(self.base_bits)?)) + } + + fn base_f64(&self) -> VortexResult { + Ok(f64::from_bits(self.base_bits)) + } +} + +pub trait FloatMultArrayExt: TypedArrayRef + FloatMultArraySlotsExt { + /// Return the multiplier base as an f64 value. + fn base(&self) -> f64 { + self.deref() + .base( + match PType::try_from(self.primary().dtype()) + .vortex_expect("validated FloatMult primary dtype") + { + PType::U32 => PType::F32, + PType::U64 => PType::F64, + _ => vortex_panic!("validated FloatMult primary dtype must be unsigned"), + }, + ) + .vortex_expect("validated FloatMult base") + } +} + +impl> FloatMultArrayExt for T {} + +impl FloatMult { + /// Construct a FloatMult array from two latent children. + pub fn try_new( + primary: ArrayRef, + secondary: ArrayRef, + float_ptype: PType, + base: f64, + ) -> VortexResult { + let base_bits = match float_ptype { + PType::F32 => u64::from(narrow_f32(base).to_bits()), + PType::F64 => base.to_bits(), + _ => vortex_bail!("FloatMult requires f32 or f64, got {float_ptype}"), + }; + let dtype = DType::Primitive(float_ptype, primary.dtype().nullability()); + let len = primary.len(); + let slots = FloatMultSlots { primary, secondary }.into_slots(); + Array::try_from_parts( + ArrayParts::new(FloatMult, dtype, len, FloatMultData { base_bits }).with_slots(slots), + ) + } + + /// Split a canonical float array into two unsigned latent children. + pub fn from_primitive( + array: ArrayView<'_, Primitive>, + base: f64, + ) -> VortexResult { + let validity = array.validity()?; + match array.ptype() { + PType::F32 => { + let base = narrow_f32(base); + vortex_ensure!( + base.is_finite() && base.abs() > 0.0, + "FloatMult base must be finite and nonzero" + ); + let (primary, secondary): (Vec<_>, Vec<_>) = array + .as_slice::() + .iter() + .copied() + .map(|value| split_f32(value, base)) + .unzip(); + Self::try_new( + PrimitiveArray::new(Buffer::from(primary), validity).into_array(), + PrimitiveArray::new(Buffer::from(secondary), NonNullable.into()).into_array(), + PType::F32, + f64::from(base), + ) + } + PType::F64 => { + vortex_ensure!( + base.is_finite() && base.abs() > 0.0, + "FloatMult base must be finite and nonzero" + ); + let (primary, secondary): (Vec<_>, Vec<_>) = array + .as_slice::() + .iter() + .copied() + .map(|value| split_f64(value, base)) + .unzip(); + Self::try_new( + PrimitiveArray::new(Buffer::from(primary), validity).into_array(), + PrimitiveArray::new(Buffer::from(secondary), NonNullable.into()).into_array(), + PType::F64, + base, + ) + } + ptype => vortex_bail!("FloatMult requires f32 or f64, got {ptype}"), + } + } + + /// Split floats when every ULP adjustment is zero. + pub fn from_primitive_constant_secondary( + array: ArrayView<'_, Primitive>, + base: f64, + ) -> VortexResult> { + let validity = array.validity()?; + let len = array.len(); + match array.ptype() { + PType::F32 => { + let base = narrow_f32(base); + let mut primary = Vec::with_capacity(len); + for value in array.as_slice::() { + let (multiple, adjustment) = split_f32(*value, base); + if adjustment != 1_u32 << 31 { + return Ok(None); + } + primary.push(multiple); + } + Ok(Some(Self::try_new( + PrimitiveArray::new(Buffer::from(primary), validity).into_array(), + ConstantArray::new(Scalar::from(1_u32 << 31), len).into_array(), + PType::F32, + f64::from(base), + )?)) + } + PType::F64 => { + let mut primary = Vec::with_capacity(len); + for value in array.as_slice::() { + let (multiple, adjustment) = split_f64(*value, base); + if adjustment != 1_u64 << 63 { + return Ok(None); + } + primary.push(multiple); + } + Ok(Some(Self::try_new( + PrimitiveArray::new(Buffer::from(primary), validity).into_array(), + ConstantArray::new(Scalar::from(1_u64 << 63), len).into_array(), + PType::F64, + base, + )?)) + } + ptype => vortex_bail!("FloatMult requires f32 or f64, got {ptype}"), + } + } +} + +/// Estimate a common multiplier base from a bounded canonical float sample. +pub fn estimate_float_mult_base(array: ArrayView<'_, Primitive>) -> Option { + match array.ptype() { + PType::F32 => estimate_base( + &array + .as_slice::() + .iter() + .copied() + .map(f64::from) + .collect::>(), + u32::BITS, + ) + .map(|base| f64::from(narrow_f32(base))), + PType::F64 => estimate_base(array.as_slice::(), u64::BITS), + _ => None, + } +} + +/// Estimate a base that produces zero ULP adjustments for every sampled value. +pub fn estimate_float_mult_constant_base(array: ArrayView<'_, Primitive>) -> Option { + let values = match array.ptype() { + PType::F32 => array + .as_slice::() + .iter() + .copied() + .map(f64::from) + .collect::>(), + PType::F64 => array.as_slice::().to_vec(), + _ => return None, + }; + let finite = values + .iter() + .copied() + .filter(|value| value.is_finite()) + .collect::>(); + if finite.len() < MIN_ANALYSIS_VALUES { + return None; + } + match array.ptype() { + PType::F32 => constant_candidate_bases_f32(&finite) + .into_iter() + .filter_map(|base| { + constant_score_f32(&finite, narrow_f32(base)).map(|score| (score, base)) + }) + .filter(|(score, _)| *score + 2 < u32::BITS) + .min_by(|(left_score, left_base), (right_score, right_base)| { + left_score + .cmp(right_score) + .then_with(|| right_base.total_cmp(left_base)) + }) + .map(|(_, base)| f64::from(narrow_f32(base))), + PType::F64 => constant_candidate_bases_f64(&finite) + .into_iter() + .filter_map(|base| constant_score_f64(&finite, base).map(|score| (score, base))) + .filter(|(score, _)| *score + 2 < u64::BITS) + .min_by(|(left_score, left_base), (right_score, right_base)| { + left_score + .cmp(right_score) + .then_with(|| right_base.total_cmp(left_base)) + }) + .map(|(_, base)| base), + _ => None, + } +} + +fn constant_candidate_bases_f32(values: &[f64]) -> Vec { + let mut candidates = Vec::with_capacity(20); + for exponent in -6..=6 { + push_candidate(&mut candidates, 10.0_f64.powi(exponent)); + } + let mut binary_exponent = i32::MAX; + for value in values { + let value = narrow_f32(*value); + if value == 0.0 { + continue; + } + let exponent = (value.abs().to_bits() >> 23) as i32 - 127; + let trailing_zeros = value.to_bits().trailing_zeros(); + let divisor_exponent = exponent - 23_u32.saturating_sub(trailing_zeros) as i32; + binary_exponent = binary_exponent.min(divisor_exponent); + } + if binary_exponent != i32::MAX { + push_candidate(&mut candidates, 2.0_f64.powi(binary_exponent)); + } + push_common_divisor_candidates(&mut candidates, values); + candidates +} + +fn constant_candidate_bases_f64(values: &[f64]) -> Vec { + let mut candidates = Vec::with_capacity(32); + for exponent in -12..=12 { + push_candidate(&mut candidates, 10.0_f64.powi(exponent)); + } + let mut binary_exponent = i32::MAX; + for value in values { + if *value == 0.0 { + continue; + } + let exponent = (value.abs().to_bits() >> 52) as i32 - 1023; + let trailing_zeros = value.to_bits().trailing_zeros(); + let divisor_exponent = exponent - 52_u32.saturating_sub(trailing_zeros) as i32; + binary_exponent = binary_exponent.min(divisor_exponent); + } + if binary_exponent != i32::MAX { + push_candidate(&mut candidates, 2.0_f64.powi(binary_exponent)); + } + push_common_divisor_candidates(&mut candidates, values); + candidates +} + +fn push_common_divisor_candidates(candidates: &mut Vec, values: &[f64]) { + if let Some(base) = approximate_common_divisor(values) { + push_candidate(candidates, base); + push_candidate(candidates, snap_base(base)); + } +} + +fn estimate_base(values: &[f64], source_bits: u32) -> Option { + let finite = values + .iter() + .copied() + .filter(|value| value.is_finite()) + .collect::>(); + if finite.len() < MIN_ANALYSIS_VALUES { + return None; + } + + let candidates = candidate_bases(&finite); + + candidates + .into_iter() + .filter_map(|base| score_base(&finite, base).map(|score| (score, base))) + .filter(|(score, _)| *score + 2.0 < f64::from(source_bits)) + .min_by(|(left, _), (right, _)| left.total_cmp(right)) + .map(|(_, base)| base) +} + +fn candidate_bases(values: &[f64]) -> Vec { + let mut candidates = Vec::with_capacity(112); + for exponent in -12..=12 { + push_candidate(&mut candidates, 10.0_f64.powi(exponent)); + } + for exponent in -40..=40 { + push_candidate(&mut candidates, 2.0_f64.powi(exponent)); + } + if let Some(base) = approximate_common_divisor(values) { + push_candidate(&mut candidates, base); + push_candidate(&mut candidates, snap_base(base)); + } + candidates +} + +fn push_candidate(candidates: &mut Vec, base: f64) { + if !base.is_finite() || base <= 0.0 { + return; + } + if candidates + .iter() + .any(|candidate| ((candidate - base) / base).abs() < 1e-12) + { + return; + } + candidates.push(base); +} + +fn score_base(values: &[f64], base: f64) -> Option { + let mut primary_min = u64::MAX; + let mut primary_max = u64::MIN; + let mut secondary_min = u64::MAX; + let mut secondary_max = u64::MIN; + let mut primary_distinct = HashSet::new(); + let mut secondary_distinct = HashSet::new(); + let mut primary_runs = 0usize; + let mut secondary_runs = 0usize; + let mut previous_primary = None; + let mut previous_secondary = None; + for value in values { + let (primary, secondary) = split_f64(*value, base); + primary_min = primary_min.min(primary); + primary_max = primary_max.max(primary); + secondary_min = secondary_min.min(secondary); + secondary_max = secondary_max.max(secondary); + primary_distinct.insert(primary); + secondary_distinct.insert(secondary); + primary_runs += usize::from(previous_primary != Some(primary)); + secondary_runs += usize::from(previous_secondary != Some(secondary)); + previous_primary = Some(primary); + previous_secondary = Some(secondary); + } + let primary_bits = u64::BITS - primary_max.wrapping_sub(primary_min).leading_zeros(); + let secondary_bits = u64::BITS - secondary_max.wrapping_sub(secondary_min).leading_zeros(); + Some( + estimated_latent_cost( + primary_bits, + primary_distinct.len(), + primary_runs, + values.len(), + ) + estimated_latent_cost( + secondary_bits, + secondary_distinct.len(), + secondary_runs, + values.len(), + ), + ) +} + +fn constant_score_f32(values: &[f64], base: f32) -> Option { + let mut primary_min = u32::MAX; + let mut primary_max = u32::MIN; + for value in values { + let (primary, secondary) = split_f32(narrow_f32(*value), base); + if secondary != 1_u32 << 31 { + return None; + } + primary_min = primary_min.min(primary); + primary_max = primary_max.max(primary); + } + Some(u32::BITS - (primary_max - primary_min).leading_zeros()) +} + +fn constant_score_f64(values: &[f64], base: f64) -> Option { + let mut primary_min = u64::MAX; + let mut primary_max = u64::MIN; + for value in values { + let (primary, secondary) = split_f64(*value, base); + if secondary != 1_u64 << 63 { + return None; + } + primary_min = primary_min.min(primary); + primary_max = primary_max.max(primary); + } + Some(u64::BITS - (primary_max - primary_min).leading_zeros()) +} + +fn estimated_latent_cost( + span_bits: u32, + distinct_count: usize, + run_count: usize, + len: usize, +) -> f64 { + if span_bits == 0 { + return 0.0; + } + let span_bits = f64::from(span_bits); + let code_bits = usize::BITS - distinct_count.saturating_sub(1).leading_zeros(); + let dict_cost = f64::from(code_bits) + distinct_count as f64 / len as f64 * span_bits; + let index_bits = usize::BITS - len.saturating_sub(1).leading_zeros(); + let run_cost = run_count as f64 / len as f64 * (span_bits + f64::from(index_bits)); + span_bits.min(dict_cost).min(run_cost) +} + +fn approximate_common_divisor(values: &[f64]) -> Option { + let mut divisors = values + .chunks_exact(2) + .filter_map(|pair| { + let a = pair[0].abs(); + let b = pair[1].abs(); + approximate_pair_divisor(a.max(b), a.min(b)) + }) + .collect::>(); + if divisors.len() < 2 { + return None; + } + divisors.sort_unstable_by(f64::total_cmp); + for percentile in [1usize, 3, 5] { + let candidate = divisors[percentile * divisors.len() / 10]; + let similar = divisors + .iter() + .filter(|divisor| (**divisor - candidate).abs() < candidate * 0.01) + .count(); + if similar >= 2 { + return Some(candidate); + } + } + None +} + +fn approximate_pair_divisor(greater: f64, lesser: f64) -> Option { + if lesser == 0.0 || lesser == greater || lesser <= greater * 2.0_f64.powi(-46) { + return None; + } + let mut greater_value = greater; + let mut greater_error = 0.0; + let mut lesser_value = lesser; + let mut lesser_error = 0.0; + for _ in 0..64 { + let ratio = (greater_value / lesser_value).round(); + greater_error += ratio * lesser_error + greater_value * f64::EPSILON; + let previous = greater_value; + greater_value = (greater_value - ratio * lesser_value).abs(); + if greater_value <= previous * 2.0_f64.powi(-16) || greater_value <= greater_error { + return Some(lesser_value); + } + if greater_value <= greater * 2.0_f64.powi(-46) + || greater_value <= greater_error * 2.0_f64.powi(6) + { + return None; + } + std::mem::swap(&mut greater_value, &mut lesser_value); + std::mem::swap(&mut greater_error, &mut lesser_error); + } + None +} + +fn snap_base(base: f64) -> f64 { + let inverse = base.recip(); + let rounded = inverse.round(); + let decimal = 10.0_f64.powf(inverse.log10().round()); + if (inverse - rounded).abs() < 0.02 { + rounded.recip() + } else if (inverse - decimal).abs() / inverse < 0.01 { + decimal.recip() + } else { + base + } +} + +fn latent_ptype(ptype: PType) -> VortexResult { + match ptype { + PType::F32 => Ok(PType::U32), + PType::F64 => Ok(PType::U64), + _ => vortex_bail!("FloatMult requires f32 or f64, got {ptype}"), + } +} + +#[allow(clippy::cast_possible_truncation)] +fn int_float_to_u32(value: f32) -> u32 { + let absolute = value.abs(); + let greatest_precise_integer = 1_u32 << f32::MANTISSA_DIGITS; + let greatest_precise_float = greatest_precise_integer as f32; + let absolute_integer = if absolute < greatest_precise_float { + absolute as u32 + } else { + greatest_precise_integer + (absolute.to_bits() - greatest_precise_float.to_bits()) + }; + if value.is_sign_positive() { + (1_u32 << 31) + absolute_integer + } else { + (1_u32 << 31) - 1 - absolute_integer + } +} + +fn int_float_from_u32(value: u32) -> f32 { + let middle = 1_u32 << 31; + let (absolute, positive) = if value >= middle { + (value - middle, true) + } else { + (middle - 1 - value, false) + }; + let greatest_precise_integer = 1_u32 << f32::MANTISSA_DIGITS; + let greatest_precise_float = greatest_precise_integer as f32; + let absolute_float = if absolute < greatest_precise_integer { + absolute as f32 + } else { + f32::from_bits(greatest_precise_float.to_bits() + absolute - greatest_precise_integer) + }; + if positive { + absolute_float + } else { + -absolute_float + } +} + +#[allow(clippy::cast_possible_truncation)] +fn int_float_to_u64(value: f64) -> u64 { + let absolute = value.abs(); + let greatest_precise_integer = 1_u64 << f64::MANTISSA_DIGITS; + let greatest_precise_float = greatest_precise_integer as f64; + let absolute_integer = if absolute < greatest_precise_float { + absolute as u64 + } else { + greatest_precise_integer + (absolute.to_bits() - greatest_precise_float.to_bits()) + }; + if value.is_sign_positive() { + (1_u64 << 63) + absolute_integer + } else { + (1_u64 << 63) - 1 - absolute_integer + } +} + +#[allow(clippy::cast_possible_truncation)] +fn narrow_f32(value: f64) -> f32 { + value as f32 +} + +fn int_float_from_u64(value: u64) -> f64 { + let middle = 1_u64 << 63; + let (absolute, positive) = if value >= middle { + (value - middle, true) + } else { + (middle - 1 - value, false) + }; + let greatest_precise_integer = 1_u64 << f64::MANTISSA_DIGITS; + let greatest_precise_float = greatest_precise_integer as f64; + let absolute_float = if absolute < greatest_precise_integer { + absolute as f64 + } else { + f64::from_bits(greatest_precise_float.to_bits() + absolute - greatest_precise_integer) + }; + if positive { + absolute_float + } else { + -absolute_float + } +} + +fn ordered_u32(value: f32) -> u32 { + let bits = value.to_bits(); + if bits & (1_u32 << 31) == 0 { + bits ^ (1_u32 << 31) + } else { + !bits + } +} + +fn ordered_u64(value: f64) -> u64 { + let bits = value.to_bits(); + if bits & (1_u64 << 63) == 0 { + bits ^ (1_u64 << 63) + } else { + !bits + } +} + +fn from_ordered_u32(ordered: u32) -> f32 { + let bits = if ordered & (1_u32 << 31) == 0 { + !ordered + } else { + ordered ^ (1_u32 << 31) + }; + f32::from_bits(bits) +} + +fn from_ordered_u64(ordered: u64) -> f64 { + let bits = if ordered & (1_u64 << 63) == 0 { + !ordered + } else { + ordered ^ (1_u64 << 63) + }; + f64::from_bits(bits) +} + +fn split_f32(value: f32, base: f32) -> (u32, u32) { + let multiple = (value / base).round(); + let primary = int_float_to_u32(multiple); + let secondary = ordered_u32(value) + .wrapping_sub(ordered_u32(multiple * base)) + .wrapping_add(1_u32 << 31); + (primary, secondary) +} + +fn split_f64(value: f64, base: f64) -> (u64, u64) { + let multiple = (value / base).round(); + let primary = int_float_to_u64(multiple); + let secondary = ordered_u64(value) + .wrapping_sub(ordered_u64(multiple * base)) + .wrapping_add(1_u64 << 63); + (primary, secondary) +} + +fn join_f32(primary: u32, secondary: u32, base: f32) -> f32 { + let approximate = int_float_from_u32(primary) * base; + from_ordered_u32(ordered_u32(approximate).wrapping_add(secondary.wrapping_add(1_u32 << 31))) +} + +fn join_f64(primary: u64, secondary: u64, base: f64) -> f64 { + let approximate = int_float_from_u64(primary) * base; + from_ordered_u64(ordered_u64(approximate).wrapping_add(secondary.wrapping_add(1_u64 << 63))) +} + +fn decode(array: ArrayView<'_, FloatMult>, ctx: &mut ExecutionCtx) -> VortexResult { + let primary = array.primary().clone().execute::(ctx)?; + let validity = primary.validity()?; + if secondary_is_constant_zero(array) { + return Ok(match PType::try_from(array.dtype())? { + PType::F32 => { + let base = array.data().base_f32()?; + let values = primary + .into_buffer::() + .map_each_in_place(|primary| int_float_from_u32(primary) * base) + .freeze(); + PrimitiveArray::new(values, validity) + } + PType::F64 => { + let base = array.data().base_f64()?; + let values = primary + .into_buffer::() + .map_each_in_place(|primary| int_float_from_u64(primary) * base) + .freeze(); + PrimitiveArray::new(values, validity) + } + ptype => vortex_panic!("unsupported FloatMult ptype {ptype}"), + }); + } + + let secondary = array.secondary().clone().execute::(ctx)?; + Ok(match PType::try_from(array.dtype())? { + PType::F32 => { + let secondary_values = secondary.as_slice::(); + let base = array.data().base_f32()?; + let mut index = 0; + let values = primary + .into_buffer::() + .map_each_in_place(|primary| { + let value = join_f32(primary, secondary_values[index], base); + index += 1; + value + }) + .freeze(); + PrimitiveArray::new(values, validity) + } + PType::F64 => { + let secondary_values = secondary.as_slice::(); + let base = array.data().base_f64()?; + let mut index = 0; + let values = primary + .into_buffer::() + .map_each_in_place(|primary| { + let value = join_f64(primary, secondary_values[index], base); + index += 1; + value + }) + .freeze(); + PrimitiveArray::new(values, validity) + } + ptype => vortex_panic!("unsupported FloatMult ptype {ptype}"), + }) +} + +fn secondary_is_constant_zero(array: ArrayView<'_, FloatMult>) -> bool { + let Some(constant) = array.secondary().as_opt::() else { + return false; + }; + match PType::try_from(array.dtype()) { + Ok(PType::F32) => { + constant.scalar().as_primitive().typed_value::() == Some(1_u32 << 31) + } + Ok(PType::F64) => { + constant.scalar().as_primitive().typed_value::() == Some(1_u64 << 63) + } + _ => false, + } +} + +#[cfg(test)] +mod tests { + use std::sync::LazyLock; + + use vortex_array::ArrayContext; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::array_session; + use vortex_array::assert_arrays_eq; + use vortex_array::assert_nth_scalar; + use vortex_array::serde::SerializeOptions; + use vortex_array::serde::SerializedArray; + use vortex_array::validity::Validity; + use vortex_buffer::ByteBufferMut; + use vortex_error::VortexResult; + use vortex_session::VortexSession; + use vortex_session::registry::ReadContext; + + use super::*; + + static SESSION: LazyLock = LazyLock::new(|| { + let session = array_session(); + crate::initialize(&session); + session + }); + + #[test] + fn float_bit_patterns_roundtrip() -> VortexResult<()> { + let values = [ + f64::NEG_INFINITY, + -1.5, + -0.0, + 0.0, + 1.5, + f64::INFINITY, + f64::from_bits(0x7ff8_0000_0000_1234), + f64::from_bits(0xfff8_0000_0000_5678), + ]; + let original = PrimitiveArray::from_iter(values); + let encoded = FloatMult::from_primitive(original.as_view(), 0.01)?; + let decoded = encoded + .into_array() + .execute::(&mut SESSION.create_execution_ctx())?; + assert_eq!( + decoded + .as_slice::() + .iter() + .map(|value| value.to_bits()) + .collect::>(), + values + .iter() + .map(|value| value.to_bits()) + .collect::>() + ); + Ok(()) + } + + #[test] + fn nullable_slice_and_scalar_access() -> VortexResult<()> { + let original = PrimitiveArray::new( + Buffer::from(vec![1.25_f32, 0.0, -0.0, 42.5, -10.0]), + Validity::from_iter([true, false, true, true, false]), + ); + let encoded = FloatMult::from_primitive(original.as_view(), 0.25)?; + let mut ctx = SESSION.create_execution_ctx(); + assert_arrays_eq!(encoded, original, &mut ctx); + assert_nth_scalar!(encoded, 3, 42.5_f32, &mut ctx); + assert!(encoded.execute_scalar(1, &mut ctx)?.is_null()); + + let sliced = encoded.into_array().slice(1..4)?; + assert!(sliced.is::()); + assert_arrays_eq!(sliced, original.into_array().slice(1..4)?, &mut ctx); + Ok(()) + } + + #[test] + fn serialization_roundtrip() -> VortexResult<()> { + let original = PrimitiveArray::from_option_iter([ + Some(-10.25_f64), + None, + Some(-0.0), + Some(42.25), + Some(100.5), + ]); + let encoded = FloatMult::from_primitive(original.as_view(), 0.25)?.into_array(); + let dtype = encoded.dtype().clone(); + let len = encoded.len(); + let array_context = ArrayContext::empty(); + let serialized = + encoded.serialize(&array_context, &SESSION, &SerializeOptions::default())?; + let mut bytes = ByteBufferMut::empty(); + for buffer in serialized { + bytes.extend_from_slice(buffer.as_ref()); + } + + let decoded = SerializedArray::try_from(bytes.freeze())?.decode( + &dtype, + len, + &ReadContext::new(array_context.to_ids()), + &SESSION, + )?; + assert!(decoded.is::()); + assert_arrays_eq!(decoded, original, &mut SESSION.create_execution_ctx()); + Ok(()) + } + + #[test] + fn estimates_decimal_and_binary_bases() { + let decimals = PrimitiveArray::from_iter((0..1_000).map(|value| value as f64 * 0.01)); + assert_eq!(estimate_float_mult_base(decimals.as_view()), Some(0.01)); + + let binary = PrimitiveArray::from_iter((0..1_000).map(|value| value as f64 * 0.125)); + assert_eq!(estimate_float_mult_base(binary.as_view()), Some(0.125)); + } +} diff --git a/encodings/float-quant/src/lib.rs b/encodings/float-quant/src/lib.rs index 6efde148ff0..d84fa7c65d1 100644 --- a/encodings/float-quant/src/lib.rs +++ b/encodings/float-quant/src/lib.rs @@ -4,14 +4,17 @@ //! Lossless float quantization as two integer child arrays. mod array; +mod float_mult; mod rules; mod slice; pub use array::*; +pub use float_mult::*; use vortex_array::session::ArraySessionExt; use vortex_session::VortexSession; /// Register the float quantization encoding in one session. pub fn initialize(session: &VortexSession) { session.arrays().register(FloatQuant); + session.arrays().register(FloatMult); } diff --git a/encodings/float-quant/src/rules.rs b/encodings/float-quant/src/rules.rs index 5faedb68866..22fbb2c9788 100644 --- a/encodings/float-quant/src/rules.rs +++ b/encodings/float-quant/src/rules.rs @@ -4,7 +4,11 @@ use vortex_array::arrays::slice::SliceReduceAdaptor; use vortex_array::optimizer::rules::ParentRuleSet; +use crate::FloatMult; use crate::FloatQuant; pub(crate) static RULES: ParentRuleSet = ParentRuleSet::new(&[ParentRuleSet::lift(&SliceReduceAdaptor(FloatQuant))]); + +pub(crate) static FLOAT_MULT_RULES: ParentRuleSet = + ParentRuleSet::new(&[ParentRuleSet::lift(&SliceReduceAdaptor(FloatMult))]); diff --git a/encodings/float-quant/src/slice.rs b/encodings/float-quant/src/slice.rs index 3a83f48220d..112e4ed4971 100644 --- a/encodings/float-quant/src/slice.rs +++ b/encodings/float-quant/src/slice.rs @@ -10,6 +10,9 @@ use vortex_array::arrays::slice::SliceReduce; use vortex_array::dtype::PType; use vortex_error::VortexResult; +use crate::FloatMult; +use crate::FloatMultArrayExt; +use crate::FloatMultArraySlotsExt; use crate::FloatQuant; use crate::FloatQuantArraySlotsExt; @@ -26,3 +29,17 @@ impl SliceReduce for FloatQuant { )) } } + +impl SliceReduce for FloatMult { + fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { + Ok(Some( + FloatMult::try_new( + array.primary().slice(range.clone())?, + array.secondary().slice(range)?, + PType::try_from(array.dtype())?, + array.base(), + )? + .into_array(), + )) + } +} diff --git a/vortex-btrblocks/examples/float_quant_dataset.rs b/vortex-btrblocks/examples/float_quant_dataset.rs index 544691e90b8..41bee8eea53 100644 --- a/vortex-btrblocks/examples/float_quant_dataset.rs +++ b/vortex-btrblocks/examples/float_quant_dataset.rs @@ -33,6 +33,7 @@ use vortex_arrow::ArrowSessionExt; use vortex_btrblocks::BtrBlocksCompressor; use vortex_btrblocks::BtrBlocksCompressorBuilder; use vortex_btrblocks::SchemeExt; +use vortex_btrblocks::schemes::float::FloatMultScheme; use vortex_btrblocks::schemes::float::FloatQuantScheme; use vortex_btrblocks::schemes::float::OrderedBlockResidualScheme; use vortex_btrblocks::schemes::range_entropy::RangeEntropyScheme; @@ -40,6 +41,9 @@ use vortex_buffer::Buffer; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_err; +use vortex_float_quant::FloatMult; +use vortex_float_quant::FloatMultArrayExt; +use vortex_float_quant::estimate_float_mult_constant_base; use vortex_pco::Pco; use vortex_range_entropy::BitSplitCodec; use vortex_range_entropy::PatchedFoRCodec; @@ -99,6 +103,13 @@ enum FloatMultPrototype { } impl FloatMultPrototype { + fn base(&self) -> f64 { + match self { + Self::F32 { base, .. } => f64::from(*base), + Self::F64 { base, .. } => *base, + } + } + fn encoded_size(&self) -> u64 { let (primary, secondary) = match self { Self::F32 { @@ -1018,15 +1029,26 @@ fn main() -> VortexResult<()> { .map(|column| column.primitive.nbytes()) .sum::(); let baseline = BtrBlocksCompressorBuilder::default() - .exclude_schemes([FloatQuantScheme.id(), OrderedBlockResidualScheme.id()]) + .exclude_schemes([ + FloatMultScheme.id(), + FloatQuantScheme.id(), + OrderedBlockResidualScheme.id(), + ]) .build(); let range_candidate = BtrBlocksCompressorBuilder::default() - .exclude_schemes([FloatQuantScheme.id(), OrderedBlockResidualScheme.id()]) + .exclude_schemes([ + FloatMultScheme.id(), + FloatQuantScheme.id(), + OrderedBlockResidualScheme.id(), + ]) .with_new_scheme(&RangeEntropyScheme) .build(); let default_without_block_residual = BtrBlocksCompressorBuilder::default() .exclude_schemes([OrderedBlockResidualScheme.id()]) .build(); + let default_without_float_mult = BtrBlocksCompressorBuilder::default() + .exclude_schemes([FloatMultScheme.id()]) + .build(); let float_candidate = BtrBlocksCompressor::default(); let stacked_candidate = BtrBlocksCompressorBuilder::default() .with_new_scheme(&RangeEntropyScheme) @@ -1042,10 +1064,14 @@ fn main() -> VortexResult<()> { Encoder::BtrBlocks(&range_candidate), ), ( - "default-off", + "default-without-block-residual", Encoder::BtrBlocks(&default_without_block_residual), ), - ("default-on", Encoder::BtrBlocks(&float_candidate)), + ( + "default-without-float-mult", + Encoder::BtrBlocks(&default_without_float_mult), + ), + ("default", Encoder::BtrBlocks(&float_candidate)), ( "default+range-entropy", Encoder::BtrBlocks(&stacked_candidate), @@ -1073,6 +1099,12 @@ fn main() -> VortexResult<()> { column.name, describe_pco(column, &pco_config, &session)? ); + if let Some(base) = estimate_float_mult_constant_base(column.primitive.as_view()) { + println!( + "float-mult-analysis\t{dataset}\t{}\tconstant-base\t{base}", + column.name + ); + } } println!("structure\tdataset\tcolumn\tconfig\tencoding\tbytes"); for (config_name, arrays) in &encoded { @@ -1085,6 +1117,13 @@ fn main() -> VortexResult<()> { encoding_tree(array), array.nbytes() ); + if let Some(float_mult) = array.as_opt::() { + println!( + "float-mult-base\t{dataset}\t{}\t{config_name}\t{}", + column.name, + float_mult.base() + ); + } } } for (column, prototype) in &float_mult_prototypes { @@ -1098,6 +1137,11 @@ fn main() -> VortexResult<()> { prototype.structure(), prototype.encoded_size(), ); + println!( + "float-mult-base\t{dataset}\t{}\tpco-prototype\t{}", + column.name, + prototype.base() + ); println!( "float-mult-backends\t{dataset}\t{}\tbit-split={}\tblock-residual={}\trange-entropy={}\trange-packed={}\trange-two-level={}", column.name, diff --git a/vortex-btrblocks/src/builder.rs b/vortex-btrblocks/src/builder.rs index 3020d9ae348..370f52f148c 100644 --- a/vortex-btrblocks/src/builder.rs +++ b/vortex-btrblocks/src/builder.rs @@ -45,6 +45,7 @@ pub const ALL_SCHEMES: &[&dyn Scheme] = &[ &float::ALPScheme, &float::ALPRDScheme, &float::FloatQuantScheme, + &float::FloatMultScheme, &float::OrderedBlockResidualScheme, &float::FloatDictScheme, &float::NullDominatedSparseScheme, @@ -176,6 +177,7 @@ impl BtrBlocksCompressorBuilder { integer::IntRLEScheme.id(), float::ALPRDScheme.id(), float::FloatQuantScheme.id(), + float::FloatMultScheme.id(), float::OrderedBlockResidualScheme.id(), float::FloatRLEScheme.id(), float::NullDominatedSparseScheme.id(), diff --git a/vortex-btrblocks/src/schemes/float/float_mult.rs b/vortex-btrblocks/src/schemes/float/float_mult.rs new file mode 100644 index 00000000000..aa1d46cb292 --- /dev/null +++ b/vortex-btrblocks/src/schemes/float/float_mult.rs @@ -0,0 +1,251 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Lossless FloatMult split for exact-multiple f32 values. + +use vortex_array::ArrayId; +use vortex_array::ArrayRef; +use vortex_array::Canonical; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::VTable; +use vortex_array::arrays::Primitive; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::dtype::PType; +use vortex_compressor::scheme::CompressionEstimate; +use vortex_compressor::scheme::DeferredEstimate; +use vortex_compressor::scheme::EstimateScore; +use vortex_compressor::scheme::EstimateVerdict; +use vortex_error::VortexResult; +use vortex_float_quant::FloatMult; +use vortex_float_quant::FloatMultArraySlotsExt; +use vortex_float_quant::estimate_float_mult_constant_base; + +use crate::ArrayAndStats; +use crate::CascadingCompressor; +use crate::CompressorContext; +use crate::Scheme; +use crate::SchemeExt; + +const SAMPLE_RUNS: usize = 16; +const SAMPLE_RUN_LEN: usize = 128; +const SELECTION_PENALTY: f64 = 0.85; +const MIN_ESTIMATED_RATIO: f64 = 1.50; +const MIN_WIN_OVER_INCUMBENT: f64 = 1.05; +const MAX_PRIMARY_BIT_WIDTH: u32 = 17; + +/// FloatMult split with normal BtrBlocks compression for an integer latent child. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct FloatMultScheme; + +impl Scheme for FloatMultScheme { + fn scheme_name(&self) -> &'static str { + "vortex.float.float_mult" + } + + fn matches(&self, canonical: &Canonical) -> bool { + canonical.dtype().is_float() && canonical.dtype().as_ptype() == PType::F32 + } + + fn produced_encodings(&self) -> Vec { + vec![FloatMult.id()] + } + + fn num_children(&self) -> usize { + 2 + } + + fn expected_compression_ratio( + &self, + data: &ArrayAndStats, + compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> CompressionEstimate { + let primitive = data.array_as_primitive(); + if compress_ctx.finished_cascading() + || primitive.ptype() != PType::F32 + || primitive.len() < SAMPLE_RUN_LEN + { + return CompressionEstimate::Verdict(EstimateVerdict::Skip); + } + + CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( + |compressor, data, best_so_far, compress_ctx, exec_ctx| { + let sample = sample_primitive(data.array_as_primitive(), exec_ctx)?; + let Some(base) = estimate_float_mult_constant_base(sample.as_view()) else { + return Ok(EstimateVerdict::Skip); + }; + let Some(encoded) = + FloatMult::from_primitive_constant_secondary(sample.as_view(), base)? + else { + return Ok(EstimateVerdict::Skip); + }; + if primary_bit_width(encoded.primary().as_::()) > MAX_PRIMARY_BIT_WIDTH { + return Ok(EstimateVerdict::Skip); + } + let compressed_primary = compressor.compress_child( + encoded.primary(), + &compress_ctx, + FloatMultScheme.id(), + 0, + exec_ctx, + )?; + let candidate = FloatMult::try_new( + compressed_primary, + encoded.secondary().clone(), + sample.ptype(), + base, + )?; + let after_nbytes = candidate.nbytes(); + if after_nbytes == 0 { + return Ok(EstimateVerdict::Skip); + } + let ratio = sample.nbytes() as f64 / after_nbytes as f64 * SELECTION_PENALTY; + if ratio < MIN_ESTIMATED_RATIO { + return Ok(EstimateVerdict::Skip); + } + let incumbent = best_so_far.and_then(EstimateScore::finite_ratio); + if incumbent.is_some_and(|best| ratio < best * MIN_WIN_OVER_INCUMBENT) { + return Ok(EstimateVerdict::Skip); + } + Ok(EstimateVerdict::Ratio(ratio)) + }, + ))) + } + + fn compress( + &self, + compressor: &CascadingCompressor, + data: &ArrayAndStats, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, + ) -> VortexResult { + let primitive = data.array_as_primitive(); + let sample = sample_primitive(primitive, exec_ctx)?; + let Some(base) = estimate_float_mult_constant_base(sample.as_view()) else { + return Ok(primitive.array().clone()); + }; + let Some(encoded) = FloatMult::from_primitive_constant_secondary(primitive, base)? else { + return Ok(primitive.array().clone()); + }; + let compressed_primary = + compressor.compress_child(encoded.primary(), &compress_ctx, self.id(), 0, exec_ctx)?; + Ok(FloatMult::try_new( + compressed_primary, + encoded.secondary().clone(), + primitive.ptype(), + base, + )? + .into_array()) + } +} + +fn primary_bit_width(primary: vortex_array::ArrayView<'_, Primitive>) -> u32 { + match primary.ptype() { + PType::U32 => { + let (minimum, maximum) = primary + .as_slice::() + .iter() + .copied() + .fold((u32::MAX, u32::MIN), |(minimum, maximum), value| { + (minimum.min(value), maximum.max(value)) + }); + u32::BITS - (maximum - minimum).leading_zeros() + } + PType::U64 => { + let (minimum, maximum) = primary + .as_slice::() + .iter() + .copied() + .fold((u64::MAX, u64::MIN), |(minimum, maximum), value| { + (minimum.min(value), maximum.max(value)) + }); + u64::BITS - (maximum - minimum).leading_zeros() + } + _ => unreachable!(), + } +} + +fn sample_primitive( + primitive: vortex_array::ArrayView<'_, Primitive>, + exec_ctx: &mut ExecutionCtx, +) -> VortexResult { + let sample_len = SAMPLE_RUNS * SAMPLE_RUN_LEN; + if primitive.len() <= sample_len { + return primitive + .array() + .clone() + .execute::(exec_ctx); + } + + let validity = primitive + .validity()? + .execute_mask(primitive.len(), exec_ctx)?; + macro_rules! sample { + ($T:ty) => {{ + let values = primitive.as_slice::<$T>(); + let mut sample = Vec::with_capacity(sample_len); + for run_index in 0..SAMPLE_RUNS { + let partition_start = run_index * primitive.len() / SAMPLE_RUNS; + let next_partition = (run_index + 1) * primitive.len() / SAMPLE_RUNS; + let start = + partition_start + (next_partition - partition_start - SAMPLE_RUN_LEN) / 2; + sample.extend( + (start..start + SAMPLE_RUN_LEN) + .map(|index| validity.value(index).then_some(values[index])), + ); + } + PrimitiveArray::from_option_iter(sample) + }}; + } + Ok(match primitive.ptype() { + PType::F32 => sample!(f32), + PType::F64 => sample!(f64), + _ => unreachable!(), + }) +} + +#[cfg(test)] +mod tests { + use std::sync::LazyLock; + + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::array_session; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::assert_arrays_eq; + use vortex_error::VortexResult; + use vortex_session::VortexSession; + + use super::*; + use crate::BtrBlocksCompressorBuilder; + + static SESSION: LazyLock = LazyLock::new(|| { + let session = array_session(); + vortex_float_quant::initialize(&session); + session + }); + + #[test] + fn candidate_never_increases_noisy_value_size() -> VortexResult<()> { + let values = (0u32..65_536) + .map(|index| f32::from_bits(0x3f80_0000 | index.wrapping_mul(7_919) & 0x007f_ffff)) + .collect::>(); + let original = PrimitiveArray::from_iter(values).into_array(); + let baseline = BtrBlocksCompressorBuilder::default() + .exclude_schemes([FloatMultScheme.id()]) + .build() + .compress(&original, &mut SESSION.create_execution_ctx())?; + let candidate = BtrBlocksCompressorBuilder::default() + .build() + .compress(&original, &mut SESSION.create_execution_ctx())?; + assert!( + candidate.nbytes() <= baseline.nbytes(), + "FloatMult candidate uses {} bytes and baseline uses {} bytes", + candidate.nbytes(), + baseline.nbytes() + ); + assert_arrays_eq!(candidate, original, &mut SESSION.create_execution_ctx()); + Ok(()) + } +} diff --git a/vortex-btrblocks/src/schemes/float/mod.rs b/vortex-btrblocks/src/schemes/float/mod.rs index e2f8aecb398..31406120abe 100644 --- a/vortex-btrblocks/src/schemes/float/mod.rs +++ b/vortex-btrblocks/src/schemes/float/mod.rs @@ -5,6 +5,7 @@ mod alp; mod alprd; +mod float_mult; mod float_quant; mod ordered_block_residual; mod rle; @@ -15,6 +16,7 @@ mod pco; pub use alp::ALPScheme; pub use alprd::ALPRDScheme; +pub use float_mult::FloatMultScheme; pub use float_quant::FloatQuantScheme; pub use ordered_block_residual::OrderedBlockResidualScheme; #[cfg(feature = "pco")] diff --git a/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs index 582a48b4dfa..173e1607be7 100644 --- a/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs @@ -18,6 +18,7 @@ use vortex_array::dtype::Nullability; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_error::VortexResult; +use vortex_float_quant::FloatMult; use vortex_float_quant::FloatQuant; use vortex_range_entropy::BlockResidual; use vortex_range_entropy::OrderedFloat; @@ -106,6 +107,18 @@ fn test_f32_does_not_use_float_quant() -> VortexResult<()> { Ok(()) } +#[test] +fn test_integer_f32_uses_float_mult() -> VortexResult<()> { + let values = (0u32..65_536) + .map(|index| ((index.wrapping_mul(7_919)) % 65_537) as f32) + .collect::>(); + let array = PrimitiveArray::from_iter(values).into_array(); + let compressed = + BtrBlocksCompressor::default().compress(&array, &mut SESSION.create_execution_ctx())?; + assert!(compressed.is::()); + Ok(()) +} + #[test] fn test_repeated_f64_prefers_existing_scheme() -> VortexResult<()> { let values = (0u32..16_384) diff --git a/vortex-btrblocks/src/schemes/float/tests.rs b/vortex-btrblocks/src/schemes/float/tests.rs index 2d0a04542cc..7bb2f4748c6 100644 --- a/vortex-btrblocks/src/schemes/float/tests.rs +++ b/vortex-btrblocks/src/schemes/float/tests.rs @@ -21,6 +21,9 @@ use vortex_fastlanes::RLE; use vortex_session::VortexSession; use crate::BtrBlocksCompressor; +use crate::BtrBlocksCompressorBuilder; +use crate::SchemeExt; +use crate::schemes::float::FloatMultScheme; use crate::schemes::float::FloatRLEScheme; static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); @@ -42,15 +45,20 @@ fn test_compress() -> VortexResult<()> { } let array = values.into_array(); - let btr = BtrBlocksCompressor::default(); - let compressed = btr.compress(&array, &mut SESSION.create_execution_ctx())?; + let baseline = BtrBlocksCompressorBuilder::default() + .exclude_schemes([FloatMultScheme.id()]) + .build() + .compress(&array, &mut SESSION.create_execution_ctx())?; + let compressed = + BtrBlocksCompressor::default().compress(&array, &mut SESSION.create_execution_ctx())?; assert_eq!(compressed.len(), 1024); - - let display = compressed - .display_as(DisplayOptions::MetadataOnly) - .to_string() - .to_lowercase(); - assert_eq!(display, "vortex.dict(f32, len=1024)"); + assert!( + compressed.nbytes() <= baseline.nbytes(), + "default uses {} bytes and the prior default uses {} bytes", + compressed.nbytes(), + baseline.nbytes() + ); + assert_arrays_eq!(compressed, array, &mut SESSION.create_execution_ctx()); Ok(()) } diff --git a/vortex/src/editions/core/v2026_08.rs b/vortex/src/editions/core/v2026_08.rs index b64faf56025..a87d49ae184 100644 --- a/vortex/src/editions/core/v2026_08.rs +++ b/vortex/src/editions/core/v2026_08.rs @@ -27,6 +27,7 @@ pub static DECLARATION: EditionDeclaration = EditionDeclaration { }, added: &[ EditionMember::array(&"vortex.block_residual"), + EditionMember::array(&"vortex.float_mult"), EditionMember::array(&"vortex.float_quant"), EditionMember::array(&"vortex.map"), EditionMember::array(&"vortex.ordered_float"), diff --git a/vortex/src/editions/tests.rs b/vortex/src/editions/tests.rs index a833854d71a..7a22c9cbdad 100644 --- a/vortex/src/editions/tests.rs +++ b/vortex/src/editions/tests.rs @@ -35,6 +35,7 @@ use vortex_error::vortex_err; use vortex_file::OpenOptionsSessionExt; use vortex_file::WriteOptionsSessionExt; use vortex_file::WriteStrategyBuilder; +use vortex_float_quant::FloatMult; use vortex_float_quant::FloatQuant; use vortex_io::session::RuntimeSession; use vortex_layout::LayoutStrategy; @@ -525,6 +526,33 @@ async fn default_writer_round_trips_float_quant() -> VortexResult<()> { Ok(()) } +#[tokio::test] +async fn default_writer_round_trips_float_mult() -> VortexResult<()> { + use crate::VortexSessionDefault; + + let session = VortexSession::default(); + let values = (0u32..65_536) + .map(|index| ((index.wrapping_mul(7_919)) % 65_537) as f32) + .collect::>(); + let expected = PrimitiveArray::from_iter(values.clone()).into_array(); + let buffer = write_with(&session, expected.clone()).await?; + let actual = session + .open_options() + .open_buffer(buffer)? + .scan()? + .into_array_stream()? + .read_all() + .await?; + assert!( + actual + .depth_first_traversal() + .any(|array| array.is::()) + ); + let decoded = actual.execute::(&mut session.create_execution_ctx())?; + assert_eq!(decoded.as_slice::(), values); + Ok(()) +} + #[tokio::test] async fn default_writer_round_trips_ordered_block_residual() -> VortexResult<()> { use crate::VortexSessionDefault; From 0f79c9244ec7bed75d33c0fd21f888edccfb3fe5 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Tue, 18 Aug 2026 17:27:20 -0400 Subject: [PATCH 04/78] perf: omit zero FloatMult adjustments Signed-off-by: "Will Manning" --- docs/plans/native-pcodec.md | 35 ++- encodings/float-quant/src/float_mult.rs | 187 +++++++++------ encodings/float-quant/src/slice.rs | 5 +- .../examples/float_quant_dataset.rs | 220 +++++++++++++++++- .../src/schemes/float/float_mult.rs | 6 +- 5 files changed, 377 insertions(+), 76 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 711b282a9c4..371047104f4 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -95,7 +95,11 @@ The automatic FloatQuant scheme accepts only `f64`. Native `f32` columns remain `FloatMultArray` represents each float as an integer multiple of one base plus a signed ULP adjustment. -The array stores the base in at most nine metadata bytes. Two unsigned child arrays store the multiples and adjustments. +The array stores the base in at most nine metadata bytes. One required unsigned child stores the multiples. + +An optional final child stores the adjustments. An absent child means that every adjustment is zero. + +The child count identifies this mode. The serialized metadata contains no separate flag. The default scheme accepts only `f32` inputs whose sampled adjustments all represent zero. @@ -406,6 +410,35 @@ FloatMult reduces total Housing size by 8.9 percent. Compression throughput regr Housing decode changes by less than measurement noise. Unselected Taxi and Air Quality results remain effectively unchanged. +### Large-row throughput check + +The source Housing file contains 20,433 complete rows. That size is too small for stable throughput and scalar-access decisions. + +The larger check repeats the source distribution to two million rows. It uses 72 MB of source floats. + +This repetition preserves the value distribution. It also preserves cardinality, so it changes some outer dictionary choices. + +The size results above remain the source for compression policy. The larger check tests throughput and access costs. + +| Path | Bytes | Decode MB/s, run 1 | Decode MB/s, run 2 | Scalar access ns, run 1 | Scalar access ns, run 2 | +| --- | ---: | ---: | ---: | ---: | ---: | +| Prior default, exact selected columns | 6,275,780 | 10,806.9 | 13,495.0 | 949.90 and 1,010.57 | 790.08 and 825.45 | +| FloatMult, exact selected columns | 6,270,986 | 10,967.0 | 13,483.6 | 675.10 and 666.78 | 551.01 and 545.61 | +| Prior default, nonzero adjustment column | 3,013,120 | 10,735.9 | 13,615.1 | 204.73 | 174.87 | +| FloatMult, nonzero adjustment column | 3,219,980 | 3,916.4 | 4,847.8 | 1,551.78 | 1,310.13 | + +The absolute throughput changed between runs. The interleaved comparisons kept the relative exact-mode result near zero. + +Exact FloatMult reduced scalar-access latency by 29 to 34 percent on both selected columns. + +The nonzero-adjustment form decoded 64 percent slower. Its scalar access was more than seven times slower. + +The full default compressor encoded at 461 to 463 MB/s. The prior default encoded at 454 to 458 MB/s. + +The large check found no default encode regression. The compact compressor encoded at 225 to 226 MB/s. + +The nonzero result supports separate selection weights for the one-child and two-child forms. + ## Why Compact still wins Pco does not use FloatQuant on the measured paper inputs. diff --git a/encodings/float-quant/src/float_mult.rs b/encodings/float-quant/src/float_mult.rs index 112c743b661..a8b171d5907 100644 --- a/encodings/float-quant/src/float_mult.rs +++ b/encodings/float-quant/src/float_mult.rs @@ -19,8 +19,6 @@ use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::TypedArrayRef; use vortex_array::array_slots; -use vortex_array::arrays::Constant; -use vortex_array::arrays::ConstantArray; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::buffer::BufferHandle; @@ -60,7 +58,7 @@ pub struct FloatMultSlots { pub primary: ArrayRef, /// Signed ULP adjustments with zero moved to the unsigned midpoint. #[slot(1)] - pub secondary: ArrayRef, + pub secondary: Option, } #[derive(Clone, Debug)] @@ -116,21 +114,27 @@ impl VTable for FloatMult { let slots = FloatMultSlotsView::from_slots(slots); let expected_primary = DType::Primitive(latent_ptype, dtype.nullability()); - let expected_secondary = DType::Primitive(latent_ptype, NonNullable); vortex_ensure!( slots.primary.dtype() == &expected_primary, "expected primary dtype {expected_primary}, got {}", slots.primary.dtype() ); vortex_ensure!( - slots.secondary.dtype() == &expected_secondary, - "expected secondary dtype {expected_secondary}, got {}", - slots.secondary.dtype() - ); - vortex_ensure!( - slots.primary.len() == len && slots.secondary.len() == len, + slots.primary.len() == len, "FloatMult child length differs from {len}" ); + if let Some(secondary) = slots.secondary { + let expected_secondary = DType::Primitive(latent_ptype, NonNullable); + vortex_ensure!( + secondary.dtype() == &expected_secondary, + "expected secondary dtype {expected_secondary}, got {}", + secondary.dtype() + ); + vortex_ensure!( + secondary.len() == len, + "FloatMult child length differs from {len}" + ); + } Ok(()) } @@ -195,7 +199,10 @@ impl VTable for FloatMult { "unsupported FloatMult metadata version {}", metadata[0] ); - vortex_ensure!(children.len() == 2, "FloatMult requires two children"); + vortex_ensure!( + matches!(children.len(), 1 | 2), + "FloatMult requires one or two children" + ); let base_bits = match ptype { PType::F32 => { @@ -212,9 +219,13 @@ impl VTable for FloatMult { }; let latent_ptype = latent_ptype(ptype)?; let primary_dtype = DType::Primitive(latent_ptype, dtype.nullability()); - let secondary_dtype = DType::Primitive(latent_ptype, NonNullable); let primary = children.get(0, &primary_dtype, len)?; - let secondary = children.get(1, &secondary_dtype, len)?; + let secondary = if children.len() == 2 { + let secondary_dtype = DType::Primitive(latent_ptype, NonNullable); + Some(children.get(1, &secondary_dtype, len)?) + } else { + None + }; let slots = FloatMultSlots { primary, secondary }.into_slots(); Ok(ArrayParts::new( self.clone(), @@ -254,36 +265,47 @@ impl OperationsVTable for FloatMult { if primary.is_null() { return Ok(Scalar::null(array.dtype().clone())); } - let secondary = array.secondary().execute_scalar(index, ctx)?; Ok(match PType::try_from(array.dtype())? { - PType::F32 => Scalar::primitive( - join_f32( - primary - .as_primitive() - .typed_value::() - .vortex_expect("validated primary scalar"), - secondary - .as_primitive() - .typed_value::() - .vortex_expect("validated secondary scalar"), - array.data().base_f32()?, - ), - array.dtype().nullability(), - ), - PType::F64 => Scalar::primitive( - join_f64( - primary - .as_primitive() - .typed_value::() - .vortex_expect("validated primary scalar"), - secondary - .as_primitive() - .typed_value::() - .vortex_expect("validated secondary scalar"), - array.data().base_f64()?, - ), - array.dtype().nullability(), - ), + PType::F32 => { + let primary = primary + .as_primitive() + .typed_value::() + .vortex_expect("validated primary scalar"); + let value = if let Some(secondary) = array.secondary() { + let secondary = secondary.execute_scalar(index, ctx)?; + join_f32( + primary, + secondary + .as_primitive() + .typed_value::() + .vortex_expect("validated secondary scalar"), + array.data().base_f32()?, + ) + } else { + int_float_from_u32(primary) * array.data().base_f32()? + }; + Scalar::primitive(value, array.dtype().nullability()) + } + PType::F64 => { + let primary = primary + .as_primitive() + .typed_value::() + .vortex_expect("validated primary scalar"); + let value = if let Some(secondary) = array.secondary() { + let secondary = secondary.execute_scalar(index, ctx)?; + join_f64( + primary, + secondary + .as_primitive() + .typed_value::() + .vortex_expect("validated secondary scalar"), + array.data().base_f64()?, + ) + } else { + int_float_from_u64(primary) * array.data().base_f64()? + }; + Scalar::primitive(value, array.dtype().nullability()) + } ptype => vortex_panic!("unsupported FloatMult ptype {ptype}"), }) } @@ -333,10 +355,10 @@ pub trait FloatMultArrayExt: TypedArrayRef + FloatMultArraySlotsExt { impl> FloatMultArrayExt for T {} impl FloatMult { - /// Construct a FloatMult array from two latent children. + /// Construct a FloatMult array from one or two latent children. pub fn try_new( primary: ArrayRef, - secondary: ArrayRef, + secondary: Option, float_ptype: PType, base: f64, ) -> VortexResult { @@ -374,7 +396,10 @@ impl FloatMult { .unzip(); Self::try_new( PrimitiveArray::new(Buffer::from(primary), validity).into_array(), - PrimitiveArray::new(Buffer::from(secondary), NonNullable.into()).into_array(), + Some( + PrimitiveArray::new(Buffer::from(secondary), NonNullable.into()) + .into_array(), + ), PType::F32, f64::from(base), ) @@ -392,7 +417,10 @@ impl FloatMult { .unzip(); Self::try_new( PrimitiveArray::new(Buffer::from(primary), validity).into_array(), - PrimitiveArray::new(Buffer::from(secondary), NonNullable.into()).into_array(), + Some( + PrimitiveArray::new(Buffer::from(secondary), NonNullable.into()) + .into_array(), + ), PType::F64, base, ) @@ -421,7 +449,7 @@ impl FloatMult { } Ok(Some(Self::try_new( PrimitiveArray::new(Buffer::from(primary), validity).into_array(), - ConstantArray::new(Scalar::from(1_u32 << 31), len).into_array(), + None, PType::F32, f64::from(base), )?)) @@ -437,7 +465,7 @@ impl FloatMult { } Ok(Some(Self::try_new( PrimitiveArray::new(Buffer::from(primary), validity).into_array(), - ConstantArray::new(Scalar::from(1_u64 << 63), len).into_array(), + None, PType::F64, base, )?)) @@ -920,7 +948,7 @@ fn join_f64(primary: u64, secondary: u64, base: f64) -> f64 { fn decode(array: ArrayView<'_, FloatMult>, ctx: &mut ExecutionCtx) -> VortexResult { let primary = array.primary().clone().execute::(ctx)?; let validity = primary.validity()?; - if secondary_is_constant_zero(array) { + let Some(secondary) = array.secondary() else { return Ok(match PType::try_from(array.dtype())? { PType::F32 => { let base = array.data().base_f32()?; @@ -940,9 +968,9 @@ fn decode(array: ArrayView<'_, FloatMult>, ctx: &mut ExecutionCtx) -> VortexResu } ptype => vortex_panic!("unsupported FloatMult ptype {ptype}"), }); - } + }; - let secondary = array.secondary().clone().execute::(ctx)?; + let secondary = secondary.clone().execute::(ctx)?; Ok(match PType::try_from(array.dtype())? { PType::F32 => { let secondary_values = secondary.as_slice::(); @@ -976,21 +1004,6 @@ fn decode(array: ArrayView<'_, FloatMult>, ctx: &mut ExecutionCtx) -> VortexResu }) } -fn secondary_is_constant_zero(array: ArrayView<'_, FloatMult>) -> bool { - let Some(constant) = array.secondary().as_opt::() else { - return false; - }; - match PType::try_from(array.dtype()) { - Ok(PType::F32) => { - constant.scalar().as_primitive().typed_value::() == Some(1_u32 << 31) - } - Ok(PType::F64) => { - constant.scalar().as_primitive().typed_value::() == Some(1_u64 << 63) - } - _ => false, - } -} - #[cfg(test)] mod tests { use std::sync::LazyLock; @@ -1025,12 +1038,24 @@ mod tests { -0.0, 0.0, 1.5, + 1.234_567_89, f64::INFINITY, f64::from_bits(0x7ff8_0000_0000_1234), f64::from_bits(0xfff8_0000_0000_5678), ]; let original = PrimitiveArray::from_iter(values); let encoded = FloatMult::from_primitive(original.as_view(), 0.01)?; + let secondary = encoded + .secondary() + .vortex_expect("general FloatMult must contain adjustments") + .clone() + .execute::(&mut SESSION.create_execution_ctx())?; + assert!( + secondary + .as_slice::() + .iter() + .any(|adjustment| *adjustment != 1_u64 << 63) + ); let decoded = encoded .into_array() .execute::(&mut SESSION.create_execution_ctx())?; @@ -1048,6 +1073,36 @@ mod tests { Ok(()) } + #[test] + fn implicit_zero_adjustment_roundtrip() -> VortexResult<()> { + let original = PrimitiveArray::from_iter([0.0_f32, 1.0, 42.0, 65_536.0]); + let encoded = FloatMult::from_primitive_constant_secondary(original.as_view(), 1.0)? + .vortex_expect("integer floats use an implicit zero adjustment"); + assert!(encoded.secondary().is_none()); + assert_eq!(encoded.as_ref().nchildren(), 1); + assert_arrays_eq!(encoded, original, &mut SESSION.create_execution_ctx()); + + let encoded = encoded.into_array(); + let dtype = encoded.dtype().clone(); + let len = encoded.len(); + let array_context = ArrayContext::empty(); + let serialized = + encoded.serialize(&array_context, &SESSION, &SerializeOptions::default())?; + let mut bytes = ByteBufferMut::empty(); + for buffer in serialized { + bytes.extend_from_slice(buffer.as_ref()); + } + let decoded = SerializedArray::try_from(bytes.freeze())?.decode( + &dtype, + len, + &ReadContext::new(array_context.to_ids()), + &SESSION, + )?; + assert!(decoded.as_::().secondary().is_none()); + assert_arrays_eq!(decoded, original, &mut SESSION.create_execution_ctx()); + Ok(()) + } + #[test] fn nullable_slice_and_scalar_access() -> VortexResult<()> { let original = PrimitiveArray::new( diff --git a/encodings/float-quant/src/slice.rs b/encodings/float-quant/src/slice.rs index 112e4ed4971..0442a271d8b 100644 --- a/encodings/float-quant/src/slice.rs +++ b/encodings/float-quant/src/slice.rs @@ -35,7 +35,10 @@ impl SliceReduce for FloatMult { Ok(Some( FloatMult::try_new( array.primary().slice(range.clone())?, - array.secondary().slice(range)?, + array + .secondary() + .map(|secondary| secondary.slice(range)) + .transpose()?, PType::try_from(array.dtype())?, array.base(), )? diff --git a/vortex-btrblocks/examples/float_quant_dataset.rs b/vortex-btrblocks/examples/float_quant_dataset.rs index 41bee8eea53..63eddced766 100644 --- a/vortex-btrblocks/examples/float_quant_dataset.rs +++ b/vortex-btrblocks/examples/float_quant_dataset.rs @@ -150,6 +150,35 @@ impl FloatMultPrototype { } } + fn has_nonzero_adjustments(&self) -> bool { + let zero = match self { + Self::F32 { .. } => 1_u64 << 31, + Self::F64 { .. } => 1_u64 << 63, + }; + self.latents() + .secondary + .iter() + .any(|adjustment| *adjustment != zero) + } + + fn to_array(&self) -> VortexResult { + let (primary, secondary, ptype, base) = match self { + Self::F32 { + primary, + secondary, + base, + .. + } => (primary, secondary, PType::F32, f64::from(*base)), + Self::F64 { + primary, + secondary, + base, + .. + } => (primary, secondary, PType::F64, *base), + }; + Ok(FloatMult::try_new(primary.clone(), Some(secondary.clone()), ptype, base)?.into_array()) + } + #[allow(clippy::cast_possible_truncation)] fn reconstruct(&self, primary: &[u64], secondary: &[u64]) -> VortexResult<()> { vortex_ensure!( @@ -369,7 +398,10 @@ impl Encoder<'_> { } } -fn read_california_housing(path: &Path) -> VortexResult> { +fn read_california_housing( + path: &Path, + target_row_count: Option, +) -> VortexResult> { let reader = BufReader::new(File::open(path)?); let mut values = std::array::from_fn::<_, 9, _>(|_| Vec::::new()); for (line_index, line) in reader.lines().enumerate() { @@ -394,6 +426,17 @@ fn read_california_housing(path: &Path) -> VortexResult> { } } + if let Some(target_row_count) = target_row_count { + for column in &mut values { + vortex_ensure!(!column.is_empty(), "California Housing input is empty"); + column.truncate(target_row_count); + while column.len() < target_row_count { + let copy_len = (target_row_count - column.len()).min(column.len()); + column.extend_from_within(..copy_len); + } + } + } + Ok(COLUMN_NAMES .into_iter() .zip(values) @@ -872,6 +915,152 @@ fn measure_decoders( Ok(()) } +fn measure_selected_float_mult( + dataset: &str, + columns: &[Column], + baseline: &[ArrayRef], + candidate: &[ArrayRef], + session: &VortexSession, +) -> VortexResult<()> { + let selected = candidate + .iter() + .enumerate() + .filter_map(|(index, array)| contains_float_mult(array).then_some(index)) + .collect::>(); + if selected.is_empty() { + return Ok(()); + } + let input_bytes = selected + .iter() + .map(|index| columns[*index].primitive.nbytes()) + .sum::(); + let selected_configs = [ + ( + "prior-default", + selected + .iter() + .map(|index| baseline[*index].clone()) + .collect::>(), + ), + ( + "float-mult", + selected + .iter() + .map(|index| candidate[*index].clone()) + .collect::>(), + ), + ]; + measure_decoders( + &format!("{dataset}-float-mult-selected"), + &selected_configs, + input_bytes, + session, + )?; + + for index in selected { + measure_scalar_pair( + dataset, + &columns[index].name, + [ + ("prior-default", &baseline[index]), + ("float-mult", &candidate[index]), + ], + session, + )?; + } + Ok(()) +} + +fn contains_float_mult(array: &ArrayRef) -> bool { + array.is::() || array.children().iter().any(contains_float_mult) +} + +fn measure_scalar_pair( + dataset: &str, + column: &str, + arrays: [(&str, &ArrayRef); 2], + session: &VortexSession, +) -> VortexResult<()> { + const ACCESSES: usize = 200_000; + const REPETITIONS: usize = 7; + let mut durations = [ + Vec::with_capacity(REPETITIONS), + Vec::with_capacity(REPETITIONS), + ]; + for repetition in 0..REPETITIONS { + for offset in 0..arrays.len() { + let config_index = (repetition + offset) % arrays.len(); + let array = arrays[config_index].1; + let mut ctx = session.create_execution_ctx(); + let start = Instant::now(); + let mut checksum = 0_u32; + for access in 0..ACCESSES { + let row = access.wrapping_mul(0x9e37_79b1) % array.len(); + let value = array + .execute_scalar(row, &mut ctx)? + .as_primitive() + .typed_value::() + .ok_or_else(|| vortex_err!("benchmark value is null"))?; + checksum ^= black_box(value.to_bits()); + } + durations[config_index].push(start.elapsed()); + black_box(checksum); + } + } + for (config_index, (config, array)) in arrays.into_iter().enumerate() { + let median = percentile(&mut durations[config_index], 1, 2); + let nanoseconds = median.as_secs_f64() * 1_000_000_000.0 / ACCESSES as f64; + let accesses_per_second = ACCESSES as f64 / median.as_secs_f64() / 1_000_000.0; + println!( + "scalar-at\t{dataset}\t{column}\t{config}\t{}\t{}\t{nanoseconds:.2}\t{accesses_per_second:.1}", + array.encoding_id(), + array.nbytes(), + ); + } + Ok(()) +} + +fn measure_nonzero_float_mult( + dataset: &str, + columns: &[Column], + prototypes: &[(&Column, FloatMultPrototype)], + baseline: &[ArrayRef], + session: &VortexSession, +) -> VortexResult<()> { + for (column, prototype) in prototypes + .iter() + .filter(|(_, prototype)| prototype.has_nonzero_adjustments()) + { + let index = columns + .iter() + .position(|candidate| candidate.name == column.name) + .ok_or_else(|| vortex_err!("missing benchmark column {}", column.name))?; + let candidate = prototype.to_array()?; + let mut verify_ctx = session.create_execution_ctx(); + assert_arrays_eq!(candidate, column.array, &mut verify_ctx); + let configs = [ + ("prior-default", vec![baseline[index].clone()]), + ("float-mult-nonzero", vec![candidate.clone()]), + ]; + measure_decoders( + &format!("{dataset}-{}-nonzero", column.name), + &configs, + column.primitive.nbytes(), + session, + )?; + measure_scalar_pair( + dataset, + &column.name, + [ + ("prior-default", &baseline[index]), + ("float-mult-nonzero", &candidate), + ], + session, + )?; + } + Ok(()) +} + fn measure_float_mult_decoder( dataset: &str, prototypes: &[(&Column, FloatMultPrototype)], @@ -1008,8 +1197,7 @@ fn main() -> VortexResult<()> { .parse::() .map_err(|error| vortex_err!("invalid row limit: {error}")) }) - .transpose()? - .unwrap_or(DEFAULT_ROW_LIMIT); + .transpose()?; let path = Path::new(&path); let dataset = path .file_stem() @@ -1020,9 +1208,9 @@ fn main() -> VortexResult<()> { .extension() .is_some_and(|extension| extension == "parquet") { - read_parquet_numeric(path, row_limit, &session)? + read_parquet_numeric(path, row_limit.unwrap_or(DEFAULT_ROW_LIMIT), &session)? } else { - read_california_housing(path)? + read_california_housing(path, row_limit)? }; let input_bytes = columns .iter() @@ -1154,6 +1342,28 @@ fn main() -> VortexResult<()> { } println!("operation\tdataset\tconfig\tbytes\tp10-ms\tmedian-ms\tp90-ms\tMB/s"); measure_decoders(dataset, &encoded, input_bytes, &session)?; + let default_without_float_mult = encoded + .iter() + .find_map(|(name, arrays)| (*name == "default-without-float-mult").then_some(arrays)) + .ok_or_else(|| vortex_err!("missing FloatMult baseline"))?; + let default = encoded + .iter() + .find_map(|(name, arrays)| (*name == "default").then_some(arrays)) + .ok_or_else(|| vortex_err!("missing default candidate"))?; + measure_selected_float_mult( + dataset, + &columns, + default_without_float_mult, + default, + &session, + )?; + measure_nonzero_float_mult( + dataset, + &columns, + &float_mult_prototypes, + default_without_float_mult, + &session, + )?; measure_float_mult_decoder(dataset, &float_mult_prototypes, &session)?; measure_float_mult_backends(dataset, &float_mult_prototypes)?; measure_compressors(dataset, &configs, &columns, input_bytes, &session)?; diff --git a/vortex-btrblocks/src/schemes/float/float_mult.rs b/vortex-btrblocks/src/schemes/float/float_mult.rs index aa1d46cb292..64372a62fc1 100644 --- a/vortex-btrblocks/src/schemes/float/float_mult.rs +++ b/vortex-btrblocks/src/schemes/float/float_mult.rs @@ -52,7 +52,7 @@ impl Scheme for FloatMultScheme { } fn num_children(&self) -> usize { - 2 + 1 } fn expected_compression_ratio( @@ -92,7 +92,7 @@ impl Scheme for FloatMultScheme { )?; let candidate = FloatMult::try_new( compressed_primary, - encoded.secondary().clone(), + encoded.secondary().cloned(), sample.ptype(), base, )?; @@ -132,7 +132,7 @@ impl Scheme for FloatMultScheme { compressor.compress_child(encoded.primary(), &compress_ctx, self.id(), 0, exec_ctx)?; Ok(FloatMult::try_new( compressed_primary, - encoded.secondary().clone(), + encoded.secondary().cloned(), primitive.ptype(), base, )? From 607508b2808db6038ec1883a2be31f7d2d4d2aa4 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Tue, 18 Aug 2026 18:16:15 -0400 Subject: [PATCH 05/78] refactor: focus native float compression prototypes Signed-off-by: Will Manning --- Cargo.lock | 28 +- Cargo.toml | 4 +- docs/plans/native-pcodec.md | 480 +--- .../Cargo.toml | 4 +- .../src/block_residual_array.rs | 7 +- .../src/codec.rs} | 248 +- encodings/block-residual/src/lib.rs | 21 + .../src/ordered_float_array.rs | 0 encodings/float-quant/src/float_mult.rs | 1163 -------- encodings/float-quant/src/lib.rs | 3 - encodings/float-quant/src/rules.rs | 4 - encodings/float-quant/src/slice.rs | 20 - .../range-entropy/examples/encode_probe.rs | 99 - encodings/range-entropy/src/array.rs | 857 ------ encodings/range-entropy/src/bit_split.rs | 264 -- encodings/range-entropy/src/codec.rs | 2337 ----------------- encodings/range-entropy/src/lib.rs | 33 - encodings/range-entropy/src/rules.rs | 10 - vortex-btrblocks/Cargo.toml | 10 +- .../examples/float_quant_dataset.rs | 1371 ---------- vortex-btrblocks/examples/pco_probe.rs | 36 - .../examples/range_entropy_throughput.rs | 994 ------- vortex-btrblocks/src/builder.rs | 2 - .../src/schemes/float/float_mult.rs | 251 -- vortex-btrblocks/src/schemes/float/mod.rs | 2 - .../schemes/float/ordered_block_residual.rs | 6 +- .../schemes/float/scheme_selection_tests.rs | 17 +- vortex-btrblocks/src/schemes/float/tests.rs | 13 - vortex-btrblocks/src/schemes/mod.rs | 1 - vortex-btrblocks/src/schemes/range_entropy.rs | 62 - vortex-file/Cargo.toml | 2 +- vortex-file/src/lib.rs | 2 +- vortex/Cargo.toml | 2 +- vortex/src/editions/core/v2026_08.rs | 1 - vortex/src/editions/tests.rs | 32 +- vortex/src/lib.rs | 6 +- 36 files changed, 187 insertions(+), 8205 deletions(-) rename encodings/{range-entropy => block-residual}/Cargo.toml (86%) rename encodings/{range-entropy => block-residual}/src/block_residual_array.rs (99%) rename encodings/{range-entropy/src/patched_for.rs => block-residual/src/codec.rs} (69%) create mode 100644 encodings/block-residual/src/lib.rs rename encodings/{range-entropy => block-residual}/src/ordered_float_array.rs (100%) delete mode 100644 encodings/float-quant/src/float_mult.rs delete mode 100644 encodings/range-entropy/examples/encode_probe.rs delete mode 100644 encodings/range-entropy/src/array.rs delete mode 100644 encodings/range-entropy/src/bit_split.rs delete mode 100644 encodings/range-entropy/src/codec.rs delete mode 100644 encodings/range-entropy/src/lib.rs delete mode 100644 encodings/range-entropy/src/rules.rs delete mode 100644 vortex-btrblocks/examples/float_quant_dataset.rs delete mode 100644 vortex-btrblocks/examples/range_entropy_throughput.rs delete mode 100644 vortex-btrblocks/src/schemes/float/float_mult.rs delete mode 100644 vortex-btrblocks/src/schemes/range_entropy.rs diff --git a/Cargo.lock b/Cargo.lock index 8126733f397..e46d9497ab1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9535,6 +9535,7 @@ dependencies = [ "vortex-alp", "vortex-array", "vortex-arrow", + "vortex-block-residual", "vortex-btrblocks", "vortex-buffer", "vortex-bytebool", @@ -9555,7 +9556,6 @@ dependencies = [ "vortex-metrics", "vortex-pco", "vortex-proto", - "vortex-range-entropy", "vortex-runend", "vortex-scan", "vortex-sequence", @@ -9749,6 +9749,17 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "vortex-block-residual" +version = "0.1.0" +dependencies = [ + "fastlanes", + "vortex-array", + "vortex-buffer", + "vortex-error", + "vortex-session", +] + [[package]] name = "vortex-btrblocks" version = "0.1.0" @@ -9769,6 +9780,7 @@ dependencies = [ "vortex-alp", "vortex-array", "vortex-arrow", + "vortex-block-residual", "vortex-buffer", "vortex-compressor", "vortex-datetime-parts", @@ -9780,7 +9792,6 @@ dependencies = [ "vortex-mask", "vortex-onpair", "vortex-pco", - "vortex-range-entropy", "vortex-runend", "vortex-sequence", "vortex-session", @@ -10159,6 +10170,7 @@ dependencies = [ "url", "vortex-alp", "vortex-array", + "vortex-block-residual", "vortex-btrblocks", "vortex-buffer", "vortex-bytebool", @@ -10176,7 +10188,6 @@ dependencies = [ "vortex-metrics", "vortex-onpair", "vortex-pco", - "vortex-range-entropy", "vortex-runend", "vortex-scan", "vortex-sequence", @@ -10529,17 +10540,6 @@ dependencies = [ "vortex-python-abi", ] -[[package]] -name = "vortex-range-entropy" -version = "0.1.0" -dependencies = [ - "fastlanes", - "vortex-array", - "vortex-buffer", - "vortex-error", - "vortex-session", -] - [[package]] name = "vortex-row" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index f715d9b6d7e..1192d89e1a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,7 +62,7 @@ members = [ "encodings/bytebool", "encodings/parquet-variant", "encodings/onpair", - "encodings/range-entropy", + "encodings/block-residual", "encodings/float-quant", # Benchmarks "benchmarks/bench-support", @@ -323,7 +323,7 @@ vortex-metrics = { version = "0.1.0", path = "./vortex-metrics", default-feature vortex-onpair = { version = "0.1.0", path = "./encodings/onpair", default-features = false } vortex-parquet-variant = { version = "0.1.0", path = "./encodings/parquet-variant" } vortex-pco = { version = "0.1.0", path = "./encodings/pco", default-features = false } -vortex-range-entropy = { version = "0.1.0", path = "./encodings/range-entropy", default-features = false } +vortex-block-residual = { version = "0.1.0", path = "./encodings/block-residual", default-features = false } vortex-float-quant = { version = "0.1.0", path = "./encodings/float-quant", default-features = false } vortex-proto = { version = "0.1.0", path = "./vortex-proto", default-features = false } vortex-row = { version = "0.1.0", path = "./vortex-row", default-features = false } diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 371047104f4..f64ad97c17e 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -4,80 +4,59 @@ Improve default numeric compression where ALP and ALP-RD lose to Pco. -Keep native Vortex arrays, bounded metadata, child arrays, and fast canonical decode. +Keep native Vortex arrays, bounded metadata, fast canonical decode, and bounded random access. Pco remains a compression oracle. The new arrays do not preserve Pco byte compatibility. -## Decision +## Current decision -Add `FloatQuantArray` and `FloatQuantScheme` to default BtrBlocks compression for `f64` arrays. +Focus the production work on these encodings: -Add `FloatMultArray` and a restricted `FloatMultScheme` to default BtrBlocks compression for `f32` arrays. +- `OrderedFloatArray`. +- `BlockResidualArray` with one reference per 1,024-value block. +- `FloatQuantArray`. -Add `OrderedFloatArray`, `BlockResidualArray`, and their BtrBlocks scheme to the default compressor. +Keep `FloatQuantScheme` and `OrderedBlockResidualScheme` as BtrBlocks candidates. -Keep `BitSplitArray` as a prototype. +Remove `FloatMultArray` and `FloatMultScheme` from the focused branch. -Keep `RangeEntropyArray` and `RangeEntropyScheme` experimental. Their current full-decode and compression costs fail the throughput gate. +Remove `RangeEntropyArray`, `RangeEntropyScheme`, and `BitSplitCodec` from the focused branch. -Keep FastLanes Delta unchanged. Its current lane-stride transform does not replace Pco adjacent Delta on the measured data. +The `wm/pcodec-entropy-experiments` branch preserves the complete entropy and bit-split prototypes. -Do not add Delta-of-delta, Delta with lookback, convolution Delta, or Pco byte compatibility. +Do not add adjacent Delta, Delta-of-delta, Delta with lookback, or convolution Delta. -## Implemented arrays - -### OrderedFloatArray +## OrderedFloatArray `OrderedFloatArray` maps IEEE float bits to unsigned integers with the same order. The transform preserves every bit pattern. It also preserves nulls and signed zero values. -The array stores one unsigned child. An empty metadata record identifies the transform. +The array stores one unsigned child. Empty metadata identifies the transform. The array supports canonical decode, scalar access, slice reduction, serialization, and validation. -### BlockResidualArray +## BlockResidualArray `BlockResidualArray` divides unsigned integers into independent blocks of 1,024 values. Each block stores one minimum value. Packed residuals store the difference from that minimum. -Rare wide residuals use sorted positions and packed high bits. Scalar access uses one packed read and one binary search. - -Nine child arrays store references, widths, offsets, packed words, patch positions, and patch high bits. - -The fixed metadata record stores only lengths and slice bounds. Variable tables do not use metadata. - -The array supports canonical decode, scalar access, slice reduction, serialization, and validation. - -The `OrderedFloat(BlockResidual)` execute kernel fuses residual decode with the inverse float transform. - -### RangeEntropyArray - -`RangeEntropyArray` stores entropy-coded range-bin identifiers and fixed-width offsets. +Rare wide residuals use sorted positions and packed high bits. -The logical dtype remains the source primitive dtype. The codec maps each value to an ordered unsigned latent. +Scalar access uses one packed read and one binary search over the patch positions. -The fixed metadata record stores these fields: +Child arrays store references, widths, offsets, packed words, patch positions, and patch high bits. -- Physical type. -- ANS table log. -- Restart block length. -- Logical slice bounds. +Fixed metadata stores only lengths and slice bounds. Variable tables remain in child arrays. -Child arrays store these variable tables: - -- Bin lower bounds. -- Bin offset widths. -- Quantized ANS weights. -- Restart block byte offsets. -- Validity. +The array supports canonical decode, scalar access, slice reduction, serialization, and validation. -One payload buffer stores the ANS stream and packed offsets. Independent restart blocks bound scalar access work. +The `OrderedFloat(BlockResidual)` execute kernel combines residual decode with the inverse float transform. -The array supports canonical decode, slice reduction, scalar access, serialization, and validation. +The production codec uses one reference per block. The multi-reference prototype did not justify its encode and decode costs. -### FloatQuantArray +## FloatQuantArray `FloatQuantArray` splits each float into a primary quantum and a secondary adjustment. @@ -85,104 +64,47 @@ Bounded metadata stores the source type and split width. Two child arrays store The BtrBlocks scheme compresses both children through the normal cascade. -A common path uses `FloatQuant(FoR(BitPacked), Constant)`. This path targets `f32` values stored in `f64` columns. +A common path uses `FloatQuant(FoR(BitPacked), Constant)` for `f32` values stored in `f64` columns. The array supports exact IEEE bit-pattern round trips, nulls, slices, scalar access, and serialization. -The automatic FloatQuant scheme accepts only `f64`. Native `f32` columns remain eligible for the separate FloatMult scheme. - -### FloatMultArray - -`FloatMultArray` represents each float as an integer multiple of one base plus a signed ULP adjustment. - -The array stores the base in at most nine metadata bytes. One required unsigned child stores the multiples. - -An optional final child stores the adjustments. An absent child means that every adjustment is zero. - -The child count identifies this mode. The serialized metadata contains no separate flag. - -The default scheme accepts only `f32` inputs whose sampled adjustments all represent zero. - -This restriction lets the encoder omit the adjustment vector. It also lets the decoder skip adjustment materialization. - -The array retains general `f32` and `f64` support for experiments with nonconstant adjustments. - -The array supports exact IEEE bit-pattern round trips, nulls, slices, scalar access, and serialization. +The automatic scheme accepts only `f64`. Native `f32` columns remain with ALP, ALP-RD, and existing schemes. ## Selection policy -FloatQuant uses normal sample comparison against ALP, ALP-RD, dictionary, sparse, and RLE schemes. - -The selected encoding can displace ALP, ALP-RD, dictionary, sparse, or RLE. - -The selector compares estimated size after an encoding-specific speed penalty. It does not prefer an encoding identity. - -The full FloatQuant analysis runs only after the sample selects the scheme. Unselected `f64` columns pay only the sample cost. - -Callers can disable the default scheme with this builder configuration: - -```rust -let compressor = BtrBlocksCompressorBuilder::default() - .exclude_schemes([FloatQuantScheme.id()]) - .build(); -``` - -BlockResidual uses eight locality-preserving sample blocks. The ordinary BtrBlocks sample destroys the block-local float structure. - -The scheme requires a 5 percent compression ratio. It also requires a 2 percent win over the current best scheme. - -Callers can disable this scheme with this builder configuration: - -```rust -let compressor = BtrBlocksCompressorBuilder::default() - .exclude_schemes([OrderedBlockResidualScheme.id()]) - .build(); -``` +`FloatQuantScheme` uses normal sample comparison against ALP, ALP-RD, dictionary, sparse, and RLE schemes. -RangeEntropy also uses sample comparison. It remains outside `ALL_SCHEMES`. +A strong constant split can use a direct `FoR(BitPacked)` primary and a constant secondary. -FloatMult tests decimal bases, one exact binary divisor, and one approximate common divisor. +Other splits compress both children through the normal integer cascade. -The default scheme requires these sample properties: +`OrderedBlockResidualScheme` uses eight locality-preserving sample blocks. -- The source type is `f32`. -- Every adjustment represents zero. -- The primary span uses at most 17 bits. -- The estimated compression ratio is at least 1.5. -- The estimate beats the incumbent by at least 5 percent. +The ordinary BtrBlocks sample does not preserve the block-local float structure. -The selector applies a 15 percent penalty to the FloatMult ratio before comparison. +The residual scheme requires a 1.05 compression ratio and a 1.02 win over the incumbent. -This penalty represents its measured encode cost. A sufficient size win can still displace ALP or another incumbent. +Both schemes remain eligible to displace ALP or ALP-RD when their adjusted size scores win. -The first implementation compresses the primary child through the normal integer cascade. +## Performance requirements -A fused scheme can select a fixed integer layout for both latent children. Its internal layout does not need to consume the generic cascade depth. +A default candidate must meet these limits: -This fused design remains a benchmark candidate. It can trade more internal work for a better size and speed point. - -Callers can disable this scheme with this builder configuration: - -```rust -let compressor = BtrBlocksCompressorBuilder::default() - .exclude_schemes([FloatMultScheme.id()]) - .build(); -``` - -## Performance gate +- Compression throughput can regress by at most 20 percent on a selected column. +- Full-decode throughput can regress by at most 20 percent. +- Scalar access must remain bounded and competitive with the displaced encoding. +- The selected candidate must reduce size materially. +- Rejected candidates must add little analysis cost. -The default candidate must meet these limits: +Use at least two million logical rows for throughput and scalar-access decisions. -- Compression throughput can regress by at most 20 percent. -- Full-decode throughput can regress by at most 20 percent. -- A selected candidate must reduce size materially. -- The result must approach Compact size on its target data. +Exclude source-array construction and storage input from codec throughput measurements. -The release benchmark excludes source-array construction and storage I/O. +## Evidence for the retained candidates -## Synthetic target result +### FloatQuant on widened f32 values -The widened-f32 input contains 262,144 arbitrary `f32` values stored as `f64`. +The input contains 262,144 arbitrary `f32` values stored in an `f64` column. | Configuration | Bytes | Compression MB/s | Decode MB/s | | --- | ---: | ---: | ---: | @@ -190,135 +112,15 @@ The widened-f32 input contains 262,144 arbitrary `f32` values stored as `f64`. | Default with FloatQuant | 688,130 | 635.0 | 13,636.4 | | Compact | 658,758 | 336.8 | 5,117.1 | -FloatQuant reduces size by 58.0 percent. It remains 4.5 percent larger than Compact. - -Compression throughput regresses by 16.7 percent. Decode throughput improves by 5.4 percent. - -The selected tree is `FloatQuant(FoR(BitPacked), Constant)`. - -### FloatQuant and RangeEntropy composition - -The stacked configuration leaves the widened-f32 tree and byte size unchanged. - -RangeEntropy loses selection for both FloatQuant children. The primary already uses FoR and bitpacking, while the secondary is constant zero. - -A repeated synthetic run reduced compression throughput by 4.9 percent because RangeEntropy still incurred sample work. - -No measured dataset selected RangeEntropy beneath FloatQuant. RangeEntropy sometimes wins as a root alternative when FloatQuant loses. - -## Pco paper data - -The benchmark reads source columns before timing and limits Parquet inputs to two million rows. - -The aggregate values below cover every supported numeric column in each downloaded input. - -### Size - -| Dataset | Without new float schemes | Default | Default plus RangeEntropy | Compact | Pco Auto | -| --- | ---: | ---: | ---: | ---: | ---: | -| California Housing | 339,682 | 339,682 | 319,705 | 230,197 | 242,414 | -| April 2023 HVFHV Taxi | 28,540,793 | 28,540,793 | 23,331,922 | 21,127,765 | 21,994,290 | -| Air Quality | 6,135,414 | 6,135,414 | 5,149,775 | 2,311,526 | 2,303,523 | - -FloatQuant and OrderedFloat with BlockResidual make no selection on these inputs. Their size and decode paths remain unchanged. - -RangeEntropy reduces Taxi size by 18.3 percent and Air Quality size by 16.1 percent. - -RangeEntropy remains 10.4 percent larger than Compact on Taxi. It remains 122.8 percent larger on Air Quality. - -### Compression throughput - -| Dataset | Without new float schemes | Without BlockResidual | Default | Default plus RangeEntropy | Compact | Pco Auto | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| California Housing | 205.8 MB/s | 208.5 MB/s | 210.0 MB/s | 114.8 MB/s | 72.1 MB/s | 206.3 MB/s | -| April 2023 HVFHV Taxi | 886.2 MB/s | 838.4 MB/s | 828.8 MB/s | 421.7 MB/s | 432.3 MB/s | 574.0 MB/s | -| Air Quality | 548.2 MB/s | 549.1 MB/s | 550.5 MB/s | 318.3 MB/s | 303.7 MB/s | 415.2 MB/s | - -The BlockResidual locality probe changes compression throughput by 1.2 percent or less on these inputs. - -RangeEntropy reduces compression throughput by 42 to 49 percent. This result fails the gate. - -### Decode throughput - -| Dataset | Without new float schemes | Without BlockResidual | Default | Default plus RangeEntropy | Compact | Pco Auto | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| California Housing | 12,392.7 MB/s | 12,531.9 MB/s | 12,558.4 MB/s | 2,221.6 MB/s | 2,336.9 MB/s | 2,413.8 MB/s | -| April 2023 HVFHV Taxi | 23,963.4 MB/s | 23,879.0 MB/s | 24,200.2 MB/s | 3,809.3 MB/s | 4,825.0 MB/s | 3,644.8 MB/s | -| Air Quality | 25,013.7 MB/s | 24,807.0 MB/s | 25,371.2 MB/s | 1,471.8 MB/s | 2,088.0 MB/s | 2,092.8 MB/s | - -RangeEntropy decode is 5.7 to 17.2 times slower than the current lightweight default. - -RangeEntropy beats Pco Auto decode on Taxi. It loses to Pco Auto decode on Housing and Air Quality. - -The native implementation does not meet the default full-decode requirement. +FloatQuant reduced size by 58.0 percent. It remained 4.5 percent larger than Compact. -## Pco encode cost +Compression throughput decreased by 16.7 percent. Decode throughput increased by 5.4 percent. -The probe separates Pco into a prepare stage and an emit stage. +The selected tree was `FloatQuant(FoR(BitPacked), Constant)`. -The prepare stage selects the mode and Delta transform. It also builds the final bins and ANS model. +This result needs a larger throughput test. The compression-ratio result remains strong. -The emit stage finds each value's bin, creates offsets, runs reverse ANS, and writes the bit stream. - -The forced test supplies the mode and Delta transform that `Auto` selected. Both paths produce identical bytes. - -| Dataset | Auto MB/s | Forced MB/s | Auto time used for search | -| --- | ---: | ---: | ---: | -| Gaussian | 647.2 | 872.0 | 25.8 percent | -| Lognormal | 686.5 | 860.3 | 20.2 percent | -| Decimal | 604.5 | 747.3 | 19.1 percent | -| Widened f32 | 604.5 | 747.5 | 19.1 percent | -| Random walk | 572.3 | 702.6 | 18.5 percent | - -Mode and Delta search uses 18 to 26 percent of Pco's default encode time on these inputs. - -The forced path still trains the full bin model. This model and final emission use most of the time. - -The complete prepare stage uses 58 to 71 percent of automatic encode time. Final emission uses the remaining 29 to 42 percent. - -Preparation includes the selected transform, Delta application, histograms, bin optimization, weight quantization, and ANS model construction. - -Compression level 4 keeps most of level 8's size benefit with much higher throughput. - -| Dataset | Level 0 bytes | Level 0 MB/s | Level 4 bytes | Level 4 MB/s | Level 8 bytes | Level 8 MB/s | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| Gaussian | 1,671,181 | 5,195.8 | 1,618,359 | 1,226.0 | 1,609,324 | 872.0 | -| Random walk | 2,097,165 | 3,322.7 | 1,611,991 | 956.8 | 1,551,142 | 702.6 | - -Level 0 proves that Pco's mode transforms do not require a slow encoder. - -The expensive part is the richer bin model. Random-walk data still needs that model or an adjacent Delta transform for good size. - -## Block residual prototype - -The block residual codec uses one or more references for each 1,024-value block. - -The single-reference form stores the block minimum. It subtracts that minimum from every ordered integer. - -FastLanes stores the low `w` residual bits for every value. A width histogram selects `w` without repeated residual scans. - -Patches store sorted 16-bit positions and the remaining high bits. Scalar access uses one packed read and one binary search. - -The multi-reference form also tests two and four references. It stores a one-bit or two-bit reference selector for each value. - -| Dataset | Form | Bytes | Encode MB/s | Decode MB/s | Scalar access ns | -| --- | --- | ---: | ---: | ---: | ---: | -| Gaussian | One reference | 1,643,520 | 4,194.7 | 15,942.8 | 3.99 | -| Gaussian | Up to four references | 1,643,520 | 1,399.9 | 15,983.3 | 4.05 | -| Random walk | One reference | 1,663,307 | 3,752.7 | 15,582.5 | 7.38 | -| Random walk | Up to four references | 1,660,912 | 1,267.9 | 14,716.9 | 7.52 | -| Four clusters | One reference | 2,102,272 | 4,667.7 | 14,492.3 | 3.01 | -| Four clusters | Up to four references | 1,978,898 | 1,151.3 | 9,348.4 | 17.21 | - -Multiple references save 0.14 percent on random-walk data and 5.9 percent on four-cluster data. - -The four-cluster result remains 17.9 percent larger than ALP-RD. It remains 44.2 percent larger than RangeEntropy. - -One local reference plus patches is the current block-residual winner. It is not the current winner for all float distributions. - -This result narrows the role of multiple references. A prefix split or range tag still handles distinct float clusters better. - -### Default candidate on random-walk floats +### OrderedFloat with BlockResidual on random walks | Configuration | Bytes | Encode MB/s | Decode MB/s | Scalar access ns | | --- | ---: | ---: | ---: | ---: | @@ -326,181 +128,99 @@ This result narrows the role of multiple references. A prefix split or range tag | Default with the scheme | 1,663,831 | 635.6 | 17,355.8 | 129.06 | | Compact Pco | 1,551,142 | 296.2 | 3,777.2 | Not measured | -The default scheme saves 10.2 percent against ALP-RD. It remains 7.3 percent larger than Pco. - -Decode throughput improves by 39.0 percent. Random access latency improves by 20.0 percent. +The residual scheme saved 10.2 percent against ALP-RD. It remained 7.3 percent larger than Pco. -Compression throughput regresses by 10.9 percent on the selected column. +Decode throughput increased by 39.0 percent. Random access latency decreased by 20.0 percent. -The direct decoder reads child buffers without copies. It also fuses the inverse ordered-float transform. +Compression throughput decreased by 10.9 percent on the selected column. The selector rejected the scheme on Gaussian, lognormal, decimal, widened-f32, and four-cluster inputs. -The locality probe reduced compression throughput by 0.5 to 2.2 percent on those unselected inputs. - -## BitSplit prototype - -The `BitSplit` prototype uses one prefix dictionary per 1,024-value block. Fixed-width suffixes store the remaining bits. - -The encoder sorts each block once. Adjacent XOR widths evaluate every split point without repeated value scans. - -Scalar access uses two packed reads and one prefix lookup. No patch search is necessary. - -| Dataset | Bytes | Encode MB/s | Decode MB/s | Scalar access ns | -| --- | ---: | ---: | ---: | ---: | -| Gaussian | 1,648,800 | 1,046.7 | 5,989.0 | 5.33 | -| Random walk | 1,664,696 | 1,057.8 | 6,260.2 | 9.85 | -| Four clusters | 1,702,464 | 900.3 | 5,898.5 | 6.19 | - -The four-cluster result is 1.4 percent larger than ALP-RD. It is 14.0 percent smaller than the multiple-reference prototype. - -`BitSplit` handles separated clusters better than residual references. Full decode remains much slower than the current ALP-RD path. - -## FloatMult composition - -Pco selects FloatMult on seven Taxi columns. The selected source columns contain 112 MB of logical values. - -| Backend for FloatMult children | Aggregate bytes | Encode MB/s | Decode MB/s | -| --- | ---: | ---: | ---: | -| Normal BtrBlocks integer compression | 22,340,941 | Not isolated | 5,079.1 | -| Block residual packing | 19,555,263 | 1,826.0 | 3,628.8 | -| BitSplit | 24,105,632 | 814.7 | 2,487.2 | -| Fixed range tags | 20,860,940 | 703.5 | 3,057.3 | -| Two-level range tags | 19,052,586 | 477.3 | 2,088.3 | -| RangeEntropy | 16,730,631 | 461.9 | 1,884.1 | -| Current default on source floats | 20,104,147 | Not isolated | Not isolated | -| Pco Auto on source floats | 15,061,783 | Not isolated | Not isolated | - -FloatMult does not create conventional integer distributions. Normal integer compression loses 11.1 percent against the current default. - -Block residual packing saves only 2.7 percent against the current default on the Taxi source columns. - -RangeEntropy saves 16.8 percent against the current default. It remains 11.1 percent larger than Pco on these columns. - -The direct decode test includes child decode, FloatMult reconstruction, and output allocation. - -The isolated encode test excludes FloatMult base search and the split from floats to latent integers. - -California Housing gives a different result. Block residual packing uses 170,645 bytes for five FloatMult columns. - -The current default uses 209,265 bytes for those source columns. Block residual packing saves 18.5 percent. - -The full block residual path decodes at 2,941.6 MB/s and encodes its existing latent integers at 937.6 MB/s. +The rejected locality probe reduced compression throughput by 0.5 to 2.2 percent. -Normal BtrBlocks children use 172,046 bytes and decode at 6,991.6 MB/s on those columns. +This result needs several larger random walks and real time-series columns. -The current `RangeEntropy` decoder is too slow for the default compressor. FloatMult therefore needs a faster entropy-like child codec. +## Removed candidates -Block residual packing is the fastest specialized encoder. Its decode path is still too slow for the default compressor. +### FloatMult -The native exact-multiple scheme uses normal integer compression for its primary child. +The useful exact-multiple form duplicated an ALP candidate that the upstream Rust search omitted. -It selects five California Housing columns. It rejects all Taxi and Air Quality columns. +The upstream ALP fix now includes equal exponent and factor pairs such as `{0,0}`. -| Dataset | Configuration | Bytes | Encode MB/s | Decode MB/s | -| --- | --- | ---: | ---: | ---: | -| California Housing | Without FloatMult | 339,682 | 200.8 | 12,298.5 | -| California Housing | Default | 309,499 | 164.5 | 12,728.7 | -| April 2023 HVFHV Taxi | Without FloatMult | 28,540,793 | 873.4 | 21,651.0 | -| April 2023 HVFHV Taxi | Default | 28,540,793 | 872.0 | 21,514.7 | -| Air Quality | Without FloatMult | 6,135,414 | 536.4 | 23,568.2 | -| Air Quality | Default | 6,135,414 | 530.4 | 23,548.0 | +The two-child form with nonzero adjustments decoded 64 percent slower than the prior default. -FloatMult reduces total Housing size by 8.9 percent. Compression throughput regresses by 16.5 percent. +Its scalar access took more than seven times longer in the large-row test. -Housing decode changes by less than measurement noise. Unselected Taxi and Air Quality results remain effectively unchanged. +Vortex will take the corrected ALP dependency after its release. This release does not block other work. -### Large-row throughput check +### RangeEntropy overlap -The source Housing file contains 20,433 complete rows. That size is too small for stable throughput and scalar-access decisions. +`RangeEntropy` and `OrderedFloat(BlockResidual)` both compress ordered unsigned latents. -The larger check repeats the source distribution to two million rows. It uses 72 MB of source floats. +`BlockResidual` models one local range per block. It uses a minimum, packed residuals, and sparse high-bit patches. -This repetition preserves the value distribution. It also preserves cardinality, so it changes some outer dictionary choices. +`RangeEntropy` models many ranges across the input. It entropy-codes range identifiers and stores fixed-width offsets inside each range. -The size results above remain the source for compression policy. The larger check tests throughput and access costs. +The entropy model can represent separated clusters better than one local reference. -| Path | Bytes | Decode MB/s, run 1 | Decode MB/s, run 2 | Scalar access ns, run 1 | Scalar access ns, run 2 | -| --- | ---: | ---: | ---: | ---: | ---: | -| Prior default, exact selected columns | 6,275,780 | 10,806.9 | 13,495.0 | 949.90 and 1,010.57 | 790.08 and 825.45 | -| FloatMult, exact selected columns | 6,270,986 | 10,967.0 | 13,483.6 | 675.10 and 666.78 | 551.01 and 545.61 | -| Prior default, nonzero adjustment column | 3,013,120 | 10,735.9 | 13,615.1 | 204.73 | 174.87 | -| FloatMult, nonzero adjustment column | 3,219,980 | 3,916.4 | 4,847.8 | 1,551.78 | 1,310.13 | +That benefit requires bin search, model construction, two encoded streams, and ANS state transitions. -The absolute throughput changed between runs. The interleaved comparisons kept the relative exact-mode result near zero. +The measured size gains did not compensate for those costs under the default writer priorities. -Exact FloatMult reduced scalar-access latency by 29 to 34 percent on both selected columns. - -The nonzero-adjustment form decoded 64 percent slower. Its scalar access was more than seven times slower. - -The full default compressor encoded at 461 to 463 MB/s. The prior default encoded at 454 to 458 MB/s. - -The large check found no default encode regression. The compact compressor encoded at 225 to 226 MB/s. - -The nonzero result supports separate selection weights for the one-child and two-child forms. - -## Why Compact still wins - -Pco does not use FloatQuant on the measured paper inputs. - -Pco selects FloatMult on five Housing columns and seven Taxi columns. - -Pco selects adjacent Delta on three Housing columns and thirteen Air Quality columns. - -These transforms explain most of the remaining Compact size gap. - -The restricted FloatMult scheme does not capture Pco's Taxi gains. Those gains require useful nonconstant adjustments and a stronger child codec. - -Pco adjacent Delta also differs from current FastLanes lane-stride Delta. - -The FastLanes Delta diagnostic selected one Air Quality column. It reduced aggregate size by 2.6 percent. - -That diagnostic reduced compression throughput by 26 percent and decode throughput by 36 percent. +| Dataset | Default bytes | RangeEntropy bytes | Compact bytes | +| --- | ---: | ---: | ---: | +| California Housing | 339,682 | 319,705 | 230,197 | +| April 2023 HVFHV Taxi | 28,540,793 | 23,331,922 | 21,127,765 | +| Air Quality | 6,135,414 | 5,149,775 | 2,311,526 | -An adjacent-Delta array requires a separate design decision. This work does not add it. +RangeEntropy reduced Taxi size by 18.3 percent and Air Quality size by 16.1 percent. -## Dataset status +It remained larger than Compact on every measured dataset. -The completed matrix includes these paper inputs: +RangeEntropy reduced compression throughput by 42 to 49 percent. -- Air Quality. -- California Housing. -- April 2023 high-volume for-hire Taxi. +Its decode path was 5.7 to 17.2 times slower than the lightweight default. -The matrix does not include CMS Payments, r/place, or Twitter. +Pco already occupies the compact, slower-encode design point with equal or better measured size. -The RangeEntropy throughput failure appears on synthetic floats, Taxi floats, Housing floats, and Air Quality integers. +RangeEntropy therefore has no current production role in the default or Compact compressor. -More datasets cannot satisfy the gate without a faster codec implementation. +### BitSplit -## Default writer integration +The prototype used one prefix dictionary per block and fixed-width suffixes. -`FloatQuantScheme` and `FloatMultScheme` belong to `ALL_SCHEMES`. +It handled separated clusters better than multiple residual references. -`BtrBlocksCompressor::default()` evaluates FloatQuant for `f64` and FloatMult for `f32`. +It remained 1.4 percent larger than ALP-RD on the four-cluster input. -The default file session registers both arrays. The August 2026 core edition permits both arrays. +Its full decode was much slower than the current ALP-RD path. -The CUDA-compatible builder excludes both schemes because no CUDA decode kernels exist. +The experimental branch retains the prototype for possible ALP-RD decomposition research. -The default writer tests write, read, and verify actual FloatQuant and FloatMult trees. +## Revalidation after the ALP release -## Current prototype scope +After the corrected ALP release, perform these steps: -Do not start RangeEntropy pushdowns before its full-decode cost improves substantially. +1. Update the workspace ALP dependency. +2. Remove any remaining FloatMult compatibility code. +3. Confirm that corrected ALP replaces the historical Housing FloatMult selections. +4. Recheck FloatQuant on widened-f32 columns. +5. Recheck OrderedFloat with BlockResidual on random walks and time-series columns. +6. Repeat throughput and scalar-access measurements with at least two million rows. +7. Compare each candidate against corrected default, Compact, and Pco Auto. +8. Measure rejected-candidate analysis cost on every dataset. +9. Add CMS Payments, r/place, and Twitter to the paper dataset matrix. -The current prototype scope includes these transforms: +Record size, encode throughput, decode throughput, and scalar-access latency for each selected column. -- Ordered float bits. -- A high-bit and low-bit split. -- FloatMult with an arbitrary common multiplier. -- Block residual packing with optional patches. +## Pull request structure -The current scope excludes these transforms: +Prepare one focused stack for `OrderedFloatArray`, `BlockResidualArray`, and `OrderedBlockResidualScheme`. -- Block-local adjacent Delta with fast random access. +Prepare a separate stack for `FloatQuantArray` and `FloatQuantScheme`. -Delta-of-delta, Delta with lookback, and convolution Delta also remain excluded. +Keep entropy, bit-split, and alternate residual models on `wm/pcodec-entropy-experiments`. ## References diff --git a/encodings/range-entropy/Cargo.toml b/encodings/block-residual/Cargo.toml similarity index 86% rename from encodings/range-entropy/Cargo.toml rename to encodings/block-residual/Cargo.toml index 60fb6151d86..42e7c06fbd5 100644 --- a/encodings/range-entropy/Cargo.toml +++ b/encodings/block-residual/Cargo.toml @@ -1,8 +1,8 @@ [package] -name = "vortex-range-entropy" +name = "vortex-block-residual" authors = { workspace = true } categories = { workspace = true } -description = "Vortex range entropy array" +description = "Vortex ordered-float and block-residual array encodings" edition = { workspace = true } homepage = { workspace = true } include = { workspace = true } diff --git a/encodings/range-entropy/src/block_residual_array.rs b/encodings/block-residual/src/block_residual_array.rs similarity index 99% rename from encodings/range-entropy/src/block_residual_array.rs rename to encodings/block-residual/src/block_residual_array.rs index 2f247e5be19..a662eee0c88 100644 --- a/encodings/range-entropy/src/block_residual_array.rs +++ b/encodings/block-residual/src/block_residual_array.rs @@ -47,9 +47,9 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use vortex_session::registry::CachedId; +use crate::BlockResidualCodec; use crate::BlockResidualParts; -use crate::PatchedFoRCodec; -use crate::patched_for::read_wide_bits; +use crate::codec::read_wide_bits; const BLOCK_LEN: usize = 1024; const METADATA_VERSION: u8 = 1; @@ -315,8 +315,7 @@ impl BlockResidual { "BlockResidual requires u64 values" ); let validity = array.validity()?; - let parts = PatchedFoRCodec::encode_single_base(array.as_slice::())? - .into_single_base_parts()?; + let parts = BlockResidualCodec::encode(array.as_slice::())?.into_parts()?; Self::try_new(parts, validity) } diff --git a/encodings/range-entropy/src/patched_for.rs b/encodings/block-residual/src/codec.rs similarity index 69% rename from encodings/range-entropy/src/patched_for.rs rename to encodings/block-residual/src/codec.rs index deecaac75e2..0570030abb0 100644 --- a/encodings/range-entropy/src/patched_for.rs +++ b/encodings/block-residual/src/codec.rs @@ -5,15 +5,14 @@ use fastlanes::BitPacking; use vortex_error::VortexResult; const CHUNK_LEN: usize = 1024; -const BASE_SAMPLE_LEN: usize = 64; const HIGH_PADDING: usize = 15; const SERIALIZED_BLOCK_METADATA_BYTES: usize = 12; -/// A prototype patched frame-of-reference codec for ordered unsigned latents. +/// Block-local residual codec for ordered unsigned latents. #[derive(Clone, Debug, PartialEq, Eq)] -pub struct PatchedFoRCodec { +pub struct BlockResidualCodec { len: usize, - blocks: Vec, + blocks: Vec, } /// Serialized children for the one-reference block residual codec. @@ -32,19 +31,17 @@ pub struct BlockResidualParts { } #[derive(Clone, Debug, PartialEq, Eq)] -struct PatchedFoRBlock { +struct BlockResidualBlock { len: u16, - bases: Vec, - cluster_width: u8, + base: u64, residual_width: u8, high_width: u8, - clusters: Vec, residuals: Vec, patch_positions: Vec, patch_highs: Vec, } -impl PatchedFoRCodec { +impl BlockResidualCodec { /// Encode ordered unsigned latents in independent 1024-value blocks. pub fn encode(values: &[u64]) -> VortexResult { let blocks = values @@ -57,20 +54,8 @@ impl PatchedFoRCodec { }) } - /// Encode with one base and a residual-width histogram per block. - pub fn encode_single_base(values: &[u64]) -> VortexResult { - let blocks = values - .chunks(CHUNK_LEN) - .map(encode_single_base_block) - .collect::>>()?; - Ok(Self { - len: values.len(), - blocks, - }) - } - - /// Convert a one-reference codec into serialized array children. - pub fn into_single_base_parts(self) -> VortexResult { + /// Convert the codec into serialized array children. + pub fn into_parts(self) -> VortexResult { let mut parts = BlockResidualParts { len: self.len, bases: Vec::with_capacity(self.blocks.len()), @@ -87,11 +72,7 @@ impl PatchedFoRCodec { parts.patch_starts.push(0); parts.high_starts.push(0); for block in self.blocks { - vortex_error::vortex_ensure!( - block.bases.len() == 1 && block.cluster_width == 0, - "block residual parts require one reference per block" - ); - parts.bases.push(block.bases[0]); + parts.bases.push(block.base); parts.residual_widths.push(block.residual_width); parts.high_widths.push(block.high_width); parts.residual_words.extend(block.residuals); @@ -110,8 +91,8 @@ impl PatchedFoRCodec { Ok(parts) } - /// Reconstruct a one-reference codec from serialized array children. - pub fn try_from_single_base_parts(parts: BlockResidualParts) -> VortexResult { + /// Reconstruct the codec from serialized array children. + pub fn try_from_parts(parts: BlockResidualParts) -> VortexResult { let block_count = parts.len.div_ceil(CHUNK_LEN); vortex_error::vortex_ensure!( parts.bases.len() == block_count @@ -178,13 +159,11 @@ impl PatchedFoRCodec { high_stop - high_start == expected_high_len, "block residual patch high payload length is invalid" ); - blocks.push(PatchedFoRBlock { + blocks.push(BlockResidualBlock { len: u16::try_from(block_len)?, - bases: vec![parts.bases[block_index]], - cluster_width: 0, + base: parts.bases[block_index], residual_width, high_width, - clusters: Vec::new(), residuals: parts.residual_words[residual_start..residual_stop].to_vec(), patch_positions: parts.patch_positions[patch_start..patch_stop].to_vec(), patch_highs: parts.patch_highs[high_start..high_stop].to_vec(), @@ -199,21 +178,9 @@ impl PatchedFoRCodec { /// Decode all values. pub fn decode(&self) -> VortexResult> { let mut values = Vec::with_capacity(self.len); - let mut clusters = [0u64; CHUNK_LEN]; let mut residuals = [0u64; CHUNK_LEN]; for block in &self.blocks { - clusters.fill(0); residuals.fill(0); - if block.cluster_width > 0 { - // SAFETY: The encoder creates one complete FastLanes chunk. - unsafe { - u64::unchecked_unpack( - usize::from(block.cluster_width), - &block.clusters, - &mut clusters, - ); - } - } if block.residual_width > 0 { // SAFETY: The encoder creates one complete FastLanes chunk. unsafe { @@ -236,18 +203,8 @@ impl PatchedFoRCodec { } let block_len = usize::from(block.len); - match block.bases.as_slice() { - [base] => { - for residual in &mut residuals[..block_len] { - *residual = residual.wrapping_add(*base); - } - } - bases => { - for (index, residual) in residuals[..block_len].iter_mut().enumerate() { - let cluster = usize::try_from(clusters[index])?; - *residual = residual.wrapping_add(bases[cluster]); - } - } + for residual in &mut residuals[..block_len] { + *residual = residual.wrapping_add(block.base); } values.extend_from_slice(&residuals[..block_len]); } @@ -257,21 +214,9 @@ impl PatchedFoRCodec { /// Decode ordered `f64` latents with a fused inverse transform. pub fn decode_ordered_f64(&self) -> VortexResult> { let mut values = Vec::with_capacity(self.len); - let mut clusters = [0_u64; CHUNK_LEN]; let mut residuals = [0_u64; CHUNK_LEN]; for block in &self.blocks { - clusters.fill(0); residuals.fill(0); - if block.cluster_width > 0 { - // SAFETY: The encoder creates one complete FastLanes chunk. - unsafe { - u64::unchecked_unpack( - usize::from(block.cluster_width), - &block.clusters, - &mut clusters, - ); - } - } if block.residual_width > 0 { // SAFETY: The encoder creates one complete FastLanes chunk. unsafe { @@ -292,18 +237,8 @@ impl PatchedFoRCodec { high_bit_position += usize::from(block.high_width); } let block_len = usize::from(block.len); - match block.bases.as_slice() { - [base] => { - for residual in &mut residuals[..block_len] { - *residual = residual.wrapping_add(*base); - } - } - bases => { - for (index, residual) in residuals[..block_len].iter_mut().enumerate() { - let cluster = usize::try_from(clusters[index])?; - *residual = residual.wrapping_add(bases[cluster]); - } - } + for residual in &mut residuals[..block_len] { + *residual = residual.wrapping_add(block.base); } values.extend(residuals[..block_len].iter().map(|&ordered| { let bits = if ordered & (1_u64 << 63) == 0 { @@ -326,18 +261,6 @@ impl PatchedFoRCodec { ); let block = &self.blocks[index / CHUNK_LEN]; let index_in_block = index % CHUNK_LEN; - let cluster = if block.cluster_width == 0 { - 0 - } else { - // SAFETY: The encoder creates one complete FastLanes chunk. - unsafe { - usize::try_from(u64::unchecked_unpack_single( - usize::from(block.cluster_width), - &block.clusters, - index_in_block, - ))? - } - }; let mut residual = if block.residual_width == 0 { 0 } else { @@ -364,7 +287,7 @@ impl PatchedFoRCodec { }; residual |= high << block.residual_width; } - Ok(block.bases[cluster].wrapping_add(residual)) + Ok(block.base.wrapping_add(residual)) } /// Return the logical value count. @@ -383,8 +306,7 @@ impl PatchedFoRCodec { .iter() .map(|block| { SERIALIZED_BLOCK_METADATA_BYTES - + block.bases.len() * size_of::() - + block.clusters.len() * size_of::() + + size_of::() + block.residuals.len() * size_of::() + block.patch_positions.len() * size_of::() + block.patch_highs.len() @@ -405,11 +327,6 @@ impl PatchedFoRCodec { .sum() } - /// Return the sum of base counts across all blocks. - pub fn total_base_count(&self) -> usize { - self.blocks.iter().map(|block| block.bases.len()).sum() - } - /// Return the sum of main residual widths across all blocks. pub fn total_residual_width(&self) -> usize { self.blocks @@ -419,7 +336,7 @@ impl PatchedFoRCodec { } } -fn encode_single_base_block(values: &[u64]) -> VortexResult { +fn encode_block(values: &[u64]) -> VortexResult { let base = values.iter().copied().min().unwrap_or(0); let mut residuals = Vec::with_capacity(CHUNK_LEN); let mut width_counts = [0usize; 65]; @@ -455,14 +372,11 @@ fn encode_single_base_block(values: &[u64]) -> VortexResult { materialize_block( values, BlockPlan { - bases: vec![base], - cluster_width: 0, + base, residual_width: best.1, high_width: best.2, - clusters: vec![0; CHUNK_LEN], residuals, patch_count: best.3, - cost_bits: best.0, }, ) } @@ -488,113 +402,15 @@ fn validate_starts( Ok(()) } -fn encode_block(values: &[u64]) -> VortexResult { - let sampled = sampled_values(values); - let mut best: Option = None; - for requested_base_count in [1, 2, 4] { - let bases = quantile_bases(&sampled, requested_base_count); - let plan = plan_block(values, bases)?; - if best - .as_ref() - .is_none_or(|current| plan.cost_bits < current.cost_bits) - { - best = Some(plan); - } - } - materialize_block( - values, - best.ok_or_else(|| vortex_error::vortex_err!("patched FoR block has no plan"))?, - ) -} - -fn sampled_values(values: &[u64]) -> Vec { - if values.len() <= BASE_SAMPLE_LEN { - let mut sampled = values.to_vec(); - sampled.sort_unstable(); - return sampled; - } - - let mut sampled = (0..BASE_SAMPLE_LEN) - .map(|index| values[index * values.len() / BASE_SAMPLE_LEN]) - .collect::>(); - sampled.push(values.iter().copied().min().unwrap_or(0)); - sampled.sort_unstable(); - sampled -} - struct BlockPlan { - bases: Vec, - cluster_width: u8, + base: u64, residual_width: u8, high_width: u8, - clusters: Vec, residuals: Vec, patch_count: usize, - cost_bits: usize, -} - -fn quantile_bases(sorted: &[u64], requested_count: usize) -> Vec { - let mut bases = (0..requested_count) - .map(|index| sorted[index * sorted.len() / requested_count]) - .collect::>(); - bases.dedup(); - bases -} - -fn plan_block(values: &[u64], bases: Vec) -> VortexResult { - let cluster_width = bit_width(u64::try_from(bases.len() - 1)?); - let mut clusters = Vec::with_capacity(CHUNK_LEN); - let mut residuals = Vec::with_capacity(CHUNK_LEN); - let mut width_counts = [0usize; 65]; - let mut maximum_width = 0u8; - for &value in values { - let cluster = bases - .partition_point(|&base| base <= value) - .saturating_sub(1); - let residual = value - bases[cluster]; - clusters.push(u64::try_from(cluster)?); - residuals.push(residual); - let width = bit_width(residual); - width_counts[usize::from(width)] += 1; - maximum_width = maximum_width.max(width); - } - clusters.resize(CHUNK_LEN, 0); - residuals.resize(CHUNK_LEN, 0); - - let mut patch_count = values.len(); - let mut best = (usize::MAX, maximum_width, 0u8, patch_count); - for residual_width in 0..=maximum_width { - patch_count -= width_counts[usize::from(residual_width)]; - let high_width = if patch_count == 0 { - 0 - } else { - maximum_width - residual_width - }; - let cost_bits = usize::from(cluster_width) * CHUNK_LEN - + usize::from(residual_width) * CHUNK_LEN - + patch_count * (u16::BITS as usize + usize::from(high_width)) - + bases.len() * u64::BITS as usize - + SERIALIZED_BLOCK_METADATA_BYTES * 8 - + usize::from(patch_count > 0) * HIGH_PADDING * 8; - if cost_bits < best.0 { - best = (cost_bits, residual_width, high_width, patch_count); - } - } - - Ok(BlockPlan { - bases, - cluster_width, - residual_width: best.1, - high_width: best.2, - clusters, - residuals, - patch_count: best.3, - cost_bits: best.0, - }) } -fn materialize_block(values: &[u64], plan: BlockPlan) -> VortexResult { - let clusters = fast_pack(&plan.clusters, plan.cluster_width); +fn materialize_block(values: &[u64], plan: BlockPlan) -> VortexResult { let residual_mask = low_mask(plan.residual_width); let low_residuals = plan .residuals @@ -621,13 +437,11 @@ fn materialize_block(values: &[u64], plan: BlockPlan) -> VortexResult VortexResult<()> { @@ -738,7 +552,7 @@ mod tests { _ => 42, }) .collect::>(); - let codec = PatchedFoRCodec::encode(&values)?; + let codec = BlockResidualCodec::encode(&values)?; assert_eq!(codec.decode()?, values); for (index, &value) in values.iter().enumerate() { assert_eq!(codec.scalar_at(index)?, value); @@ -750,22 +564,22 @@ mod tests { #[test] fn constant_roundtrip() -> VortexResult<()> { let values = vec![42; 10_000]; - let codec = PatchedFoRCodec::encode(&values)?; + let codec = BlockResidualCodec::encode(&values)?; assert_eq!(codec.decode()?, values); assert_eq!(codec.scalar_at(4_321)?, 42); Ok(()) } #[test] - fn single_base_parts_roundtrip() -> VortexResult<()> { + fn parts_roundtrip() -> VortexResult<()> { let values = (0..4_099) .map(|index| { let value = u64::try_from(index)?; Ok(1_000_000_u64.wrapping_add(value * value)) }) .collect::>>()?; - let parts = PatchedFoRCodec::encode_single_base(&values)?.into_single_base_parts()?; - let codec = PatchedFoRCodec::try_from_single_base_parts(parts)?; + let parts = BlockResidualCodec::encode(&values)?.into_parts()?; + let codec = BlockResidualCodec::try_from_parts(parts)?; assert_eq!(codec.decode()?, values); Ok(()) } diff --git a/encodings/block-residual/src/lib.rs b/encodings/block-residual/src/lib.rs new file mode 100644 index 00000000000..f1d0b7b3070 --- /dev/null +++ b/encodings/block-residual/src/lib.rs @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Ordered-float and block-residual array encodings. + +mod block_residual_array; +mod codec; +mod ordered_float_array; + +pub use block_residual_array::*; +pub use codec::BlockResidualCodec; +pub use codec::BlockResidualParts; +pub use ordered_float_array::*; +use vortex_array::session::ArraySessionExt; +use vortex_session::VortexSession; + +/// Register the ordered-float and block-residual encodings in one session. +pub fn initialize(session: &VortexSession) { + session.arrays().register(BlockResidual); + session.arrays().register(OrderedFloat); +} diff --git a/encodings/range-entropy/src/ordered_float_array.rs b/encodings/block-residual/src/ordered_float_array.rs similarity index 100% rename from encodings/range-entropy/src/ordered_float_array.rs rename to encodings/block-residual/src/ordered_float_array.rs diff --git a/encodings/float-quant/src/float_mult.rs b/encodings/float-quant/src/float_mult.rs deleted file mode 100644 index a8b171d5907..00000000000 --- a/encodings/float-quant/src/float_mult.rs +++ /dev/null @@ -1,1163 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::fmt::Display; -use std::fmt::Formatter; -use std::hash::Hash; -use std::hash::Hasher; - -use vortex_array::Array; -use vortex_array::ArrayEq; -use vortex_array::ArrayHash; -use vortex_array::ArrayId; -use vortex_array::ArrayParts; -use vortex_array::ArrayRef; -use vortex_array::ArrayView; -use vortex_array::EqMode; -use vortex_array::ExecutionCtx; -use vortex_array::ExecutionResult; -use vortex_array::IntoArray; -use vortex_array::TypedArrayRef; -use vortex_array::array_slots; -use vortex_array::arrays::Primitive; -use vortex_array::arrays::PrimitiveArray; -use vortex_array::buffer::BufferHandle; -use vortex_array::dtype::DType; -use vortex_array::dtype::Nullability::NonNullable; -use vortex_array::dtype::PType; -use vortex_array::scalar::Scalar; -use vortex_array::serde::ArrayChildren; -use vortex_array::vtable::OperationsVTable; -use vortex_array::vtable::VTable; -use vortex_array::vtable::ValidityChild; -use vortex_array::vtable::ValidityVTableFromChild; -use vortex_buffer::Buffer; -use vortex_error::VortexExpect; -use vortex_error::VortexResult; -use vortex_error::vortex_bail; -use vortex_error::vortex_ensure; -use vortex_error::vortex_panic; -use vortex_session::VortexSession; -use vortex_session::registry::CachedId; -use vortex_utils::aliases::hash_set::HashSet; - -use crate::rules::FLOAT_MULT_RULES; - -const METADATA_VERSION: u8 = 1; -const F32_METADATA_LEN: usize = 5; -const F64_METADATA_LEN: usize = 9; -const MIN_ANALYSIS_VALUES: usize = 32; - -/// A lossless float split into approximate multiples and ULP adjustments. -pub type FloatMultArray = Array; - -#[array_slots(FloatMult)] -pub struct FloatMultSlots { - /// Signed integer multiples encoded in float order. - #[slot(0)] - pub primary: ArrayRef, - /// Signed ULP adjustments with zero moved to the unsigned midpoint. - #[slot(1)] - pub secondary: Option, -} - -#[derive(Clone, Debug)] -pub struct FloatMultData { - base_bits: u64, -} - -impl Display for FloatMultData { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "base_bits: {}", self.base_bits) - } -} - -impl ArrayHash for FloatMultData { - fn array_hash(&self, state: &mut H, _accuracy: EqMode) { - self.base_bits.hash(state); - } -} - -impl ArrayEq for FloatMultData { - fn array_eq(&self, other: &Self, _accuracy: EqMode) -> bool { - self.base_bits == other.base_bits - } -} - -#[derive(Clone, Debug)] -pub struct FloatMult; - -impl VTable for FloatMult { - type TypedArrayData = FloatMultData; - type OperationsVTable = Self; - type ValidityVTable = ValidityVTableFromChild; - - fn id(&self) -> ArrayId { - static ID: CachedId = CachedId::new("vortex.float_mult"); - *ID - } - - fn validate( - &self, - data: &Self::TypedArrayData, - dtype: &DType, - len: usize, - slots: &[Option], - ) -> VortexResult<()> { - let ptype = PType::try_from(dtype)?; - let latent_ptype = latent_ptype(ptype)?; - let base = data.base(ptype)?; - vortex_ensure!( - base.is_finite() && base.abs() > 0.0, - "FloatMult base must be finite and nonzero" - ); - - let slots = FloatMultSlotsView::from_slots(slots); - let expected_primary = DType::Primitive(latent_ptype, dtype.nullability()); - vortex_ensure!( - slots.primary.dtype() == &expected_primary, - "expected primary dtype {expected_primary}, got {}", - slots.primary.dtype() - ); - vortex_ensure!( - slots.primary.len() == len, - "FloatMult child length differs from {len}" - ); - if let Some(secondary) = slots.secondary { - let expected_secondary = DType::Primitive(latent_ptype, NonNullable); - vortex_ensure!( - secondary.dtype() == &expected_secondary, - "expected secondary dtype {expected_secondary}, got {}", - secondary.dtype() - ); - vortex_ensure!( - secondary.len() == len, - "FloatMult child length differs from {len}" - ); - } - Ok(()) - } - - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { - 0 - } - - fn buffer(_array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { - vortex_panic!("FloatMultArray buffer index {idx} out of bounds") - } - - fn buffer_name(_array: ArrayView<'_, Self>, idx: usize) -> Option { - vortex_panic!("FloatMultArray buffer_name index {idx} out of bounds") - } - - fn with_buffers( - &self, - array: ArrayView<'_, Self>, - buffers: &[BufferHandle], - ) -> VortexResult> { - vortex_array::vtable::with_empty_buffers(self, array, buffers) - } - - fn serialize( - array: ArrayView<'_, Self>, - _session: &VortexSession, - ) -> VortexResult>> { - let mut metadata = vec![METADATA_VERSION]; - match PType::try_from(array.dtype())? { - PType::F32 => metadata.extend_from_slice( - &u32::try_from(array.data().base_bits) - .vortex_expect("validated f32 FloatMult base bits") - .to_le_bytes(), - ), - PType::F64 => metadata.extend_from_slice(&array.data().base_bits.to_le_bytes()), - ptype => vortex_bail!("FloatMult requires f32 or f64, got {ptype}"), - } - Ok(Some(metadata)) - } - - fn deserialize( - &self, - dtype: &DType, - len: usize, - metadata: &[u8], - _buffers: &[BufferHandle], - children: &dyn ArrayChildren, - _session: &VortexSession, - ) -> VortexResult> { - let ptype = PType::try_from(dtype)?; - let expected_metadata_len = match ptype { - PType::F32 => F32_METADATA_LEN, - PType::F64 => F64_METADATA_LEN, - _ => vortex_bail!("FloatMult requires f32 or f64, got {ptype}"), - }; - vortex_ensure!( - metadata.len() == expected_metadata_len, - "FloatMult metadata requires {expected_metadata_len} bytes" - ); - vortex_ensure!( - metadata[0] == METADATA_VERSION, - "unsupported FloatMult metadata version {}", - metadata[0] - ); - vortex_ensure!( - matches!(children.len(), 1 | 2), - "FloatMult requires one or two children" - ); - - let base_bits = match ptype { - PType::F32 => { - let mut bytes = [0; 4]; - bytes.copy_from_slice(&metadata[1..5]); - u64::from(u32::from_le_bytes(bytes)) - } - PType::F64 => { - let mut bytes = [0; 8]; - bytes.copy_from_slice(&metadata[1..9]); - u64::from_le_bytes(bytes) - } - _ => unreachable!(), - }; - let latent_ptype = latent_ptype(ptype)?; - let primary_dtype = DType::Primitive(latent_ptype, dtype.nullability()); - let primary = children.get(0, &primary_dtype, len)?; - let secondary = if children.len() == 2 { - let secondary_dtype = DType::Primitive(latent_ptype, NonNullable); - Some(children.get(1, &secondary_dtype, len)?) - } else { - None - }; - let slots = FloatMultSlots { primary, secondary }.into_slots(); - Ok(ArrayParts::new( - self.clone(), - dtype.clone(), - len, - FloatMultData { base_bits }, - ) - .with_slots(slots)) - } - - fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { - FloatMultSlots::NAMES[idx].to_string() - } - - fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { - Ok(ExecutionResult::done( - decode(array.as_view(), ctx)?.into_array(), - )) - } - - fn reduce_parent( - array: ArrayView<'_, Self>, - parent: &ArrayRef, - child_idx: usize, - ) -> VortexResult> { - FLOAT_MULT_RULES.evaluate(array, parent, child_idx) - } -} - -impl OperationsVTable for FloatMult { - fn scalar_at( - array: ArrayView<'_, FloatMult>, - index: usize, - ctx: &mut ExecutionCtx, - ) -> VortexResult { - let primary = array.primary().execute_scalar(index, ctx)?; - if primary.is_null() { - return Ok(Scalar::null(array.dtype().clone())); - } - Ok(match PType::try_from(array.dtype())? { - PType::F32 => { - let primary = primary - .as_primitive() - .typed_value::() - .vortex_expect("validated primary scalar"); - let value = if let Some(secondary) = array.secondary() { - let secondary = secondary.execute_scalar(index, ctx)?; - join_f32( - primary, - secondary - .as_primitive() - .typed_value::() - .vortex_expect("validated secondary scalar"), - array.data().base_f32()?, - ) - } else { - int_float_from_u32(primary) * array.data().base_f32()? - }; - Scalar::primitive(value, array.dtype().nullability()) - } - PType::F64 => { - let primary = primary - .as_primitive() - .typed_value::() - .vortex_expect("validated primary scalar"); - let value = if let Some(secondary) = array.secondary() { - let secondary = secondary.execute_scalar(index, ctx)?; - join_f64( - primary, - secondary - .as_primitive() - .typed_value::() - .vortex_expect("validated secondary scalar"), - array.data().base_f64()?, - ) - } else { - int_float_from_u64(primary) * array.data().base_f64()? - }; - Scalar::primitive(value, array.dtype().nullability()) - } - ptype => vortex_panic!("unsupported FloatMult ptype {ptype}"), - }) - } -} - -impl ValidityChild for FloatMult { - fn validity_child(array: ArrayView<'_, FloatMult>) -> ArrayRef { - array.primary().clone() - } -} - -impl FloatMultData { - fn base(&self, ptype: PType) -> VortexResult { - match ptype { - PType::F32 => Ok(f64::from(self.base_f32()?)), - PType::F64 => self.base_f64(), - _ => vortex_bail!("FloatMult requires f32 or f64, got {ptype}"), - } - } - - fn base_f32(&self) -> VortexResult { - Ok(f32::from_bits(u32::try_from(self.base_bits)?)) - } - - fn base_f64(&self) -> VortexResult { - Ok(f64::from_bits(self.base_bits)) - } -} - -pub trait FloatMultArrayExt: TypedArrayRef + FloatMultArraySlotsExt { - /// Return the multiplier base as an f64 value. - fn base(&self) -> f64 { - self.deref() - .base( - match PType::try_from(self.primary().dtype()) - .vortex_expect("validated FloatMult primary dtype") - { - PType::U32 => PType::F32, - PType::U64 => PType::F64, - _ => vortex_panic!("validated FloatMult primary dtype must be unsigned"), - }, - ) - .vortex_expect("validated FloatMult base") - } -} - -impl> FloatMultArrayExt for T {} - -impl FloatMult { - /// Construct a FloatMult array from one or two latent children. - pub fn try_new( - primary: ArrayRef, - secondary: Option, - float_ptype: PType, - base: f64, - ) -> VortexResult { - let base_bits = match float_ptype { - PType::F32 => u64::from(narrow_f32(base).to_bits()), - PType::F64 => base.to_bits(), - _ => vortex_bail!("FloatMult requires f32 or f64, got {float_ptype}"), - }; - let dtype = DType::Primitive(float_ptype, primary.dtype().nullability()); - let len = primary.len(); - let slots = FloatMultSlots { primary, secondary }.into_slots(); - Array::try_from_parts( - ArrayParts::new(FloatMult, dtype, len, FloatMultData { base_bits }).with_slots(slots), - ) - } - - /// Split a canonical float array into two unsigned latent children. - pub fn from_primitive( - array: ArrayView<'_, Primitive>, - base: f64, - ) -> VortexResult { - let validity = array.validity()?; - match array.ptype() { - PType::F32 => { - let base = narrow_f32(base); - vortex_ensure!( - base.is_finite() && base.abs() > 0.0, - "FloatMult base must be finite and nonzero" - ); - let (primary, secondary): (Vec<_>, Vec<_>) = array - .as_slice::() - .iter() - .copied() - .map(|value| split_f32(value, base)) - .unzip(); - Self::try_new( - PrimitiveArray::new(Buffer::from(primary), validity).into_array(), - Some( - PrimitiveArray::new(Buffer::from(secondary), NonNullable.into()) - .into_array(), - ), - PType::F32, - f64::from(base), - ) - } - PType::F64 => { - vortex_ensure!( - base.is_finite() && base.abs() > 0.0, - "FloatMult base must be finite and nonzero" - ); - let (primary, secondary): (Vec<_>, Vec<_>) = array - .as_slice::() - .iter() - .copied() - .map(|value| split_f64(value, base)) - .unzip(); - Self::try_new( - PrimitiveArray::new(Buffer::from(primary), validity).into_array(), - Some( - PrimitiveArray::new(Buffer::from(secondary), NonNullable.into()) - .into_array(), - ), - PType::F64, - base, - ) - } - ptype => vortex_bail!("FloatMult requires f32 or f64, got {ptype}"), - } - } - - /// Split floats when every ULP adjustment is zero. - pub fn from_primitive_constant_secondary( - array: ArrayView<'_, Primitive>, - base: f64, - ) -> VortexResult> { - let validity = array.validity()?; - let len = array.len(); - match array.ptype() { - PType::F32 => { - let base = narrow_f32(base); - let mut primary = Vec::with_capacity(len); - for value in array.as_slice::() { - let (multiple, adjustment) = split_f32(*value, base); - if adjustment != 1_u32 << 31 { - return Ok(None); - } - primary.push(multiple); - } - Ok(Some(Self::try_new( - PrimitiveArray::new(Buffer::from(primary), validity).into_array(), - None, - PType::F32, - f64::from(base), - )?)) - } - PType::F64 => { - let mut primary = Vec::with_capacity(len); - for value in array.as_slice::() { - let (multiple, adjustment) = split_f64(*value, base); - if adjustment != 1_u64 << 63 { - return Ok(None); - } - primary.push(multiple); - } - Ok(Some(Self::try_new( - PrimitiveArray::new(Buffer::from(primary), validity).into_array(), - None, - PType::F64, - base, - )?)) - } - ptype => vortex_bail!("FloatMult requires f32 or f64, got {ptype}"), - } - } -} - -/// Estimate a common multiplier base from a bounded canonical float sample. -pub fn estimate_float_mult_base(array: ArrayView<'_, Primitive>) -> Option { - match array.ptype() { - PType::F32 => estimate_base( - &array - .as_slice::() - .iter() - .copied() - .map(f64::from) - .collect::>(), - u32::BITS, - ) - .map(|base| f64::from(narrow_f32(base))), - PType::F64 => estimate_base(array.as_slice::(), u64::BITS), - _ => None, - } -} - -/// Estimate a base that produces zero ULP adjustments for every sampled value. -pub fn estimate_float_mult_constant_base(array: ArrayView<'_, Primitive>) -> Option { - let values = match array.ptype() { - PType::F32 => array - .as_slice::() - .iter() - .copied() - .map(f64::from) - .collect::>(), - PType::F64 => array.as_slice::().to_vec(), - _ => return None, - }; - let finite = values - .iter() - .copied() - .filter(|value| value.is_finite()) - .collect::>(); - if finite.len() < MIN_ANALYSIS_VALUES { - return None; - } - match array.ptype() { - PType::F32 => constant_candidate_bases_f32(&finite) - .into_iter() - .filter_map(|base| { - constant_score_f32(&finite, narrow_f32(base)).map(|score| (score, base)) - }) - .filter(|(score, _)| *score + 2 < u32::BITS) - .min_by(|(left_score, left_base), (right_score, right_base)| { - left_score - .cmp(right_score) - .then_with(|| right_base.total_cmp(left_base)) - }) - .map(|(_, base)| f64::from(narrow_f32(base))), - PType::F64 => constant_candidate_bases_f64(&finite) - .into_iter() - .filter_map(|base| constant_score_f64(&finite, base).map(|score| (score, base))) - .filter(|(score, _)| *score + 2 < u64::BITS) - .min_by(|(left_score, left_base), (right_score, right_base)| { - left_score - .cmp(right_score) - .then_with(|| right_base.total_cmp(left_base)) - }) - .map(|(_, base)| base), - _ => None, - } -} - -fn constant_candidate_bases_f32(values: &[f64]) -> Vec { - let mut candidates = Vec::with_capacity(20); - for exponent in -6..=6 { - push_candidate(&mut candidates, 10.0_f64.powi(exponent)); - } - let mut binary_exponent = i32::MAX; - for value in values { - let value = narrow_f32(*value); - if value == 0.0 { - continue; - } - let exponent = (value.abs().to_bits() >> 23) as i32 - 127; - let trailing_zeros = value.to_bits().trailing_zeros(); - let divisor_exponent = exponent - 23_u32.saturating_sub(trailing_zeros) as i32; - binary_exponent = binary_exponent.min(divisor_exponent); - } - if binary_exponent != i32::MAX { - push_candidate(&mut candidates, 2.0_f64.powi(binary_exponent)); - } - push_common_divisor_candidates(&mut candidates, values); - candidates -} - -fn constant_candidate_bases_f64(values: &[f64]) -> Vec { - let mut candidates = Vec::with_capacity(32); - for exponent in -12..=12 { - push_candidate(&mut candidates, 10.0_f64.powi(exponent)); - } - let mut binary_exponent = i32::MAX; - for value in values { - if *value == 0.0 { - continue; - } - let exponent = (value.abs().to_bits() >> 52) as i32 - 1023; - let trailing_zeros = value.to_bits().trailing_zeros(); - let divisor_exponent = exponent - 52_u32.saturating_sub(trailing_zeros) as i32; - binary_exponent = binary_exponent.min(divisor_exponent); - } - if binary_exponent != i32::MAX { - push_candidate(&mut candidates, 2.0_f64.powi(binary_exponent)); - } - push_common_divisor_candidates(&mut candidates, values); - candidates -} - -fn push_common_divisor_candidates(candidates: &mut Vec, values: &[f64]) { - if let Some(base) = approximate_common_divisor(values) { - push_candidate(candidates, base); - push_candidate(candidates, snap_base(base)); - } -} - -fn estimate_base(values: &[f64], source_bits: u32) -> Option { - let finite = values - .iter() - .copied() - .filter(|value| value.is_finite()) - .collect::>(); - if finite.len() < MIN_ANALYSIS_VALUES { - return None; - } - - let candidates = candidate_bases(&finite); - - candidates - .into_iter() - .filter_map(|base| score_base(&finite, base).map(|score| (score, base))) - .filter(|(score, _)| *score + 2.0 < f64::from(source_bits)) - .min_by(|(left, _), (right, _)| left.total_cmp(right)) - .map(|(_, base)| base) -} - -fn candidate_bases(values: &[f64]) -> Vec { - let mut candidates = Vec::with_capacity(112); - for exponent in -12..=12 { - push_candidate(&mut candidates, 10.0_f64.powi(exponent)); - } - for exponent in -40..=40 { - push_candidate(&mut candidates, 2.0_f64.powi(exponent)); - } - if let Some(base) = approximate_common_divisor(values) { - push_candidate(&mut candidates, base); - push_candidate(&mut candidates, snap_base(base)); - } - candidates -} - -fn push_candidate(candidates: &mut Vec, base: f64) { - if !base.is_finite() || base <= 0.0 { - return; - } - if candidates - .iter() - .any(|candidate| ((candidate - base) / base).abs() < 1e-12) - { - return; - } - candidates.push(base); -} - -fn score_base(values: &[f64], base: f64) -> Option { - let mut primary_min = u64::MAX; - let mut primary_max = u64::MIN; - let mut secondary_min = u64::MAX; - let mut secondary_max = u64::MIN; - let mut primary_distinct = HashSet::new(); - let mut secondary_distinct = HashSet::new(); - let mut primary_runs = 0usize; - let mut secondary_runs = 0usize; - let mut previous_primary = None; - let mut previous_secondary = None; - for value in values { - let (primary, secondary) = split_f64(*value, base); - primary_min = primary_min.min(primary); - primary_max = primary_max.max(primary); - secondary_min = secondary_min.min(secondary); - secondary_max = secondary_max.max(secondary); - primary_distinct.insert(primary); - secondary_distinct.insert(secondary); - primary_runs += usize::from(previous_primary != Some(primary)); - secondary_runs += usize::from(previous_secondary != Some(secondary)); - previous_primary = Some(primary); - previous_secondary = Some(secondary); - } - let primary_bits = u64::BITS - primary_max.wrapping_sub(primary_min).leading_zeros(); - let secondary_bits = u64::BITS - secondary_max.wrapping_sub(secondary_min).leading_zeros(); - Some( - estimated_latent_cost( - primary_bits, - primary_distinct.len(), - primary_runs, - values.len(), - ) + estimated_latent_cost( - secondary_bits, - secondary_distinct.len(), - secondary_runs, - values.len(), - ), - ) -} - -fn constant_score_f32(values: &[f64], base: f32) -> Option { - let mut primary_min = u32::MAX; - let mut primary_max = u32::MIN; - for value in values { - let (primary, secondary) = split_f32(narrow_f32(*value), base); - if secondary != 1_u32 << 31 { - return None; - } - primary_min = primary_min.min(primary); - primary_max = primary_max.max(primary); - } - Some(u32::BITS - (primary_max - primary_min).leading_zeros()) -} - -fn constant_score_f64(values: &[f64], base: f64) -> Option { - let mut primary_min = u64::MAX; - let mut primary_max = u64::MIN; - for value in values { - let (primary, secondary) = split_f64(*value, base); - if secondary != 1_u64 << 63 { - return None; - } - primary_min = primary_min.min(primary); - primary_max = primary_max.max(primary); - } - Some(u64::BITS - (primary_max - primary_min).leading_zeros()) -} - -fn estimated_latent_cost( - span_bits: u32, - distinct_count: usize, - run_count: usize, - len: usize, -) -> f64 { - if span_bits == 0 { - return 0.0; - } - let span_bits = f64::from(span_bits); - let code_bits = usize::BITS - distinct_count.saturating_sub(1).leading_zeros(); - let dict_cost = f64::from(code_bits) + distinct_count as f64 / len as f64 * span_bits; - let index_bits = usize::BITS - len.saturating_sub(1).leading_zeros(); - let run_cost = run_count as f64 / len as f64 * (span_bits + f64::from(index_bits)); - span_bits.min(dict_cost).min(run_cost) -} - -fn approximate_common_divisor(values: &[f64]) -> Option { - let mut divisors = values - .chunks_exact(2) - .filter_map(|pair| { - let a = pair[0].abs(); - let b = pair[1].abs(); - approximate_pair_divisor(a.max(b), a.min(b)) - }) - .collect::>(); - if divisors.len() < 2 { - return None; - } - divisors.sort_unstable_by(f64::total_cmp); - for percentile in [1usize, 3, 5] { - let candidate = divisors[percentile * divisors.len() / 10]; - let similar = divisors - .iter() - .filter(|divisor| (**divisor - candidate).abs() < candidate * 0.01) - .count(); - if similar >= 2 { - return Some(candidate); - } - } - None -} - -fn approximate_pair_divisor(greater: f64, lesser: f64) -> Option { - if lesser == 0.0 || lesser == greater || lesser <= greater * 2.0_f64.powi(-46) { - return None; - } - let mut greater_value = greater; - let mut greater_error = 0.0; - let mut lesser_value = lesser; - let mut lesser_error = 0.0; - for _ in 0..64 { - let ratio = (greater_value / lesser_value).round(); - greater_error += ratio * lesser_error + greater_value * f64::EPSILON; - let previous = greater_value; - greater_value = (greater_value - ratio * lesser_value).abs(); - if greater_value <= previous * 2.0_f64.powi(-16) || greater_value <= greater_error { - return Some(lesser_value); - } - if greater_value <= greater * 2.0_f64.powi(-46) - || greater_value <= greater_error * 2.0_f64.powi(6) - { - return None; - } - std::mem::swap(&mut greater_value, &mut lesser_value); - std::mem::swap(&mut greater_error, &mut lesser_error); - } - None -} - -fn snap_base(base: f64) -> f64 { - let inverse = base.recip(); - let rounded = inverse.round(); - let decimal = 10.0_f64.powf(inverse.log10().round()); - if (inverse - rounded).abs() < 0.02 { - rounded.recip() - } else if (inverse - decimal).abs() / inverse < 0.01 { - decimal.recip() - } else { - base - } -} - -fn latent_ptype(ptype: PType) -> VortexResult { - match ptype { - PType::F32 => Ok(PType::U32), - PType::F64 => Ok(PType::U64), - _ => vortex_bail!("FloatMult requires f32 or f64, got {ptype}"), - } -} - -#[allow(clippy::cast_possible_truncation)] -fn int_float_to_u32(value: f32) -> u32 { - let absolute = value.abs(); - let greatest_precise_integer = 1_u32 << f32::MANTISSA_DIGITS; - let greatest_precise_float = greatest_precise_integer as f32; - let absolute_integer = if absolute < greatest_precise_float { - absolute as u32 - } else { - greatest_precise_integer + (absolute.to_bits() - greatest_precise_float.to_bits()) - }; - if value.is_sign_positive() { - (1_u32 << 31) + absolute_integer - } else { - (1_u32 << 31) - 1 - absolute_integer - } -} - -fn int_float_from_u32(value: u32) -> f32 { - let middle = 1_u32 << 31; - let (absolute, positive) = if value >= middle { - (value - middle, true) - } else { - (middle - 1 - value, false) - }; - let greatest_precise_integer = 1_u32 << f32::MANTISSA_DIGITS; - let greatest_precise_float = greatest_precise_integer as f32; - let absolute_float = if absolute < greatest_precise_integer { - absolute as f32 - } else { - f32::from_bits(greatest_precise_float.to_bits() + absolute - greatest_precise_integer) - }; - if positive { - absolute_float - } else { - -absolute_float - } -} - -#[allow(clippy::cast_possible_truncation)] -fn int_float_to_u64(value: f64) -> u64 { - let absolute = value.abs(); - let greatest_precise_integer = 1_u64 << f64::MANTISSA_DIGITS; - let greatest_precise_float = greatest_precise_integer as f64; - let absolute_integer = if absolute < greatest_precise_float { - absolute as u64 - } else { - greatest_precise_integer + (absolute.to_bits() - greatest_precise_float.to_bits()) - }; - if value.is_sign_positive() { - (1_u64 << 63) + absolute_integer - } else { - (1_u64 << 63) - 1 - absolute_integer - } -} - -#[allow(clippy::cast_possible_truncation)] -fn narrow_f32(value: f64) -> f32 { - value as f32 -} - -fn int_float_from_u64(value: u64) -> f64 { - let middle = 1_u64 << 63; - let (absolute, positive) = if value >= middle { - (value - middle, true) - } else { - (middle - 1 - value, false) - }; - let greatest_precise_integer = 1_u64 << f64::MANTISSA_DIGITS; - let greatest_precise_float = greatest_precise_integer as f64; - let absolute_float = if absolute < greatest_precise_integer { - absolute as f64 - } else { - f64::from_bits(greatest_precise_float.to_bits() + absolute - greatest_precise_integer) - }; - if positive { - absolute_float - } else { - -absolute_float - } -} - -fn ordered_u32(value: f32) -> u32 { - let bits = value.to_bits(); - if bits & (1_u32 << 31) == 0 { - bits ^ (1_u32 << 31) - } else { - !bits - } -} - -fn ordered_u64(value: f64) -> u64 { - let bits = value.to_bits(); - if bits & (1_u64 << 63) == 0 { - bits ^ (1_u64 << 63) - } else { - !bits - } -} - -fn from_ordered_u32(ordered: u32) -> f32 { - let bits = if ordered & (1_u32 << 31) == 0 { - !ordered - } else { - ordered ^ (1_u32 << 31) - }; - f32::from_bits(bits) -} - -fn from_ordered_u64(ordered: u64) -> f64 { - let bits = if ordered & (1_u64 << 63) == 0 { - !ordered - } else { - ordered ^ (1_u64 << 63) - }; - f64::from_bits(bits) -} - -fn split_f32(value: f32, base: f32) -> (u32, u32) { - let multiple = (value / base).round(); - let primary = int_float_to_u32(multiple); - let secondary = ordered_u32(value) - .wrapping_sub(ordered_u32(multiple * base)) - .wrapping_add(1_u32 << 31); - (primary, secondary) -} - -fn split_f64(value: f64, base: f64) -> (u64, u64) { - let multiple = (value / base).round(); - let primary = int_float_to_u64(multiple); - let secondary = ordered_u64(value) - .wrapping_sub(ordered_u64(multiple * base)) - .wrapping_add(1_u64 << 63); - (primary, secondary) -} - -fn join_f32(primary: u32, secondary: u32, base: f32) -> f32 { - let approximate = int_float_from_u32(primary) * base; - from_ordered_u32(ordered_u32(approximate).wrapping_add(secondary.wrapping_add(1_u32 << 31))) -} - -fn join_f64(primary: u64, secondary: u64, base: f64) -> f64 { - let approximate = int_float_from_u64(primary) * base; - from_ordered_u64(ordered_u64(approximate).wrapping_add(secondary.wrapping_add(1_u64 << 63))) -} - -fn decode(array: ArrayView<'_, FloatMult>, ctx: &mut ExecutionCtx) -> VortexResult { - let primary = array.primary().clone().execute::(ctx)?; - let validity = primary.validity()?; - let Some(secondary) = array.secondary() else { - return Ok(match PType::try_from(array.dtype())? { - PType::F32 => { - let base = array.data().base_f32()?; - let values = primary - .into_buffer::() - .map_each_in_place(|primary| int_float_from_u32(primary) * base) - .freeze(); - PrimitiveArray::new(values, validity) - } - PType::F64 => { - let base = array.data().base_f64()?; - let values = primary - .into_buffer::() - .map_each_in_place(|primary| int_float_from_u64(primary) * base) - .freeze(); - PrimitiveArray::new(values, validity) - } - ptype => vortex_panic!("unsupported FloatMult ptype {ptype}"), - }); - }; - - let secondary = secondary.clone().execute::(ctx)?; - Ok(match PType::try_from(array.dtype())? { - PType::F32 => { - let secondary_values = secondary.as_slice::(); - let base = array.data().base_f32()?; - let mut index = 0; - let values = primary - .into_buffer::() - .map_each_in_place(|primary| { - let value = join_f32(primary, secondary_values[index], base); - index += 1; - value - }) - .freeze(); - PrimitiveArray::new(values, validity) - } - PType::F64 => { - let secondary_values = secondary.as_slice::(); - let base = array.data().base_f64()?; - let mut index = 0; - let values = primary - .into_buffer::() - .map_each_in_place(|primary| { - let value = join_f64(primary, secondary_values[index], base); - index += 1; - value - }) - .freeze(); - PrimitiveArray::new(values, validity) - } - ptype => vortex_panic!("unsupported FloatMult ptype {ptype}"), - }) -} - -#[cfg(test)] -mod tests { - use std::sync::LazyLock; - - use vortex_array::ArrayContext; - use vortex_array::IntoArray; - use vortex_array::VortexSessionExecute; - use vortex_array::array_session; - use vortex_array::assert_arrays_eq; - use vortex_array::assert_nth_scalar; - use vortex_array::serde::SerializeOptions; - use vortex_array::serde::SerializedArray; - use vortex_array::validity::Validity; - use vortex_buffer::ByteBufferMut; - use vortex_error::VortexResult; - use vortex_session::VortexSession; - use vortex_session::registry::ReadContext; - - use super::*; - - static SESSION: LazyLock = LazyLock::new(|| { - let session = array_session(); - crate::initialize(&session); - session - }); - - #[test] - fn float_bit_patterns_roundtrip() -> VortexResult<()> { - let values = [ - f64::NEG_INFINITY, - -1.5, - -0.0, - 0.0, - 1.5, - 1.234_567_89, - f64::INFINITY, - f64::from_bits(0x7ff8_0000_0000_1234), - f64::from_bits(0xfff8_0000_0000_5678), - ]; - let original = PrimitiveArray::from_iter(values); - let encoded = FloatMult::from_primitive(original.as_view(), 0.01)?; - let secondary = encoded - .secondary() - .vortex_expect("general FloatMult must contain adjustments") - .clone() - .execute::(&mut SESSION.create_execution_ctx())?; - assert!( - secondary - .as_slice::() - .iter() - .any(|adjustment| *adjustment != 1_u64 << 63) - ); - let decoded = encoded - .into_array() - .execute::(&mut SESSION.create_execution_ctx())?; - assert_eq!( - decoded - .as_slice::() - .iter() - .map(|value| value.to_bits()) - .collect::>(), - values - .iter() - .map(|value| value.to_bits()) - .collect::>() - ); - Ok(()) - } - - #[test] - fn implicit_zero_adjustment_roundtrip() -> VortexResult<()> { - let original = PrimitiveArray::from_iter([0.0_f32, 1.0, 42.0, 65_536.0]); - let encoded = FloatMult::from_primitive_constant_secondary(original.as_view(), 1.0)? - .vortex_expect("integer floats use an implicit zero adjustment"); - assert!(encoded.secondary().is_none()); - assert_eq!(encoded.as_ref().nchildren(), 1); - assert_arrays_eq!(encoded, original, &mut SESSION.create_execution_ctx()); - - let encoded = encoded.into_array(); - let dtype = encoded.dtype().clone(); - let len = encoded.len(); - let array_context = ArrayContext::empty(); - let serialized = - encoded.serialize(&array_context, &SESSION, &SerializeOptions::default())?; - let mut bytes = ByteBufferMut::empty(); - for buffer in serialized { - bytes.extend_from_slice(buffer.as_ref()); - } - let decoded = SerializedArray::try_from(bytes.freeze())?.decode( - &dtype, - len, - &ReadContext::new(array_context.to_ids()), - &SESSION, - )?; - assert!(decoded.as_::().secondary().is_none()); - assert_arrays_eq!(decoded, original, &mut SESSION.create_execution_ctx()); - Ok(()) - } - - #[test] - fn nullable_slice_and_scalar_access() -> VortexResult<()> { - let original = PrimitiveArray::new( - Buffer::from(vec![1.25_f32, 0.0, -0.0, 42.5, -10.0]), - Validity::from_iter([true, false, true, true, false]), - ); - let encoded = FloatMult::from_primitive(original.as_view(), 0.25)?; - let mut ctx = SESSION.create_execution_ctx(); - assert_arrays_eq!(encoded, original, &mut ctx); - assert_nth_scalar!(encoded, 3, 42.5_f32, &mut ctx); - assert!(encoded.execute_scalar(1, &mut ctx)?.is_null()); - - let sliced = encoded.into_array().slice(1..4)?; - assert!(sliced.is::()); - assert_arrays_eq!(sliced, original.into_array().slice(1..4)?, &mut ctx); - Ok(()) - } - - #[test] - fn serialization_roundtrip() -> VortexResult<()> { - let original = PrimitiveArray::from_option_iter([ - Some(-10.25_f64), - None, - Some(-0.0), - Some(42.25), - Some(100.5), - ]); - let encoded = FloatMult::from_primitive(original.as_view(), 0.25)?.into_array(); - let dtype = encoded.dtype().clone(); - let len = encoded.len(); - let array_context = ArrayContext::empty(); - let serialized = - encoded.serialize(&array_context, &SESSION, &SerializeOptions::default())?; - let mut bytes = ByteBufferMut::empty(); - for buffer in serialized { - bytes.extend_from_slice(buffer.as_ref()); - } - - let decoded = SerializedArray::try_from(bytes.freeze())?.decode( - &dtype, - len, - &ReadContext::new(array_context.to_ids()), - &SESSION, - )?; - assert!(decoded.is::()); - assert_arrays_eq!(decoded, original, &mut SESSION.create_execution_ctx()); - Ok(()) - } - - #[test] - fn estimates_decimal_and_binary_bases() { - let decimals = PrimitiveArray::from_iter((0..1_000).map(|value| value as f64 * 0.01)); - assert_eq!(estimate_float_mult_base(decimals.as_view()), Some(0.01)); - - let binary = PrimitiveArray::from_iter((0..1_000).map(|value| value as f64 * 0.125)); - assert_eq!(estimate_float_mult_base(binary.as_view()), Some(0.125)); - } -} diff --git a/encodings/float-quant/src/lib.rs b/encodings/float-quant/src/lib.rs index d84fa7c65d1..6efde148ff0 100644 --- a/encodings/float-quant/src/lib.rs +++ b/encodings/float-quant/src/lib.rs @@ -4,17 +4,14 @@ //! Lossless float quantization as two integer child arrays. mod array; -mod float_mult; mod rules; mod slice; pub use array::*; -pub use float_mult::*; use vortex_array::session::ArraySessionExt; use vortex_session::VortexSession; /// Register the float quantization encoding in one session. pub fn initialize(session: &VortexSession) { session.arrays().register(FloatQuant); - session.arrays().register(FloatMult); } diff --git a/encodings/float-quant/src/rules.rs b/encodings/float-quant/src/rules.rs index 22fbb2c9788..5faedb68866 100644 --- a/encodings/float-quant/src/rules.rs +++ b/encodings/float-quant/src/rules.rs @@ -4,11 +4,7 @@ use vortex_array::arrays::slice::SliceReduceAdaptor; use vortex_array::optimizer::rules::ParentRuleSet; -use crate::FloatMult; use crate::FloatQuant; pub(crate) static RULES: ParentRuleSet = ParentRuleSet::new(&[ParentRuleSet::lift(&SliceReduceAdaptor(FloatQuant))]); - -pub(crate) static FLOAT_MULT_RULES: ParentRuleSet = - ParentRuleSet::new(&[ParentRuleSet::lift(&SliceReduceAdaptor(FloatMult))]); diff --git a/encodings/float-quant/src/slice.rs b/encodings/float-quant/src/slice.rs index 0442a271d8b..3a83f48220d 100644 --- a/encodings/float-quant/src/slice.rs +++ b/encodings/float-quant/src/slice.rs @@ -10,9 +10,6 @@ use vortex_array::arrays::slice::SliceReduce; use vortex_array::dtype::PType; use vortex_error::VortexResult; -use crate::FloatMult; -use crate::FloatMultArrayExt; -use crate::FloatMultArraySlotsExt; use crate::FloatQuant; use crate::FloatQuantArraySlotsExt; @@ -29,20 +26,3 @@ impl SliceReduce for FloatQuant { )) } } - -impl SliceReduce for FloatMult { - fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { - Ok(Some( - FloatMult::try_new( - array.primary().slice(range.clone())?, - array - .secondary() - .map(|secondary| secondary.slice(range)) - .transpose()?, - PType::try_from(array.dtype())?, - array.base(), - )? - .into_array(), - )) - } -} diff --git a/encodings/range-entropy/examples/encode_probe.rs b/encodings/range-entropy/examples/encode_probe.rs deleted file mode 100644 index 43766f18f4a..00000000000 --- a/encodings/range-entropy/examples/encode_probe.rs +++ /dev/null @@ -1,99 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::f64::consts::TAU; -use std::hint::black_box; -use std::time::Duration; -use std::time::Instant; - -use vortex_error::VortexResult; -use vortex_range_entropy::RangeEntropyCodec; - -const N: usize = 1 << 18; -const BLOCK_LEN: usize = 8192; -const ITERATIONS: usize = 200; - -struct Rng(u64); - -impl Rng { - fn next_u64(&mut self) -> u64 { - self.0 ^= self.0 << 13; - self.0 ^= self.0 >> 7; - self.0 ^= self.0 << 17; - self.0 - } - - fn uniform(&mut self) -> f64 { - ((self.next_u64() >> 11) as f64 + 0.5) * (1.0 / ((1_u64 << 53) as f64)) - } - - fn normal(&mut self) -> f64 { - let radius = (-2.0 * self.uniform().ln()).sqrt(); - radius * (TAU * self.uniform()).cos() - } -} - -fn ordered_f64(value: f64) -> u64 { - let bits = value.to_bits(); - if bits & (1_u64 << 63) == 0 { - bits ^ (1_u64 << 63) - } else { - !bits - } -} - -fn random_walk() -> Vec { - let mut rng = Rng(0x4d59_5df4_d0f3_3173); - let mut value = 0.0; - (0..N) - .map(|_| { - value += rng.normal() * 0.01; - ordered_f64(value) - }) - .collect() -} - -fn main() -> VortexResult<()> { - let values = random_walk(); - black_box(RangeEntropyCodec::encode(&values, BLOCK_LEN)?); - - let mut durations = Vec::with_capacity(ITERATIONS); - let mut encoded_size = 0usize; - let mut bin_count = 0usize; - for _ in 0..ITERATIONS { - let start = Instant::now(); - let encoded = RangeEntropyCodec::encode(black_box(&values), BLOCK_LEN)?; - durations.push(start.elapsed()); - encoded_size = black_box(encoded.encoded_size()); - bin_count = black_box(encoded.bin_lowers().len()); - } - - durations.sort_unstable(); - let p10 = durations[ITERATIONS / 10]; - let median = durations[ITERATIONS / 2]; - let p90 = durations[ITERATIONS * 9 / 10]; - let input_bytes = (N * size_of::()) as f64; - let throughput = input_bytes / median.as_secs_f64() / 1_000_000.0; - print_result(encoded_size, bin_count, p10, median, p90, throughput); - Ok(()) -} - -fn print_result( - encoded_size: usize, - bin_count: usize, - p10: Duration, - median: Duration, - p90: Duration, - throughput: f64, -) { - println!( - "random-walk values={N} blocks={} bins={bin_count} bytes={encoded_size}", - N.div_ceil(BLOCK_LEN), - ); - println!( - "encode p10={:.3}ms median={:.3}ms p90={:.3}ms throughput={throughput:.1}MB/s", - p10.as_secs_f64() * 1_000.0, - median.as_secs_f64() * 1_000.0, - p90.as_secs_f64() * 1_000.0, - ); -} diff --git a/encodings/range-entropy/src/array.rs b/encodings/range-entropy/src/array.rs deleted file mode 100644 index b0c0985b683..00000000000 --- a/encodings/range-entropy/src/array.rs +++ /dev/null @@ -1,857 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::fmt::Display; -use std::fmt::Formatter; -use std::hash::Hash; -use std::hash::Hasher; -use std::ops::Range; - -use vortex_array::Array; -use vortex_array::ArrayEq; -use vortex_array::ArrayHash; -use vortex_array::ArrayId; -use vortex_array::ArrayParts; -use vortex_array::ArrayRef; -use vortex_array::ArrayView; -use vortex_array::EqMode; -use vortex_array::ExecutionCtx; -use vortex_array::ExecutionResult; -use vortex_array::IntoArray; -use vortex_array::TypedArrayRef; -use vortex_array::array_slots; -use vortex_array::arrays::Primitive; -use vortex_array::arrays::PrimitiveArray; -use vortex_array::arrays::slice::SliceReduce; -use vortex_array::buffer::BufferHandle; -use vortex_array::dtype::DType; -use vortex_array::dtype::Nullability; -use vortex_array::dtype::Nullability::NonNullable; -use vortex_array::dtype::PType; -use vortex_array::dtype::half::f16; -use vortex_array::scalar::Scalar; -use vortex_array::serde::ArrayChildren; -use vortex_array::validity::Validity; -use vortex_array::vtable::OperationsVTable; -use vortex_array::vtable::VTable; -use vortex_array::vtable::ValidityVTable; -use vortex_array::vtable::child_to_validity; -use vortex_array::vtable::validity_to_child; -use vortex_buffer::Buffer; -use vortex_error::VortexResult; -use vortex_error::vortex_bail; -use vortex_error::vortex_ensure; -use vortex_error::vortex_panic; -use vortex_session::VortexSession; -use vortex_session::registry::CachedId; - -use crate::RangeEntropyCodec; -use crate::RangeEntropyParts; -use crate::rules::RULES; - -const METADATA_VERSION: u8 = 1; -const METADATA_LEN: usize = 23; -const MAX_SCALE_BITS: u8 = 10; -const MAX_BINS: usize = 64; - -/// A Vortex array with range-bin identifiers and fixed-width offsets. -pub type RangeEntropyArray = Array; - -#[array_slots(RangeEntropy)] -pub struct RangeEntropySlots { - /// Lower bound for each range bin. - #[slot(0)] - pub bin_lowers: ArrayRef, - /// Fixed offset width for each range bin. - #[slot(1)] - pub offset_widths: ArrayRef, - /// Quantized tANS weight for each range bin. - #[slot(2)] - pub weights: ArrayRef, - /// Byte offset for each restart block. - #[slot(3)] - pub block_offsets: ArrayRef, - /// Validity for the unsliced logical values. - #[slot(4)] - pub validity: Option, -} - -#[derive(Clone, Debug)] -pub struct RangeEntropyData { - payload: BufferHandle, - block_len: usize, - scale_bits: u8, - unsliced_len: usize, - slice_start: usize, - slice_stop: usize, -} - -impl Display for RangeEntropyData { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "block_len: {}, scale_bits: {}, slice: {}..{}", - self.block_len, self.scale_bits, self.slice_start, self.slice_stop - ) - } -} - -impl ArrayHash for RangeEntropyData { - fn array_hash(&self, state: &mut H, accuracy: EqMode) { - self.payload.array_hash(state, accuracy); - self.block_len.hash(state); - self.scale_bits.hash(state); - self.unsliced_len.hash(state); - self.slice_start.hash(state); - self.slice_stop.hash(state); - } -} - -impl ArrayEq for RangeEntropyData { - fn array_eq(&self, other: &Self, accuracy: EqMode) -> bool { - self.payload.array_eq(&other.payload, accuracy) - && self.block_len == other.block_len - && self.scale_bits == other.scale_bits - && self.unsliced_len == other.unsliced_len - && self.slice_start == other.slice_start - && self.slice_stop == other.slice_stop - } -} - -#[derive(Clone, Debug)] -pub struct RangeEntropy; - -impl VTable for RangeEntropy { - type TypedArrayData = RangeEntropyData; - - type OperationsVTable = Self; - type ValidityVTable = Self; - - fn id(&self) -> ArrayId { - static ID: CachedId = CachedId::new("vortex.range_entropy"); - *ID - } - - fn validate( - &self, - data: &Self::TypedArrayData, - dtype: &DType, - len: usize, - slots: &[Option], - ) -> VortexResult<()> { - let slots = RangeEntropySlotsView::from_slots(slots); - let validity = child_to_validity(slots.validity, dtype.nullability()); - data.validate(dtype, len, slots, &validity) - } - - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { - 1 - } - - fn buffer(array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { - match idx { - 0 => array.payload.clone(), - _ => vortex_panic!("RangeEntropyArray buffer index {idx} out of bounds"), - } - } - - fn buffer_name(_array: ArrayView<'_, Self>, idx: usize) -> Option { - match idx { - 0 => Some("payload".to_string()), - _ => vortex_panic!("RangeEntropyArray buffer index {idx} out of bounds"), - } - } - - fn with_buffers( - &self, - array: ArrayView<'_, Self>, - buffers: &[BufferHandle], - ) -> VortexResult> { - vortex_ensure!( - buffers.len() == 1, - "RangeEntropyArray expects one buffer, got {}", - buffers.len() - ); - let mut data = array.data().clone(); - data.payload = buffers[0].clone(); - Ok( - ArrayParts::new(self.clone(), array.dtype().clone(), array.len(), data) - .with_slots(array.slots().iter().cloned().collect()), - ) - } - - fn serialize( - array: ArrayView<'_, Self>, - _session: &VortexSession, - ) -> VortexResult>> { - let bin_count = u8::try_from(array.bin_lowers().len())?; - Ok(Some( - RangeEntropyMetadata { - scale_bits: array.scale_bits, - bin_count, - block_len: u32::try_from(array.block_len)?, - unsliced_len: u64::try_from(array.unsliced_len)?, - slice_start: u64::try_from(array.slice_start)?, - } - .encode(), - )) - } - - fn deserialize( - &self, - dtype: &DType, - len: usize, - metadata: &[u8], - buffers: &[BufferHandle], - children: &dyn ArrayChildren, - _session: &VortexSession, - ) -> VortexResult> { - vortex_ensure!( - buffers.len() == 1, - "RangeEntropyArray expects one buffer, got {}", - buffers.len() - ); - let metadata = RangeEntropyMetadata::decode(metadata)?; - let unsliced_len = usize::try_from(metadata.unsliced_len)?; - let slice_start = usize::try_from(metadata.slice_start)?; - let slice_stop = slice_start - .checked_add(len) - .ok_or_else(|| vortex_error::vortex_err!("range entropy slice length overflows"))?; - let block_len = usize::try_from(metadata.block_len)?; - vortex_ensure!(block_len > 0, "range entropy block length must be positive"); - let block_offsets_len = unsliced_len.div_ceil(block_len) + 1; - let bin_count = usize::from(metadata.bin_count); - - let bin_lowers = children.get(0, &primitive_dtype(PType::U64), bin_count)?; - let offset_widths = children.get(1, &primitive_dtype(PType::U8), bin_count)?; - let weights = children.get(2, &primitive_dtype(PType::U16), bin_count)?; - let block_offsets = children.get(3, &primitive_dtype(PType::U32), block_offsets_len)?; - let validity = match children.len() { - 4 => Validity::from(dtype.nullability()), - 5 => Validity::Array(children.get(4, &Validity::DTYPE, unsliced_len)?), - count => vortex_bail!("RangeEntropyArray expects four or five children, got {count}"), - }; - let slots = RangeEntropySlots { - bin_lowers, - offset_widths, - weights, - block_offsets, - validity: validity_to_child(&validity, unsliced_len), - } - .into_slots(); - let data = RangeEntropyData { - payload: buffers[0].clone(), - block_len, - scale_bits: metadata.scale_bits, - unsliced_len, - slice_start, - slice_stop, - }; - Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) - } - - fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { - RangeEntropySlots::NAMES[idx].to_string() - } - - fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { - Ok(ExecutionResult::done( - decompress_array(array.as_view(), ctx)?.into_array(), - )) - } - - fn reduce_parent( - array: ArrayView<'_, Self>, - parent: &ArrayRef, - child_idx: usize, - ) -> VortexResult> { - RULES.evaluate(array, parent, child_idx) - } -} - -impl OperationsVTable for RangeEntropy { - fn scalar_at( - array: ArrayView<'_, RangeEntropy>, - index: usize, - ctx: &mut ExecutionCtx, - ) -> VortexResult { - let codec = codec_from_array(array, ctx)?; - scalar_from_latent( - codec.value(array.slice_start + index)?, - array.dtype().as_ptype(), - array.dtype().nullability(), - ) - } -} - -impl ValidityVTable for RangeEntropy { - fn validity(array: ArrayView<'_, RangeEntropy>) -> VortexResult { - array - .unsliced_validity() - .slice(array.slice_start..array.slice_stop) - } -} - -impl SliceReduce for RangeEntropy { - fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { - let data = array.data().slice(range); - Ok(Some( - Array::try_from_parts( - ArrayParts::new(RangeEntropy, array.dtype().clone(), data.len(), data) - .with_slots(array.slots().iter().cloned().collect()), - )? - .into_array(), - )) - } -} - -pub trait RangeEntropyArrayExt: TypedArrayRef + RangeEntropyArraySlotsExt { - /// Return the validity for all values before a logical slice. - fn unsliced_validity(&self) -> Validity { - child_to_validity( - self.as_ref().slots()[RangeEntropySlots::VALIDITY].as_ref(), - self.as_ref().dtype().nullability(), - ) - } - - /// Decode the logical slice to a canonical primitive array. - fn decompress(&self, ctx: &mut ExecutionCtx) -> VortexResult { - decompress_array(self.to_owned().as_view(), ctx) - } -} - -impl> RangeEntropyArrayExt for T {} - -impl RangeEntropy { - /// Encode a canonical primitive array. - pub fn from_primitive( - array: ArrayView<'_, Primitive>, - block_len: usize, - ) -> VortexResult { - let dtype = array.dtype().clone(); - let validity = array.validity()?; - let codec = RangeEntropyCodec::encode(&ordered_latents(array), block_len)?; - let parts = codec.into_parts(); - let unsliced_len = parts.len; - let data = RangeEntropyData { - payload: BufferHandle::new_host(parts.payload), - block_len: parts.block_len, - scale_bits: parts.scale_bits, - unsliced_len, - slice_start: 0, - slice_stop: unsliced_len, - }; - let slots = RangeEntropySlots { - bin_lowers: PrimitiveArray::from_iter(parts.bin_lowers).into_array(), - offset_widths: PrimitiveArray::from_iter(parts.offset_widths).into_array(), - weights: PrimitiveArray::from_iter(parts.weights).into_array(), - block_offsets: PrimitiveArray::from_iter(parts.block_offsets).into_array(), - validity: validity_to_child(&validity, unsliced_len), - } - .into_slots(); - Array::try_from_parts( - ArrayParts::new(RangeEntropy, dtype, unsliced_len, data).with_slots(slots), - ) - } -} - -impl RangeEntropyData { - fn validate( - &self, - dtype: &DType, - len: usize, - slots: RangeEntropySlotsView<'_>, - validity: &Validity, - ) -> VortexResult<()> { - vortex_ensure!( - matches!(dtype, DType::Primitive(..)), - "RangeEntropyArray requires a primitive dtype" - ); - vortex_ensure!( - self.block_len > 0, - "range entropy block length must be positive" - ); - vortex_ensure!( - self.scale_bits <= MAX_SCALE_BITS, - "tANS table log {} exceeds {MAX_SCALE_BITS}", - self.scale_bits, - ); - vortex_ensure!( - self.slice_start <= self.slice_stop && self.slice_stop <= self.unsliced_len, - "range entropy slice exceeds its source length" - ); - vortex_ensure!( - len == self.len(), - "range entropy length does not match its slice" - ); - vortex_ensure!( - slots.bin_lowers.dtype() == &primitive_dtype(PType::U64), - "range entropy bin lowers require non-nullable u64 values" - ); - vortex_ensure!( - slots.offset_widths.dtype() == &primitive_dtype(PType::U8), - "range entropy offset widths require non-nullable u8 values" - ); - vortex_ensure!( - slots.weights.dtype() == &primitive_dtype(PType::U16), - "range entropy weights require non-nullable u16 values" - ); - vortex_ensure!( - slots.block_offsets.dtype() == &primitive_dtype(PType::U32), - "range entropy block offsets require non-nullable u32 values" - ); - let bin_count = slots.bin_lowers.len(); - vortex_ensure!( - bin_count <= MAX_BINS, - "range entropy bin count exceeds {MAX_BINS}" - ); - vortex_ensure!( - slots.offset_widths.len() == bin_count && slots.weights.len() == bin_count, - "range entropy bin children have different lengths" - ); - vortex_ensure!( - slots.block_offsets.len() == self.unsliced_len.div_ceil(self.block_len) + 1, - "range entropy block offset count is invalid" - ); - if let Some(validity_len) = validity.maybe_len() { - vortex_ensure!( - validity_len == self.unsliced_len, - "range entropy validity length is invalid" - ); - } - Ok(()) - } - - fn len(&self) -> usize { - self.slice_stop - self.slice_start - } - - fn slice(&self, range: Range) -> Self { - Self { - slice_start: self.slice_start + range.start, - slice_stop: self.slice_start + range.end, - ..self.clone() - } - } -} - -fn codec_from_array( - array: ArrayView<'_, RangeEntropy>, - ctx: &mut ExecutionCtx, -) -> VortexResult { - let bin_lowers = array.bin_lowers().clone().execute::(ctx)?; - let offset_widths = array - .offset_widths() - .clone() - .execute::(ctx)?; - let weights = array.weights().clone().execute::(ctx)?; - let block_offsets = array - .block_offsets() - .clone() - .execute::(ctx)?; - let payload = array.payload.clone().try_to_host_sync()?; - RangeEntropyCodec::try_from_parts(RangeEntropyParts { - len: array.unsliced_len, - block_len: array.block_len, - scale_bits: array.scale_bits, - bin_lowers: bin_lowers.as_slice::().to_vec(), - offset_widths: offset_widths.as_slice::().to_vec(), - weights: weights.as_slice::().to_vec(), - block_offsets: block_offsets.as_slice::().to_vec(), - payload, - }) -} - -fn decompress_array( - array: ArrayView<'_, RangeEntropy>, - ctx: &mut ExecutionCtx, -) -> VortexResult { - let latents = - codec_from_array(array, ctx)?.decode_range(array.slice_start..array.slice_stop)?; - primitive_from_latents(latents, array.dtype().as_ptype(), array.validity()?) -} - -#[derive(Clone, Copy)] -struct RangeEntropyMetadata { - scale_bits: u8, - bin_count: u8, - block_len: u32, - unsliced_len: u64, - slice_start: u64, -} - -impl RangeEntropyMetadata { - fn encode(self) -> Vec { - let mut bytes = Vec::with_capacity(METADATA_LEN); - bytes.push(METADATA_VERSION); - bytes.push(self.scale_bits); - bytes.push(self.bin_count); - bytes.extend_from_slice(&self.block_len.to_le_bytes()); - bytes.extend_from_slice(&self.unsliced_len.to_le_bytes()); - bytes.extend_from_slice(&self.slice_start.to_le_bytes()); - bytes - } - - fn decode(bytes: &[u8]) -> VortexResult { - vortex_ensure!( - bytes.len() == METADATA_LEN, - "RangeEntropyArray metadata requires {METADATA_LEN} bytes" - ); - vortex_ensure!( - bytes[0] == METADATA_VERSION, - "unsupported RangeEntropyArray metadata version {}", - bytes[0] - ); - Ok(Self { - scale_bits: bytes[1], - bin_count: bytes[2], - block_len: u32::from_le_bytes([bytes[3], bytes[4], bytes[5], bytes[6]]), - unsliced_len: u64::from_le_bytes([ - bytes[7], bytes[8], bytes[9], bytes[10], bytes[11], bytes[12], bytes[13], bytes[14], - ]), - slice_start: u64::from_le_bytes([ - bytes[15], bytes[16], bytes[17], bytes[18], bytes[19], bytes[20], bytes[21], - bytes[22], - ]), - }) - } -} - -fn primitive_dtype(ptype: PType) -> DType { - DType::Primitive(ptype, NonNullable) -} - -fn ordered_latents(array: ArrayView<'_, Primitive>) -> Vec { - match array.ptype() { - PType::U8 => array - .as_slice::() - .iter() - .map(|&value| u64::from(value)) - .collect(), - PType::U16 => array - .as_slice::() - .iter() - .map(|&value| u64::from(value)) - .collect(), - PType::U32 => array - .as_slice::() - .iter() - .map(|&value| u64::from(value)) - .collect(), - PType::U64 => array.as_slice::().to_vec(), - PType::I8 => array - .as_slice::() - .iter() - .map(|&value| u64::from(value.cast_unsigned() ^ (1_u8 << 7))) - .collect(), - PType::I16 => array - .as_slice::() - .iter() - .map(|&value| u64::from(value.cast_unsigned() ^ (1_u16 << 15))) - .collect(), - PType::I32 => array - .as_slice::() - .iter() - .map(|&value| u64::from(value.cast_unsigned() ^ (1_u32 << 31))) - .collect(), - PType::I64 => array - .as_slice::() - .iter() - .map(|&value| value.cast_unsigned() ^ (1_u64 << 63)) - .collect(), - PType::F16 => array - .as_slice::() - .iter() - .map(|&value| ordered_float_bits(u64::from(value.to_bits()), 16)) - .collect(), - PType::F32 => array - .as_slice::() - .iter() - .map(|&value| ordered_float_bits(u64::from(value.to_bits()), 32)) - .collect(), - PType::F64 => array - .as_slice::() - .iter() - .map(|&value| ordered_float_bits(value.to_bits(), 64)) - .collect(), - } -} - -fn ordered_float_bits(bits: u64, width: u8) -> u64 { - let sign = 1_u64 << (width - 1); - if bits & sign == 0 { - bits ^ sign - } else { - !bits & width_mask(width) - } -} - -fn float_bits_from_ordered(value: u64, width: u8) -> u64 { - let sign = 1_u64 << (width - 1); - if value & sign == 0 { - !value & width_mask(width) - } else { - value ^ sign - } -} - -fn width_mask(width: u8) -> u64 { - if width == 64 { - u64::MAX - } else { - (1_u64 << width) - 1 - } -} - -fn primitive_from_latents( - latents: Vec, - ptype: PType, - validity: Validity, -) -> VortexResult { - Ok(match ptype { - PType::U8 => PrimitiveArray::new( - Buffer::from(convert_latents(&latents, u8::try_from)?), - validity, - ), - PType::U16 => PrimitiveArray::new( - Buffer::from(convert_latents(&latents, u16::try_from)?), - validity, - ), - PType::U32 => PrimitiveArray::new( - Buffer::from(convert_latents(&latents, u32::try_from)?), - validity, - ), - PType::U64 => PrimitiveArray::new(Buffer::from(latents), validity), - PType::I8 => PrimitiveArray::new( - Buffer::from( - latents - .into_iter() - .map(|value| Ok((u8::try_from(value)? ^ (1_u8 << 7)).cast_signed())) - .collect::>>()?, - ), - validity, - ), - PType::I16 => PrimitiveArray::new( - Buffer::from( - latents - .into_iter() - .map(|value| Ok((u16::try_from(value)? ^ (1_u16 << 15)).cast_signed())) - .collect::>>()?, - ), - validity, - ), - PType::I32 => PrimitiveArray::new( - Buffer::from( - latents - .into_iter() - .map(|value| Ok((u32::try_from(value)? ^ (1_u32 << 31)).cast_signed())) - .collect::>>()?, - ), - validity, - ), - PType::I64 => PrimitiveArray::new( - Buffer::from(latents) - .map_each_in_place(|value| (value ^ (1_u64 << 63)).cast_signed()) - .freeze(), - validity, - ), - PType::F16 => PrimitiveArray::new( - Buffer::from( - latents - .into_iter() - .map(|value| { - Ok(f16::from_bits(u16::try_from(float_bits_from_ordered( - value, 16, - ))?)) - }) - .collect::>>()?, - ), - validity, - ), - PType::F32 => PrimitiveArray::new( - Buffer::from( - latents - .into_iter() - .map(|value| { - Ok(f32::from_bits(u32::try_from(float_bits_from_ordered( - value, 32, - ))?)) - }) - .collect::>>()?, - ), - validity, - ), - PType::F64 => PrimitiveArray::new( - Buffer::from(latents) - .map_each_in_place(|value| f64::from_bits(float_bits_from_ordered(value, 64))) - .freeze(), - validity, - ), - }) -} - -fn convert_latents( - latents: &[u64], - convert: impl Fn(u64) -> Result, -) -> VortexResult> { - latents - .iter() - .copied() - .map(convert) - .collect::, _>>() - .map_err(Into::into) -} - -fn scalar_from_latent(latent: u64, ptype: PType, nullability: Nullability) -> VortexResult { - Ok(match ptype { - PType::U8 => Scalar::primitive(u8::try_from(latent)?, nullability), - PType::U16 => Scalar::primitive(u16::try_from(latent)?, nullability), - PType::U32 => Scalar::primitive(u32::try_from(latent)?, nullability), - PType::U64 => Scalar::primitive(latent, nullability), - PType::I8 => Scalar::primitive( - (u8::try_from(latent)? ^ (1_u8 << 7)).cast_signed(), - nullability, - ), - PType::I16 => Scalar::primitive( - (u16::try_from(latent)? ^ (1_u16 << 15)).cast_signed(), - nullability, - ), - PType::I32 => Scalar::primitive( - (u32::try_from(latent)? ^ (1_u32 << 31)).cast_signed(), - nullability, - ), - PType::I64 => Scalar::primitive((latent ^ (1_u64 << 63)).cast_signed(), nullability), - PType::F16 => Scalar::primitive( - f16::from_bits(u16::try_from(float_bits_from_ordered(latent, 16))?), - nullability, - ), - PType::F32 => Scalar::primitive( - f32::from_bits(u32::try_from(float_bits_from_ordered(latent, 32))?), - nullability, - ), - PType::F64 => Scalar::primitive( - f64::from_bits(float_bits_from_ordered(latent, 64)), - nullability, - ), - }) -} - -#[cfg(test)] -mod tests { - use std::sync::LazyLock; - - use vortex_array::ArrayContext; - use vortex_array::IntoArray; - use vortex_array::VortexSessionExecute; - use vortex_array::assert_arrays_eq; - use vortex_array::assert_nth_scalar; - use vortex_array::serde::SerializeOptions; - use vortex_array::serde::SerializedArray; - use vortex_buffer::ByteBufferMut; - use vortex_error::VortexResult; - use vortex_session::VortexSession; - use vortex_session::registry::ReadContext; - - use super::*; - - static SESSION: LazyLock = LazyLock::new(|| { - let session = vortex_array::array_session(); - crate::initialize(&session); - session - }); - - fn assert_roundtrip(array: PrimitiveArray) -> VortexResult<()> { - let encoded = RangeEntropy::from_primitive(array.as_view(), 3)?; - let mut ctx = SESSION.create_execution_ctx(); - assert_arrays_eq!(encoded, array, &mut ctx); - - let slice_stop = array.len() - 1; - let slice = encoded.slice(1..slice_stop)?; - assert!(slice.is::()); - assert_arrays_eq!(slice, array.into_array().slice(1..slice_stop)?, &mut ctx); - Ok(()) - } - - #[test] - fn primitive_types_roundtrip() -> VortexResult<()> { - for array in [ - PrimitiveArray::from_iter([0_u8, 1, u8::MAX, 7]), - PrimitiveArray::from_iter([0_u16, 1, u16::MAX, 7]), - PrimitiveArray::from_iter([0_u32, 1, u32::MAX, 7]), - PrimitiveArray::from_iter([0_u64, 1, u64::MAX, 7]), - PrimitiveArray::from_iter([i8::MIN, -1, 0, i8::MAX]), - PrimitiveArray::from_iter([i16::MIN, -1, 0, i16::MAX]), - PrimitiveArray::from_iter([i32::MIN, -1, 0, i32::MAX]), - PrimitiveArray::from_iter([i64::MIN, -1, 0, i64::MAX]), - PrimitiveArray::from_iter([ - f16::NEG_INFINITY, - f16::from_f32(-0.0), - f16::from_f32(1.5), - f16::INFINITY, - ]), - PrimitiveArray::from_iter([f32::NEG_INFINITY, -0.0, 1.5, f32::INFINITY]), - PrimitiveArray::from_iter([f64::NEG_INFINITY, -0.0, 1.5, f64::INFINITY]), - ] { - assert_roundtrip(array)?; - } - Ok(()) - } - - #[test] - fn nullable_slice_and_scalar_access() -> VortexResult<()> { - let array = PrimitiveArray::from_option_iter([ - Some(-10.5_f64), - None, - Some(-0.0), - Some(42.25), - None, - Some(1_000.0), - ]); - let encoded = RangeEntropy::from_primitive(array.as_view(), 2)?; - let mut ctx = SESSION.create_execution_ctx(); - assert_arrays_eq!(encoded, array, &mut ctx); - assert_nth_scalar!(encoded, 3, 42.25_f64, &mut ctx); - assert!(encoded.execute_scalar(1, &mut ctx)?.is_null()); - - let sliced = encoded.slice(1..5)?; - assert!(sliced.is::()); - assert_arrays_eq!(sliced, array.into_array().slice(1..5)?, &mut ctx); - Ok(()) - } - - #[test] - fn serialization_roundtrip() -> VortexResult<()> { - let original = PrimitiveArray::from_option_iter([ - Some(i64::MIN), - None, - Some(-1), - Some(0), - Some(i64::MAX), - ]); - let encoded = RangeEntropy::from_primitive(original.as_view(), 2)?; - let sliced = encoded.slice(1..5)?; - let dtype = sliced.dtype().clone(); - let len = sliced.len(); - let array_context = ArrayContext::empty(); - let serialized = - sliced.serialize(&array_context, &SESSION, &SerializeOptions::default())?; - let mut bytes = ByteBufferMut::empty(); - for buffer in serialized { - bytes.extend_from_slice(buffer.as_ref()); - } - - let decoded = SerializedArray::try_from(bytes.freeze())?.decode( - &dtype, - len, - &ReadContext::new(array_context.to_ids()), - &SESSION, - )?; - assert!(decoded.is::()); - assert_arrays_eq!( - decoded, - original.into_array().slice(1..5)?, - &mut SESSION.create_execution_ctx() - ); - Ok(()) - } -} diff --git a/encodings/range-entropy/src/bit_split.rs b/encodings/range-entropy/src/bit_split.rs deleted file mode 100644 index 22095099cd7..00000000000 --- a/encodings/range-entropy/src/bit_split.rs +++ /dev/null @@ -1,264 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use fastlanes::BitPacking; -use vortex_error::VortexResult; -use vortex_error::vortex_ensure; -use vortex_error::vortex_err; - -const CHUNK_LEN: usize = 1024; -const SERIALIZED_BLOCK_METADATA_BYTES: usize = 8; - -/// A prototype block-local prefix dictionary with fixed-width suffixes. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct BitSplitCodec { - len: usize, - blocks: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -struct BitSplitBlock { - len: u16, - suffix_width: u8, - code_width: u8, - prefixes: Vec, - codes: Vec, - suffixes: Vec, -} - -impl BitSplitCodec { - /// Encode ordered unsigned latents in independent 1,024-value blocks. - pub fn encode(values: &[u64]) -> VortexResult { - let blocks = values - .chunks(CHUNK_LEN) - .map(encode_block) - .collect::>>()?; - Ok(Self { - len: values.len(), - blocks, - }) - } - - /// Decode all values. - pub fn decode(&self) -> VortexResult> { - let mut values = Vec::with_capacity(self.len); - let mut codes = [0_u64; CHUNK_LEN]; - let mut suffixes = [0_u64; CHUNK_LEN]; - for block in &self.blocks { - codes.fill(0); - suffixes.fill(0); - if block.code_width > 0 { - // SAFETY: The encoder creates one complete FastLanes chunk. - unsafe { - u64::unchecked_unpack(usize::from(block.code_width), &block.codes, &mut codes); - } - } - if block.suffix_width > 0 { - // SAFETY: The encoder creates one complete FastLanes chunk. - unsafe { - u64::unchecked_unpack( - usize::from(block.suffix_width), - &block.suffixes, - &mut suffixes, - ); - } - } - - for index in 0..usize::from(block.len) { - let prefix = block.prefixes[usize::try_from(codes[index])?]; - values.push(join(prefix, suffixes[index], block.suffix_width)); - } - } - Ok(values) - } - - /// Decode one value with two direct packed reads and one dictionary lookup. - pub fn scalar_at(&self, index: usize) -> VortexResult { - vortex_ensure!( - index < self.len, - "index {index} is out of bounds for length {}", - self.len - ); - let block = &self.blocks[index / CHUNK_LEN]; - let index_in_block = index % CHUNK_LEN; - let code = if block.code_width == 0 { - 0 - } else { - // SAFETY: The encoder creates one complete FastLanes chunk. - unsafe { - usize::try_from(u64::unchecked_unpack_single( - usize::from(block.code_width), - &block.codes, - index_in_block, - ))? - } - }; - let suffix = if block.suffix_width == 0 { - 0 - } else { - // SAFETY: The encoder creates one complete FastLanes chunk. - unsafe { - u64::unchecked_unpack_single( - usize::from(block.suffix_width), - &block.suffixes, - index_in_block, - ) - } - }; - Ok(join(block.prefixes[code], suffix, block.suffix_width)) - } - - /// Return the encoded bytes, including estimated serialized metadata. - pub fn encoded_size(&self) -> usize { - self.blocks - .iter() - .map(|block| { - SERIALIZED_BLOCK_METADATA_BYTES - + block.prefixes.len() * size_of::() - + block.codes.len() * size_of::() - + block.suffixes.len() * size_of::() - }) - .sum() - } - - /// Return the average suffix width across all blocks. - pub fn average_suffix_width(&self) -> f64 { - self.blocks - .iter() - .map(|block| f64::from(block.suffix_width)) - .sum::() - / self.blocks.len().max(1) as f64 - } - - /// Return the average prefix count across all blocks. - pub fn average_prefix_count(&self) -> f64 { - self.blocks - .iter() - .map(|block| block.prefixes.len() as f64) - .sum::() - / self.blocks.len().max(1) as f64 - } -} - -fn encode_block(values: &[u64]) -> VortexResult { - let mut sorted = values.to_vec(); - sorted.sort_unstable(); - sorted.dedup(); - - let mut divergence_counts = [0_usize; 65]; - for adjacent in sorted.windows(2) { - divergence_counts[usize::from(bit_width(adjacent[0] ^ adjacent[1]))] += 1; - } - let mut prefix_counts = [0_usize; 65]; - let mut active_divergences = 0_usize; - for suffix_width in (0..=64_usize).rev() { - prefix_counts[suffix_width] = usize::from(!sorted.is_empty()) + active_divergences; - active_divergences += divergence_counts[suffix_width]; - } - - let mut best_suffix_width = 64_u8; - let mut best_prefix_count = 1_usize; - let mut best_size = CHUNK_LEN * size_of::(); - for suffix_width in 0..=64_u8 { - let prefix_count = prefix_counts[usize::from(suffix_width)]; - let code_width = bit_width(u64::try_from(prefix_count.saturating_sub(1))?); - let size = prefix_count * size_of::() - + (CHUNK_LEN * usize::from(code_width)).div_ceil(8) - + (CHUNK_LEN * usize::from(suffix_width)).div_ceil(8); - if size < best_size { - best_size = size; - best_suffix_width = suffix_width; - best_prefix_count = prefix_count; - } - } - - let mut prefixes = Vec::with_capacity(best_prefix_count); - for value in sorted { - let prefix = prefix(value, best_suffix_width); - if prefixes.last() != Some(&prefix) { - prefixes.push(prefix); - } - } - let code_width = bit_width(u64::try_from(prefixes.len().saturating_sub(1))?); - let suffix_mask = low_mask(best_suffix_width); - let mut code_values = [0_u64; CHUNK_LEN]; - let mut suffix_values = [0_u64; CHUNK_LEN]; - for (index, &value) in values.iter().enumerate() { - let prefix = prefix(value, best_suffix_width); - let code = prefixes - .binary_search(&prefix) - .map_err(|_| vortex_err!("prefix dictionary does not contain {prefix}"))?; - code_values[index] = u64::try_from(code)?; - suffix_values[index] = value & suffix_mask; - } - - Ok(BitSplitBlock { - len: u16::try_from(values.len())?, - suffix_width: best_suffix_width, - code_width, - prefixes, - codes: fast_pack(&code_values, code_width), - suffixes: fast_pack(&suffix_values, best_suffix_width), - }) -} - -fn prefix(value: u64, suffix_width: u8) -> u64 { - if suffix_width == 64 { - 0 - } else { - value >> suffix_width - } -} - -fn join(prefix: u64, suffix: u64, suffix_width: u8) -> u64 { - if suffix_width == 64 { - suffix - } else { - (prefix << suffix_width) | suffix - } -} - -fn fast_pack(values: &[u64; CHUNK_LEN], width: u8) -> Vec { - if width == 0 { - return Vec::new(); - } - let mut packed = vec![0_u64; CHUNK_LEN * usize::from(width) / u64::BITS as usize]; - // SAFETY: Both slices have the exact lengths required for one FastLanes chunk. - unsafe { u64::unchecked_pack(usize::from(width), values, &mut packed) }; - packed -} - -fn bit_width(value: u64) -> u8 { - u8::try_from(u64::BITS - value.leading_zeros()).unwrap_or(64) -} - -fn low_mask(bits: u8) -> u64 { - match bits { - 0 => 0, - 64 => u64::MAX, - _ => (1_u64 << bits) - 1, - } -} - -#[cfg(test)] -mod tests { - use vortex_error::VortexResult; - - use super::BitSplitCodec; - - #[test] - fn roundtrip_and_scalar_access() -> VortexResult<()> { - let values = (0..2_051) - .map(|index| { - let prefix = [11_u64, 1_000, 5_000, 90_000][index % 4]; - Ok((prefix << 19) | u64::try_from(index * 17)?) - }) - .collect::>>()?; - let codec = BitSplitCodec::encode(&values)?; - assert_eq!(codec.decode()?, values); - for (index, &value) in values.iter().enumerate() { - assert_eq!(codec.scalar_at(index)?, value); - } - Ok(()) - } -} diff --git a/encodings/range-entropy/src/codec.rs b/encodings/range-entropy/src/codec.rs deleted file mode 100644 index 05ec8f09531..00000000000 --- a/encodings/range-entropy/src/codec.rs +++ /dev/null @@ -1,2337 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::cmp::Ordering; -use std::cmp::Reverse; -use std::ops::Deref; -use std::ops::DerefMut; -use std::ops::Range; - -use vortex_buffer::ByteBuffer; -use vortex_error::VortexResult; -use vortex_error::vortex_ensure; - -const MAX_SCALE_BITS: u8 = 10; -const ANS_INTERLEAVING: usize = 4; -const ANS_STATE_BYTES: usize = ANS_INTERLEAVING * size_of::(); -const ANS_PADDING: usize = size_of::(); -const DECODE_BATCH_SIZE: usize = 256; -const OFFSET_PADDING: usize = 15; -const MAX_BINS: usize = 64; -const SPLIT_CANDIDATES: usize = 64; -const TRAINING_SAMPLE_SIZE: usize = 8192; -const BIN_TABLE_COST_BITS: f64 = 128.0; - -#[repr(align(64))] -struct DecodeScratch([T; DECODE_BATCH_SIZE]); - -impl Deref for DecodeScratch { - type Target = [T; DECODE_BATCH_SIZE]; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl DerefMut for DecodeScratch { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} - -/// A restart-block range entropy stream for ordered unsigned latents. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RangeEntropyCodec { - len: usize, - block_len: usize, - scale_bits: u8, - bin_lowers: Vec, - offset_widths: Vec, - weights: Vec, - block_offsets: Vec, - payload: ByteBuffer, -} - -/// Owned components of a range entropy stream. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RangeEntropyParts { - /// Logical value count. - pub len: usize, - /// Logical values per restart block. - pub block_len: usize, - /// Base-two logarithm of the ANS table size. - pub scale_bits: u8, - /// Lower bound for each range bin. - pub bin_lowers: Vec, - /// Fixed offset width for each range bin. - pub offset_widths: Vec, - /// Quantized ANS weight for each range bin. - pub weights: Vec, - /// Byte offsets for restart blocks. - pub block_offsets: Vec, - /// Encoded ANS and offset streams. - pub payload: ByteBuffer, -} - -/// A restart-block range codec with fixed-width bin identifiers. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RangePackedCodec { - len: usize, - block_len: usize, - symbol_width: u8, - bin_lowers: Vec, - offset_widths: Vec, - block_offsets: Vec, - payload: ByteBuffer, -} - -/// A restart-block range codec with hot-bin tags and a cold-bin escape stream. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RangeTwoLevelCodec { - len: usize, - block_len: usize, - tag_width: u8, - cold_width: u8, - bin_lowers: Vec, - offset_widths: Vec, - block_offsets: Vec, - payload: ByteBuffer, -} - -/// A restart-block range codec with one fixed-width residual stream per bin. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RangeGroupedCodec { - len: usize, - block_len: usize, - symbol_width: u8, - bin_lowers: Vec, - offset_widths: Vec, - block_offsets: Vec, - payload: ByteBuffer, -} - -impl RangeEntropyCodec { - /// Encode ordered unsigned latents into independent restart blocks. - pub fn encode(values: &[u64], block_len: usize) -> VortexResult { - vortex_ensure!(block_len > 0, "block length must be greater than zero"); - - if values.is_empty() { - return Ok(Self { - len: 0, - block_len, - scale_bits: 0, - bin_lowers: Vec::new(), - offset_widths: Vec::new(), - weights: Vec::new(), - block_offsets: vec![0], - payload: ByteBuffer::empty(), - }); - } - - let (training_sample, minimum, maximum) = training_sample(values); - let bins = cover_domain(optimize_bins(&training_sample), minimum, maximum); - let (bins, symbols, counts) = assign_bins(values, bins)?; - let bin_lowers: Vec<_> = bins.iter().map(|bin| bin.lower).collect(); - let offset_widths: Vec<_> = bins.iter().map(|bin| bin.offset_bits).collect(); - - let scale_bits = choose_scale_bits(values.len(), bins.len()); - let weights = quantize_weights(&counts, scale_bits)?; - let table = AnsEncoderTable::new(&weights, scale_bits)?; - - let n_blocks = values.len().div_ceil(block_len); - let mut block_offsets = Vec::with_capacity(n_blocks + 1); - let mut payload = Vec::new(); - block_offsets.push(0); - - for block_idx in 0..n_blocks { - let start = block_idx * block_len; - let stop = (start + block_len).min(values.len()); - encode_block( - &values[start..stop], - &symbols[start..stop], - &bins, - &table, - &mut payload, - )?; - block_offsets.push(u32::try_from(payload.len())?); - } - - let codec = Self { - len: values.len(), - block_len, - scale_bits, - bin_lowers, - offset_widths, - weights, - block_offsets, - payload: ByteBuffer::from(payload), - }; - codec.validate()?; - Ok(codec) - } - - /// Construct a codec from stored components. - pub fn try_from_parts(parts: RangeEntropyParts) -> VortexResult { - let codec = Self { - len: parts.len, - block_len: parts.block_len, - scale_bits: parts.scale_bits, - bin_lowers: parts.bin_lowers, - offset_widths: parts.offset_widths, - weights: parts.weights, - block_offsets: parts.block_offsets, - payload: parts.payload, - }; - codec.validate()?; - Ok(codec) - } - - /// Consume the codec and return its stored components. - pub fn into_parts(self) -> RangeEntropyParts { - RangeEntropyParts { - len: self.len, - block_len: self.block_len, - scale_bits: self.scale_bits, - bin_lowers: self.bin_lowers, - offset_widths: self.offset_widths, - weights: self.weights, - block_offsets: self.block_offsets, - payload: self.payload, - } - } - - /// Decode all values. - pub fn decode(&self) -> VortexResult> { - self.decode_range(0..self.len) - } - - /// Decode one logical range through the intersecting restart blocks. - pub fn decode_range(&self, range: Range) -> VortexResult> { - self.validate()?; - vortex_ensure!( - range.start <= range.end && range.end <= self.len, - "decode range {:?} exceeds length {}", - range, - self.len - ); - if range.is_empty() { - return Ok(Vec::new()); - } - - let first_block = range.start / self.block_len; - let last_block = (range.end - 1) / self.block_len; - let bins = self.bins()?; - let table = AnsDecoderTable::new(&self.weights, self.scale_bits, &bins)?; - let mut values = Vec::with_capacity(range.len()); - for block_idx in first_block..=last_block { - let block_start = block_idx * self.block_len; - let start = range.start.saturating_sub(block_start); - let stop = (range.end - block_start).min(self.block_value_count(block_idx)); - self.decode_block_into(block_idx, start..stop, &table, &mut values)?; - } - Ok(values) - } - - /// Decode one restart block. - pub fn decode_block(&self, block_idx: usize) -> VortexResult> { - self.validate()?; - vortex_ensure!( - block_idx < self.n_blocks(), - "block index {block_idx} is out of bounds for {} blocks", - self.n_blocks() - ); - - let bins = self.bins()?; - let table = AnsDecoderTable::new(&self.weights, self.scale_bits, &bins)?; - let n_values = self.block_value_count(block_idx); - let mut values = Vec::with_capacity(n_values); - self.decode_block_into(block_idx, 0..n_values, &table, &mut values)?; - Ok(values) - } - - fn decode_block_into( - &self, - block_idx: usize, - output_range: Range, - table: &AnsDecoderTable, - values: &mut Vec, - ) -> VortexResult<()> { - let start = usize::try_from(self.block_offsets[block_idx])?; - let stop = usize::try_from(self.block_offsets[block_idx + 1])?; - let n_values = self.block_value_count(block_idx); - decode_block_into( - &self.payload.as_slice()[start..stop], - n_values, - output_range, - table, - values, - ) - } - - /// Return one value after one restart-block decode. - pub fn value(&self, index: usize) -> VortexResult { - vortex_ensure!( - index < self.len, - "value index {index} is out of bounds for length {}", - self.len - ); - let block_idx = index / self.block_len; - let within_block = index % self.block_len; - Ok(self.decode_block(block_idx)?[within_block]) - } - - /// Return the logical value count. - pub fn len(&self) -> usize { - self.len - } - - /// Return true when the stream has no values. - pub fn is_empty(&self) -> bool { - self.len == 0 - } - - /// Return the restart block length. - pub fn block_len(&self) -> usize { - self.block_len - } - - /// Return the ANS table size log. - pub fn scale_bits(&self) -> u8 { - self.scale_bits - } - - /// Return the bin lower bounds. - pub fn bin_lowers(&self) -> &[u64] { - &self.bin_lowers - } - - /// Return the fixed offset width for each bin. - pub fn offset_widths(&self) -> &[u8] { - &self.offset_widths - } - - /// Return the quantized ANS weight for each bin. - pub fn weights(&self) -> &[u16] { - &self.weights - } - - /// Return the byte offsets for restart blocks. - pub fn block_offsets(&self) -> &[u32] { - &self.block_offsets - } - - /// Return the encoded payload. - pub fn payload(&self) -> &[u8] { - self.payload.as_slice() - } - - /// Return the total bytes in the variable tables and payload. - pub fn encoded_size(&self) -> usize { - self.bin_lowers.len() * size_of::() - + self.offset_widths.len() * size_of::() - + self.weights.len() * size_of::() - + self.block_offsets.len() * size_of::() - + self.payload.len() - } - - fn validate(&self) -> VortexResult<()> { - vortex_ensure!(self.block_len > 0, "block length must be greater than zero"); - vortex_ensure!( - self.scale_bits <= MAX_SCALE_BITS, - "ANS table log {} exceeds {MAX_SCALE_BITS}", - self.scale_bits - ); - vortex_ensure!( - self.bin_lowers.len() == self.offset_widths.len() - && self.bin_lowers.len() == self.weights.len(), - "range entropy bin tables have different lengths" - ); - - if self.len == 0 { - vortex_ensure!(self.bin_lowers.is_empty(), "empty streams cannot have bins"); - vortex_ensure!( - self.payload.is_empty(), - "empty streams cannot have payload bytes" - ); - vortex_ensure!( - self.block_offsets == [0], - "empty streams require one zero block offset" - ); - return Ok(()); - } - - vortex_ensure!( - !self.bin_lowers.is_empty(), - "non-empty streams require bins" - ); - vortex_ensure!( - self.bin_lowers.len() <= MAX_BINS, - "bin count {} exceeds maximum {MAX_BINS}", - self.bin_lowers.len() - ); - vortex_ensure!( - self.offset_widths.iter().all(|&width| width <= 64), - "offset width exceeds 64 bits" - ); - vortex_ensure!( - self.weights.iter().all(|&weight| weight > 0), - "ANS weights must be positive" - ); - vortex_ensure!( - self.weights - .iter() - .map(|&weight| u32::from(weight)) - .sum::() - == 1_u32 << self.scale_bits, - "ANS weights do not sum to the table size" - ); - vortex_ensure!( - self.block_offsets.len() == self.n_blocks() + 1, - "expected {} block offsets, got {}", - self.n_blocks() + 1, - self.block_offsets.len() - ); - vortex_ensure!( - self.block_offsets[0] == 0, - "first block offset must be zero" - ); - vortex_ensure!( - self.block_offsets.windows(2).all(|pair| pair[0] <= pair[1]), - "block offsets must be monotonic" - ); - vortex_ensure!( - usize::try_from(*self.block_offsets.last().unwrap_or(&0))? == self.payload.len(), - "last block offset does not match payload length" - ); - Ok(()) - } - - fn bins(&self) -> VortexResult> { - self.bin_lowers - .iter() - .copied() - .zip(self.offset_widths.iter().copied()) - .map(|(lower, offset_bits)| Bin::try_new(lower, offset_bits)) - .collect() - } - - fn n_blocks(&self) -> usize { - self.len.div_ceil(self.block_len) - } - - fn block_value_count(&self, block_idx: usize) -> usize { - let start = block_idx * self.block_len; - (self.len - start).min(self.block_len) - } -} - -impl RangePackedCodec { - /// Encode ordered unsigned latents with fixed-width bin identifiers. - pub fn encode(values: &[u64], block_len: usize) -> VortexResult { - vortex_ensure!(block_len > 0, "block length must be greater than zero"); - - if values.is_empty() { - return Ok(Self { - len: 0, - block_len, - symbol_width: 0, - bin_lowers: Vec::new(), - offset_widths: Vec::new(), - block_offsets: vec![0], - payload: ByteBuffer::empty(), - }); - } - - let (training_sample, minimum, maximum) = training_sample(values); - let bins = cover_domain(optimize_bins(&training_sample), minimum, maximum); - let (bins, symbols, _) = assign_bins(values, bins)?; - let symbol_width = bit_width(u64::try_from(bins.len() - 1)?); - let bin_lowers = bins.iter().map(|bin| bin.lower).collect(); - let offset_widths = bins.iter().map(|bin| bin.offset_bits).collect(); - - let n_blocks = values.len().div_ceil(block_len); - let mut block_offsets = Vec::with_capacity(n_blocks + 1); - let mut payload = Vec::new(); - block_offsets.push(0); - for block_idx in 0..n_blocks { - let start = block_idx * block_len; - let stop = (start + block_len).min(values.len()); - encode_packed_block( - &values[start..stop], - &symbols[start..stop], - &bins, - symbol_width, - &mut payload, - ); - block_offsets.push(u32::try_from(payload.len())?); - } - - let codec = Self { - len: values.len(), - block_len, - symbol_width, - bin_lowers, - offset_widths, - block_offsets, - payload: ByteBuffer::from(payload), - }; - codec.validate()?; - Ok(codec) - } - - /// Decode all values. - pub fn decode(&self) -> VortexResult> { - self.validate()?; - let bins = self.bins()?; - let mut values = Vec::with_capacity(self.len); - for block_idx in 0..self.n_blocks() { - let start = usize::try_from(self.block_offsets[block_idx])?; - let stop = usize::try_from(self.block_offsets[block_idx + 1])?; - decode_packed_block_into( - &self.payload.as_slice()[start..stop], - self.block_value_count(block_idx), - self.symbol_width, - &bins, - &mut values, - )?; - } - Ok(values) - } - - /// Return the logical value count. - pub fn len(&self) -> usize { - self.len - } - - /// Return true when the stream has no values. - pub fn is_empty(&self) -> bool { - self.len == 0 - } - - /// Return the fixed bin identifier width. - pub fn symbol_width(&self) -> u8 { - self.symbol_width - } - - /// Return the bin lower bounds. - pub fn bin_lowers(&self) -> &[u64] { - &self.bin_lowers - } - - /// Return the total bytes in the variable tables and payload. - pub fn encoded_size(&self) -> usize { - self.bin_lowers.len() * size_of::() - + self.offset_widths.len() * size_of::() - + self.block_offsets.len() * size_of::() - + self.payload.len() - } - - fn validate(&self) -> VortexResult<()> { - vortex_ensure!(self.block_len > 0, "block length must be greater than zero"); - vortex_ensure!( - self.bin_lowers.len() == self.offset_widths.len(), - "range packed bin tables have different lengths" - ); - - if self.len == 0 { - vortex_ensure!(self.bin_lowers.is_empty(), "empty streams cannot have bins"); - vortex_ensure!( - self.payload.is_empty(), - "empty streams cannot have payload bytes" - ); - vortex_ensure!( - self.block_offsets == [0], - "empty streams require one zero block offset" - ); - return Ok(()); - } - - vortex_ensure!( - !self.bin_lowers.is_empty() && self.bin_lowers.len() <= MAX_BINS, - "range packed bin count is invalid" - ); - vortex_ensure!( - self.symbol_width == bit_width(u64::try_from(self.bin_lowers.len() - 1)?), - "range packed symbol width does not match its bin count" - ); - vortex_ensure!( - self.offset_widths.iter().all(|&width| width <= 64), - "offset width exceeds 64 bits" - ); - vortex_ensure!( - self.block_offsets.len() == self.n_blocks() + 1, - "range packed block offset count is invalid" - ); - vortex_ensure!( - self.block_offsets.first() == Some(&0) - && self.block_offsets.windows(2).all(|pair| pair[0] <= pair[1]), - "range packed block offsets are invalid" - ); - vortex_ensure!( - usize::try_from(*self.block_offsets.last().unwrap_or(&0))? == self.payload.len(), - "last block offset does not match payload length" - ); - Ok(()) - } - - fn bins(&self) -> VortexResult> { - self.bin_lowers - .iter() - .copied() - .zip(self.offset_widths.iter().copied()) - .map(|(lower, offset_bits)| Bin::try_new(lower, offset_bits)) - .collect() - } - - fn n_blocks(&self) -> usize { - self.len.div_ceil(self.block_len) - } - - fn block_value_count(&self, block_idx: usize) -> usize { - let start = block_idx * self.block_len; - (self.len - start).min(self.block_len) - } -} - -impl RangeTwoLevelCodec { - /// Encode ordered unsigned latents with hot-bin tags and cold-bin escapes. - pub fn encode(values: &[u64], block_len: usize) -> VortexResult { - vortex_ensure!(block_len > 0, "block length must be greater than zero"); - if values.is_empty() { - return Ok(Self { - len: 0, - block_len, - tag_width: 0, - cold_width: 0, - bin_lowers: Vec::new(), - offset_widths: Vec::new(), - block_offsets: vec![0], - payload: ByteBuffer::empty(), - }); - } - - let (training_sample, minimum, maximum) = training_sample(values); - let bins = cover_domain(optimize_bins(&training_sample), minimum, maximum); - let (bins, symbols, counts) = assign_bins(values, bins)?; - let (tag_width, direct_count, order) = choose_two_level_layout(&counts); - let cold_count = bins.len() - direct_count; - let cold_width = bit_width(u64::try_from(cold_count.saturating_sub(1))?); - let mut rank_by_symbol = vec![0usize; bins.len()]; - for (rank, &symbol) in order.iter().enumerate() { - rank_by_symbol[symbol] = rank; - } - let reordered_bins = order.iter().map(|&symbol| bins[symbol]).collect::>(); - let ranks = symbols - .into_iter() - .map(|symbol| rank_by_symbol[symbol]) - .collect::>(); - - let n_blocks = values.len().div_ceil(block_len); - let mut block_offsets = Vec::with_capacity(n_blocks + 1); - let mut payload = Vec::new(); - block_offsets.push(0); - for block_idx in 0..n_blocks { - let start = block_idx * block_len; - let stop = (start + block_len).min(values.len()); - encode_two_level_block( - &values[start..stop], - &ranks[start..stop], - &reordered_bins, - tag_width, - direct_count, - cold_width, - &mut payload, - )?; - block_offsets.push(u32::try_from(payload.len())?); - } - - Ok(Self { - len: values.len(), - block_len, - tag_width, - cold_width, - bin_lowers: reordered_bins.iter().map(|bin| bin.lower).collect(), - offset_widths: reordered_bins.iter().map(|bin| bin.offset_bits).collect(), - block_offsets, - payload: ByteBuffer::from(payload), - }) - } - - /// Decode all values. - pub fn decode(&self) -> VortexResult> { - if self.len == 0 { - return Ok(Vec::new()); - } - let bins = self - .bin_lowers - .iter() - .copied() - .zip(self.offset_widths.iter().copied()) - .map(|(lower, offset_bits)| Bin::try_new(lower, offset_bits)) - .collect::>>()?; - let direct_count = ((1usize << self.tag_width) - 1).min(bins.len()); - let mut values = Vec::with_capacity(self.len); - for block_idx in 0..self.len.div_ceil(self.block_len) { - let start = usize::try_from(self.block_offsets[block_idx])?; - let stop = usize::try_from(self.block_offsets[block_idx + 1])?; - let block_start = block_idx * self.block_len; - let block_value_count = (self.len - block_start).min(self.block_len); - decode_two_level_block_into( - &self.payload.as_slice()[start..stop], - block_value_count, - self.tag_width, - direct_count, - self.cold_width, - &bins, - &mut values, - )?; - } - Ok(values) - } - - /// Return the fixed hot-bin tag width. - pub fn tag_width(&self) -> u8 { - self.tag_width - } - - /// Return the fixed cold-bin identifier width. - pub fn cold_width(&self) -> u8 { - self.cold_width - } - - /// Return the total bytes in the variable tables and payload. - pub fn encoded_size(&self) -> usize { - self.bin_lowers.len() * size_of::() - + self.offset_widths.len() * size_of::() - + self.block_offsets.len() * size_of::() - + self.payload.len() - } -} - -impl RangeGroupedCodec { - /// Encode ordered unsigned latents with one residual stream per bin. - pub fn encode(values: &[u64], block_len: usize) -> VortexResult { - vortex_ensure!(block_len > 0, "block length must be greater than zero"); - if values.is_empty() { - return Ok(Self { - len: 0, - block_len, - symbol_width: 0, - bin_lowers: Vec::new(), - offset_widths: Vec::new(), - block_offsets: vec![0], - payload: ByteBuffer::empty(), - }); - } - - let (training_sample, minimum, maximum) = training_sample(values); - let bins = cover_domain(optimize_bins(&training_sample), minimum, maximum); - let (bins, symbols, _) = assign_bins(values, bins)?; - let symbol_width = bit_width(u64::try_from(bins.len() - 1)?); - let n_blocks = values.len().div_ceil(block_len); - let mut block_offsets = Vec::with_capacity(n_blocks + 1); - let mut payload = Vec::new(); - block_offsets.push(0); - for block_idx in 0..n_blocks { - let start = block_idx * block_len; - let stop = (start + block_len).min(values.len()); - encode_grouped_block( - &values[start..stop], - &symbols[start..stop], - &bins, - symbol_width, - &mut payload, - )?; - block_offsets.push(u32::try_from(payload.len())?); - } - - Ok(Self { - len: values.len(), - block_len, - symbol_width, - bin_lowers: bins.iter().map(|bin| bin.lower).collect(), - offset_widths: bins.iter().map(|bin| bin.offset_bits).collect(), - block_offsets, - payload: ByteBuffer::from(payload), - }) - } - - /// Decode all values. - pub fn decode(&self) -> VortexResult> { - if self.len == 0 { - return Ok(Vec::new()); - } - let bins = self - .bin_lowers - .iter() - .copied() - .zip(self.offset_widths.iter().copied()) - .map(|(lower, offset_bits)| Bin::try_new(lower, offset_bits)) - .collect::>>()?; - let mut values = Vec::with_capacity(self.len); - for block_idx in 0..self.len.div_ceil(self.block_len) { - let start = usize::try_from(self.block_offsets[block_idx])?; - let stop = usize::try_from(self.block_offsets[block_idx + 1])?; - let block_start = block_idx * self.block_len; - let block_value_count = (self.len - block_start).min(self.block_len); - decode_grouped_block_into( - &self.payload.as_slice()[start..stop], - block_value_count, - self.symbol_width, - &bins, - &mut values, - )?; - } - Ok(values) - } - - /// Return the total bytes in the variable tables and payload. - pub fn encoded_size(&self) -> usize { - self.bin_lowers.len() * size_of::() - + self.offset_widths.len() * size_of::() - + self.block_offsets.len() * size_of::() - + self.payload.len() - } -} - -#[derive(Clone, Copy, Debug)] -struct Bin { - lower: u64, - offset_bits: u8, -} - -impl Bin { - fn try_new(lower: u64, offset_bits: u8) -> VortexResult { - vortex_ensure!(offset_bits <= 64, "offset width exceeds 64 bits"); - Ok(Self { lower, offset_bits }) - } - - fn from_values(lower: u64, upper: u64) -> Self { - let offset_bits = bit_width(upper - lower); - Self { lower, offset_bits } - } - - fn contains(&self, value: u64) -> bool { - if value < self.lower { - return false; - } - self.offset_bits == 64 || value - self.lower <= low_mask(self.offset_bits) - } -} - -fn bit_width(value: u64) -> u8 { - u8::try_from(u64::BITS - value.leading_zeros()).unwrap_or(64) -} - -fn low_mask(bits: u8) -> u64 { - if bits == 64 { - u64::MAX - } else if bits == 0 { - 0 - } else { - (1_u64 << bits) - 1 - } -} - -fn find_bin(value: u64, bins: &[Bin], search_order: &[usize]) -> VortexResult { - search_order - .iter() - .copied() - .find(|&index| bins[index].contains(value)) - .ok_or_else(|| vortex_error::vortex_err!("no range entropy bin contains value {value}")) -} - -fn assign_bins( - values: &[u64], - mut bins: Vec, -) -> VortexResult<(Vec, Vec, Vec)> { - loop { - let mut search_order: Vec<_> = (0..bins.len()).collect(); - search_order.sort_by_key(|&index| bins[index].offset_bits); - let mut counts = vec![0usize; bins.len()]; - let mut symbols = Vec::with_capacity(values.len()); - for &value in values { - let symbol = find_bin(value, &bins, &search_order)?; - counts[symbol] += 1; - symbols.push(symbol); - } - if counts.iter().all(|&count| count > 0) { - return Ok((bins, symbols, counts)); - } - bins = bins - .into_iter() - .zip(counts) - .filter_map(|(bin, count)| (count > 0).then_some(bin)) - .collect(); - } -} - -fn choose_two_level_layout(counts: &[usize]) -> (u8, usize, Vec) { - let mut order = (0..counts.len()).collect::>(); - order.sort_unstable_by_key(|&symbol| Reverse(counts[symbol])); - if counts.len() <= 1 { - return (0, counts.len(), order); - } - - let flat_width = bit_width((counts.len() - 1) as u64); - let mut best = (usize::MAX, flat_width, counts.len()); - let minimum_tag_width = flat_width.min(2); - for tag_width in minimum_tag_width..=flat_width { - let direct_count = ((1usize << tag_width) - 1).min(counts.len()); - let cold_count = counts.len() - direct_count; - let cold_width = bit_width(cold_count.saturating_sub(1) as u64); - let cold_values = order[direct_count..] - .iter() - .map(|&symbol| counts[symbol]) - .sum::(); - let cost = counts.iter().sum::() * usize::from(tag_width) - + cold_values * usize::from(cold_width); - if cost < best.0 { - best = (cost, tag_width, direct_count); - } - } - (best.1, best.2, order) -} - -fn optimize_bins(sorted: &[u64]) -> Vec { - debug_assert!(!sorted.is_empty()); - let mut segments = Vec::with_capacity(MAX_BINS); - segments.push(0..sorted.len()); - - while segments.len() < MAX_BINS { - let best = segments - .iter() - .enumerate() - .filter_map(|(segment_idx, segment)| { - best_split(sorted, segment).map(|split| (segment_idx, split)) - }) - .max_by(|(_, left), (_, right)| { - left.gain - .partial_cmp(&right.gain) - .unwrap_or(Ordering::Equal) - }); - - let Some((segment_idx, split)) = best else { - break; - }; - if split.gain <= BIN_TABLE_COST_BITS { - break; - } - - let segment = segments.remove(segment_idx); - segments.push(segment.start..split.at); - segments.push(split.at..segment.end); - segments.sort_unstable_by_key(|range| range.start); - } - - segments - .into_iter() - .map(|range| Bin::from_values(sorted[range.start], sorted[range.end - 1])) - .collect() -} - -fn training_sample(values: &[u64]) -> (Vec, u64, u64) { - let mut minimum = u64::MAX; - let mut maximum = u64::MIN; - for &value in values { - minimum = minimum.min(value); - maximum = maximum.max(value); - } - - let mut sample = if values.len() <= TRAINING_SAMPLE_SIZE { - values.to_vec() - } else { - (0..TRAINING_SAMPLE_SIZE) - .map(|index| values[index * values.len() / TRAINING_SAMPLE_SIZE]) - .collect() - }; - sample.push(minimum); - sample.push(maximum); - sample.sort_unstable(); - (sample, minimum, maximum) -} - -fn cover_domain(mut bins: Vec, minimum: u64, maximum: u64) -> Vec { - if let Some(first) = bins.first_mut() { - first.lower = minimum; - } - for index in 0..bins.len().saturating_sub(1) { - let upper = bins[index + 1].lower - 1; - bins[index].offset_bits = bit_width(upper - bins[index].lower); - } - if let Some(last) = bins.last_mut() { - last.offset_bits = bit_width(maximum - last.lower); - } - bins -} - -#[derive(Clone, Copy)] -struct Split { - at: usize, - gain: f64, -} - -fn best_split(sorted: &[u64], segment: &Range) -> Option { - let len = segment.len(); - if len < 2 || sorted[segment.start] == sorted[segment.end - 1] { - return None; - } - - let whole_width = f64::from(bit_width(sorted[segment.end - 1] - sorted[segment.start])); - let whole_cost = len as f64 * whole_width; - let mut best: Option = None; - - for candidate in 1..SPLIT_CANDIDATES { - let mut at = segment.start + len * candidate / SPLIT_CANDIDATES; - if at <= segment.start || at >= segment.end { - continue; - } - while at < segment.end && sorted[at - 1] == sorted[at] { - at += 1; - } - if at >= segment.end { - continue; - } - - let left_len = at - segment.start; - let right_len = segment.end - at; - let left_width = f64::from(bit_width(sorted[at - 1] - sorted[segment.start])); - let right_width = f64::from(bit_width(sorted[segment.end - 1] - sorted[at])); - let symbol_cost = entropy_split_cost(left_len, right_len); - let split_cost = - left_len as f64 * left_width + right_len as f64 * right_width + symbol_cost; - let gain = whole_cost - split_cost; - - if best.is_none_or(|current| gain > current.gain) { - best = Some(Split { at, gain }); - } - } - best -} - -fn entropy_split_cost(left: usize, right: usize) -> f64 { - let total = (left + right) as f64; - let left = left as f64; - let right = right as f64; - left * (total / left).log2() + right * (total / right).log2() -} - -fn choose_scale_bits(value_count: usize, bin_count: usize) -> u8 { - if bin_count <= 1 { - return 0; - } - - let required_bits = usize::BITS - (bin_count - 1).leading_zeros(); - let useful_bits = usize::BITS - (value_count - 1).leading_zeros(); - u8::try_from(required_bits.max(useful_bits.min(u32::from(MAX_SCALE_BITS)))) - .unwrap_or(MAX_SCALE_BITS) -} - -fn quantize_weights(counts: &[usize], scale_bits: u8) -> VortexResult> { - let total_weight = 1_u32 << scale_bits; - let used: Vec<_> = counts - .iter() - .enumerate() - .filter(|(_, count)| **count > 0) - .map(|(index, &count)| (index, count)) - .collect(); - vortex_ensure!(!used.is_empty(), "cannot quantize an empty histogram"); - vortex_ensure!( - used.len() <= total_weight as usize, - "symbol count exceeds the ANS table size" - ); - - let total_count: usize = used.iter().map(|(_, count)| count).sum(); - let distributable = usize::try_from(total_weight)? - used.len(); - let mut weights = vec![0u16; counts.len()]; - let mut remainders = Vec::with_capacity(used.len()); - let mut allocated = 0usize; - - for &(index, count) in &used { - let scaled = count * distributable; - let extra = scaled / total_count; - let remainder = scaled % total_count; - weights[index] = u16::try_from(1 + extra)?; - allocated += 1 + extra; - remainders.push((index, remainder)); - } - - remainders.sort_unstable_by_key(|entry| Reverse(entry.1)); - for &(index, _) in remainders - .iter() - .take(usize::try_from(total_weight)? - allocated) - { - weights[index] += 1; - } - Ok(weights) -} - -struct AnsEncoderSymbol { - renorm_bit_cutoff: u32, - min_renorm_bits: u8, - next_states: Vec, -} - -#[derive(Clone, Copy)] -struct AnsDecoderNode { - next_state_idx_base: u16, - offset_bits: u8, - bits_to_read: u8, -} - -struct AnsEncoderTable { - table_size: u32, - encoder_symbols: Vec, -} - -struct AnsDecoderTable { - table_size: u32, - decoder_nodes: Vec, - lowers: Vec, -} - -fn spread_symbols(weights: &[u16], scale_bits: u8) -> VortexResult> { - let table_size = 1_u32 << scale_bits; - let mut state_symbols = vec![usize::MAX; usize::try_from(table_size)?]; - let mut stride = 3 * table_size / 5; - if stride.is_multiple_of(2) { - stride += 1; - } - let mask = table_size - 1; - let mut step = 0u32; - for (symbol, &weight) in weights.iter().enumerate() { - for _ in 0..weight { - let state_idx = (stride * step) & mask; - state_symbols[usize::try_from(state_idx)?] = symbol; - step += 1; - } - } - vortex_ensure!( - state_symbols.iter().all(|&symbol| symbol != usize::MAX), - "ANS table has unassigned states" - ); - Ok(state_symbols) -} - -fn validate_weights(weights: &[u16], scale_bits: u8) -> VortexResult { - let table_size = 1_u32 << scale_bits; - vortex_ensure!( - weights.iter().map(|&weight| u32::from(weight)).sum::() == table_size, - "ANS weights do not sum to the table size" - ); - vortex_ensure!( - weights.iter().all(|&weight| weight > 0), - "ANS weights must be positive" - ); - Ok(table_size) -} - -impl AnsEncoderTable { - fn new(weights: &[u16], scale_bits: u8) -> VortexResult { - let table_size = 1_u32 << scale_bits; - validate_weights(weights, scale_bits)?; - let state_symbols = spread_symbols(weights, scale_bits)?; - - let mut encoder_symbols = Vec::with_capacity(weights.len()); - for &weight in weights { - let frequency = u32::from(weight); - let max_x_s = 2 * frequency - 1; - let min_renorm_bits = scale_bits - u8::try_from(max_x_s.ilog2())?; - encoder_symbols.push(AnsEncoderSymbol { - renorm_bit_cutoff: 2 * frequency * (1_u32 << min_renorm_bits), - min_renorm_bits, - next_states: Vec::with_capacity(usize::from(weight)), - }); - } - for (state_idx, &symbol) in state_symbols.iter().enumerate() { - encoder_symbols[symbol] - .next_states - .push(table_size + u32::try_from(state_idx)?); - } - - Ok(Self { - table_size, - encoder_symbols, - }) - } -} - -impl AnsDecoderTable { - fn new(weights: &[u16], scale_bits: u8, bins: &[Bin]) -> VortexResult { - let table_size = validate_weights(weights, scale_bits)?; - vortex_ensure!( - weights.len() == bins.len(), - "ANS weights and range bins have different lengths" - ); - let state_symbols = spread_symbols(weights, scale_bits)?; - let mut symbol_x_s: Vec<_> = weights.iter().map(|&weight| u32::from(weight)).collect(); - let mut decoder_nodes = Vec::with_capacity(usize::try_from(table_size)?); - let mut lowers = Vec::with_capacity(usize::try_from(table_size)?); - for symbol in state_symbols { - let next_state_base = symbol_x_s[symbol]; - let bits_to_read = - u8::try_from(next_state_base.leading_zeros() - table_size.leading_zeros())?; - let next_state_idx_base = (next_state_base << bits_to_read) - table_size; - let bin = bins[symbol]; - decoder_nodes.push(AnsDecoderNode { - next_state_idx_base: u16::try_from(next_state_idx_base)?, - offset_bits: bin.offset_bits, - bits_to_read, - }); - lowers.push(bin.lower); - symbol_x_s[symbol] += 1; - } - - Ok(Self { - table_size, - decoder_nodes, - lowers, - }) - } -} - -fn encode_block( - values: &[u64], - symbols: &[usize], - bins: &[Bin], - table: &AnsEncoderTable, - payload: &mut Vec, -) -> VortexResult<()> { - let ans = encode_symbols(symbols, table)?; - payload.extend_from_slice(&u32::try_from(ans.len())?.to_le_bytes()); - payload.extend_from_slice(&ans); - - let mut offsets = BitWriter::with_capacity(size_of_val(values)); - for (&value, &symbol) in values.iter().zip(symbols) { - let bin = &bins[symbol]; - offsets.write(value - bin.lower, bin.offset_bits); - } - payload.extend_from_slice(&offsets.finish()); - payload.extend_from_slice(&[0; OFFSET_PADDING]); - Ok(()) -} - -fn encode_packed_block( - values: &[u64], - symbols: &[usize], - bins: &[Bin], - symbol_width: u8, - payload: &mut Vec, -) { - let mut encoded_symbols = BitWriter::with_capacity(symbols.len()); - for &symbol in symbols { - encoded_symbols.write(symbol as u64, symbol_width); - } - payload.extend_from_slice(&encoded_symbols.finish()); - payload.extend_from_slice(&[0; ANS_PADDING]); - - let mut offsets = BitWriter::with_capacity(size_of_val(values)); - for (&value, &symbol) in values.iter().zip(symbols) { - let bin = &bins[symbol]; - offsets.write(value - bin.lower, bin.offset_bits); - } - payload.extend_from_slice(&offsets.finish()); - payload.extend_from_slice(&[0; OFFSET_PADDING]); -} - -fn encode_two_level_block( - values: &[u64], - ranks: &[usize], - bins: &[Bin], - tag_width: u8, - direct_count: usize, - cold_width: u8, - payload: &mut Vec, -) -> VortexResult<()> { - let escape_tag = direct_count; - let cold_value_count = ranks.iter().filter(|&&rank| rank >= direct_count).count(); - payload.extend_from_slice(&u32::try_from(cold_value_count)?.to_le_bytes()); - - let mut tags = BitWriter::with_capacity(ranks.len()); - let mut cold = BitWriter::with_capacity(cold_value_count); - for &rank in ranks { - if rank < direct_count { - tags.write(rank as u64, tag_width); - } else { - tags.write(escape_tag as u64, tag_width); - cold.write((rank - direct_count) as u64, cold_width); - } - } - payload.extend_from_slice(&tags.finish()); - payload.extend_from_slice(&[0; ANS_PADDING]); - payload.extend_from_slice(&cold.finish()); - payload.extend_from_slice(&[0; ANS_PADDING]); - - let mut offsets = BitWriter::with_capacity(size_of_val(values)); - for (&value, &rank) in values.iter().zip(ranks) { - let bin = &bins[rank]; - offsets.write(value - bin.lower, bin.offset_bits); - } - payload.extend_from_slice(&offsets.finish()); - payload.extend_from_slice(&[0; OFFSET_PADDING]); - Ok(()) -} - -fn encode_grouped_block( - values: &[u64], - symbols: &[usize], - bins: &[Bin], - symbol_width: u8, - payload: &mut Vec, -) -> VortexResult<()> { - let mut encoded_symbols = BitWriter::with_capacity(symbols.len()); - let mut streams = (0..bins.len()) - .map(|_| BitWriter::with_capacity(values.len() / bins.len().max(1))) - .collect::>(); - for (&value, &symbol) in values.iter().zip(symbols) { - let bin = &bins[symbol]; - encoded_symbols.write(symbol as u64, symbol_width); - streams[symbol].write(value - bin.lower, bin.offset_bits); - } - payload.extend_from_slice(&encoded_symbols.finish()); - payload.extend_from_slice(&[0; ANS_PADDING]); - - let mut stream_offsets = Vec::with_capacity(bins.len() + 1); - let mut encoded_streams = Vec::new(); - stream_offsets.push(0u32); - for stream in streams { - encoded_streams.extend_from_slice(&stream.finish()); - encoded_streams.extend_from_slice(&[0; OFFSET_PADDING]); - stream_offsets.push(u32::try_from(encoded_streams.len())?); - } - for offset in stream_offsets { - payload.extend_from_slice(&offset.to_le_bytes()); - } - payload.extend_from_slice(&encoded_streams); - Ok(()) -} - -#[expect( - clippy::cast_possible_truncation, - reason = "bin identifiers and stream positions fit their destination types" -)] -fn decode_grouped_block_into( - payload: &[u8], - n_values: usize, - symbol_width: u8, - bins: &[Bin], - values: &mut Vec, -) -> VortexResult<()> { - let symbol_bytes = (n_values * usize::from(symbol_width)).div_ceil(8); - let offset_table_start = symbol_bytes + ANS_PADDING; - let offset_table_bytes = (bins.len() + 1) * size_of::(); - let streams_start = offset_table_start + offset_table_bytes; - vortex_ensure!( - payload.len() >= streams_start, - "grouped restart block is too short" - ); - let mut stream_starts = [0usize; MAX_BINS]; - let mut stream_stops = [0usize; MAX_BINS]; - for index in 0..bins.len() { - let read_offset = |entry: usize| { - let start = offset_table_start + entry * size_of::(); - u32::from_le_bytes([ - payload[start], - payload[start + 1], - payload[start + 2], - payload[start + 3], - ]) - }; - stream_starts[index] = streams_start + usize::try_from(read_offset(index))?; - stream_stops[index] = streams_start + usize::try_from(read_offset(index + 1))?; - vortex_ensure!( - stream_starts[index] <= stream_stops[index] - && stream_stops[index] <= payload.len() - && stream_stops[index] - stream_starts[index] >= OFFSET_PADDING, - "grouped residual stream is invalid" - ); - } - - let mut symbols = BitReader::new(payload, symbol_bytes)?; - let mut table = [PackedDecodeNode::default(); MAX_BINS]; - for (index, bin) in bins.iter().enumerate() { - table[index] = PackedDecodeNode { - lower: bin.lower, - offset_bits: u32::from(bin.offset_bits), - }; - } - let mut stream_positions = [0u32; MAX_BINS]; - let symbol_mask = low_mask(symbol_width); - let mut seen_symbols = 0u64; - - for _ in (0..n_values / 8 * 8).step_by(8) { - // SAFETY: The padded symbol stream provides eight readable bytes. - let packed = unsafe { symbols.read_unchecked(symbol_width * 8) }; - for lane in 0..8 { - let symbol = ((packed >> (lane * usize::from(symbol_width))) & symbol_mask) as usize; - seen_symbols |= 1_u64 << symbol; - let node = table[symbol]; - let bit_position = stream_positions[symbol]; - let byte_position = stream_starts[symbol] + bit_position as usize / 8; - let bits_past_byte = bit_position % 8; - // SAFETY: Each stream contains fifteen padding bytes. - let first = unsafe { read_u64_unaligned(payload.as_ptr().add(byte_position)) }; - let offset = if node.offset_bits <= 57 { - (first >> bits_past_byte) & ((1_u64 << node.offset_bits) - 1) - } else { - // SAFETY: Each stream contains fifteen padding bytes. - let second = unsafe { read_u64_unaligned(payload.as_ptr().add(byte_position + 7)) }; - let processed = 56 - bits_past_byte; - ((first >> bits_past_byte) | (second << processed)) - & low_mask(node.offset_bits as u8) - }; - stream_positions[symbol] += node.offset_bits; - values.push(node.lower.wrapping_add(offset)); - } - } - for _ in n_values / 8 * 8..n_values { - let symbol = symbols.read(symbol_width)? as usize; - seen_symbols |= 1_u64 << symbol; - let node = table[symbol]; - let bit_position = stream_positions[symbol]; - let byte_position = stream_starts[symbol] + bit_position as usize / 8; - let bits_past_byte = bit_position % 8; - // SAFETY: Each stream contains fifteen padding bytes. - let first = unsafe { read_u64_unaligned(payload.as_ptr().add(byte_position)) }; - let offset = if node.offset_bits <= 57 { - (first >> bits_past_byte) & ((1_u64 << node.offset_bits) - 1) - } else { - // SAFETY: Each stream contains fifteen padding bytes. - let second = unsafe { read_u64_unaligned(payload.as_ptr().add(byte_position + 7)) }; - let processed = 56 - bits_past_byte; - ((first >> bits_past_byte) | (second << processed)) & low_mask(node.offset_bits as u8) - }; - stream_positions[symbol] += node.offset_bits; - values.push(node.lower.wrapping_add(offset)); - } - - let valid_symbols = low_mask(u8::try_from(bins.len())?); - vortex_ensure!( - seen_symbols & !valid_symbols == 0, - "grouped symbol exceeds bin table" - ); - symbols.finish()?; - for index in 0..bins.len() { - let used_bytes = usize::try_from(stream_positions[index])?.div_ceil(8); - vortex_ensure!( - stream_starts[index] + used_bytes + OFFSET_PADDING == stream_stops[index], - "grouped residual stream has unused bytes" - ); - } - Ok(()) -} - -#[derive(Clone, Copy, Default)] -struct PackedDecodeNode { - lower: u64, - offset_bits: u32, -} - -#[expect( - clippy::cast_possible_truncation, - reason = "bin identifiers and block offset positions fit their destination types" -)] -fn decode_two_level_block_into( - payload: &[u8], - n_values: usize, - tag_width: u8, - direct_count: usize, - cold_width: u8, - bins: &[Bin], - values: &mut Vec, -) -> VortexResult<()> { - vortex_ensure!(payload.len() >= 4, "two-level restart block is too short"); - let cold_value_count = usize::try_from(u32::from_le_bytes([ - payload[0], payload[1], payload[2], payload[3], - ]))?; - let tag_bytes = (n_values * usize::from(tag_width)).div_ceil(8); - let cold_bytes = (cold_value_count * usize::from(cold_width)).div_ceil(8); - let tags_start = 4; - let cold_start = tags_start + tag_bytes + ANS_PADDING; - let offsets_start = cold_start + cold_bytes + ANS_PADDING; - vortex_ensure!( - payload.len() >= offsets_start + OFFSET_PADDING, - "two-level restart block is too short" - ); - let mut tags = BitReader::new(&payload[tags_start..], tag_bytes)?; - let mut cold = BitReader::new(&payload[cold_start..], cold_bytes)?; - let offsets = &payload[offsets_start..]; - let mut table = [PackedDecodeNode::default(); MAX_BINS]; - for (index, bin) in bins.iter().enumerate() { - table[index] = PackedDecodeNode { - lower: bin.lower, - offset_bits: u32::from(bin.offset_bits), - }; - } - - let escape_tag = direct_count; - let tag_mask = low_mask(tag_width); - let mut offset_bit_position = 0u32; - let full_len = n_values / 8 * 8; - for _ in (0..full_len).step_by(8) { - // SAFETY: The encoded codec invariant provides eight tag padding bytes. - let packed = unsafe { tags.read_unchecked(tag_width * 8) }; - for lane in 0..8 { - let tag = ((packed >> (lane * usize::from(tag_width))) & tag_mask) as usize; - let rank = if tag == escape_tag { - // SAFETY: The encoded codec invariant provides eight cold padding bytes. - direct_count + unsafe { cold.read_unchecked(cold_width) } as usize - } else { - tag - }; - let node = table[rank]; - // SAFETY: The private codec fields come from the validated encoder. - let offset = unsafe { read_packed_offset(offsets, offset_bit_position, node) }; - offset_bit_position += node.offset_bits; - values.push(node.lower.wrapping_add(offset)); - } - } - for _ in full_len..n_values { - let tag = tags.read(tag_width)? as usize; - let rank = if tag == escape_tag { - direct_count + cold.read(cold_width)? as usize - } else { - tag - }; - let node = table[rank]; - // SAFETY: The private codec fields come from the validated encoder. - let offset = unsafe { read_packed_offset(offsets, offset_bit_position, node) }; - offset_bit_position += node.offset_bits; - values.push(node.lower.wrapping_add(offset)); - } - - tags.finish()?; - cold.finish()?; - let offset_bytes = offset_bit_position.div_ceil(8) as usize; - vortex_ensure!( - offsets.len() == offset_bytes + OFFSET_PADDING, - "offset stream has unused bytes" - ); - Ok(()) -} - -#[inline(always)] -#[expect( - clippy::cast_possible_truncation, - reason = "validated offset widths do not exceed 64 bits" -)] -unsafe fn read_packed_offset(offsets: &[u8], bit_position: u32, node: PackedDecodeNode) -> u64 { - let byte_position = bit_position as usize / 8; - let bits_past_byte = bit_position % 8; - // SAFETY: The encoded codec invariant provides fifteen readable padding bytes. - let first = unsafe { read_u64_unaligned(offsets.as_ptr().add(byte_position)) }; - if node.offset_bits <= 57 { - (first >> bits_past_byte) & ((1_u64 << node.offset_bits) - 1) - } else { - // SAFETY: The encoded codec invariant provides fifteen readable padding bytes. - let second = unsafe { read_u64_unaligned(offsets.as_ptr().add(byte_position + 7)) }; - let processed = 56 - bits_past_byte; - ((first >> bits_past_byte) | (second << processed)) & low_mask(node.offset_bits as u8) - } -} - -#[expect( - clippy::cast_possible_truncation, - reason = "bin identifiers and block offset positions fit their destination types" -)] -fn decode_packed_block_into( - payload: &[u8], - n_values: usize, - symbol_width: u8, - bins: &[Bin], - values: &mut Vec, -) -> VortexResult<()> { - let symbol_bits = n_values * usize::from(symbol_width); - let symbol_bytes = symbol_bits.div_ceil(8); - vortex_ensure!( - payload.len() >= symbol_bytes + ANS_PADDING + OFFSET_PADDING, - "range packed restart block is too short" - ); - let offsets = &payload[symbol_bytes + ANS_PADDING..]; - let mut symbols = BitReader::new(payload, symbol_bytes)?; - let mut table = [PackedDecodeNode::default(); MAX_BINS]; - for (index, bin) in bins.iter().enumerate() { - table[index] = PackedDecodeNode { - lower: bin.lower, - offset_bits: u32::from(bin.offset_bits), - }; - } - - let symbol_mask = low_mask(symbol_width); - let mut offset_bit_position = 0u32; - let full_len = n_values / 8 * 8; - for _ in (0..full_len).step_by(8) { - // SAFETY: The encoded codec invariant provides eight symbol padding bytes. - let packed = unsafe { symbols.read_unchecked(symbol_width * 8) }; - for lane in 0..8 { - let symbol = ((packed >> (lane * usize::from(symbol_width))) & symbol_mask) as usize; - let node = table[symbol]; - // SAFETY: The private codec fields come from the validated encoder. - let offset = unsafe { read_packed_offset(offsets, offset_bit_position, node) }; - offset_bit_position += node.offset_bits; - values.push(node.lower.wrapping_add(offset)); - } - } - for _ in full_len..n_values { - let symbol = symbols.read(symbol_width)? as usize; - let node = table[symbol]; - // SAFETY: The private codec fields come from the validated encoder. - let offset = unsafe { read_packed_offset(offsets, offset_bit_position, node) }; - offset_bit_position += node.offset_bits; - values.push(node.lower.wrapping_add(offset)); - } - - symbols.finish()?; - let offset_bytes = offset_bit_position.div_ceil(8) as usize; - vortex_ensure!( - offsets.len() == offset_bytes + OFFSET_PADDING, - "offset stream has unused bytes" - ); - vortex_ensure!( - offsets[offset_bytes..].iter().all(|&byte| byte == 0), - "offset stream has nonzero padding" - ); - Ok(()) -} - -#[inline(never)] -#[expect( - clippy::cast_possible_truncation, - reason = "ANS transitions read at most MAX_SCALE_BITS bits" -)] -unsafe fn read_full_ans_batch( - symbols: &mut BitReader<'_>, - table: &AnsDecoderTable, - states: &mut [u32; ANS_INTERLEAVING], - lowers: &mut DecodeScratch, - widths: &mut DecodeScratch, - bit_positions: &mut DecodeScratch, -) -> u32 { - let [mut state_0, mut state_1, mut state_2, mut state_3] = *states; - let mut offset_bit_position = 0u32; - - for base_index in (0..DECODE_BATCH_SIZE).step_by(ANS_INTERLEAVING) { - // SAFETY: Initial states are validated, and each ANS transition stays in the table. - let node_0 = unsafe { table.decoder_nodes.get_unchecked(state_0 as usize) }; - // SAFETY: Initial states are validated, and each ANS transition stays in the table. - let node_1 = unsafe { table.decoder_nodes.get_unchecked(state_1 as usize) }; - // SAFETY: Initial states are validated, and each ANS transition stays in the table. - let node_2 = unsafe { table.decoder_nodes.get_unchecked(state_2 as usize) }; - // SAFETY: Initial states are validated, and each ANS transition stays in the table. - let node_3 = unsafe { table.decoder_nodes.get_unchecked(state_3 as usize) }; - let width_0 = node_0.bits_to_read; - let width_1 = node_1.bits_to_read; - let width_2 = node_2.bits_to_read; - let width_3 = node_3.bits_to_read; - let packed_width = width_0 + width_1 + width_2 + width_3; - // SAFETY: The caller verifies enough readable bytes for the worst-case batch. - let packed = unsafe { symbols.read_unchecked(packed_width) }; - let shift_1 = width_0; - let shift_2 = shift_1 + width_1; - let shift_3 = shift_2 + width_2; - - macro_rules! write_symbol { - ($index:expr, $state:ident, $node:ident, $shift:expr, $width:ident) => {{ - // SAFETY: The decoder node uses the same validated state index. - let lower = unsafe { *table.lowers.get_unchecked($state as usize) }; - // SAFETY: Every scratch index is within this full batch. - unsafe { *lowers.get_unchecked_mut($index) = lower }; - // SAFETY: Every scratch index is within this full batch. - unsafe { *bit_positions.get_unchecked_mut($index) = offset_bit_position }; - // SAFETY: Every scratch index is within this full batch. - unsafe { *widths.get_unchecked_mut($index) = u32::from($node.offset_bits) }; - offset_bit_position += u32::from($node.offset_bits); - $state = u32::from($node.next_state_idx_base) - + ((packed >> $shift) & ((1_u64 << $width) - 1)) as u32; - }}; - } - - write_symbol!(base_index, state_0, node_0, 0, width_0); - write_symbol!(base_index + 1, state_1, node_1, shift_1, width_1); - write_symbol!(base_index + 2, state_2, node_2, shift_2, width_2); - write_symbol!(base_index + 3, state_3, node_3, shift_3, width_3); - } - - *states = [state_0, state_1, state_2, state_3]; - offset_bit_position -} - -#[expect( - clippy::cast_possible_truncation, - reason = "validated ANS and offset widths fit their destination types" -)] -fn decode_block_into( - payload: &[u8], - n_values: usize, - output_range: Range, - table: &AnsDecoderTable, - values: &mut Vec, -) -> VortexResult<()> { - vortex_ensure!( - output_range.start <= output_range.end && output_range.end <= n_values, - "output range exceeds restart block" - ); - vortex_ensure!( - payload.len() >= 4 + ANS_STATE_BYTES, - "restart block is too short" - ); - let ans_len = usize::try_from(u32::from_le_bytes([ - payload[0], payload[1], payload[2], payload[3], - ]))?; - vortex_ensure!( - ans_len >= ANS_STATE_BYTES + ANS_PADDING, - "ANS stream is too short" - ); - vortex_ensure!( - 4 + ans_len <= payload.len(), - "ANS stream exceeds restart block" - ); - - let encoded_symbols = &payload[4..4 + ans_len]; - let mut state_idxs = [0u32; ANS_INTERLEAVING]; - for (index, state_idx) in state_idxs.iter_mut().enumerate() { - let byte_idx = index * size_of::(); - *state_idx = u32::from(u16::from_le_bytes([ - encoded_symbols[byte_idx], - encoded_symbols[byte_idx + 1], - ])); - } - vortex_ensure!( - state_idxs - .iter() - .all(|&state_idx| state_idx < table.table_size), - "ANS state index exceeds the table" - ); - let offsets = &payload[4 + ans_len..]; - let symbol_bytes = &payload[4 + ANS_STATE_BYTES..]; - let symbol_unpadded_len = ans_len - ANS_STATE_BYTES - ANS_PADDING; - let mut symbols = BitReader::new(symbol_bytes, symbol_unpadded_len)?; - let [mut state_0, mut state_1, mut state_2, mut state_3] = state_idxs; - let mut offset_bit_position = 0u32; - let mut lowers = DecodeScratch([0u64; DECODE_BATCH_SIZE]); - let mut widths = DecodeScratch([0u32; DECODE_BATCH_SIZE]); - let mut bit_positions = DecodeScratch([0u32; DECODE_BATCH_SIZE]); - let mut decoded = DecodeScratch([0u64; DECODE_BATCH_SIZE]); - - for batch_start in (0..n_values).step_by(DECODE_BATCH_SIZE) { - let batch_len = (n_values - batch_start).min(DECODE_BATCH_SIZE); - let full_batch_len = batch_len / ANS_INTERLEAVING * ANS_INTERLEAVING; - let batch_offset_start = offset_bit_position; - let mut batch_offset_bits = 0u32; - let fast_full_batch = batch_len == DECODE_BATCH_SIZE - && symbols.can_read_unchecked(DECODE_BATCH_SIZE * usize::from(MAX_SCALE_BITS)); - if fast_full_batch { - let mut states = [state_0, state_1, state_2, state_3]; - // SAFETY: The size check provides readable bytes for the worst-case batch. - batch_offset_bits = unsafe { - read_full_ans_batch( - &mut symbols, - table, - &mut states, - &mut lowers, - &mut widths, - &mut bit_positions, - ) - }; - [state_0, state_1, state_2, state_3] = states; - symbols.check_in_bounds()?; - } else { - for base_index in (0..full_batch_len).step_by(ANS_INTERLEAVING) { - // SAFETY: Initial states are validated, and each ANS transition stays in the table. - let node_0 = unsafe { table.decoder_nodes.get_unchecked(state_0 as usize) }; - // SAFETY: Initial states are validated, and each ANS transition stays in the table. - let node_1 = unsafe { table.decoder_nodes.get_unchecked(state_1 as usize) }; - // SAFETY: Initial states are validated, and each ANS transition stays in the table. - let node_2 = unsafe { table.decoder_nodes.get_unchecked(state_2 as usize) }; - // SAFETY: Initial states are validated, and each ANS transition stays in the table. - let node_3 = unsafe { table.decoder_nodes.get_unchecked(state_3 as usize) }; - let width_0 = node_0.bits_to_read; - let width_1 = node_1.bits_to_read; - let width_2 = node_2.bits_to_read; - let width_3 = node_3.bits_to_read; - let packed_width = width_0 + width_1 + width_2 + width_3; - let packed = symbols.read(packed_width)?; - let shift_1 = width_0; - let shift_2 = shift_1 + width_1; - let shift_3 = shift_2 + width_2; - // SAFETY: The corresponding decoder nodes use the same validated state indexes. - lowers[base_index] = unsafe { *table.lowers.get_unchecked(state_0 as usize) }; - // SAFETY: The corresponding decoder nodes use the same validated state indexes. - lowers[base_index + 1] = unsafe { *table.lowers.get_unchecked(state_1 as usize) }; - // SAFETY: The corresponding decoder nodes use the same validated state indexes. - lowers[base_index + 2] = unsafe { *table.lowers.get_unchecked(state_2 as usize) }; - // SAFETY: The corresponding decoder nodes use the same validated state indexes. - lowers[base_index + 3] = unsafe { *table.lowers.get_unchecked(state_3 as usize) }; - for (index, width) in [ - node_0.offset_bits, - node_1.offset_bits, - node_2.offset_bits, - node_3.offset_bits, - ] - .into_iter() - .enumerate() - { - bit_positions[base_index + index] = batch_offset_bits; - widths[base_index + index] = u32::from(width); - batch_offset_bits += u32::from(width); - } - state_0 = u32::from(node_0.next_state_idx_base) - + (packed & ((1_u64 << width_0) - 1)) as u32; - state_1 = u32::from(node_1.next_state_idx_base) - + ((packed >> shift_1) & ((1_u64 << width_1) - 1)) as u32; - state_2 = u32::from(node_2.next_state_idx_base) - + ((packed >> shift_2) & ((1_u64 << width_2) - 1)) as u32; - state_3 = u32::from(node_3.next_state_idx_base) - + ((packed >> shift_3) & ((1_u64 << width_3) - 1)) as u32; - } - } - - state_idxs = [state_0, state_1, state_2, state_3]; - for index in full_batch_len..batch_len { - let lane = index - full_batch_len; - let state_idx = &mut state_idxs[lane]; - let current_state = *state_idx; - // SAFETY: Initial states are validated, and each ANS transition stays in the table. - let node = unsafe { table.decoder_nodes.get_unchecked(current_state as usize) }; - *state_idx = - u32::from(node.next_state_idx_base) + symbols.read(node.bits_to_read)? as u32; - // SAFETY: The corresponding decoder node was read from the same state index. - lowers[index] = unsafe { *table.lowers.get_unchecked(current_state as usize) }; - bit_positions[index] = batch_offset_bits; - widths[index] = u32::from(node.offset_bits); - batch_offset_bits += u32::from(node.offset_bits); - } - [state_0, state_1, state_2, state_3] = state_idxs; - - let last_bit_position = batch_offset_start + bit_positions[batch_len - 1]; - let last_byte_position = last_bit_position as usize / 8; - vortex_ensure!( - last_byte_position + OFFSET_PADDING <= offsets.len(), - "offset stream is truncated" - ); - decoded[..batch_len].copy_from_slice(&lowers[..batch_len]); - if widths[..batch_len].iter().all(|&width| width <= 57) { - // SAFETY: The caller verified eight readable bytes after each start position. - unsafe { - read_offsets::<8>( - offsets, - batch_offset_start, - &bit_positions.0, - &widths.0, - &mut decoded.0, - batch_len, - ) - } - } else { - // SAFETY: The caller verified fifteen readable bytes after each start position. - unsafe { - read_offsets::<15>( - offsets, - batch_offset_start, - &bit_positions.0, - &widths.0, - &mut decoded.0, - batch_len, - ) - } - } - offset_bit_position += batch_offset_bits; - - let output_start = output_range.start.max(batch_start); - let output_stop = output_range.end.min(batch_start + batch_len); - if output_start < output_stop { - values - .extend_from_slice(&decoded[output_start - batch_start..output_stop - batch_start]); - } - } - symbols.finish()?; - let offset_bytes = offset_bit_position.div_ceil(8) as usize; - vortex_ensure!( - offsets.len() == offset_bytes + OFFSET_PADDING, - "offset stream has unused bytes" - ); - vortex_ensure!( - offsets[offset_bytes..].iter().all(|&byte| byte == 0), - "offset stream has nonzero padding" - ); - if !offset_bit_position.is_multiple_of(8) { - let used_bits = offset_bit_position % 8; - let trailing_mask = !low_mask(u8::try_from(used_bits)?).to_le_bytes()[0]; - vortex_ensure!( - offsets[offset_bytes - 1] & trailing_mask == 0, - "offset stream has nonzero trailing bits" - ); - } - state_idxs = [state_0, state_1, state_2, state_3]; - vortex_ensure!( - state_idxs.iter().all(|&state_idx| state_idx == 0), - "ANS stream ended in an invalid state" - ); - Ok(()) -} - -#[inline(never)] -#[expect( - clippy::cast_possible_truncation, - reason = "validated offset widths do not exceed 64 bits" -)] -unsafe fn read_offsets( - bytes: &[u8], - base_bit_position: u32, - bit_positions: &[u32], - widths: &[u32], - values: &mut [u64], - len: usize, -) { - for index in 0..len { - let bit_position = base_bit_position + bit_positions[index]; - let byte_position = bit_position as usize / 8; - let bits_past_byte = bit_position % 8; - let first = unsafe { read_u64_unaligned(bytes.as_ptr().add(byte_position)) }; - let offset = match READ_BYTES { - 8 => { - debug_assert!(widths[index] <= 57); - (first >> bits_past_byte) & ((1_u64 << widths[index]) - 1) - } - 15 => { - let second = unsafe { read_u64_unaligned(bytes.as_ptr().add(byte_position + 7)) }; - let processed = 56 - bits_past_byte; - ((first >> bits_past_byte) | (second << processed)) & low_mask(widths[index] as u8) - } - _ => unreachable!("invalid offset read width {READ_BYTES}"), - }; - values[index] = values[index].wrapping_add(offset); - } -} - -type ReadOffsetsFn = unsafe fn(&[u8], u32, &[u32], &[u32], &mut [u64], usize); - -#[used] -static FORCE_EXPORT_READ_OFFSETS_8: ReadOffsetsFn = read_offsets::<8>; - -#[used] -static FORCE_EXPORT_READ_OFFSETS_15: ReadOffsetsFn = read_offsets::<15>; - -unsafe fn read_u64_unaligned(pointer: *const u8) -> u64 { - // SAFETY: The caller provides eight readable bytes at the pointer. - u64::from_le(unsafe { pointer.cast::().read_unaligned() }) -} - -fn encode_symbols(symbols: &[usize], table: &AnsEncoderTable) -> VortexResult> { - let mut states = [table.table_size; ANS_INTERLEAVING]; - let mut writes = vec![(0u32, 0u8); symbols.len()]; - let full_len = symbols.len() / ANS_INTERLEAVING * ANS_INTERLEAVING; - - let mut encode_one = |index: usize, lane: usize| -> VortexResult<()> { - let symbol = symbols[index]; - let state = states[lane]; - let symbol_info = table - .encoder_symbols - .get(symbol) - .ok_or_else(|| vortex_error::vortex_err!("symbol {symbol} exceeds ANS table"))?; - let renorm_bits = if state >= symbol_info.renorm_bit_cutoff { - symbol_info.min_renorm_bits + 1 - } else { - symbol_info.min_renorm_bits - }; - let frequency = u32::try_from(symbol_info.next_states.len())?; - let x_s = state >> renorm_bits; - let next_state_idx = usize::try_from(x_s - frequency)?; - let next_state = *symbol_info - .next_states - .get(next_state_idx) - .ok_or_else(|| vortex_error::vortex_err!("ANS state index exceeds the symbol table"))?; - writes[index] = (state, renorm_bits); - states[lane] = next_state; - Ok(()) - }; - - for lane in (0..symbols.len() - full_len).rev() { - encode_one(full_len + lane, lane)?; - } - for base_index in (0..full_len).step_by(ANS_INTERLEAVING).rev() { - for lane in (0..ANS_INTERLEAVING).rev() { - encode_one(base_index + lane, lane)?; - } - } - - let mut encoded = Vec::with_capacity(ANS_STATE_BYTES + symbols.len() / 2); - for state in states { - encoded.extend_from_slice(&u16::try_from(state - table.table_size)?.to_le_bytes()); - } - let mut writer = BitWriter::with_capacity(symbols.len() * 2); - for (value, width) in writes { - writer.write(u64::from(value) & low_mask(width), width); - } - encoded.extend_from_slice(&writer.finish()); - encoded.extend_from_slice(&[0; ANS_PADDING]); - Ok(encoded) -} - -struct BitWriter { - bytes: Vec, - pending: u64, - pending_bits: u8, -} - -impl BitWriter { - fn with_capacity(capacity: usize) -> Self { - Self { - bytes: Vec::with_capacity(capacity), - pending: 0, - pending_bits: 0, - } - } - - fn write(&mut self, value: u64, width: u8) { - debug_assert!(width <= 64); - debug_assert!(width == 64 || value <= low_mask(width)); - if width == 0 { - return; - } - - let available = 64 - self.pending_bits; - self.pending |= value << self.pending_bits; - if width < available { - self.pending_bits += width; - return; - } - - self.bytes.extend_from_slice(&self.pending.to_le_bytes()); - let remaining = width - available; - self.pending = if remaining == 0 { - 0 - } else { - value >> available - }; - self.pending_bits = remaining; - } - - fn finish(mut self) -> Vec { - if self.pending_bits > 0 { - let byte_count = usize::from(self.pending_bits).div_ceil(8); - self.bytes - .extend_from_slice(&self.pending.to_le_bytes()[..byte_count]); - } - self.bytes - } -} - -struct BitReader<'a> { - bytes: &'a [u8], - unpadded_len: usize, - bit_position: usize, -} - -impl<'a> BitReader<'a> { - fn new(bytes: &'a [u8], unpadded_len: usize) -> VortexResult { - vortex_ensure!( - bytes.len() >= unpadded_len + ANS_PADDING, - "ANS stream is too short" - ); - Ok(Self { - bytes, - unpadded_len, - bit_position: 0, - }) - } - - fn can_read_unchecked(&self, max_bits: usize) -> bool { - let byte_position = self.bit_position / 8; - let bits_past_byte = self.bit_position % 8; - let max_bytes = (bits_past_byte + max_bits).div_ceil(8) + size_of::(); - byte_position + max_bytes <= self.bytes.len() - } - - fn check_in_bounds(&self) -> VortexResult<()> { - vortex_ensure!( - self.bit_position <= self.unpadded_len * 8, - "ANS stream is truncated" - ); - Ok(()) - } - - #[inline(always)] - unsafe fn read_unchecked(&mut self, width: u8) -> u64 { - let byte_position = self.bit_position / 8; - let bits_past_byte = self.bit_position % 8; - // SAFETY: The caller verifies eight readable bytes after the current position. - let packed = unsafe { read_u64_unaligned(self.bytes.as_ptr().add(byte_position)) }; - self.bit_position += usize::from(width); - (packed >> bits_past_byte) & ((1_u64 << width) - 1) - } - - #[inline(always)] - fn read(&mut self, width: u8) -> VortexResult { - vortex_ensure!(width <= 57, "ANS read width exceeds 57 bits"); - let required = self.bit_position + usize::from(width); - vortex_ensure!(required <= self.unpadded_len * 8, "ANS stream is truncated"); - // SAFETY: The bounds check and stream padding provide eight readable bytes. - Ok(unsafe { self.read_unchecked(width) }) - } - - fn finish(&self) -> VortexResult<()> { - let used_bytes = self.bit_position.div_ceil(8); - vortex_ensure!( - used_bytes == self.unpadded_len, - "ANS stream has unused bytes" - ); - vortex_ensure!( - self.bytes[self.unpadded_len..self.unpadded_len + ANS_PADDING] - .iter() - .all(|&byte| byte == 0), - "ANS stream has nonzero padding" - ); - if !self.bit_position.is_multiple_of(8) { - let used_bits = self.bit_position % 8; - let trailing_mask = !low_mask(u8::try_from(used_bits)?).to_le_bytes()[0]; - vortex_ensure!( - self.bytes[used_bytes - 1] & trailing_mask == 0, - "ANS stream has nonzero trailing bits" - ); - } - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use vortex_buffer::ByteBuffer; - use vortex_error::VortexResult; - - use super::RangeEntropyCodec; - use super::RangeEntropyParts; - use super::RangeGroupedCodec; - use super::RangePackedCodec; - use super::RangeTwoLevelCodec; - - #[test] - fn empty_roundtrip() -> VortexResult<()> { - let encoded = RangeEntropyCodec::encode(&[], 256)?; - assert!(encoded.is_empty()); - assert_eq!(encoded.decode()?, Vec::::new()); - Ok(()) - } - - #[test] - fn constant_roundtrip() -> VortexResult<()> { - let values = vec![42; 10_000]; - let encoded = RangeEntropyCodec::encode(&values, 256)?; - assert_eq!(encoded.bin_lowers(), [42]); - assert_eq!(encoded.offset_widths(), [0]); - assert_eq!(encoded.decode()?, values); - Ok(()) - } - - #[test] - fn clustered_roundtrip_across_blocks() -> VortexResult<()> { - let values: Vec<_> = (0..20_000) - .map(|index| match index % 4 { - 0 => 1_000 + index % 17, - 1 => 1_000_000 + index % 31, - 2 => u64::MAX - index % 13, - _ => index % 7, - }) - .collect(); - let encoded = RangeEntropyCodec::encode(&values, 1_024)?; - assert!(encoded.bin_lowers().len() > 1); - assert_eq!( - encoded.block_offsets().len(), - values.len().div_ceil(1_024) + 1 - ); - assert_eq!(encoded.decode()?, values); - assert_eq!(encoded.value(10_777)?, values[10_777]); - Ok(()) - } - - #[test] - fn full_width_roundtrip() -> VortexResult<()> { - let values = [0, u64::MAX, 1, u64::MAX - 1]; - let encoded = RangeEntropyCodec::encode(&values, 3)?; - assert_eq!(encoded.decode()?, values); - Ok(()) - } - - #[test] - fn rejects_zero_block_length() { - assert!(RangeEntropyCodec::encode(&[1, 2, 3], 0).is_err()); - } - - #[test] - fn restart_blocks_are_independent() -> VortexResult<()> { - let values: Vec<_> = (0_u64..1_000).map(|value| value * value).collect(); - let encoded = RangeEntropyCodec::encode(&values, 100)?; - for block_idx in 0..10 { - assert_eq!( - encoded.decode_block(block_idx)?, - values[block_idx * 100..(block_idx + 1) * 100] - ); - } - Ok(()) - } - - #[test] - fn range_decode_touches_boundary_blocks() -> VortexResult<()> { - let values: Vec<_> = (0_u64..1_000).map(|value| value * value).collect(); - let encoded = RangeEntropyCodec::encode(&values, 100)?; - assert_eq!(encoded.decode_range(95..205)?, values[95..205]); - assert_eq!(encoded.decode_range(500..500)?, Vec::::new()); - assert!(encoded.decode_range(999..1_001).is_err()); - Ok(()) - } - - #[test] - fn stored_parts_roundtrip() -> VortexResult<()> { - let values: Vec<_> = (0_u64..4_000).map(|value| value % 97).collect(); - let parts = RangeEntropyCodec::encode(&values, 256)?.into_parts(); - let restored = RangeEntropyCodec::try_from_parts(parts)?; - assert_eq!(restored.decode()?, values); - Ok(()) - } - - #[test] - fn corrupt_block_offset_is_rejected() -> VortexResult<()> { - let values: Vec<_> = (0_u64..1_000).collect(); - let mut parts = RangeEntropyCodec::encode(&values, 100)?.into_parts(); - parts.block_offsets.pop(); - assert!(RangeEntropyCodec::try_from_parts(parts).is_err()); - Ok(()) - } - - #[test] - fn corrupt_payload_is_rejected_during_decode() -> VortexResult<()> { - let values: Vec<_> = (0_u64..1_000).map(|value| value % 31).collect(); - let mut parts = RangeEntropyCodec::encode(&values, 100)?.into_parts(); - let mut payload = parts.payload.to_vec(); - payload.pop(); - parts.payload = ByteBuffer::from(payload); - if let Some(last) = parts.block_offsets.last_mut() { - *last = u32::try_from(parts.payload.len())?; - } - let restored = RangeEntropyCodec::try_from_parts(parts)?; - assert!(restored.decode().is_err()); - Ok(()) - } - - #[test] - fn pseudo_random_roundtrip() -> VortexResult<()> { - let mut state = 0x4d59_5df4_d0f3_3173_u64; - let values: Vec<_> = (0..8_192) - .map(|index| { - state ^= state << 13; - state ^= state >> 7; - state ^= state << 17; - match index % 3 { - 0 => state, - 1 => state % 1_000, - _ => u64::MAX - state % 257, - } - }) - .collect(); - for block_len in [1, 7, 256, 1_024, 8_192] { - assert_eq!( - RangeEntropyCodec::encode(&values, block_len)?.decode()?, - values - ); - } - Ok(()) - } - - #[test] - fn range_packed_roundtrip() -> VortexResult<()> { - let mut state = 0x4d59_5df4_d0f3_3173_u64; - let values: Vec<_> = (0..20_000) - .map(|index| { - state ^= state << 13; - state ^= state >> 7; - state ^= state << 17; - match index % 4 { - 0 => state, - 1 => state % 1_000, - 2 => u64::MAX - state % 257, - _ => 42, - } - }) - .collect(); - for block_len in [1, 7, 256, 1_024, 8_192] { - assert_eq!( - RangePackedCodec::encode(&values, block_len)?.decode()?, - values - ); - } - Ok(()) - } - - #[test] - fn range_packed_empty_and_constant_roundtrip() -> VortexResult<()> { - let empty = RangePackedCodec::encode(&[], 256)?; - assert!(empty.is_empty()); - assert_eq!(empty.decode()?, Vec::::new()); - - let values = vec![42; 10_000]; - let constant = RangePackedCodec::encode(&values, 256)?; - assert_eq!(constant.symbol_width(), 0); - assert_eq!(constant.decode()?, values); - Ok(()) - } - - #[test] - fn range_two_level_roundtrip() -> VortexResult<()> { - let mut state = 0x4d59_5df4_d0f3_3173_u64; - let values: Vec<_> = (0..20_000) - .map(|index| { - state ^= state << 13; - state ^= state >> 7; - state ^= state << 17; - match index % 4 { - 0 => state, - 1 => state % 1_000, - 2 => u64::MAX - state % 257, - _ => 42, - } - }) - .collect(); - for block_len in [1, 7, 256, 1_024, 8_192] { - assert_eq!( - RangeTwoLevelCodec::encode(&values, block_len)?.decode()?, - values - ); - } - Ok(()) - } - - #[test] - fn range_grouped_roundtrip() -> VortexResult<()> { - let values = (0_u64..20_000) - .map(|index| match index % 4 { - 0 => index, - 1 => 1_000_000 + index % 31, - 2 => u64::MAX - index % 13, - _ => 42, - }) - .collect::>(); - for block_len in [1, 7, 256, 1_024, 8_192] { - assert_eq!( - RangeGroupedCodec::encode(&values, block_len)?.decode()?, - values - ); - } - Ok(()) - } - - #[test] - fn invalid_weight_sum_is_rejected() -> VortexResult<()> { - let parts = RangeEntropyParts { - len: 1, - block_len: 1, - scale_bits: 12, - bin_lowers: vec![0], - offset_widths: vec![0], - weights: vec![1], - block_offsets: vec![0, 8], - payload: ByteBuffer::from(vec![0; 8]), - }; - assert!(RangeEntropyCodec::try_from_parts(parts).is_err()); - Ok(()) - } -} diff --git a/encodings/range-entropy/src/lib.rs b/encodings/range-entropy/src/lib.rs deleted file mode 100644 index d3c8ffe6371..00000000000 --- a/encodings/range-entropy/src/lib.rs +++ /dev/null @@ -1,33 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! Native range-bin and ANS entropy codec for Vortex numeric arrays. - -mod array; -mod bit_split; -mod block_residual_array; -mod codec; -mod ordered_float_array; -mod patched_for; -mod rules; - -pub use array::*; -pub use bit_split::BitSplitCodec; -pub use block_residual_array::*; -pub use codec::RangeEntropyCodec; -pub use codec::RangeEntropyParts; -pub use codec::RangeGroupedCodec; -pub use codec::RangePackedCodec; -pub use codec::RangeTwoLevelCodec; -pub use ordered_float_array::*; -pub use patched_for::BlockResidualParts; -pub use patched_for::PatchedFoRCodec; -use vortex_array::session::ArraySessionExt; -use vortex_session::VortexSession; - -/// Register the range entropy encoding in one session. -pub fn initialize(session: &VortexSession) { - session.arrays().register(RangeEntropy); - session.arrays().register(BlockResidual); - session.arrays().register(OrderedFloat); -} diff --git a/encodings/range-entropy/src/rules.rs b/encodings/range-entropy/src/rules.rs deleted file mode 100644 index 060e7943666..00000000000 --- a/encodings/range-entropy/src/rules.rs +++ /dev/null @@ -1,10 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use vortex_array::arrays::slice::SliceReduceAdaptor; -use vortex_array::optimizer::rules::ParentRuleSet; - -use crate::RangeEntropy; - -pub(crate) static RULES: ParentRuleSet = - ParentRuleSet::new(&[ParentRuleSet::lift(&SliceReduceAdaptor(RangeEntropy))]); diff --git a/vortex-btrblocks/Cargo.toml b/vortex-btrblocks/Cargo.toml index 6312637c6e8..f517a92d942 100644 --- a/vortex-btrblocks/Cargo.toml +++ b/vortex-btrblocks/Cargo.toml @@ -22,6 +22,7 @@ pco = { workspace = true, optional = true } rand = { workspace = true } vortex-alp = { workspace = true } vortex-array = { workspace = true } +vortex-block-residual = { workspace = true } vortex-buffer = { workspace = true } vortex-compressor = { workspace = true } vortex-datetime-parts = { workspace = true } @@ -32,7 +33,6 @@ vortex-float-quant = { workspace = true } vortex-fsst = { workspace = true } vortex-onpair = { workspace = true, optional = true } vortex-pco = { workspace = true, optional = true } -vortex-range-entropy = { workspace = true } vortex-runend = { workspace = true } vortex-sequence = { workspace = true } vortex-sparse = { workspace = true } @@ -78,11 +78,3 @@ test = false [[example]] name = "pco_probe" required-features = ["pco"] - -[[example]] -name = "range_entropy_throughput" -required-features = ["pco", "zstd"] - -[[example]] -name = "float_quant_dataset" -required-features = ["pco", "zstd"] diff --git a/vortex-btrblocks/examples/float_quant_dataset.rs b/vortex-btrblocks/examples/float_quant_dataset.rs deleted file mode 100644 index 63eddced766..00000000000 --- a/vortex-btrblocks/examples/float_quant_dataset.rs +++ /dev/null @@ -1,1371 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::fs::File; -use std::hint::black_box; -use std::io::BufRead; -use std::io::BufReader; -use std::path::Path; -use std::sync::Arc; -use std::time::Duration; -use std::time::Instant; - -use arrow_array::ArrayRef as ArrowArrayRef; -use arrow_schema::DataType; -use arrow_select::concat::concat; -use parquet::arrow::ProjectionMask; -use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; -use pco::ChunkConfig; -use pco::DeltaSpec; -use pco::ModeSpec; -use pco::data_types::Number; -use pco::metadata::DeltaEncoding; -use pco::metadata::Mode; -use pco::wrapped::FileCompressor; -use vortex_array::ArrayRef; -use vortex_array::IntoArray; -use vortex_array::VortexSessionExecute; -use vortex_array::array_session; -use vortex_array::arrays::PrimitiveArray; -use vortex_array::assert_arrays_eq; -use vortex_array::dtype::PType; -use vortex_arrow::ArrowSessionExt; -use vortex_btrblocks::BtrBlocksCompressor; -use vortex_btrblocks::BtrBlocksCompressorBuilder; -use vortex_btrblocks::SchemeExt; -use vortex_btrblocks::schemes::float::FloatMultScheme; -use vortex_btrblocks::schemes::float::FloatQuantScheme; -use vortex_btrblocks::schemes::float::OrderedBlockResidualScheme; -use vortex_btrblocks::schemes::range_entropy::RangeEntropyScheme; -use vortex_buffer::Buffer; -use vortex_error::VortexResult; -use vortex_error::vortex_ensure; -use vortex_error::vortex_err; -use vortex_float_quant::FloatMult; -use vortex_float_quant::FloatMultArrayExt; -use vortex_float_quant::estimate_float_mult_constant_base; -use vortex_pco::Pco; -use vortex_range_entropy::BitSplitCodec; -use vortex_range_entropy::PatchedFoRCodec; -use vortex_range_entropy::RangeEntropyCodec; -use vortex_range_entropy::RangePackedCodec; -use vortex_range_entropy::RangeTwoLevelCodec; -use vortex_session::VortexSession; - -const COLUMN_NAMES: [&str; 9] = [ - "longitude", - "latitude", - "housingMedianAge", - "totalRooms", - "totalBedrooms", - "population", - "households", - "medianIncome", - "medianHouseValue", -]; -const DEFAULT_ROW_LIMIT: usize = 2_000_000; - -struct Column { - name: String, - primitive: PrimitiveArray, - array: ArrayRef, -} - -#[derive(Clone, Copy)] -struct FloatMultBackendSizes { - bit_split: usize, - block_residual: usize, - range_entropy: usize, - range_packed: usize, - range_two_level: usize, -} - -struct FloatMultLatents { - primary: Vec, - secondary: Vec, -} - -enum FloatMultPrototype { - F32 { - base: f32, - primary: ArrayRef, - secondary: ArrayRef, - backend_sizes: FloatMultBackendSizes, - latents: FloatMultLatents, - }, - F64 { - base: f64, - primary: ArrayRef, - secondary: ArrayRef, - backend_sizes: FloatMultBackendSizes, - latents: FloatMultLatents, - }, -} - -impl FloatMultPrototype { - fn base(&self) -> f64 { - match self { - Self::F32 { base, .. } => f64::from(*base), - Self::F64 { base, .. } => *base, - } - } - - fn encoded_size(&self) -> u64 { - let (primary, secondary) = match self { - Self::F32 { - primary, secondary, .. - } - | Self::F64 { - primary, secondary, .. - } => (primary, secondary), - }; - primary.nbytes() + secondary.nbytes() + 16 - } - - fn structure(&self) -> String { - let (primary, secondary) = match self { - Self::F32 { - primary, secondary, .. - } - | Self::F64 { - primary, secondary, .. - } => (primary, secondary), - }; - format!( - "float-mult({},{})", - encoding_tree(primary), - encoding_tree(secondary) - ) - } - - fn backend_sizes(&self) -> FloatMultBackendSizes { - match self { - Self::F32 { backend_sizes, .. } | Self::F64 { backend_sizes, .. } => *backend_sizes, - } - } - - fn latents(&self) -> &FloatMultLatents { - match self { - Self::F32 { latents, .. } | Self::F64 { latents, .. } => latents, - } - } - - fn has_nonzero_adjustments(&self) -> bool { - let zero = match self { - Self::F32 { .. } => 1_u64 << 31, - Self::F64 { .. } => 1_u64 << 63, - }; - self.latents() - .secondary - .iter() - .any(|adjustment| *adjustment != zero) - } - - fn to_array(&self) -> VortexResult { - let (primary, secondary, ptype, base) = match self { - Self::F32 { - primary, - secondary, - base, - .. - } => (primary, secondary, PType::F32, f64::from(*base)), - Self::F64 { - primary, - secondary, - base, - .. - } => (primary, secondary, PType::F64, *base), - }; - Ok(FloatMult::try_new(primary.clone(), Some(secondary.clone()), ptype, base)?.into_array()) - } - - #[allow(clippy::cast_possible_truncation)] - fn reconstruct(&self, primary: &[u64], secondary: &[u64]) -> VortexResult<()> { - vortex_ensure!( - primary.len() == secondary.len(), - "FloatMult latent lengths differ" - ); - match self { - Self::F32 { base, .. } => { - let values = primary - .iter() - .copied() - .zip(secondary.iter().copied()) - .map(|(primary, secondary)| { - join_float_mult_f32(primary as u32, secondary as u32, *base) - }) - .collect::>(); - black_box(values); - } - Self::F64 { base, .. } => { - let values = primary - .iter() - .copied() - .zip(secondary.iter().copied()) - .map(|(primary, secondary)| join_float_mult_f64(primary, secondary, *base)) - .collect::>(); - black_box(values); - } - } - Ok(()) - } - - fn decode(&self, session: &VortexSession) -> VortexResult { - match self { - Self::F32 { - base, - primary, - secondary, - .. - } => { - let primary = primary - .clone() - .execute::(&mut session.create_execution_ctx())?; - let secondary = secondary - .clone() - .execute::(&mut session.create_execution_ctx())?; - let validity = primary.as_view().validity()?; - Ok(PrimitiveArray::new( - Buffer::from( - primary - .as_slice::() - .iter() - .copied() - .zip(secondary.as_slice::().iter().copied()) - .map(|(primary, secondary)| { - join_float_mult_f32(primary, secondary, *base) - }) - .collect::>(), - ), - validity, - ) - .into_array()) - } - Self::F64 { - base, - primary, - secondary, - .. - } => { - let primary = primary - .clone() - .execute::(&mut session.create_execution_ctx())?; - let secondary = secondary - .clone() - .execute::(&mut session.create_execution_ctx())?; - let validity = primary.as_view().validity()?; - Ok(PrimitiveArray::new( - Buffer::from( - primary - .as_slice::() - .iter() - .copied() - .zip(secondary.as_slice::().iter().copied()) - .map(|(primary, secondary)| { - join_float_mult_f64(primary, secondary, *base) - }) - .collect::>(), - ), - validity, - ) - .into_array()) - } - } - } -} - -#[derive(Clone, Copy)] -enum FloatMultLatentBackend { - BitSplit, - BlockResidual, - RangeEntropy, - RangePacked, - RangeTwoLevel, -} - -impl FloatMultLatentBackend { - const ALL: [Self; 5] = [ - Self::BitSplit, - Self::BlockResidual, - Self::RangeEntropy, - Self::RangePacked, - Self::RangeTwoLevel, - ]; - - fn name(self) -> &'static str { - match self { - Self::BitSplit => "bit-split", - Self::BlockResidual => "block-residual", - Self::RangeEntropy => "range-entropy", - Self::RangePacked => "range-packed", - Self::RangeTwoLevel => "range-two-level", - } - } - - fn encode(self, latents: &FloatMultLatents) -> VortexResult { - Ok(match self { - Self::BitSplit => FloatMultCodecPair::BitSplit( - BitSplitCodec::encode(&latents.primary)?, - BitSplitCodec::encode(&latents.secondary)?, - ), - Self::BlockResidual => FloatMultCodecPair::BlockResidual( - PatchedFoRCodec::encode_single_base(&latents.primary)?, - PatchedFoRCodec::encode_single_base(&latents.secondary)?, - ), - Self::RangeEntropy => FloatMultCodecPair::RangeEntropy( - RangeEntropyCodec::encode(&latents.primary, 8_192)?, - RangeEntropyCodec::encode(&latents.secondary, 8_192)?, - ), - Self::RangePacked => FloatMultCodecPair::RangePacked( - RangePackedCodec::encode(&latents.primary, 8_192)?, - RangePackedCodec::encode(&latents.secondary, 8_192)?, - ), - Self::RangeTwoLevel => FloatMultCodecPair::RangeTwoLevel( - RangeTwoLevelCodec::encode(&latents.primary, 8_192)?, - RangeTwoLevelCodec::encode(&latents.secondary, 8_192)?, - ), - }) - } -} - -enum FloatMultCodecPair { - BitSplit(BitSplitCodec, BitSplitCodec), - BlockResidual(PatchedFoRCodec, PatchedFoRCodec), - RangeEntropy(RangeEntropyCodec, RangeEntropyCodec), - RangePacked(RangePackedCodec, RangePackedCodec), - RangeTwoLevel(RangeTwoLevelCodec, RangeTwoLevelCodec), -} - -impl FloatMultCodecPair { - fn encoded_size(&self) -> usize { - match self { - Self::BitSplit(primary, secondary) => { - primary.encoded_size() + secondary.encoded_size() + 16 - } - Self::BlockResidual(primary, secondary) => { - primary.encoded_size() + secondary.encoded_size() + 16 - } - Self::RangeEntropy(primary, secondary) => { - primary.encoded_size() + secondary.encoded_size() + 16 - } - Self::RangePacked(primary, secondary) => { - primary.encoded_size() + secondary.encoded_size() + 16 - } - Self::RangeTwoLevel(primary, secondary) => { - primary.encoded_size() + secondary.encoded_size() + 16 - } - } - } - - fn decode(&self) -> VortexResult<(Vec, Vec)> { - match self { - Self::BitSplit(primary, secondary) => Ok((primary.decode()?, secondary.decode()?)), - Self::BlockResidual(primary, secondary) => Ok((primary.decode()?, secondary.decode()?)), - Self::RangeEntropy(primary, secondary) => Ok((primary.decode()?, secondary.decode()?)), - Self::RangePacked(primary, secondary) => Ok((primary.decode()?, secondary.decode()?)), - Self::RangeTwoLevel(primary, secondary) => Ok((primary.decode()?, secondary.decode()?)), - } - } -} - -enum Encoder<'a> { - BtrBlocks(&'a BtrBlocksCompressor), - Pco(&'a ChunkConfig), -} - -impl Encoder<'_> { - fn encode(&self, columns: &[Column], session: &VortexSession) -> VortexResult> { - match self { - Self::BtrBlocks(compressor) => columns - .iter() - .map(|column| { - compressor.compress(&column.array, &mut session.create_execution_ctx()) - }) - .collect(), - Self::Pco(config) => columns - .iter() - .map(|column| { - Ok(Pco::from_primitive_with_config( - column.primitive.as_view(), - config, - pco::DEFAULT_MAX_PAGE_N, - &mut session.create_execution_ctx(), - )? - .into_array()) - }) - .collect(), - } - } -} - -fn read_california_housing( - path: &Path, - target_row_count: Option, -) -> VortexResult> { - let reader = BufReader::new(File::open(path)?); - let mut values = std::array::from_fn::<_, 9, _>(|_| Vec::::new()); - for (line_index, line) in reader.lines().enumerate() { - let line = line?; - let fields = line.split(',').collect::>(); - vortex_ensure!( - fields.len() == COLUMN_NAMES.len(), - "line {} contains {} fields instead of {}", - line_index + 1, - fields.len(), - COLUMN_NAMES.len() - ); - for (column_index, field) in fields.into_iter().enumerate() { - values[column_index].push(field.parse::().map_err(|error| { - vortex_err!( - "cannot parse line {} column {} as f32: {}", - line_index + 1, - column_index + 1, - error - ) - })?); - } - } - - if let Some(target_row_count) = target_row_count { - for column in &mut values { - vortex_ensure!(!column.is_empty(), "California Housing input is empty"); - column.truncate(target_row_count); - while column.len() < target_row_count { - let copy_len = (target_row_count - column.len()).min(column.len()); - column.extend_from_within(..copy_len); - } - } - } - - Ok(COLUMN_NAMES - .into_iter() - .zip(values) - .map(|(name, values)| { - let primitive = PrimitiveArray::from_iter(values); - let array = primitive.clone().into_array(); - Column { - name: name.to_string(), - primitive, - array, - } - }) - .collect::>()) -} - -fn read_parquet_numeric( - path: &Path, - row_limit: usize, - session: &VortexSession, -) -> VortexResult> { - let file = File::open(path)?; - let builder = ParquetRecordBatchReaderBuilder::try_new(file) - .map_err(|error| vortex_err!("cannot read Parquet metadata: {error}"))?; - let schema = Arc::clone(builder.schema()); - let selected = schema - .fields() - .iter() - .enumerate() - .filter_map(|(index, field)| { - matches!( - field.data_type(), - DataType::Int16 - | DataType::Int32 - | DataType::Int64 - | DataType::UInt16 - | DataType::UInt32 - | DataType::UInt64 - | DataType::Float32 - | DataType::Float64 - ) - .then_some(index) - }) - .collect::>(); - vortex_ensure!( - !selected.is_empty(), - "Parquet file contains no numeric columns" - ); - let mask = ProjectionMask::roots(builder.parquet_schema(), selected.iter().copied()); - let reader = builder - .with_projection(mask) - .with_batch_size(65_536) - .build() - .map_err(|error| vortex_err!("cannot build Parquet reader: {error}"))?; - let mut chunks = (0..selected.len()) - .map(|_| Vec::::new()) - .collect::>(); - let mut row_count = 0usize; - for batch in reader { - let batch = batch.map_err(|error| vortex_err!("cannot read Parquet batch: {error}"))?; - let batch_len = batch.num_rows().min(row_limit - row_count); - for (column_chunks, array) in chunks.iter_mut().zip(batch.columns()) { - column_chunks.push(array.slice(0, batch_len)); - } - row_count += batch_len; - if row_count == row_limit { - break; - } - } - vortex_ensure!(row_count > 0, "Parquet file contains no rows"); - - selected - .into_iter() - .zip(chunks) - .map(|(field_index, chunks)| { - let field = &schema.fields()[field_index]; - let chunk_refs = chunks - .iter() - .map(|chunk| chunk.as_ref()) - .collect::>(); - let combined = concat(&chunk_refs) - .map_err(|error| vortex_err!("cannot concatenate {}: {error}", field.name()))?; - let array = session.arrow().from_arrow_array(combined, field.as_ref())?; - let primitive = array.execute::(&mut session.create_execution_ctx())?; - let array = primitive.clone().into_array(); - Ok(Column { - name: field.name().clone(), - primitive, - array, - }) - }) - .collect() -} - -fn percentile(durations: &mut [Duration], numerator: usize, denominator: usize) -> Duration { - durations.sort_unstable(); - durations[durations.len() * numerator / denominator] -} - -fn encoding_tree(array: &ArrayRef) -> String { - let children = array.children(); - if children.is_empty() { - return array.encoding_id().to_string(); - } - let children = children - .iter() - .map(encoding_tree) - .collect::>() - .join(","); - format!("{}({children})", array.encoding_id()) -} - -fn describe_pco_values(values: &[T], config: &ChunkConfig) -> VortexResult { - let compressor = FileCompressor::default(); - let chunk = compressor - .chunk_compressor(values, config) - .map_err(|error| vortex_err!("cannot inspect Pco selection: {error}"))?; - let mode = match &chunk.meta().mode { - Mode::Classic => "classic".to_string(), - Mode::IntMult(_) => "int-mult".to_string(), - Mode::FloatMult(_) => "float-mult".to_string(), - Mode::FloatQuant(bits) => format!("float-quant-{bits}"), - Mode::Dict(_) => "dict".to_string(), - _ => "unknown".to_string(), - }; - let delta = match &chunk.meta().delta_encoding { - DeltaEncoding::NoOp => "none".to_string(), - DeltaEncoding::Consecutive { order, .. } => format!("consecutive-{order}"), - DeltaEncoding::Lookback { .. } => "lookback".to_string(), - DeltaEncoding::Conv1(_) => "conv1".to_string(), - _ => "unknown".to_string(), - }; - Ok(format!("{mode}\t{delta}")) -} - -fn describe_pco( - column: &Column, - config: &ChunkConfig, - session: &VortexSession, -) -> VortexResult { - let mask = column - .primitive - .as_view() - .validity()? - .execute_mask(column.primitive.len(), &mut session.create_execution_ctx())?; - macro_rules! describe_typed { - ($T:ty) => {{ - let values = column - .primitive - .as_slice::<$T>() - .iter() - .copied() - .zip(mask.iter()) - .filter_map(|(value, valid)| valid.then_some(value)) - .collect::>(); - describe_pco_values(&values, config) - }}; - } - match column.primitive.ptype() { - PType::I16 => describe_typed!(i16), - PType::I32 => describe_typed!(i32), - PType::I64 => describe_typed!(i64), - PType::U16 => describe_typed!(u16), - PType::U32 => describe_typed!(u32), - PType::U64 => describe_typed!(u64), - PType::F32 => describe_typed!(f32), - PType::F64 => describe_typed!(f64), - ptype => Err(vortex_err!("unsupported numeric type {ptype}")), - } -} - -fn ordered_f32(value: f32) -> u32 { - let bits = value.to_bits(); - if bits & (1_u32 << 31) == 0 { - bits ^ (1_u32 << 31) - } else { - !bits - } -} - -fn from_ordered_f32(value: u32) -> f32 { - if value & (1_u32 << 31) == 0 { - f32::from_bits(!value) - } else { - f32::from_bits(value ^ (1_u32 << 31)) - } -} - -fn ordered_f64(value: f64) -> u64 { - let bits = value.to_bits(); - if bits & (1_u64 << 63) == 0 { - bits ^ (1_u64 << 63) - } else { - !bits - } -} - -fn from_ordered_f64(value: u64) -> f64 { - if value & (1_u64 << 63) == 0 { - f64::from_bits(!value) - } else { - f64::from_bits(value ^ (1_u64 << 63)) - } -} - -#[expect( - clippy::cast_possible_truncation, - reason = "the branch limits the float to the exact u32 integer range" -)] -fn int_float_to_u32(value: f32) -> u32 { - let absolute = value.abs(); - let greatest_precise_integer = 1_u32 << f32::MANTISSA_DIGITS; - let greatest_precise_float = greatest_precise_integer as f32; - let absolute_integer = if absolute < greatest_precise_float { - absolute as u32 - } else { - greatest_precise_integer + (absolute.to_bits() - greatest_precise_float.to_bits()) - }; - if value.is_sign_positive() { - (1_u32 << 31) + absolute_integer - } else { - (1_u32 << 31) - 1 - absolute_integer - } -} - -fn int_float_from_u32(value: u32) -> f32 { - let middle = 1_u32 << 31; - let (negative, absolute_integer) = if value >= middle { - (false, value - middle) - } else { - (true, middle - 1 - value) - }; - let greatest_precise_integer = 1_u32 << f32::MANTISSA_DIGITS; - let absolute = if absolute_integer < greatest_precise_integer { - absolute_integer as f32 - } else { - f32::from_bits( - (greatest_precise_integer as f32).to_bits() - + (absolute_integer - greatest_precise_integer), - ) - }; - if negative { -absolute } else { absolute } -} - -#[expect( - clippy::cast_possible_truncation, - reason = "the branch limits the float to the exact u64 integer range" -)] -fn int_float_to_u64(value: f64) -> u64 { - let absolute = value.abs(); - let greatest_precise_integer = 1_u64 << f64::MANTISSA_DIGITS; - let greatest_precise_float = greatest_precise_integer as f64; - let absolute_integer = if absolute < greatest_precise_float { - absolute as u64 - } else { - greatest_precise_integer + (absolute.to_bits() - greatest_precise_float.to_bits()) - }; - if value.is_sign_positive() { - (1_u64 << 63) + absolute_integer - } else { - (1_u64 << 63) - 1 - absolute_integer - } -} - -fn int_float_from_u64(value: u64) -> f64 { - let middle = 1_u64 << 63; - let (negative, absolute_integer) = if value >= middle { - (false, value - middle) - } else { - (true, middle - 1 - value) - }; - let greatest_precise_integer = 1_u64 << f64::MANTISSA_DIGITS; - let absolute = if absolute_integer < greatest_precise_integer { - absolute_integer as f64 - } else { - f64::from_bits( - (greatest_precise_integer as f64).to_bits() - + (absolute_integer - greatest_precise_integer), - ) - }; - if negative { -absolute } else { absolute } -} - -fn split_float_mult_f32(value: f32, base: f32) -> (u32, u32) { - let multiple = (value / base).round(); - let primary = int_float_to_u32(multiple); - let secondary = ordered_f32(value) - .wrapping_sub(ordered_f32(multiple * base)) - .wrapping_add(1_u32 << 31); - (primary, secondary) -} - -fn split_float_mult_f64(value: f64, base: f64) -> (u64, u64) { - let multiple = (value / base).round(); - let primary = int_float_to_u64(multiple); - let secondary = ordered_f64(value) - .wrapping_sub(ordered_f64(multiple * base)) - .wrapping_add(1_u64 << 63); - (primary, secondary) -} - -fn join_float_mult_f32(primary: u32, secondary: u32, base: f32) -> f32 { - let approximate = int_float_from_u32(primary) * base; - from_ordered_f32(ordered_f32(approximate).wrapping_add(secondary.wrapping_add(1_u32 << 31))) -} - -fn join_float_mult_f64(primary: u64, secondary: u64, base: f64) -> f64 { - let approximate = int_float_from_u64(primary) * base; - from_ordered_f64(ordered_f64(approximate).wrapping_add(secondary.wrapping_add(1_u64 << 63))) -} - -fn float_mult_prototype( - column: &Column, - pco_config: &ChunkConfig, - compressor: &BtrBlocksCompressor, - session: &VortexSession, -) -> VortexResult> { - let mask = column - .primitive - .as_view() - .validity()? - .execute_mask(column.primitive.len(), &mut session.create_execution_ctx())?; - macro_rules! build { - ($T:ty, $L:ty, $variant:ident, $ordered:ident, $split:ident) => {{ - let values = column.primitive.as_slice::<$T>(); - let valid_values = values - .iter() - .copied() - .zip(mask.iter()) - .filter_map(|(value, valid)| valid.then_some(value)) - .collect::>(); - let chunk = FileCompressor::default() - .chunk_compressor(&valid_values, pco_config) - .map_err(|error| vortex_err!("cannot inspect Pco selection: {error}"))?; - let Mode::FloatMult(base) = &chunk.meta().mode else { - return Ok(None); - }; - let base = $ordered( - *base - .downcast_ref::<$L>() - .ok_or_else(|| vortex_err!("Pco FloatMult base has the wrong latent type"))?, - ); - let (primary, secondary): (Vec<$L>, Vec<$L>) = values - .iter() - .copied() - .map(|value| $split(value, base)) - .unzip(); - let primary_u64 = primary.iter().copied().map(u64::from).collect::>(); - let secondary_u64 = secondary.iter().copied().map(u64::from).collect::>(); - let backend_sizes = FloatMultBackendSizes { - bit_split: BitSplitCodec::encode(&primary_u64)?.encoded_size() - + BitSplitCodec::encode(&secondary_u64)?.encoded_size() - + 16, - block_residual: PatchedFoRCodec::encode_single_base(&primary_u64)?.encoded_size() - + PatchedFoRCodec::encode_single_base(&secondary_u64)?.encoded_size() - + 16, - range_entropy: RangeEntropyCodec::encode(&primary_u64, 8_192)?.encoded_size() - + RangeEntropyCodec::encode(&secondary_u64, 8_192)?.encoded_size() - + 16, - range_packed: RangePackedCodec::encode(&primary_u64, 8_192)?.encoded_size() - + RangePackedCodec::encode(&secondary_u64, 8_192)?.encoded_size() - + 16, - range_two_level: RangeTwoLevelCodec::encode(&primary_u64, 8_192)?.encoded_size() - + RangeTwoLevelCodec::encode(&secondary_u64, 8_192)?.encoded_size() - + 16, - }; - let latents = FloatMultLatents { - primary: primary_u64, - secondary: secondary_u64, - }; - let validity = column.primitive.as_view().validity()?; - let primary = PrimitiveArray::new(Buffer::from(primary), validity.clone()).into_array(); - let secondary = PrimitiveArray::new(Buffer::from(secondary), validity).into_array(); - let primary = compressor.compress(&primary, &mut session.create_execution_ctx())?; - let secondary = compressor.compress(&secondary, &mut session.create_execution_ctx())?; - Ok(Some(FloatMultPrototype::$variant { - base, - primary, - secondary, - backend_sizes, - latents, - })) - }}; - } - match column.primitive.ptype() { - PType::F32 => build!(f32, u32, F32, from_ordered_f32, split_float_mult_f32), - PType::F64 => build!(f64, u64, F64, from_ordered_f64, split_float_mult_f64), - _ => Ok(None), - } -} - -fn decode_all(arrays: &[ArrayRef], session: &VortexSession) -> VortexResult<()> { - for array in arrays { - let decoded = array - .clone() - .execute::(&mut session.create_execution_ctx())?; - black_box(decoded.nbytes()); - } - Ok(()) -} - -fn measure_compressors( - dataset: &str, - configs: &[(&str, Encoder<'_>)], - columns: &[Column], - input_bytes: u64, - session: &VortexSession, -) -> VortexResult<()> { - let iterations = (200_000_000u64 / input_bytes).clamp(3, 100) as usize; - for (_, encoder) in configs { - black_box(encoder.encode(columns, session)?); - } - - let mut durations = (0..configs.len()) - .map(|_| Vec::with_capacity(iterations)) - .collect::>(); - for iteration in 0..iterations { - for offset in 0..configs.len() { - let index = (iteration + offset) % configs.len(); - let start = Instant::now(); - black_box(configs[index].1.encode(columns, session)?); - durations[index].push(start.elapsed()); - } - } - - for (index, (name, _)) in configs.iter().enumerate() { - let p10 = percentile(&mut durations[index].clone(), 1, 10); - let p90 = percentile(&mut durations[index].clone(), 9, 10); - let median = percentile(&mut durations[index], 1, 2); - let throughput = input_bytes as f64 / median.as_secs_f64() / 1_000_000.0; - println!( - "compress\t{dataset}\t{name}\t{input_bytes}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", - p10.as_secs_f64() * 1_000.0, - median.as_secs_f64() * 1_000.0, - p90.as_secs_f64() * 1_000.0, - ); - } - Ok(()) -} - -fn measure_decoders( - dataset: &str, - configs: &[(&str, Vec)], - input_bytes: u64, - session: &VortexSession, -) -> VortexResult<()> { - let iterations = (1_000_000_000u64 / input_bytes).clamp(5, 200) as usize; - for (_, arrays) in configs { - black_box(decode_all(arrays, session)?); - } - - let mut durations = (0..configs.len()) - .map(|_| Vec::with_capacity(iterations)) - .collect::>(); - for iteration in 0..iterations { - for offset in 0..configs.len() { - let index = (iteration + offset) % configs.len(); - let start = Instant::now(); - black_box(decode_all(&configs[index].1, session)?); - durations[index].push(start.elapsed()); - } - } - - for (index, (name, arrays)) in configs.iter().enumerate() { - let output_bytes = arrays.iter().map(ArrayRef::nbytes).sum::(); - let p10 = percentile(&mut durations[index].clone(), 1, 10); - let p90 = percentile(&mut durations[index].clone(), 9, 10); - let median = percentile(&mut durations[index], 1, 2); - let throughput = input_bytes as f64 / median.as_secs_f64() / 1_000_000.0; - println!( - "decode\t{dataset}\t{name}\t{output_bytes}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", - p10.as_secs_f64() * 1_000.0, - median.as_secs_f64() * 1_000.0, - p90.as_secs_f64() * 1_000.0, - ); - } - Ok(()) -} - -fn measure_selected_float_mult( - dataset: &str, - columns: &[Column], - baseline: &[ArrayRef], - candidate: &[ArrayRef], - session: &VortexSession, -) -> VortexResult<()> { - let selected = candidate - .iter() - .enumerate() - .filter_map(|(index, array)| contains_float_mult(array).then_some(index)) - .collect::>(); - if selected.is_empty() { - return Ok(()); - } - let input_bytes = selected - .iter() - .map(|index| columns[*index].primitive.nbytes()) - .sum::(); - let selected_configs = [ - ( - "prior-default", - selected - .iter() - .map(|index| baseline[*index].clone()) - .collect::>(), - ), - ( - "float-mult", - selected - .iter() - .map(|index| candidate[*index].clone()) - .collect::>(), - ), - ]; - measure_decoders( - &format!("{dataset}-float-mult-selected"), - &selected_configs, - input_bytes, - session, - )?; - - for index in selected { - measure_scalar_pair( - dataset, - &columns[index].name, - [ - ("prior-default", &baseline[index]), - ("float-mult", &candidate[index]), - ], - session, - )?; - } - Ok(()) -} - -fn contains_float_mult(array: &ArrayRef) -> bool { - array.is::() || array.children().iter().any(contains_float_mult) -} - -fn measure_scalar_pair( - dataset: &str, - column: &str, - arrays: [(&str, &ArrayRef); 2], - session: &VortexSession, -) -> VortexResult<()> { - const ACCESSES: usize = 200_000; - const REPETITIONS: usize = 7; - let mut durations = [ - Vec::with_capacity(REPETITIONS), - Vec::with_capacity(REPETITIONS), - ]; - for repetition in 0..REPETITIONS { - for offset in 0..arrays.len() { - let config_index = (repetition + offset) % arrays.len(); - let array = arrays[config_index].1; - let mut ctx = session.create_execution_ctx(); - let start = Instant::now(); - let mut checksum = 0_u32; - for access in 0..ACCESSES { - let row = access.wrapping_mul(0x9e37_79b1) % array.len(); - let value = array - .execute_scalar(row, &mut ctx)? - .as_primitive() - .typed_value::() - .ok_or_else(|| vortex_err!("benchmark value is null"))?; - checksum ^= black_box(value.to_bits()); - } - durations[config_index].push(start.elapsed()); - black_box(checksum); - } - } - for (config_index, (config, array)) in arrays.into_iter().enumerate() { - let median = percentile(&mut durations[config_index], 1, 2); - let nanoseconds = median.as_secs_f64() * 1_000_000_000.0 / ACCESSES as f64; - let accesses_per_second = ACCESSES as f64 / median.as_secs_f64() / 1_000_000.0; - println!( - "scalar-at\t{dataset}\t{column}\t{config}\t{}\t{}\t{nanoseconds:.2}\t{accesses_per_second:.1}", - array.encoding_id(), - array.nbytes(), - ); - } - Ok(()) -} - -fn measure_nonzero_float_mult( - dataset: &str, - columns: &[Column], - prototypes: &[(&Column, FloatMultPrototype)], - baseline: &[ArrayRef], - session: &VortexSession, -) -> VortexResult<()> { - for (column, prototype) in prototypes - .iter() - .filter(|(_, prototype)| prototype.has_nonzero_adjustments()) - { - let index = columns - .iter() - .position(|candidate| candidate.name == column.name) - .ok_or_else(|| vortex_err!("missing benchmark column {}", column.name))?; - let candidate = prototype.to_array()?; - let mut verify_ctx = session.create_execution_ctx(); - assert_arrays_eq!(candidate, column.array, &mut verify_ctx); - let configs = [ - ("prior-default", vec![baseline[index].clone()]), - ("float-mult-nonzero", vec![candidate.clone()]), - ]; - measure_decoders( - &format!("{dataset}-{}-nonzero", column.name), - &configs, - column.primitive.nbytes(), - session, - )?; - measure_scalar_pair( - dataset, - &column.name, - [ - ("prior-default", &baseline[index]), - ("float-mult-nonzero", &candidate), - ], - session, - )?; - } - Ok(()) -} - -fn measure_float_mult_decoder( - dataset: &str, - prototypes: &[(&Column, FloatMultPrototype)], - session: &VortexSession, -) -> VortexResult<()> { - if prototypes.is_empty() { - return Ok(()); - } - let input_bytes = prototypes - .iter() - .map(|(column, _)| column.primitive.nbytes()) - .sum::(); - let encoded_bytes = prototypes - .iter() - .map(|(_, prototype)| prototype.encoded_size()) - .sum::(); - let iterations = (1_000_000_000_u64 / input_bytes).clamp(5, 200) as usize; - for (_, prototype) in prototypes { - black_box(prototype.decode(session)?); - } - let mut durations = Vec::with_capacity(iterations); - for _ in 0..iterations { - let start = Instant::now(); - for (_, prototype) in prototypes { - black_box(prototype.decode(session)?); - } - durations.push(start.elapsed()); - } - let p10 = percentile(&mut durations.clone(), 1, 10); - let p90 = percentile(&mut durations.clone(), 9, 10); - let median = percentile(&mut durations, 1, 2); - let throughput = input_bytes as f64 / median.as_secs_f64() / 1_000_000.0; - println!( - "decode\t{dataset}\tfloat-mult-prototype\t{encoded_bytes}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", - p10.as_secs_f64() * 1_000.0, - median.as_secs_f64() * 1_000.0, - p90.as_secs_f64() * 1_000.0, - ); - Ok(()) -} - -fn measure_float_mult_backends( - dataset: &str, - prototypes: &[(&Column, FloatMultPrototype)], -) -> VortexResult<()> { - if prototypes.is_empty() { - return Ok(()); - } - let input_bytes = prototypes - .iter() - .map(|(column, _)| column.primitive.nbytes()) - .sum::(); - let encoded = FloatMultLatentBackend::ALL - .into_iter() - .map(|backend| { - let codecs = prototypes - .iter() - .map(|(_, prototype)| backend.encode(prototype.latents())) - .collect::>>()?; - Ok((backend, codecs)) - }) - .collect::>>()?; - - let decode_iterations = (1_000_000_000_u64 / input_bytes).clamp(5, 50) as usize; - for (backend, codecs) in &encoded { - for ((_, prototype), codec) in prototypes.iter().zip(codecs) { - let (primary, secondary) = codec.decode()?; - prototype.reconstruct(&primary, &secondary)?; - } - let mut durations = Vec::with_capacity(decode_iterations); - for _ in 0..decode_iterations { - let start = Instant::now(); - for ((_, prototype), codec) in prototypes.iter().zip(codecs) { - let (primary, secondary) = codec.decode()?; - prototype.reconstruct(&primary, &secondary)?; - } - durations.push(start.elapsed()); - } - let encoded_bytes = codecs - .iter() - .map(FloatMultCodecPair::encoded_size) - .sum::(); - let p10 = percentile(&mut durations.clone(), 1, 10); - let p90 = percentile(&mut durations.clone(), 9, 10); - let median = percentile(&mut durations, 1, 2); - let throughput = input_bytes as f64 / median.as_secs_f64() / 1_000_000.0; - println!( - "float-mult-backend-decode\t{dataset}\t{}\t{encoded_bytes}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", - backend.name(), - p10.as_secs_f64() * 1_000.0, - median.as_secs_f64() * 1_000.0, - p90.as_secs_f64() * 1_000.0, - ); - } - - let encode_iterations = (200_000_000_u64 / input_bytes).clamp(3, 20) as usize; - for backend in FloatMultLatentBackend::ALL { - let mut durations = Vec::with_capacity(encode_iterations); - for _ in 0..encode_iterations { - let start = Instant::now(); - let encoded_bytes = prototypes - .iter() - .try_fold(0_usize, |size, (_, prototype)| { - Ok::<_, vortex_error::VortexError>( - size + backend.encode(prototype.latents())?.encoded_size(), - ) - })?; - durations.push(start.elapsed()); - black_box(encoded_bytes); - } - let p10 = percentile(&mut durations.clone(), 1, 10); - let p90 = percentile(&mut durations.clone(), 9, 10); - let median = percentile(&mut durations, 1, 2); - let throughput = input_bytes as f64 / median.as_secs_f64() / 1_000_000.0; - println!( - "float-mult-backend-encode\t{dataset}\t{}\t{input_bytes}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", - backend.name(), - p10.as_secs_f64() * 1_000.0, - median.as_secs_f64() * 1_000.0, - p90.as_secs_f64() * 1_000.0, - ); - } - Ok(()) -} - -fn main() -> VortexResult<()> { - let path = std::env::args() - .nth(1) - .ok_or_else(|| vortex_err!("usage: float_quant_dataset [row limit]"))?; - let row_limit = std::env::args() - .nth(2) - .map(|value| { - value - .parse::() - .map_err(|error| vortex_err!("invalid row limit: {error}")) - }) - .transpose()?; - let path = Path::new(&path); - let dataset = path - .file_stem() - .and_then(|name| name.to_str()) - .ok_or_else(|| vortex_err!("data path has no valid file name"))?; - let session = array_session(); - let columns = if path - .extension() - .is_some_and(|extension| extension == "parquet") - { - read_parquet_numeric(path, row_limit.unwrap_or(DEFAULT_ROW_LIMIT), &session)? - } else { - read_california_housing(path, row_limit)? - }; - let input_bytes = columns - .iter() - .map(|column| column.primitive.nbytes()) - .sum::(); - let baseline = BtrBlocksCompressorBuilder::default() - .exclude_schemes([ - FloatMultScheme.id(), - FloatQuantScheme.id(), - OrderedBlockResidualScheme.id(), - ]) - .build(); - let range_candidate = BtrBlocksCompressorBuilder::default() - .exclude_schemes([ - FloatMultScheme.id(), - FloatQuantScheme.id(), - OrderedBlockResidualScheme.id(), - ]) - .with_new_scheme(&RangeEntropyScheme) - .build(); - let default_without_block_residual = BtrBlocksCompressorBuilder::default() - .exclude_schemes([OrderedBlockResidualScheme.id()]) - .build(); - let default_without_float_mult = BtrBlocksCompressorBuilder::default() - .exclude_schemes([FloatMultScheme.id()]) - .build(); - let float_candidate = BtrBlocksCompressor::default(); - let stacked_candidate = BtrBlocksCompressorBuilder::default() - .with_new_scheme(&RangeEntropyScheme) - .build(); - let compact = BtrBlocksCompressorBuilder::default().with_compact().build(); - let pco_config = ChunkConfig::default() - .with_mode_spec(ModeSpec::Auto) - .with_delta_spec(DeltaSpec::Auto); - let configs = [ - ("default-without-new-float", Encoder::BtrBlocks(&baseline)), - ( - "default-without-new-float+range-entropy", - Encoder::BtrBlocks(&range_candidate), - ), - ( - "default-without-block-residual", - Encoder::BtrBlocks(&default_without_block_residual), - ), - ( - "default-without-float-mult", - Encoder::BtrBlocks(&default_without_float_mult), - ), - ("default", Encoder::BtrBlocks(&float_candidate)), - ( - "default+range-entropy", - Encoder::BtrBlocks(&stacked_candidate), - ), - ("compact", Encoder::BtrBlocks(&compact)), - ("pco-auto", Encoder::Pco(&pco_config)), - ]; - let encoded = configs - .iter() - .map(|(name, encoder)| Ok((*name, encoder.encode(&columns, &session)?))) - .collect::>>()?; - let float_mult_prototypes = columns - .iter() - .filter_map(|column| { - float_mult_prototype(column, &pco_config, &baseline, &session) - .transpose() - .map(|result| result.map(|prototype| (column, prototype))) - }) - .collect::>>()?; - - println!("pco-selection\tdataset\tcolumn\tmode\tdelta"); - for column in &columns { - println!( - "pco-selection\t{dataset}\t{}\t{}", - column.name, - describe_pco(column, &pco_config, &session)? - ); - if let Some(base) = estimate_float_mult_constant_base(column.primitive.as_view()) { - println!( - "float-mult-analysis\t{dataset}\t{}\tconstant-base\t{base}", - column.name - ); - } - } - println!("structure\tdataset\tcolumn\tconfig\tencoding\tbytes"); - for (config_name, arrays) in &encoded { - for (column, array) in columns.iter().zip(arrays) { - let mut verify_ctx = session.create_execution_ctx(); - assert_arrays_eq!(array, column.array, &mut verify_ctx); - println!( - "structure\t{dataset}\t{}\t{config_name}\t{}\t{}", - column.name, - encoding_tree(array), - array.nbytes() - ); - if let Some(float_mult) = array.as_opt::() { - println!( - "float-mult-base\t{dataset}\t{}\t{config_name}\t{}", - column.name, - float_mult.base() - ); - } - } - } - for (column, prototype) in &float_mult_prototypes { - let decoded = prototype.decode(&session)?; - let mut verify_ctx = session.create_execution_ctx(); - assert_arrays_eq!(decoded, column.array, &mut verify_ctx); - let backend_sizes = prototype.backend_sizes(); - println!( - "structure\t{dataset}\t{}\tfloat-mult-prototype\t{}\t{}", - column.name, - prototype.structure(), - prototype.encoded_size(), - ); - println!( - "float-mult-base\t{dataset}\t{}\tpco-prototype\t{}", - column.name, - prototype.base() - ); - println!( - "float-mult-backends\t{dataset}\t{}\tbit-split={}\tblock-residual={}\trange-entropy={}\trange-packed={}\trange-two-level={}", - column.name, - backend_sizes.bit_split, - backend_sizes.block_residual, - backend_sizes.range_entropy, - backend_sizes.range_packed, - backend_sizes.range_two_level, - ); - } - println!("operation\tdataset\tconfig\tbytes\tp10-ms\tmedian-ms\tp90-ms\tMB/s"); - measure_decoders(dataset, &encoded, input_bytes, &session)?; - let default_without_float_mult = encoded - .iter() - .find_map(|(name, arrays)| (*name == "default-without-float-mult").then_some(arrays)) - .ok_or_else(|| vortex_err!("missing FloatMult baseline"))?; - let default = encoded - .iter() - .find_map(|(name, arrays)| (*name == "default").then_some(arrays)) - .ok_or_else(|| vortex_err!("missing default candidate"))?; - measure_selected_float_mult( - dataset, - &columns, - default_without_float_mult, - default, - &session, - )?; - measure_nonzero_float_mult( - dataset, - &columns, - &float_mult_prototypes, - default_without_float_mult, - &session, - )?; - measure_float_mult_decoder(dataset, &float_mult_prototypes, &session)?; - measure_float_mult_backends(dataset, &float_mult_prototypes)?; - measure_compressors(dataset, &configs, &columns, input_bytes, &session)?; - Ok(()) -} diff --git a/vortex-btrblocks/examples/pco_probe.rs b/vortex-btrblocks/examples/pco_probe.rs index acd6c96e8c3..02f1feea8d8 100644 --- a/vortex-btrblocks/examples/pco_probe.rs +++ b/vortex-btrblocks/examples/pco_probe.rs @@ -20,11 +20,9 @@ use vortex_array::arrays::PrimitiveArray; use vortex_btrblocks::BtrBlocksCompressor; use vortex_btrblocks::BtrBlocksCompressorBuilder; use vortex_btrblocks::schemes::float::PcoScheme; -use vortex_btrblocks::schemes::range_entropy::RangeEntropyScheme; use vortex_error::VortexResult; use vortex_error::vortex_err; use vortex_pco::Pco; -use vortex_range_entropy::RangeEntropyCodec; const N: usize = 1 << 18; const VALUES_PER_PAGE: usize = 8192; @@ -213,40 +211,12 @@ fn report( Ok(()) } -fn ordered_f64(value: f64) -> u64 { - let bits = value.to_bits(); - if bits & (1_u64 << 63) == 0 { - bits ^ (1_u64 << 63) - } else { - !bits - } -} - -fn report_native(dataset: &str, values: &[f64], input_bytes: u64) -> VortexResult<()> { - let latents: Vec<_> = values.iter().copied().map(ordered_f64).collect(); - let start = Instant::now(); - let compressed = RangeEntropyCodec::encode(&latents, VALUES_PER_PAGE)?; - let elapsed = start.elapsed(); - let encoded_bytes = u64::try_from(compressed.encoded_size())?; - let bits_per_value = 8.0 * encoded_bytes as f64 / N as f64; - let throughput = input_bytes as f64 / elapsed.as_secs_f64() / 1_000_000.0; - eprintln!( - "{dataset}: native range entropy used {} bins", - compressed.bin_lowers().len() - ); - println!("{dataset}\trange-entropy\t{encoded_bytes}\t{bits_per_value:.3}\t{throughput:.1}"); - Ok(()) -} - fn main() -> VortexResult<()> { let session = array_session(); let default = BtrBlocksCompressor::default(); let with_auto = BtrBlocksCompressorBuilder::default() .with_new_scheme(&PcoScheme) .build(); - let with_range_entropy = BtrBlocksCompressorBuilder::default() - .with_new_scheme(&RangeEntropyScheme) - .build(); println!("dataset\tencoder\tbytes\tbits/value\tMB/s"); println!("pco-stage columns: dataset, config, bytes, prepare-ms, emit-ms, total-ms, MB/s"); @@ -265,18 +235,12 @@ fn main() -> VortexResult<()> { let config = selected_config(name).with_compression_level(level); report_pco_stages(name, &format!("selected-level-{level}"), &values, &config)?; } - report_native(name, &values, input_bytes)?; - report(name, "btrblocks", input_bytes, || { default.compress(&array_ref, &mut session.create_execution_ctx()) })?; report(name, "btrblocks+pco-auto", input_bytes, || { with_auto.compress(&array_ref, &mut session.create_execution_ctx()) })?; - report(name, "btrblocks+range-entropy", input_bytes, || { - with_range_entropy.compress(&array_ref, &mut session.create_execution_ctx()) - })?; - for (encoder, config) in [ ( "pco-classic", diff --git a/vortex-btrblocks/examples/range_entropy_throughput.rs b/vortex-btrblocks/examples/range_entropy_throughput.rs deleted file mode 100644 index c4886fc63fe..00000000000 --- a/vortex-btrblocks/examples/range_entropy_throughput.rs +++ /dev/null @@ -1,994 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::f64::consts::TAU; -use std::hint::black_box; -use std::time::Duration; -use std::time::Instant; - -use pco::ChunkConfig; -use pco::DeltaSpec; -use pco::ModeSpec; -use pco::PagingSpec; -use vortex_array::ArrayRef; -use vortex_array::IntoArray; -use vortex_array::VortexSessionExecute; -use vortex_array::array_session; -use vortex_array::arrays::PrimitiveArray; -use vortex_array::assert_arrays_eq; -use vortex_btrblocks::BtrBlocksCompressor; -use vortex_btrblocks::BtrBlocksCompressorBuilder; -use vortex_btrblocks::SchemeExt; -use vortex_btrblocks::schemes::float::FloatQuantScheme; -use vortex_btrblocks::schemes::float::OrderedBlockResidualScheme; -use vortex_btrblocks::schemes::range_entropy::RangeEntropyScheme; -use vortex_error::VortexResult; -use vortex_float_quant::FloatQuant; -use vortex_float_quant::FloatQuantArraySlotsExt; -use vortex_float_quant::estimate_k; -use vortex_pco::Pco; -use vortex_range_entropy::BitSplitCodec; -use vortex_range_entropy::PatchedFoRCodec; -use vortex_range_entropy::RangeEntropy; -use vortex_range_entropy::RangeEntropyCodec; -use vortex_range_entropy::RangeGroupedCodec; -use vortex_range_entropy::RangePackedCodec; -use vortex_range_entropy::RangeTwoLevelCodec; -use vortex_session::VortexSession; - -const N: usize = 1 << 18; -const VALUES_PER_PAGE: usize = 8192; -const DECODE_ITERATIONS: usize = 100; -const COMPRESS_ITERATIONS: usize = 40; -const SCALAR_ACCESS_ITERATIONS: usize = N * 8; -const ARRAY_SCALAR_ACCESS_ITERATIONS: usize = N; - -struct Rng(u64); - -impl Rng { - fn next_u64(&mut self) -> u64 { - self.0 ^= self.0 << 13; - self.0 ^= self.0 >> 7; - self.0 ^= self.0 << 17; - self.0 - } - - fn uniform(&mut self) -> f64 { - ((self.next_u64() >> 11) as f64 + 0.5) * (1.0 / ((1_u64 << 53) as f64)) - } - - fn normal(&mut self) -> f64 { - let radius = (-2.0 * self.uniform().ln()).sqrt(); - radius * (TAU * self.uniform()).cos() - } -} - -#[expect( - clippy::cast_possible_truncation, - reason = "the benchmark models f32 values stored as f64" -)] -fn widen_f32(value: f64) -> f64 { - value as f32 as f64 -} - -fn datasets() -> Vec<(&'static str, Vec)> { - let mut rng = Rng(0x4d59_5df4_d0f3_3173); - let gaussian = (0..N).map(|_| 1_000.0 + 17.0 * rng.normal()).collect(); - let lognormal = (0..N).map(|_| rng.normal().exp()).collect(); - let decimal = (0..N) - .map(|_| (rng.next_u64() % 1_000_000) as f64 / 100.0) - .collect(); - let widened_f32 = (0..N) - .map(|_| widen_f32(1_000.0 + 17.0 * rng.normal())) - .collect(); - let mut value = 0.0; - let random_walk = (0..N) - .map(|_| { - value += rng.normal() * 0.01; - value - }) - .collect(); - let four_clusters = (0..N) - .map(|_| { - let center = match rng.next_u64() & 3 { - 0 => -1_000_000_000.0, - 1 => -1_000.0, - 2 => 1_000.0, - _ => 1_000_000_000.0, - }; - center + 17.0 * rng.normal() - }) - .collect(); - - vec![ - ("gaussian", gaussian), - ("lognormal", lognormal), - ("decimal", decimal), - ("widened-f32", widened_f32), - ("random-walk", random_walk), - ("four-clusters", four_clusters), - ] -} - -fn pco_config(mode: ModeSpec, delta: DeltaSpec) -> ChunkConfig { - ChunkConfig::default() - .with_mode_spec(mode) - .with_delta_spec(delta) - .with_paging_spec(PagingSpec::EqualPagesUpTo(VALUES_PER_PAGE)) -} - -fn percentile(durations: &mut [Duration], numerator: usize, denominator: usize) -> Duration { - durations.sort_unstable(); - durations[durations.len() * numerator / denominator] -} - -fn encoding_tree(array: &ArrayRef) -> String { - let children = array.children(); - if children.is_empty() { - return array.encoding_id().to_string(); - } - let children = children - .iter() - .map(encoding_tree) - .collect::>() - .join(","); - format!("{}({children})", array.encoding_id()) -} - -fn decode_once(array: &ArrayRef, session: &VortexSession) -> VortexResult { - let start = Instant::now(); - let decoded = array - .clone() - .execute::(&mut session.create_execution_ctx())?; - black_box(decoded.nbytes()); - Ok(start.elapsed()) -} - -fn report_decode( - dataset: &str, - encoder: &str, - array: &ArrayRef, - durations: &mut [Duration], -) -> VortexResult<()> { - let mut for_p10 = durations.to_vec(); - let mut for_median = durations.to_vec(); - let p10 = percentile(&mut for_p10, 1, 10); - let median = percentile(&mut for_median, 1, 2); - let p90 = percentile(durations, 9, 10); - let throughput = (N * size_of::()) as f64 / median.as_secs_f64() / 1_000_000.0; - println!( - "decode\t{dataset}\t{encoder}\t{}\t{}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", - array.encoding_id(), - array.nbytes(), - p10.as_secs_f64() * 1_000.0, - median.as_secs_f64() * 1_000.0, - p90.as_secs_f64() * 1_000.0, - ); - Ok(()) -} - -fn measure_decoders( - dataset: &str, - configs: &[(&str, &ArrayRef)], - session: &VortexSession, -) -> VortexResult<()> { - for _ in 0..3 { - for (_, array) in configs { - black_box(decode_once(array, session)?); - } - } - - let mut durations = vec![Vec::with_capacity(DECODE_ITERATIONS); configs.len()]; - for iteration in 0..DECODE_ITERATIONS { - for offset in 0..configs.len() { - let config_index = (iteration + offset) % configs.len(); - durations[config_index].push(decode_once(configs[config_index].1, session)?); - } - } - - for (config_index, (config, array)) in configs.iter().enumerate() { - report_decode(dataset, config, array, &mut durations[config_index])?; - } - Ok(()) -} - -fn measure_range_entropy_codec( - dataset: &str, - values: &[f64], - codec: &RangeEntropyCodec, -) -> VortexResult<()> { - for _ in 0..3 { - black_box(codec.decode()?); - } - let mut durations = Vec::with_capacity(DECODE_ITERATIONS); - for _ in 0..DECODE_ITERATIONS { - let start = Instant::now(); - black_box(codec.decode()?); - durations.push(start.elapsed()); - } - let p10 = percentile(&mut durations.to_vec(), 1, 10); - let p90 = percentile(&mut durations.to_vec(), 9, 10); - let median = percentile(&mut durations, 1, 2); - let throughput = - values.len() as f64 * size_of::() as f64 / median.as_secs_f64() / 1_000_000.0; - println!( - "decode\t{dataset}\trange-entropy-codec\tvortex.range_entropy_codec\t{}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", - codec.encoded_size(), - p10.as_secs_f64() * 1_000.0, - median.as_secs_f64() * 1_000.0, - p90.as_secs_f64() * 1_000.0, - ); - Ok(()) -} - -fn measure_range_packed_codec( - dataset: &str, - values: &[f64], - codec: &RangePackedCodec, -) -> VortexResult<()> { - for _ in 0..3 { - black_box(codec.decode()?); - } - let mut durations = Vec::with_capacity(DECODE_ITERATIONS); - for _ in 0..DECODE_ITERATIONS { - let start = Instant::now(); - black_box(codec.decode()?); - durations.push(start.elapsed()); - } - let p10 = percentile(&mut durations.to_vec(), 1, 10); - let p90 = percentile(&mut durations.to_vec(), 9, 10); - let median = percentile(&mut durations, 1, 2); - let throughput = - values.len() as f64 * size_of::() as f64 / median.as_secs_f64() / 1_000_000.0; - println!( - "decode\t{dataset}\trange-packed-codec\tvortex.range_packed_codec\t{}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", - codec.encoded_size(), - p10.as_secs_f64() * 1_000.0, - median.as_secs_f64() * 1_000.0, - p90.as_secs_f64() * 1_000.0, - ); - Ok(()) -} - -fn measure_range_two_level_codec( - dataset: &str, - values: &[f64], - codec: &RangeTwoLevelCodec, -) -> VortexResult<()> { - for _ in 0..3 { - black_box(codec.decode()?); - } - let mut durations = Vec::with_capacity(DECODE_ITERATIONS); - for _ in 0..DECODE_ITERATIONS { - let start = Instant::now(); - black_box(codec.decode()?); - durations.push(start.elapsed()); - } - let p10 = percentile(&mut durations.to_vec(), 1, 10); - let p90 = percentile(&mut durations.to_vec(), 9, 10); - let median = percentile(&mut durations, 1, 2); - let throughput = - values.len() as f64 * size_of::() as f64 / median.as_secs_f64() / 1_000_000.0; - println!( - "decode\t{dataset}\trange-two-level-codec\tvortex.range_two_level_codec\t{}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", - codec.encoded_size(), - p10.as_secs_f64() * 1_000.0, - median.as_secs_f64() * 1_000.0, - p90.as_secs_f64() * 1_000.0, - ); - Ok(()) -} - -fn measure_range_grouped_codec( - dataset: &str, - values: &[f64], - codec: &RangeGroupedCodec, -) -> VortexResult<()> { - for _ in 0..3 { - black_box(codec.decode()?); - } - let mut durations = Vec::with_capacity(DECODE_ITERATIONS); - for _ in 0..DECODE_ITERATIONS { - let start = Instant::now(); - black_box(codec.decode()?); - durations.push(start.elapsed()); - } - let p10 = percentile(&mut durations.to_vec(), 1, 10); - let p90 = percentile(&mut durations.to_vec(), 9, 10); - let median = percentile(&mut durations, 1, 2); - let throughput = - values.len() as f64 * size_of::() as f64 / median.as_secs_f64() / 1_000_000.0; - println!( - "decode\t{dataset}\trange-grouped-codec\tvortex.range_grouped_codec\t{}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", - codec.encoded_size(), - p10.as_secs_f64() * 1_000.0, - median.as_secs_f64() * 1_000.0, - p90.as_secs_f64() * 1_000.0, - ); - Ok(()) -} - -fn measure_patched_for_codec( - dataset: &str, - config: &str, - values: &[f64], - codec: &PatchedFoRCodec, -) -> VortexResult<()> { - for _ in 0..3 { - black_box(codec.decode()?); - } - let mut durations = Vec::with_capacity(DECODE_ITERATIONS); - for _ in 0..DECODE_ITERATIONS { - let start = Instant::now(); - black_box(codec.decode()?); - durations.push(start.elapsed()); - } - let p10 = percentile(&mut durations.to_vec(), 1, 10); - let p90 = percentile(&mut durations.to_vec(), 9, 10); - let median = percentile(&mut durations, 1, 2); - let throughput = - values.len() as f64 * size_of::() as f64 / median.as_secs_f64() / 1_000_000.0; - println!( - "decode\t{dataset}\t{config}\tvortex.patched_for_codec\t{}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", - codec.encoded_size(), - p10.as_secs_f64() * 1_000.0, - median.as_secs_f64() * 1_000.0, - p90.as_secs_f64() * 1_000.0, - ); - Ok(()) -} - -fn measure_ordered_float_patched_for_codec( - dataset: &str, - values: &[f64], - codec: &PatchedFoRCodec, -) -> VortexResult<()> { - let decode = || -> VortexResult> { - Ok(codec - .decode()? - .into_iter() - .map(|ordered| { - let bits = if ordered & (1_u64 << 63) == 0 { - !ordered - } else { - ordered ^ (1_u64 << 63) - }; - f64::from_bits(bits) - }) - .collect::>()) - }; - for _ in 0..3 { - black_box(decode()?); - } - let mut durations = Vec::with_capacity(DECODE_ITERATIONS); - for _ in 0..DECODE_ITERATIONS { - let start = Instant::now(); - black_box(decode()?); - durations.push(start.elapsed()); - } - let p10 = percentile(&mut durations.to_vec(), 1, 10); - let p90 = percentile(&mut durations.to_vec(), 9, 10); - let median = percentile(&mut durations, 1, 2); - let throughput = - values.len() as f64 * size_of::() as f64 / median.as_secs_f64() / 1_000_000.0; - println!( - "decode\t{dataset}\tordered-float+patched-for-single-base\tvortex.prototype_stack\t{}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", - codec.encoded_size(), - p10.as_secs_f64() * 1_000.0, - median.as_secs_f64() * 1_000.0, - p90.as_secs_f64() * 1_000.0, - ); - - for _ in 0..3 { - black_box(codec.decode_ordered_f64()?); - } - let mut durations = Vec::with_capacity(DECODE_ITERATIONS); - for _ in 0..DECODE_ITERATIONS { - let start = Instant::now(); - black_box(codec.decode_ordered_f64()?); - durations.push(start.elapsed()); - } - let p10 = percentile(&mut durations.to_vec(), 1, 10); - let p90 = percentile(&mut durations.to_vec(), 9, 10); - let median = percentile(&mut durations, 1, 2); - let throughput = - values.len() as f64 * size_of::() as f64 / median.as_secs_f64() / 1_000_000.0; - println!( - "decode\t{dataset}\tordered-float+patched-for-single-base-fused\tvortex.prototype_fused\t{}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", - codec.encoded_size(), - p10.as_secs_f64() * 1_000.0, - median.as_secs_f64() * 1_000.0, - p90.as_secs_f64() * 1_000.0, - ); - Ok(()) -} - -fn measure_patched_for_scalar_at( - dataset: &str, - config: &str, - codec: &PatchedFoRCodec, -) -> VortexResult<()> { - let mut checksum = 0_u64; - for access in 0..N { - let index = access.wrapping_mul(0x9e37_79b1) & (N - 1); - checksum ^= black_box(codec.scalar_at(index)?); - } - black_box(checksum); - - let mut durations = Vec::with_capacity(9); - for _ in 0..9 { - let start = Instant::now(); - let mut checksum = 0_u64; - for access in 0..SCALAR_ACCESS_ITERATIONS { - let index = access.wrapping_mul(0x9e37_79b1) & (N - 1); - checksum ^= black_box(codec.scalar_at(index)?); - } - durations.push(start.elapsed()); - black_box(checksum); - } - let median = percentile(&mut durations, 1, 2); - let nanoseconds = median.as_secs_f64() * 1_000_000_000.0 / SCALAR_ACCESS_ITERATIONS as f64; - let accesses_per_second = SCALAR_ACCESS_ITERATIONS as f64 / median.as_secs_f64() / 1_000_000.0; - println!( - "scalar-at\t{dataset}\t{config}\tvortex.patched_for_codec\t{}\t{nanoseconds:.2}\t{accesses_per_second:.1}", - codec.encoded_size(), - ); - Ok(()) -} - -fn measure_array_scalar_at( - dataset: &str, - config: &str, - array: &ArrayRef, - session: &VortexSession, -) -> VortexResult<()> { - let mut durations = Vec::with_capacity(5); - for _ in 0..5 { - let mut ctx = session.create_execution_ctx(); - let start = Instant::now(); - let mut checksum = 0_u64; - for access in 0..ARRAY_SCALAR_ACCESS_ITERATIONS { - let index = access.wrapping_mul(0x9e37_79b1) & (N - 1); - let value = array - .execute_scalar(index, &mut ctx)? - .as_primitive() - .typed_value::() - .ok_or_else(|| vortex_error::vortex_err!("benchmark value is null"))?; - checksum ^= black_box(value.to_bits()); - } - durations.push(start.elapsed()); - black_box(checksum); - } - let median = percentile(&mut durations, 1, 2); - let nanoseconds = - median.as_secs_f64() * 1_000_000_000.0 / ARRAY_SCALAR_ACCESS_ITERATIONS as f64; - let accesses_per_second = - ARRAY_SCALAR_ACCESS_ITERATIONS as f64 / median.as_secs_f64() / 1_000_000.0; - println!( - "scalar-at\t{dataset}\t{config}\t{}\t{}\t{nanoseconds:.2}\t{accesses_per_second:.1}", - array.encoding_id(), - array.nbytes(), - ); - Ok(()) -} - -fn measure_bit_split_codec( - dataset: &str, - values: &[f64], - codec: &BitSplitCodec, -) -> VortexResult<()> { - for _ in 0..3 { - black_box(codec.decode()?); - } - let mut durations = Vec::with_capacity(DECODE_ITERATIONS); - for _ in 0..DECODE_ITERATIONS { - let start = Instant::now(); - black_box(codec.decode()?); - durations.push(start.elapsed()); - } - let p10 = percentile(&mut durations.to_vec(), 1, 10); - let p90 = percentile(&mut durations.to_vec(), 9, 10); - let median = percentile(&mut durations, 1, 2); - let throughput = - values.len() as f64 * size_of::() as f64 / median.as_secs_f64() / 1_000_000.0; - println!( - "decode\t{dataset}\tbit-split-codec\tvortex.bit_split_codec\t{}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", - codec.encoded_size(), - p10.as_secs_f64() * 1_000.0, - median.as_secs_f64() * 1_000.0, - p90.as_secs_f64() * 1_000.0, - ); - Ok(()) -} - -fn measure_bit_split_scalar_at(dataset: &str, codec: &BitSplitCodec) -> VortexResult<()> { - let mut durations = Vec::with_capacity(9); - for _ in 0..9 { - let start = Instant::now(); - let mut checksum = 0_u64; - for access in 0..SCALAR_ACCESS_ITERATIONS { - let index = access.wrapping_mul(0x9e37_79b1) & (N - 1); - checksum ^= black_box(codec.scalar_at(index)?); - } - durations.push(start.elapsed()); - black_box(checksum); - } - let median = percentile(&mut durations, 1, 2); - let nanoseconds = median.as_secs_f64() * 1_000_000_000.0 / SCALAR_ACCESS_ITERATIONS as f64; - let accesses_per_second = SCALAR_ACCESS_ITERATIONS as f64 / median.as_secs_f64() / 1_000_000.0; - println!( - "scalar-at\t{dataset}\tbit-split-codec\tvortex.bit_split_codec\t{}\t{nanoseconds:.2}\t{accesses_per_second:.1}", - codec.encoded_size(), - ); - Ok(()) -} - -fn measure_native_codec_encoders(dataset: &str, values: &[u64]) -> VortexResult<()> { - for (name, encode) in [( - "range-entropy-codec", - RangeEntropyCodec::encode as fn(&[u64], usize) -> VortexResult, - )] { - let mut durations = Vec::with_capacity(COMPRESS_ITERATIONS); - let mut size = 0; - for _ in 0..COMPRESS_ITERATIONS { - let start = Instant::now(); - let codec = encode(black_box(values), VALUES_PER_PAGE)?; - durations.push(start.elapsed()); - size = black_box(codec.encoded_size()); - } - report_native_codec_encoder(dataset, name, size, &mut durations); - } - - let mut durations = Vec::with_capacity(COMPRESS_ITERATIONS); - let mut size = 0; - for _ in 0..COMPRESS_ITERATIONS { - let start = Instant::now(); - let codec = RangePackedCodec::encode(black_box(values), VALUES_PER_PAGE)?; - durations.push(start.elapsed()); - size = black_box(codec.encoded_size()); - } - report_native_codec_encoder(dataset, "range-packed-codec", size, &mut durations); - - let mut durations = Vec::with_capacity(COMPRESS_ITERATIONS); - let mut size = 0; - for _ in 0..COMPRESS_ITERATIONS { - let start = Instant::now(); - let codec = RangeTwoLevelCodec::encode(black_box(values), VALUES_PER_PAGE)?; - durations.push(start.elapsed()); - size = black_box(codec.encoded_size()); - } - report_native_codec_encoder(dataset, "range-two-level-codec", size, &mut durations); - - let mut durations = Vec::with_capacity(COMPRESS_ITERATIONS); - let mut size = 0; - for _ in 0..COMPRESS_ITERATIONS { - let start = Instant::now(); - let codec = RangeGroupedCodec::encode(black_box(values), VALUES_PER_PAGE)?; - durations.push(start.elapsed()); - size = black_box(codec.encoded_size()); - } - report_native_codec_encoder(dataset, "range-grouped-codec", size, &mut durations); - - let mut durations = Vec::with_capacity(COMPRESS_ITERATIONS); - let mut size = 0; - for _ in 0..COMPRESS_ITERATIONS { - let start = Instant::now(); - let codec = PatchedFoRCodec::encode(black_box(values))?; - durations.push(start.elapsed()); - size = black_box(codec.encoded_size()); - } - report_native_codec_encoder(dataset, "patched-for-codec", size, &mut durations); - - let mut durations = Vec::with_capacity(COMPRESS_ITERATIONS); - let mut size = 0; - for _ in 0..COMPRESS_ITERATIONS { - let start = Instant::now(); - let codec = PatchedFoRCodec::encode_single_base(black_box(values))?; - durations.push(start.elapsed()); - size = black_box(codec.encoded_size()); - } - report_native_codec_encoder(dataset, "patched-for-single-base", size, &mut durations); - - let mut durations = Vec::with_capacity(COMPRESS_ITERATIONS); - let mut size = 0; - for _ in 0..COMPRESS_ITERATIONS { - let start = Instant::now(); - let codec = BitSplitCodec::encode(black_box(values))?; - durations.push(start.elapsed()); - size = black_box(codec.encoded_size()); - } - report_native_codec_encoder(dataset, "bit-split-codec", size, &mut durations); - Ok(()) -} - -fn measure_ordered_float_patched_for_encoder(dataset: &str, values: &[f64]) -> VortexResult<()> { - let mut durations = Vec::with_capacity(COMPRESS_ITERATIONS); - let mut size = 0; - for _ in 0..COMPRESS_ITERATIONS { - let start = Instant::now(); - let ordered = values - .iter() - .map(|value| { - let bits = value.to_bits(); - if bits & (1_u64 << 63) == 0 { - bits ^ (1_u64 << 63) - } else { - !bits - } - }) - .collect::>(); - let codec = PatchedFoRCodec::encode_single_base(black_box(&ordered))?; - durations.push(start.elapsed()); - size = black_box(codec.encoded_size()); - } - report_native_codec_encoder( - dataset, - "ordered-float+patched-for-single-base", - size, - &mut durations, - ); - Ok(()) -} - -fn report_native_codec_encoder(dataset: &str, name: &str, size: usize, durations: &mut [Duration]) { - let p10 = percentile(&mut durations.to_vec(), 1, 10); - let p90 = percentile(&mut durations.to_vec(), 9, 10); - let median = percentile(durations, 1, 2); - let throughput = (N * size_of::()) as f64 / median.as_secs_f64() / 1_000_000.0; - println!( - "codec-encode\t{dataset}\t{name}\tnative-codec\t{size}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", - p10.as_secs_f64() * 1_000.0, - median.as_secs_f64() * 1_000.0, - p90.as_secs_f64() * 1_000.0, - ); -} - -fn report_symbol_models(dataset: &str, codec: &RangeEntropyCodec) { - let total_weight = (1_u64 << codec.scale_bits()) as f64; - let mut probabilities = codec - .weights() - .iter() - .map(|&weight| f64::from(weight) / total_weight) - .collect::>(); - let entropy = probabilities - .iter() - .map(|&probability| -probability * probability.log2()) - .sum::(); - probabilities.sort_by(|left, right| right.total_cmp(left)); - let bin_count = probabilities.len(); - let flat_width = usize::from(u8::try_from(codec.bin_lowers().len().ilog2()).unwrap_or(u8::MAX)) - + usize::from(!codec.bin_lowers().len().is_power_of_two()); - let mut best = (f64::INFINITY, 0usize, 0usize, 0.0); - let minimum_tag_width = flat_width.min(2); - for tag_width in minimum_tag_width..=flat_width { - let direct_count = ((1usize << tag_width) - 1).min(bin_count); - let escape_probability = 1.0 - probabilities[..direct_count].iter().sum::(); - let cold_count = bin_count - direct_count; - let cold_width = if cold_count <= 1 { - 0 - } else { - usize::from(u8::try_from(cold_count.ilog2()).unwrap_or(u8::MAX)) - + usize::from(!cold_count.is_power_of_two()) - }; - let cost = tag_width as f64 + escape_probability * cold_width as f64; - if cost < best.0 { - best = (cost, tag_width, cold_width, escape_probability); - } - } - println!( - "symbol-model\t{dataset}\tbins={bin_count}\tentropy={entropy:.3}\tflat={flat_width}\ttwo-level={:.3}\ttag={}\tcold={}\tescape={:.3}", - best.0, best.1, best.2, best.3 - ); -} - -fn measure_float_quant_stages( - dataset: &str, - primitive: &PrimitiveArray, - compressor: &BtrBlocksCompressor, - session: &VortexSession, -) -> VortexResult<()> { - let Some(k) = estimate_k(primitive.as_view()) else { - return Ok(()); - }; - let encoded = FloatQuant::from_primitive(primitive.as_view(), k)?; - let primary = encoded.primary().clone(); - let secondary = encoded.secondary().clone(); - let mut estimate_times = Vec::with_capacity(COMPRESS_ITERATIONS); - let mut split_times = Vec::with_capacity(COMPRESS_ITERATIONS); - let mut primary_times = Vec::with_capacity(COMPRESS_ITERATIONS); - let mut secondary_times = Vec::with_capacity(COMPRESS_ITERATIONS); - for _ in 0..COMPRESS_ITERATIONS { - let start = Instant::now(); - black_box(estimate_k(primitive.as_view())); - estimate_times.push(start.elapsed()); - - let start = Instant::now(); - black_box(FloatQuant::from_primitive(primitive.as_view(), k)?); - split_times.push(start.elapsed()); - - let start = Instant::now(); - black_box(compressor.compress(&primary, &mut session.create_execution_ctx())?); - primary_times.push(start.elapsed()); - - let start = Instant::now(); - black_box(compressor.compress(&secondary, &mut session.create_execution_ctx())?); - secondary_times.push(start.elapsed()); - } - - for (stage, durations) in [ - ("estimate-k", &mut estimate_times), - ("split", &mut split_times), - ("primary-child", &mut primary_times), - ("secondary-child", &mut secondary_times), - ] { - let median = percentile(durations, 1, 2); - println!( - "stage\t{dataset}\t{stage}\tk={k}\t{:.3}", - median.as_secs_f64() * 1_000.0 - ); - } - Ok(()) -} - -fn compress_once( - compressor: &BtrBlocksCompressor, - values: &[f64], - session: &VortexSession, -) -> VortexResult<(Duration, ArrayRef)> { - let array = PrimitiveArray::from_iter(values.iter().copied()).into_array(); - let start = Instant::now(); - let compressed = compressor.compress(&array, &mut session.create_execution_ctx())?; - Ok((start.elapsed(), compressed)) -} - -fn measure_compressors( - dataset: &str, - values: &[f64], - configs: &[(&str, &BtrBlocksCompressor)], - session: &VortexSession, -) -> VortexResult<()> { - for _ in 0..2 { - for (_, compressor) in configs { - black_box(compress_once(compressor, values, session)?); - } - } - - let mut durations = (0..configs.len()) - .map(|_| Vec::with_capacity(COMPRESS_ITERATIONS)) - .collect::>(); - let mut outputs = vec![None; configs.len()]; - for iteration in 0..COMPRESS_ITERATIONS { - for offset in 0..configs.len() { - let config_index = (iteration + offset) % configs.len(); - record_compression( - configs[config_index].1, - values, - session, - &mut durations[config_index], - &mut outputs[config_index], - )?; - } - } - - for (index, (name, _)) in configs.iter().enumerate() { - report_compressor(dataset, name, outputs[index].take(), &mut durations[index])?; - } - Ok(()) -} - -fn record_compression( - compressor: &BtrBlocksCompressor, - values: &[f64], - session: &VortexSession, - durations: &mut Vec, - last_output: &mut Option, -) -> VortexResult<()> { - let (elapsed, output) = compress_once(compressor, values, session)?; - durations.push(elapsed); - *last_output = Some(output); - Ok(()) -} - -fn report_compressor( - dataset: &str, - config: &str, - output: Option, - durations: &mut [Duration], -) -> VortexResult<()> { - let output = - output.ok_or_else(|| vortex_error::vortex_err!("compressor produced no output"))?; - let p10 = percentile(&mut durations.to_vec(), 1, 10); - let p90 = percentile(&mut durations.to_vec(), 9, 10); - let median = percentile(durations, 1, 2); - let throughput = (N * size_of::()) as f64 / median.as_secs_f64() / 1_000_000.0; - println!( - "compress\t{dataset}\t{config}\t{}\t{}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", - output.encoding_id(), - output.nbytes(), - p10.as_secs_f64() * 1_000.0, - median.as_secs_f64() * 1_000.0, - p90.as_secs_f64() * 1_000.0, - ); - Ok(()) -} - -#[expect( - clippy::cognitive_complexity, - reason = "the benchmark driver constructs and measures one fixed configuration matrix" -)] -fn main() -> VortexResult<()> { - let dataset_filter = std::env::args().nth(1); - let session = array_session(); - let baseline = BtrBlocksCompressorBuilder::default() - .exclude_schemes([FloatQuantScheme.id(), OrderedBlockResidualScheme.id()]) - .build(); - let range_candidate = BtrBlocksCompressorBuilder::default() - .exclude_schemes([FloatQuantScheme.id(), OrderedBlockResidualScheme.id()]) - .with_new_scheme(&RangeEntropyScheme) - .build(); - let default_without_block_residual = BtrBlocksCompressorBuilder::default() - .exclude_schemes([OrderedBlockResidualScheme.id()]) - .build(); - let float_candidate = BtrBlocksCompressor::default(); - let stacked_candidate = BtrBlocksCompressorBuilder::default() - .with_new_scheme(&RangeEntropyScheme) - .build(); - let compact = BtrBlocksCompressorBuilder::default().with_compact().build(); - - println!("operation\tdataset\tconfig\tencoding\tbytes\tp10-ms\tmedian-ms\tp90-ms\tMB/s"); - for (name, values) in datasets() { - if dataset_filter - .as_deref() - .is_some_and(|filter| filter != name) - { - continue; - } - let primitive = PrimitiveArray::from_iter(values.iter().copied()); - let expected = primitive.clone().into_array(); - let ordered_values = values - .iter() - .map(|value| { - let bits = value.to_bits(); - if bits & (1_u64 << 63) == 0 { - bits ^ (1_u64 << 63) - } else { - !bits - } - }) - .collect::>(); - let native_codec = RangeEntropyCodec::encode(&ordered_values, VALUES_PER_PAGE)?; - let range_packed_codec = RangePackedCodec::encode(&ordered_values, VALUES_PER_PAGE)?; - let range_two_level_codec = RangeTwoLevelCodec::encode(&ordered_values, VALUES_PER_PAGE)?; - let range_grouped_codec = RangeGroupedCodec::encode(&ordered_values, VALUES_PER_PAGE)?; - let patched_for_codec = PatchedFoRCodec::encode(&ordered_values)?; - let patched_for_single_base = PatchedFoRCodec::encode_single_base(&ordered_values)?; - let bit_split_codec = BitSplitCodec::encode(&ordered_values)?; - println!( - "patched-model\t{name}\tblocks={}\tavg-bases={:.2}\tavg-width={:.2}\tpatch-rate={:.4}", - patched_for_codec.block_count(), - patched_for_codec.total_base_count() as f64 - / patched_for_codec.block_count().max(1) as f64, - patched_for_codec.total_residual_width() as f64 - / patched_for_codec.block_count().max(1) as f64, - patched_for_codec.patch_count() as f64 / ordered_values.len().max(1) as f64, - ); - println!( - "bit-split-model\t{name}\tavg-prefixes={:.2}\tavg-suffix-width={:.2}", - bit_split_codec.average_prefix_count(), - bit_split_codec.average_suffix_width(), - ); - report_symbol_models(name, &native_codec); - let native = - RangeEntropy::from_primitive(primitive.as_view(), VALUES_PER_PAGE)?.into_array(); - let pco_classic = Pco::from_primitive_with_config( - primitive.as_view(), - &pco_config(ModeSpec::Classic, DeltaSpec::NoOp), - pco::DEFAULT_MAX_PAGE_N, - &mut session.create_execution_ctx(), - )? - .into_array(); - let pco_auto = Pco::from_primitive_with_config( - primitive.as_view(), - &pco_config(ModeSpec::Auto, DeltaSpec::Auto), - pco::DEFAULT_MAX_PAGE_N, - &mut session.create_execution_ctx(), - )? - .into_array(); - let default_encoded = baseline.compress(&expected, &mut session.create_execution_ctx())?; - let range_encoded = - range_candidate.compress(&expected, &mut session.create_execution_ctx())?; - let float_encoded = - float_candidate.compress(&expected, &mut session.create_execution_ctx())?; - let default_without_block_residual_encoded = default_without_block_residual - .compress(&expected, &mut session.create_execution_ctx())?; - let stacked_encoded = - stacked_candidate.compress(&expected, &mut session.create_execution_ctx())?; - let compact_encoded = compact.compress(&expected, &mut session.create_execution_ctx())?; - println!( - "structure\t{name}\twithout-new-float={}\twithout-new-float+range={}\tdefault-off={}\tdefault-on={}\tdefault+range={}\tcompact={}", - encoding_tree(&default_encoded), - encoding_tree(&range_encoded), - encoding_tree(&default_without_block_residual_encoded), - encoding_tree(&float_encoded), - encoding_tree(&stacked_encoded), - encoding_tree(&compact_encoded) - ); - - let mut verify_ctx = session.create_execution_ctx(); - assert_eq!(range_packed_codec.decode()?, ordered_values); - assert_eq!(range_two_level_codec.decode()?, ordered_values); - assert_eq!(range_grouped_codec.decode()?, ordered_values); - assert_eq!(patched_for_codec.decode()?, ordered_values); - assert_eq!(patched_for_single_base.decode()?, ordered_values); - assert_eq!(bit_split_codec.decode()?, ordered_values); - assert_arrays_eq!(native, expected, &mut verify_ctx); - assert_arrays_eq!(pco_classic, expected, &mut verify_ctx); - assert_arrays_eq!(pco_auto, expected, &mut verify_ctx); - assert_arrays_eq!(default_encoded, expected, &mut verify_ctx); - assert_arrays_eq!(range_encoded, expected, &mut verify_ctx); - assert_arrays_eq!( - default_without_block_residual_encoded, - expected, - &mut verify_ctx - ); - assert_arrays_eq!(float_encoded, expected, &mut verify_ctx); - assert_arrays_eq!(stacked_encoded, expected, &mut verify_ctx); - assert_arrays_eq!(compact_encoded, expected, &mut verify_ctx); - - measure_decoders( - name, - &[ - ("default-without-new-float", &default_encoded), - ("default-without-new-float+range-entropy", &range_encoded), - ("default-off", &default_without_block_residual_encoded), - ("default-on", &float_encoded), - ("default+range-entropy", &stacked_encoded), - ("compact", &compact_encoded), - ("range-entropy", &native), - ("pco-classic", &pco_classic), - ("pco-auto", &pco_auto), - ], - &session, - )?; - measure_array_scalar_at( - name, - "default-off", - &default_without_block_residual_encoded, - &session, - )?; - measure_array_scalar_at(name, "default-on", &float_encoded, &session)?; - measure_range_entropy_codec(name, &values, &native_codec)?; - measure_range_packed_codec(name, &values, &range_packed_codec)?; - measure_range_two_level_codec(name, &values, &range_two_level_codec)?; - measure_range_grouped_codec(name, &values, &range_grouped_codec)?; - measure_patched_for_codec(name, "patched-for-codec", &values, &patched_for_codec)?; - measure_patched_for_codec( - name, - "patched-for-single-base", - &values, - &patched_for_single_base, - )?; - measure_ordered_float_patched_for_codec(name, &values, &patched_for_single_base)?; - measure_patched_for_scalar_at(name, "patched-for-codec", &patched_for_codec)?; - measure_patched_for_scalar_at(name, "patched-for-single-base", &patched_for_single_base)?; - measure_bit_split_codec(name, &values, &bit_split_codec)?; - measure_bit_split_scalar_at(name, &bit_split_codec)?; - measure_native_codec_encoders(name, &ordered_values)?; - measure_ordered_float_patched_for_encoder(name, &values)?; - measure_float_quant_stages(name, &primitive, &baseline, &session)?; - measure_compressors( - name, - &values, - &[ - ("default-without-new-float", &baseline), - ("default-without-new-float+range-entropy", &range_candidate), - ("default-off", &default_without_block_residual), - ("default-on", &float_candidate), - ("default+range-entropy", &stacked_candidate), - ("compact", &compact), - ], - &session, - )?; - } - Ok(()) -} diff --git a/vortex-btrblocks/src/builder.rs b/vortex-btrblocks/src/builder.rs index 370f52f148c..3020d9ae348 100644 --- a/vortex-btrblocks/src/builder.rs +++ b/vortex-btrblocks/src/builder.rs @@ -45,7 +45,6 @@ pub const ALL_SCHEMES: &[&dyn Scheme] = &[ &float::ALPScheme, &float::ALPRDScheme, &float::FloatQuantScheme, - &float::FloatMultScheme, &float::OrderedBlockResidualScheme, &float::FloatDictScheme, &float::NullDominatedSparseScheme, @@ -177,7 +176,6 @@ impl BtrBlocksCompressorBuilder { integer::IntRLEScheme.id(), float::ALPRDScheme.id(), float::FloatQuantScheme.id(), - float::FloatMultScheme.id(), float::OrderedBlockResidualScheme.id(), float::FloatRLEScheme.id(), float::NullDominatedSparseScheme.id(), diff --git a/vortex-btrblocks/src/schemes/float/float_mult.rs b/vortex-btrblocks/src/schemes/float/float_mult.rs deleted file mode 100644 index 64372a62fc1..00000000000 --- a/vortex-btrblocks/src/schemes/float/float_mult.rs +++ /dev/null @@ -1,251 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! Lossless FloatMult split for exact-multiple f32 values. - -use vortex_array::ArrayId; -use vortex_array::ArrayRef; -use vortex_array::Canonical; -use vortex_array::ExecutionCtx; -use vortex_array::IntoArray; -use vortex_array::VTable; -use vortex_array::arrays::Primitive; -use vortex_array::arrays::PrimitiveArray; -use vortex_array::dtype::PType; -use vortex_compressor::scheme::CompressionEstimate; -use vortex_compressor::scheme::DeferredEstimate; -use vortex_compressor::scheme::EstimateScore; -use vortex_compressor::scheme::EstimateVerdict; -use vortex_error::VortexResult; -use vortex_float_quant::FloatMult; -use vortex_float_quant::FloatMultArraySlotsExt; -use vortex_float_quant::estimate_float_mult_constant_base; - -use crate::ArrayAndStats; -use crate::CascadingCompressor; -use crate::CompressorContext; -use crate::Scheme; -use crate::SchemeExt; - -const SAMPLE_RUNS: usize = 16; -const SAMPLE_RUN_LEN: usize = 128; -const SELECTION_PENALTY: f64 = 0.85; -const MIN_ESTIMATED_RATIO: f64 = 1.50; -const MIN_WIN_OVER_INCUMBENT: f64 = 1.05; -const MAX_PRIMARY_BIT_WIDTH: u32 = 17; - -/// FloatMult split with normal BtrBlocks compression for an integer latent child. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub struct FloatMultScheme; - -impl Scheme for FloatMultScheme { - fn scheme_name(&self) -> &'static str { - "vortex.float.float_mult" - } - - fn matches(&self, canonical: &Canonical) -> bool { - canonical.dtype().is_float() && canonical.dtype().as_ptype() == PType::F32 - } - - fn produced_encodings(&self) -> Vec { - vec![FloatMult.id()] - } - - fn num_children(&self) -> usize { - 1 - } - - fn expected_compression_ratio( - &self, - data: &ArrayAndStats, - compress_ctx: CompressorContext, - _exec_ctx: &mut ExecutionCtx, - ) -> CompressionEstimate { - let primitive = data.array_as_primitive(); - if compress_ctx.finished_cascading() - || primitive.ptype() != PType::F32 - || primitive.len() < SAMPLE_RUN_LEN - { - return CompressionEstimate::Verdict(EstimateVerdict::Skip); - } - - CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |compressor, data, best_so_far, compress_ctx, exec_ctx| { - let sample = sample_primitive(data.array_as_primitive(), exec_ctx)?; - let Some(base) = estimate_float_mult_constant_base(sample.as_view()) else { - return Ok(EstimateVerdict::Skip); - }; - let Some(encoded) = - FloatMult::from_primitive_constant_secondary(sample.as_view(), base)? - else { - return Ok(EstimateVerdict::Skip); - }; - if primary_bit_width(encoded.primary().as_::()) > MAX_PRIMARY_BIT_WIDTH { - return Ok(EstimateVerdict::Skip); - } - let compressed_primary = compressor.compress_child( - encoded.primary(), - &compress_ctx, - FloatMultScheme.id(), - 0, - exec_ctx, - )?; - let candidate = FloatMult::try_new( - compressed_primary, - encoded.secondary().cloned(), - sample.ptype(), - base, - )?; - let after_nbytes = candidate.nbytes(); - if after_nbytes == 0 { - return Ok(EstimateVerdict::Skip); - } - let ratio = sample.nbytes() as f64 / after_nbytes as f64 * SELECTION_PENALTY; - if ratio < MIN_ESTIMATED_RATIO { - return Ok(EstimateVerdict::Skip); - } - let incumbent = best_so_far.and_then(EstimateScore::finite_ratio); - if incumbent.is_some_and(|best| ratio < best * MIN_WIN_OVER_INCUMBENT) { - return Ok(EstimateVerdict::Skip); - } - Ok(EstimateVerdict::Ratio(ratio)) - }, - ))) - } - - fn compress( - &self, - compressor: &CascadingCompressor, - data: &ArrayAndStats, - compress_ctx: CompressorContext, - exec_ctx: &mut ExecutionCtx, - ) -> VortexResult { - let primitive = data.array_as_primitive(); - let sample = sample_primitive(primitive, exec_ctx)?; - let Some(base) = estimate_float_mult_constant_base(sample.as_view()) else { - return Ok(primitive.array().clone()); - }; - let Some(encoded) = FloatMult::from_primitive_constant_secondary(primitive, base)? else { - return Ok(primitive.array().clone()); - }; - let compressed_primary = - compressor.compress_child(encoded.primary(), &compress_ctx, self.id(), 0, exec_ctx)?; - Ok(FloatMult::try_new( - compressed_primary, - encoded.secondary().cloned(), - primitive.ptype(), - base, - )? - .into_array()) - } -} - -fn primary_bit_width(primary: vortex_array::ArrayView<'_, Primitive>) -> u32 { - match primary.ptype() { - PType::U32 => { - let (minimum, maximum) = primary - .as_slice::() - .iter() - .copied() - .fold((u32::MAX, u32::MIN), |(minimum, maximum), value| { - (minimum.min(value), maximum.max(value)) - }); - u32::BITS - (maximum - minimum).leading_zeros() - } - PType::U64 => { - let (minimum, maximum) = primary - .as_slice::() - .iter() - .copied() - .fold((u64::MAX, u64::MIN), |(minimum, maximum), value| { - (minimum.min(value), maximum.max(value)) - }); - u64::BITS - (maximum - minimum).leading_zeros() - } - _ => unreachable!(), - } -} - -fn sample_primitive( - primitive: vortex_array::ArrayView<'_, Primitive>, - exec_ctx: &mut ExecutionCtx, -) -> VortexResult { - let sample_len = SAMPLE_RUNS * SAMPLE_RUN_LEN; - if primitive.len() <= sample_len { - return primitive - .array() - .clone() - .execute::(exec_ctx); - } - - let validity = primitive - .validity()? - .execute_mask(primitive.len(), exec_ctx)?; - macro_rules! sample { - ($T:ty) => {{ - let values = primitive.as_slice::<$T>(); - let mut sample = Vec::with_capacity(sample_len); - for run_index in 0..SAMPLE_RUNS { - let partition_start = run_index * primitive.len() / SAMPLE_RUNS; - let next_partition = (run_index + 1) * primitive.len() / SAMPLE_RUNS; - let start = - partition_start + (next_partition - partition_start - SAMPLE_RUN_LEN) / 2; - sample.extend( - (start..start + SAMPLE_RUN_LEN) - .map(|index| validity.value(index).then_some(values[index])), - ); - } - PrimitiveArray::from_option_iter(sample) - }}; - } - Ok(match primitive.ptype() { - PType::F32 => sample!(f32), - PType::F64 => sample!(f64), - _ => unreachable!(), - }) -} - -#[cfg(test)] -mod tests { - use std::sync::LazyLock; - - use vortex_array::IntoArray; - use vortex_array::VortexSessionExecute; - use vortex_array::array_session; - use vortex_array::arrays::PrimitiveArray; - use vortex_array::assert_arrays_eq; - use vortex_error::VortexResult; - use vortex_session::VortexSession; - - use super::*; - use crate::BtrBlocksCompressorBuilder; - - static SESSION: LazyLock = LazyLock::new(|| { - let session = array_session(); - vortex_float_quant::initialize(&session); - session - }); - - #[test] - fn candidate_never_increases_noisy_value_size() -> VortexResult<()> { - let values = (0u32..65_536) - .map(|index| f32::from_bits(0x3f80_0000 | index.wrapping_mul(7_919) & 0x007f_ffff)) - .collect::>(); - let original = PrimitiveArray::from_iter(values).into_array(); - let baseline = BtrBlocksCompressorBuilder::default() - .exclude_schemes([FloatMultScheme.id()]) - .build() - .compress(&original, &mut SESSION.create_execution_ctx())?; - let candidate = BtrBlocksCompressorBuilder::default() - .build() - .compress(&original, &mut SESSION.create_execution_ctx())?; - assert!( - candidate.nbytes() <= baseline.nbytes(), - "FloatMult candidate uses {} bytes and baseline uses {} bytes", - candidate.nbytes(), - baseline.nbytes() - ); - assert_arrays_eq!(candidate, original, &mut SESSION.create_execution_ctx()); - Ok(()) - } -} diff --git a/vortex-btrblocks/src/schemes/float/mod.rs b/vortex-btrblocks/src/schemes/float/mod.rs index 31406120abe..e2f8aecb398 100644 --- a/vortex-btrblocks/src/schemes/float/mod.rs +++ b/vortex-btrblocks/src/schemes/float/mod.rs @@ -5,7 +5,6 @@ mod alp; mod alprd; -mod float_mult; mod float_quant; mod ordered_block_residual; mod rle; @@ -16,7 +15,6 @@ mod pco; pub use alp::ALPScheme; pub use alprd::ALPRDScheme; -pub use float_mult::FloatMultScheme; pub use float_quant::FloatQuantScheme; pub use ordered_block_residual::OrderedBlockResidualScheme; #[cfg(feature = "pco")] diff --git a/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs b/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs index 7afb49174e7..c5cbedc55ed 100644 --- a/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs +++ b/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs @@ -12,14 +12,14 @@ use vortex_array::VTable; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::PType; +use vortex_block_residual::BlockResidual; +use vortex_block_residual::OrderedFloat; +use vortex_block_residual::OrderedFloatArraySlotsExt; use vortex_compressor::scheme::CompressionEstimate; use vortex_compressor::scheme::DeferredEstimate; use vortex_compressor::scheme::EstimateScore; use vortex_compressor::scheme::EstimateVerdict; use vortex_error::VortexResult; -use vortex_range_entropy::BlockResidual; -use vortex_range_entropy::OrderedFloat; -use vortex_range_entropy::OrderedFloatArraySlotsExt; use crate::ArrayAndStats; use crate::CascadingCompressor; diff --git a/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs index 173e1607be7..c8470be7352 100644 --- a/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs @@ -16,12 +16,11 @@ use vortex_array::builders::ArrayBuilder; use vortex_array::builders::PrimitiveBuilder; use vortex_array::dtype::Nullability; use vortex_array::validity::Validity; +use vortex_block_residual::BlockResidual; +use vortex_block_residual::OrderedFloat; use vortex_buffer::Buffer; use vortex_error::VortexResult; -use vortex_float_quant::FloatMult; use vortex_float_quant::FloatQuant; -use vortex_range_entropy::BlockResidual; -use vortex_range_entropy::OrderedFloat; use vortex_session::VortexSession; use crate::BtrBlocksCompressor; @@ -107,18 +106,6 @@ fn test_f32_does_not_use_float_quant() -> VortexResult<()> { Ok(()) } -#[test] -fn test_integer_f32_uses_float_mult() -> VortexResult<()> { - let values = (0u32..65_536) - .map(|index| ((index.wrapping_mul(7_919)) % 65_537) as f32) - .collect::>(); - let array = PrimitiveArray::from_iter(values).into_array(); - let compressed = - BtrBlocksCompressor::default().compress(&array, &mut SESSION.create_execution_ctx())?; - assert!(compressed.is::()); - Ok(()) -} - #[test] fn test_repeated_f64_prefers_existing_scheme() -> VortexResult<()> { let values = (0u32..16_384) diff --git a/vortex-btrblocks/src/schemes/float/tests.rs b/vortex-btrblocks/src/schemes/float/tests.rs index 7bb2f4748c6..2d6e2f9900c 100644 --- a/vortex-btrblocks/src/schemes/float/tests.rs +++ b/vortex-btrblocks/src/schemes/float/tests.rs @@ -21,9 +21,6 @@ use vortex_fastlanes::RLE; use vortex_session::VortexSession; use crate::BtrBlocksCompressor; -use crate::BtrBlocksCompressorBuilder; -use crate::SchemeExt; -use crate::schemes::float::FloatMultScheme; use crate::schemes::float::FloatRLEScheme; static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); @@ -45,19 +42,9 @@ fn test_compress() -> VortexResult<()> { } let array = values.into_array(); - let baseline = BtrBlocksCompressorBuilder::default() - .exclude_schemes([FloatMultScheme.id()]) - .build() - .compress(&array, &mut SESSION.create_execution_ctx())?; let compressed = BtrBlocksCompressor::default().compress(&array, &mut SESSION.create_execution_ctx())?; assert_eq!(compressed.len(), 1024); - assert!( - compressed.nbytes() <= baseline.nbytes(), - "default uses {} bytes and the prior default uses {} bytes", - compressed.nbytes(), - baseline.nbytes() - ); assert_arrays_eq!(compressed, array, &mut SESSION.create_execution_ctx()); Ok(()) diff --git a/vortex-btrblocks/src/schemes/mod.rs b/vortex-btrblocks/src/schemes/mod.rs index 9bf88d5f732..a0e9b042a66 100644 --- a/vortex-btrblocks/src/schemes/mod.rs +++ b/vortex-btrblocks/src/schemes/mod.rs @@ -6,7 +6,6 @@ pub mod binary; pub mod float; pub mod integer; -pub mod range_entropy; pub mod string; pub mod decimal; diff --git a/vortex-btrblocks/src/schemes/range_entropy.rs b/vortex-btrblocks/src/schemes/range_entropy.rs deleted file mode 100644 index c070107e1fb..00000000000 --- a/vortex-btrblocks/src/schemes/range_entropy.rs +++ /dev/null @@ -1,62 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! Native range entropy compression for primitive numeric arrays. - -use vortex_array::ArrayId; -use vortex_array::ArrayRef; -use vortex_array::Canonical; -use vortex_array::ExecutionCtx; -use vortex_array::IntoArray; -use vortex_array::VTable; -use vortex_compressor::scheme::CompressionEstimate; -use vortex_compressor::scheme::DeferredEstimate; -use vortex_error::VortexResult; -use vortex_range_entropy::RangeEntropy; - -use crate::ArrayAndStats; -use crate::CascadingCompressor; -use crate::CompressorContext; -use crate::Scheme; - -const RESTART_BLOCK_LEN: usize = 8192; - -/// Native range bins plus tANS for primitive numeric arrays. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub struct RangeEntropyScheme; - -impl Scheme for RangeEntropyScheme { - fn scheme_name(&self) -> &'static str { - "vortex.numeric.range_entropy" - } - - fn matches(&self, canonical: &Canonical) -> bool { - canonical.dtype().is_int() || canonical.dtype().is_float() - } - - fn produced_encodings(&self) -> Vec { - vec![RangeEntropy.id()] - } - - fn expected_compression_ratio( - &self, - _data: &ArrayAndStats, - _compress_ctx: CompressorContext, - _exec_ctx: &mut ExecutionCtx, - ) -> CompressionEstimate { - CompressionEstimate::Deferred(DeferredEstimate::Sample) - } - - fn compress( - &self, - _compressor: &CascadingCompressor, - data: &ArrayAndStats, - _compress_ctx: CompressorContext, - _exec_ctx: &mut ExecutionCtx, - ) -> VortexResult { - Ok( - RangeEntropy::from_primitive(data.array_as_primitive(), RESTART_BLOCK_LEN)? - .into_array(), - ) - } -} diff --git a/vortex-file/Cargo.toml b/vortex-file/Cargo.toml index eb3663c7846..5b86c11467c 100644 --- a/vortex-file/Cargo.toml +++ b/vortex-file/Cargo.toml @@ -35,6 +35,7 @@ url = { workspace = true } vortex-alp = { workspace = true } vortex-array = { workspace = true } vortex-btrblocks = { workspace = true } +vortex-block-residual = { workspace = true } vortex-buffer = { workspace = true } vortex-bytebool = { workspace = true } @@ -52,7 +53,6 @@ vortex-mask = { workspace = true } vortex-metrics = { workspace = true } vortex-onpair = { workspace = true } vortex-pco = { workspace = true } -vortex-range-entropy = { workspace = true } vortex-runend = { workspace = true } vortex-scan = { workspace = true } vortex-sequence = { workspace = true } diff --git a/vortex-file/src/lib.rs b/vortex-file/src/lib.rs index 020792b6998..465c7fd6e3c 100644 --- a/vortex-file/src/lib.rs +++ b/vortex-file/src/lib.rs @@ -191,7 +191,7 @@ pub fn register_default_encodings(session: &VortexSession) { vortex_decimal_byte_parts::initialize(session); vortex_fastlanes::initialize(session); vortex_float_quant::initialize(session); - vortex_range_entropy::initialize(session); + vortex_block_residual::initialize(session); vortex_runend::initialize(session); vortex_sequence::initialize(session); vortex_sparse::initialize(session); diff --git a/vortex/Cargo.toml b/vortex/Cargo.toml index 2ce314106ab..3d98384ecaa 100644 --- a/vortex/Cargo.toml +++ b/vortex/Cargo.toml @@ -26,6 +26,7 @@ workspace = true vortex-alp = { workspace = true } vortex-array = { workspace = true } vortex-arrow = { workspace = true } +vortex-block-residual = { workspace = true } vortex-btrblocks = { workspace = true } vortex-buffer = { workspace = true } vortex-bytebool = { workspace = true } @@ -45,7 +46,6 @@ vortex-layout = { workspace = true } vortex-mask = { workspace = true } vortex-metrics = { workspace = true } vortex-pco = { workspace = true } -vortex-range-entropy = { workspace = true } vortex-proto = { workspace = true, default-features = true } vortex-runend = { workspace = true } vortex-scan = { workspace = true } diff --git a/vortex/src/editions/core/v2026_08.rs b/vortex/src/editions/core/v2026_08.rs index a87d49ae184..b64faf56025 100644 --- a/vortex/src/editions/core/v2026_08.rs +++ b/vortex/src/editions/core/v2026_08.rs @@ -27,7 +27,6 @@ pub static DECLARATION: EditionDeclaration = EditionDeclaration { }, added: &[ EditionMember::array(&"vortex.block_residual"), - EditionMember::array(&"vortex.float_mult"), EditionMember::array(&"vortex.float_quant"), EditionMember::array(&"vortex.map"), EditionMember::array(&"vortex.ordered_float"), diff --git a/vortex/src/editions/tests.rs b/vortex/src/editions/tests.rs index 7a22c9cbdad..732f6ed047f 100644 --- a/vortex/src/editions/tests.rs +++ b/vortex/src/editions/tests.rs @@ -18,6 +18,8 @@ use vortex_array::dtype::PType; use vortex_array::field_path; use vortex_array::session::ArraySessionExt; use vortex_array::stream::ArrayStreamExt; +use vortex_block_residual::BlockResidual; +use vortex_block_residual::OrderedFloat; use vortex_btrblocks::BtrBlocksCompressorBuilder; use vortex_buffer::ByteBufferMut; use vortex_edition::ComponentKind; @@ -35,15 +37,12 @@ use vortex_error::vortex_err; use vortex_file::OpenOptionsSessionExt; use vortex_file::WriteOptionsSessionExt; use vortex_file::WriteStrategyBuilder; -use vortex_float_quant::FloatMult; use vortex_float_quant::FloatQuant; use vortex_io::session::RuntimeSession; use vortex_layout::LayoutStrategy; use vortex_layout::layouts::compressed::CompressingStrategy; use vortex_layout::layouts::flat::writer::FlatLayoutStrategy; use vortex_layout::session::LayoutSession; -use vortex_range_entropy::BlockResidual; -use vortex_range_entropy::OrderedFloat; use vortex_sequence::Sequence; use vortex_session::VortexSession; use vortex_session::registry::Id; @@ -526,33 +525,6 @@ async fn default_writer_round_trips_float_quant() -> VortexResult<()> { Ok(()) } -#[tokio::test] -async fn default_writer_round_trips_float_mult() -> VortexResult<()> { - use crate::VortexSessionDefault; - - let session = VortexSession::default(); - let values = (0u32..65_536) - .map(|index| ((index.wrapping_mul(7_919)) % 65_537) as f32) - .collect::>(); - let expected = PrimitiveArray::from_iter(values.clone()).into_array(); - let buffer = write_with(&session, expected.clone()).await?; - let actual = session - .open_options() - .open_buffer(buffer)? - .scan()? - .into_array_stream()? - .read_all() - .await?; - assert!( - actual - .depth_first_traversal() - .any(|array| array.is::()) - ); - let decoded = actual.execute::(&mut session.create_execution_ctx())?; - assert_eq!(decoded.as_slice::(), values); - Ok(()) -} - #[tokio::test] async fn default_writer_round_trips_ordered_block_residual() -> VortexResult<()> { use crate::VortexSessionDefault; diff --git a/vortex/src/lib.rs b/vortex/src/lib.rs index 43a5188816c..cb6e8aff091 100644 --- a/vortex/src/lib.rs +++ b/vortex/src/lib.rs @@ -275,9 +275,9 @@ pub mod encodings { pub use vortex_pco::*; } - /// Ordered-float, block-residual, and range-entropy numeric encodings. - pub mod range_entropy { - pub use vortex_range_entropy::*; + /// Ordered-float and block-residual numeric encodings. + pub mod block_residual { + pub use vortex_block_residual::*; } /// Arrow-compatible run-end encoding. From d3066bf3bf24193c3907de92da70ed3b982627eb Mon Sep 17 00:00:00 2001 From: Will Manning Date: Tue, 18 Aug 2026 20:28:50 -0400 Subject: [PATCH 06/78] perf: optimize and benchmark native float compression Signed-off-by: Will Manning --- Cargo.lock | 5 +- Cargo.toml | 2 +- benchmarks/compress-bench/Cargo.toml | 1 + benchmarks/compress-bench/src/main.rs | 17 +- benchmarks/compress-bench/src/vortex.rs | 36 +- docs/plans/native-pcodec.md | 98 +++-- encodings/float-quant/src/array.rs | 178 ++++++-- encodings/float-quant/src/slice.rs | 5 +- vortex-btrblocks/Cargo.toml | 4 + .../examples/float_compressor_bench.rs | 404 ++++++++++++++++++ .../src/schemes/float/float_quant.rs | 34 +- .../schemes/float/scheme_selection_tests.rs | 2 + vortex/benches/single_encoding_throughput.rs | 249 +++++++++++ 13 files changed, 955 insertions(+), 80 deletions(-) create mode 100644 vortex-btrblocks/examples/float_compressor_bench.rs diff --git a/Cargo.lock b/Cargo.lock index e46d9497ab1..39dd8297e02 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -76,9 +76,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "alp" -version = "0.0.2" +version = "0.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41ceb950ed9f82837662e3f3c470403987f32e3a25e5b741c8f17b3b0cca394d" +checksum = "639486620c15f839f53206c75be0d1a5a79c743c3e2a097d5e9e4b67fdb966fe" dependencies = [ "fastlanes", "itertools 0.15.0", @@ -1584,6 +1584,7 @@ dependencies = [ "vortex", "vortex-arrow", "vortex-bench", + "vortex-btrblocks", "vortex-cuda", ] diff --git a/Cargo.toml b/Cargo.toml index 1192d89e1a8..4e4791a9a48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -99,7 +99,7 @@ rust-version = "1.95" version = "0.1.0" [workspace.dependencies] -alp = "0.0.2" +alp = "0.0.3" anyhow = "1.0.100" arbitrary = "1.3.2" arc-swap = "1.9" diff --git a/benchmarks/compress-bench/Cargo.toml b/benchmarks/compress-bench/Cargo.toml index 4046a12d42e..94fdf6213f1 100644 --- a/benchmarks/compress-bench/Cargo.toml +++ b/benchmarks/compress-bench/Cargo.toml @@ -33,6 +33,7 @@ tracing = { workspace = true } vortex = { workspace = true } vortex-arrow = { workspace = true } vortex-bench = { workspace = true } +vortex-btrblocks = { workspace = true } vortex-cuda = { workspace = true, optional = true } [features] diff --git a/benchmarks/compress-bench/src/main.rs b/benchmarks/compress-bench/src/main.rs index 1b1603e52c8..88d78733499 100644 --- a/benchmarks/compress-bench/src/main.rs +++ b/benchmarks/compress-bench/src/main.rs @@ -81,6 +81,9 @@ struct Args { ingest_output: Option, #[arg(long)] tracing: bool, + /// Exclude FloatQuant and OrderedBlockResidual from Vortex compression. + #[arg(long)] + vortex_without_new_float: bool, /// Format for the primary stderr log sink. `text` is the default human-readable format; /// `json` emits one JSON object per event, suitable for piping into `jq`. #[arg(long, value_enum, default_value_t = LogFormat::Text)] @@ -112,12 +115,17 @@ async fn main() -> anyhow::Result<()> { args.display_format, args.output_path, args.ingest_output, + args.vortex_without_new_float, ) .await } /// Get a compressor for the given format. -fn get_compressor(format: Format, gpu_decompress: bool) -> Box { +fn get_compressor( + format: Format, + gpu_decompress: bool, + vortex_without_new_float: bool, +) -> Box { if gpu_decompress { #[cfg(feature = "cuda")] { @@ -128,7 +136,7 @@ fn get_compressor(format: Format, gpu_decompress: bool) -> Box { } match format { - Format::OnDiskVortex => Box::new(VortexCompressor), + Format::OnDiskVortex => Box::new(VortexCompressor::new(!vortex_without_new_float)), Format::Parquet => Box::new(ParquetCompressor::new()), #[cfg(feature = "lance")] Format::Lance => Box::new(LanceCompressor), @@ -155,6 +163,7 @@ async fn run_compress( display_format: DisplayFormat, output_path: Option, ingest_output: Option, + vortex_without_new_float: bool, ) -> anyhow::Result<()> { let targets = formats .iter() @@ -233,6 +242,7 @@ async fn run_compress( iterations, dataset_handle, gpu_decompress, + vortex_without_new_float, ) .await?; measurements.push(m); @@ -276,6 +286,7 @@ async fn run_benchmark_for_dataset( iterations: usize, dataset_handle: &dyn Dataset, gpu_decompress: bool, + vortex_without_new_float: bool, ) -> anyhow::Result<(CompressMeasurements, Vec)> { let bench_name = dataset_handle.name(); let (v3_dataset, v3_variant) = dataset_handle.v3_dataset_dims(); @@ -291,7 +302,7 @@ async fn run_benchmark_for_dataset( let mut v3_records: Vec = Vec::new(); for format in formats { - let compressor = get_compressor(*format, gpu_decompress); + let compressor = get_compressor(*format, gpu_decompress, vortex_without_new_float); for op in ops { let time = match op { diff --git a/benchmarks/compress-bench/src/vortex.rs b/benchmarks/compress-bench/src/vortex.rs index 20b4b9f1402..53690f994bd 100644 --- a/benchmarks/compress-bench/src/vortex.rs +++ b/benchmarks/compress-bench/src/vortex.rs @@ -17,16 +17,44 @@ use vortex::dtype::FieldNames; use vortex::expr::root; use vortex::expr::select; use vortex::file::OpenOptionsSessionExt; +use vortex::file::VortexWriteOptions; use vortex::file::WriteOptionsSessionExt; +use vortex::file::WriteStrategyBuilder; use vortex_arrow::ArrowSessionExt; use vortex_bench::Format; use vortex_bench::SESSION; use vortex_bench::compress::Compressor; use vortex_bench::compress::read_projection; use vortex_bench::conversions::parquet_to_vortex_chunks; +use vortex_btrblocks::BtrBlocksCompressorBuilder; +use vortex_btrblocks::SchemeExt; +use vortex_btrblocks::schemes::float::FloatQuantScheme; +use vortex_btrblocks::schemes::float::OrderedBlockResidualScheme; /// Compressor implementation for Vortex format. -pub struct VortexCompressor; +pub struct VortexCompressor { + new_float_schemes: bool, +} + +impl VortexCompressor { + pub fn new(new_float_schemes: bool) -> Self { + Self { new_float_schemes } + } + + fn write_options(&self) -> VortexWriteOptions { + let options = SESSION.write_options(); + if self.new_float_schemes { + return options; + } + let compressor = BtrBlocksCompressorBuilder::default() + .exclude_schemes([FloatQuantScheme.id(), OrderedBlockResidualScheme.id()]); + options.with_strategy( + WriteStrategyBuilder::default() + .with_btrblocks_builder(compressor) + .build(), + ) + } +} #[async_trait] impl Compressor for VortexCompressor { @@ -41,8 +69,7 @@ impl Compressor for VortexCompressor { let mut buf = Vec::new(); let start = Instant::now(); let mut cursor = Cursor::new(&mut buf); - SESSION - .write_options() + self.write_options() .write(&mut cursor, uncompressed.into_array().to_array_stream()) .await?; let elapsed = start.elapsed(); @@ -55,8 +82,7 @@ impl Compressor for VortexCompressor { let uncompressed = parquet_to_vortex_chunks(parquet_path.to_path_buf()).await?; let mut buf = Vec::new(); let mut cursor = Cursor::new(&mut buf); - SESSION - .write_options() + self.write_options() .write(&mut cursor, uncompressed.into_array().to_array_stream()) .await?; diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index f64ad97c17e..2765340d0cf 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -60,11 +60,12 @@ The production codec uses one reference per block. The multi-reference prototype `FloatQuantArray` splits each float into a primary quantum and a secondary adjustment. -Bounded metadata stores the source type and split width. Two child arrays store the integer latents. +Bounded metadata stores the source type and split width. One or two child arrays store the integer latents. -The BtrBlocks scheme compresses both children through the normal cascade. +The BtrBlocks scheme compresses each present child through the normal cascade. -A common path uses `FloatQuant(FoR(BitPacked), Constant)` for `f32` values stored in `f64` columns. +A common path uses `FloatQuant(FoR(BitPacked))` for `f32` values stored in `f64` columns. +An absent secondary child represents zero low bits. The array supports exact IEEE bit-pattern round trips, nulls, slices, scalar access, and serialization. @@ -74,7 +75,7 @@ The automatic scheme accepts only `f64`. Native `f32` columns remain with ALP, A `FloatQuantScheme` uses normal sample comparison against ALP, ALP-RD, dictionary, sparse, and RLE schemes. -A strong constant split can use a direct `FoR(BitPacked)` primary and a constant secondary. +A strong constant split can use a direct `FoR(BitPacked)` primary and an implicit-zero secondary. Other splits compress both children through the normal integer cascade. @@ -104,41 +105,78 @@ Exclude source-array construction and storage input from codec throughput measur ### FloatQuant on widened f32 values -The input contains 262,144 arbitrary `f32` values stored in an `f64` column. +The input contains two million arbitrary `f32` values stored in an `f64` column. | Configuration | Bytes | Compression MB/s | Decode MB/s | | --- | ---: | ---: | ---: | -| Default without FloatQuant | 1,638,436 | 762.4 | 12,932.1 | -| Default with FloatQuant | 688,130 | 635.0 | 13,636.4 | -| Compact | 658,758 | 336.8 | 5,117.1 | +| Prior default | 14,057,966 | 550.4 | 11,310.8 | +| Default with FloatQuant | 8,753,920 | 244.9 | 14,664.3 | +| Compact | 6,051,640 | 175.4 | 4,019.0 | -FloatQuant reduced size by 58.0 percent. It remained 4.5 percent larger than Compact. +FloatQuant reduced size by 37.7 percent. It remained 44.7 percent larger than Compact. -Compression throughput decreased by 16.7 percent. Decode throughput increased by 5.4 percent. +Full compressor throughput decreased by 55.5 percent. Decode throughput increased by 29.6 percent. -The selected tree was `FloatQuant(FoR(BitPacked), Constant)`. +The selected tree was `FloatQuant(FoR(BitPacked))` with an implicit-zero secondary. -This result needs a larger throughput test. The compression-ratio result remains strong. +The isolated tree compressed at 2,503 MB/s and decoded at 15,070 MB/s. + +Pco compressed the same input at 533 MB/s and decoded it at 3,973 MB/s. + +The tree itself exceeded Pco throughput by 4.7 times during compression and 3.8 times during decode. + +The BtrBlocks analysis and candidate path caused most of the full compressor cost. + +FloatQuant cannot enter the default set until its selection path meets the compression throughput limit. ### OrderedFloat with BlockResidual on random walks | Configuration | Bytes | Encode MB/s | Decode MB/s | Scalar access ns | | --- | ---: | ---: | ---: | ---: | -| Default without the scheme | 1,853,564 | 713.4 | 12,486.1 | 161.24 | -| Default with the scheme | 1,663,831 | 635.6 | 17,355.8 | 129.06 | -| Compact Pco | 1,551,142 | 296.2 | 3,777.2 | Not measured | +| Prior default | 12,255,488 | 577.9 | 12,869.9 | 161.24 | +| Default with the scheme | 10,425,690 | 516.1 | 18,019.7 | 129.06 | +| Compact Pco | 9,342,749 | 316.3 | 4,611.7 | Not measured | + +The residual scheme saved 14.9 percent against the prior default. It remained 11.6 percent larger than Compact. -The residual scheme saved 10.2 percent against ALP-RD. It remained 7.3 percent larger than Pco. +Decode throughput increased by 40.0 percent. Random access latency decreased by 20.0 percent. -Decode throughput increased by 39.0 percent. Random access latency decreased by 20.0 percent. +Compression throughput decreased by 10.7 percent on the selected column. -Compression throughput decreased by 10.9 percent on the selected column. +The isolated tree compressed at 2,504 MB/s and decoded at 18,920 MB/s. + +Pco compressed the same input at 648 MB/s and decoded it at 4,635 MB/s. The selector rejected the scheme on Gaussian, lognormal, decimal, widened-f32, and four-cluster inputs. The rejected locality probe reduced compression throughput by 0.5 to 2.2 percent. -This result needs several larger random walks and real time-series columns. +The HashTags dataset selected this scheme for two columns. + +It reduced the numeric subset by 6.0 percent with no isolated compressor regression. + +### Broad dataset revalidation + +The file benchmark compared the corrected default against the new schemes. + +| Dataset | Prior bytes | Proposed bytes | Size change | +| --- | ---: | ---: | ---: | +| Taxi | 439,806,364 | 439,806,364 | 0.000 percent | +| Air Quality | 15,848,420 | 15,848,420 | 0.000 percent | +| Arade | 143,343,828 | 143,343,828 | 0.000 percent | +| Euro2016 | 164,673,900 | 164,674,236 | +0.0002 percent | +| Food | 38,894,360 | 38,922,408 | +0.072 percent | +| HashTags | 195,035,692 | 194,753,372 | -0.145 percent | + +Separate benchmark processes introduced low single-digit throughput noise. + +The focused two-million-row run selected no new schemes in Taxi, Air Quality, Arade, Euro2016, or Food. + +Ordered BlockResidual selected two HashTags columns. FloatQuant selected no columns in these datasets. + +The FloatQuant probe reduced HashTags compression throughput by 2.4 percent without a selection. + +This analysis cost needs optimization before FloatQuant can enter the default set. ## Removed candidates @@ -200,19 +238,19 @@ The experimental branch retains the prototype for possible ALP-RD decomposition ## Revalidation after the ALP release -After the corrected ALP release, perform these steps: +The workspace now uses ALP 0.0.3. + +The corrected ALP selected `ALP(FoR(BitPacked))` for the integer-valued float control. + +The control used 5,002,240 bytes with or without the new schemes. -1. Update the workspace ALP dependency. -2. Remove any remaining FloatMult compatibility code. -3. Confirm that corrected ALP replaces the historical Housing FloatMult selections. -4. Recheck FloatQuant on widened-f32 columns. -5. Recheck OrderedFloat with BlockResidual on random walks and time-series columns. -6. Repeat throughput and scalar-access measurements with at least two million rows. -7. Compare each candidate against corrected default, Compact, and Pco Auto. -8. Measure rejected-candidate analysis cost on every dataset. -9. Add CMS Payments, r/place, and Twitter to the paper dataset matrix. +Complete these remaining steps: -Record size, encode throughput, decode throughput, and scalar-access latency for each selected column. +1. Optimize the FloatQuant selection path. +2. Repeat the broad file benchmark after that change. +3. Measure random access on two million values for every retained tree. +4. Add CMS Payments, r/place, and Twitter to the paper dataset matrix. +5. Compare the geometric mean against Parquet with Zstd. ## Pull request structure diff --git a/encodings/float-quant/src/array.rs b/encodings/float-quant/src/array.rs index 995cfb28af3..c50dbcabd46 100644 --- a/encodings/float-quant/src/array.rs +++ b/encodings/float-quant/src/array.rs @@ -19,7 +19,6 @@ use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::TypedArrayRef; use vortex_array::array_slots; -use vortex_array::arrays::ConstantArray; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::buffer::BufferHandle; @@ -56,7 +55,7 @@ pub struct FloatQuantSlots { pub primary: ArrayRef, /// Sign-normalized low `k` bits. #[slot(1)] - pub secondary: ArrayRef, + pub secondary: Option, } #[derive(Clone, Debug)] @@ -113,21 +112,27 @@ impl VTable for FloatQuant { let slots = FloatQuantSlotsView::from_slots(slots); let expected_primary = DType::Primitive(latent_ptype, dtype.nullability()); - let expected_secondary = DType::Primitive(latent_ptype, NonNullable); vortex_ensure!( slots.primary.dtype() == &expected_primary, "expected primary dtype {expected_primary}, got {}", slots.primary.dtype() ); vortex_ensure!( - slots.secondary.dtype() == &expected_secondary, - "expected secondary dtype {expected_secondary}, got {}", - slots.secondary.dtype() - ); - vortex_ensure!( - slots.primary.len() == len && slots.secondary.len() == len, - "FloatQuant child length differs from {len}" + slots.primary.len() == len, + "FloatQuant primary length differs" ); + if let Some(secondary) = slots.secondary { + let expected_secondary = DType::Primitive(latent_ptype, NonNullable); + vortex_ensure!( + secondary.dtype() == &expected_secondary, + "expected secondary dtype {expected_secondary}, got {}", + secondary.dtype() + ); + vortex_ensure!( + secondary.len() == len, + "FloatQuant secondary length differs" + ); + } Ok(()) } @@ -176,14 +181,21 @@ impl VTable for FloatQuant { "unsupported FloatQuant metadata version {}", metadata[0] ); - vortex_ensure!(children.len() == 2, "FloatQuant requires two children"); + vortex_ensure!( + matches!(children.len(), 1 | 2), + "FloatQuant requires one or two children" + ); let ptype = PType::try_from(dtype)?; let latent_ptype = latent_ptype(ptype)?; let primary_dtype = DType::Primitive(latent_ptype, dtype.nullability()); - let secondary_dtype = DType::Primitive(latent_ptype, NonNullable); let primary = children.get(0, &primary_dtype, len)?; - let secondary = children.get(1, &secondary_dtype, len)?; + let secondary = if children.len() == 2 { + let secondary_dtype = DType::Primitive(latent_ptype, NonNullable); + Some(children.get(1, &secondary_dtype, len)?) + } else { + None + }; let slots = FloatQuantSlots { primary, secondary }.into_slots(); Ok(ArrayParts::new( self.clone(), @@ -223,7 +235,6 @@ impl OperationsVTable for FloatQuant { if primary.is_null() { return Ok(Scalar::null(array.dtype().clone())); } - let secondary = array.secondary().execute_scalar(index, ctx)?; let k = array.data().k; Ok(match PType::try_from(array.dtype())? { PType::F32 => Scalar::primitive( @@ -232,10 +243,19 @@ impl OperationsVTable for FloatQuant { .as_primitive() .typed_value::() .vortex_expect("validated primary scalar"), - secondary - .as_primitive() - .typed_value::() - .vortex_expect("validated secondary scalar"), + array + .secondary() + .map(|secondary| { + secondary + .execute_scalar(index, ctx)? + .as_primitive() + .typed_value::() + .ok_or_else(|| { + vortex_error::vortex_err!("validated secondary scalar is null") + }) + }) + .transpose()? + .unwrap_or(0), k, ), array.dtype().nullability(), @@ -246,10 +266,19 @@ impl OperationsVTable for FloatQuant { .as_primitive() .typed_value::() .vortex_expect("validated primary scalar"), - secondary - .as_primitive() - .typed_value::() - .vortex_expect("validated secondary scalar"), + array + .secondary() + .map(|secondary| { + secondary + .execute_scalar(index, ctx)? + .as_primitive() + .typed_value::() + .ok_or_else(|| { + vortex_error::vortex_err!("validated secondary scalar is null") + }) + }) + .transpose()? + .unwrap_or(0), k, ), array.dtype().nullability(), @@ -275,10 +304,10 @@ pub trait FloatQuantArrayExt: TypedArrayRef + FloatQuantArraySlotsEx impl> FloatQuantArrayExt for T {} impl FloatQuant { - /// Construct a float quantization array from two latent children. + /// Construct a float quantization array from one or two latent children. pub fn try_new( primary: ArrayRef, - secondary: ArrayRef, + secondary: Option, float_ptype: PType, k: u8, ) -> VortexResult { @@ -298,7 +327,10 @@ impl FloatQuant { let (primary, secondary) = split_f32(array.as_slice::(), k)?; Self::try_new( PrimitiveArray::new(Buffer::from(primary), validity).into_array(), - PrimitiveArray::new(Buffer::from(secondary), NonNullable.into()).into_array(), + Some( + PrimitiveArray::new(Buffer::from(secondary), NonNullable.into()) + .into_array(), + ), PType::F32, k, ) @@ -307,7 +339,10 @@ impl FloatQuant { let (primary, secondary) = split_f64(array.as_slice::(), k)?; Self::try_new( PrimitiveArray::new(Buffer::from(primary), validity).into_array(), - PrimitiveArray::new(Buffer::from(secondary), NonNullable.into()).into_array(), + Some( + PrimitiveArray::new(Buffer::from(secondary), NonNullable.into()) + .into_array(), + ), PType::F64, k, ) @@ -322,13 +357,12 @@ impl FloatQuant { k: u8, ) -> VortexResult { let validity = array.validity()?; - let len = array.len(); match array.ptype() { PType::F32 => { let primary = split_primary_f32(array.as_slice::(), k)?; Self::try_new( PrimitiveArray::new(Buffer::from(primary), validity).into_array(), - ConstantArray::new(Scalar::from(0u32), len).into_array(), + None, PType::F32, k, ) @@ -337,7 +371,7 @@ impl FloatQuant { let primary = split_primary_f64(array.as_slice::(), k)?; Self::try_new( PrimitiveArray::new(Buffer::from(primary), validity).into_array(), - ConstantArray::new(Scalar::from(0u64), len).into_array(), + None, PType::F64, k, ) @@ -657,14 +691,59 @@ fn join_f64(primary: u64, secondary: u64, k: u8) -> f64 { f64::from_bits(bits) } +fn join_zero_f32(primary: u32, k: u8) -> f32 { + let low_mask = (1_u32 << k) - 1; + let sign_cutoff = (1_u32 << 31) >> k; + let low = if primary >= sign_cutoff { 0 } else { low_mask }; + let ordered = (primary << k).wrapping_add(low); + let bits = if ordered & (1_u32 << 31) == 0 { + !ordered + } else { + ordered ^ (1_u32 << 31) + }; + f32::from_bits(bits) +} + +fn join_zero_f64(primary: u64, k: u8) -> f64 { + let low_mask = (1_u64 << k) - 1; + let sign_cutoff = (1_u64 << 63) >> k; + let low = if primary >= sign_cutoff { 0 } else { low_mask }; + let ordered = (primary << k).wrapping_add(low); + let bits = if ordered & (1_u64 << 63) == 0 { + !ordered + } else { + ordered ^ (1_u64 << 63) + }; + f64::from_bits(bits) +} + fn decode( array: ArrayView<'_, FloatQuant>, ctx: &mut ExecutionCtx, ) -> VortexResult { let primary = array.primary().clone().execute::(ctx)?; - let secondary = array.secondary().clone().execute::(ctx)?; let validity = primary.validity()?; let k = array.data().k; + let Some(secondary) = array.secondary() else { + return Ok(match PType::try_from(array.dtype())? { + PType::F32 => PrimitiveArray::new( + primary + .into_buffer::() + .map_each_in_place(|primary| join_zero_f32(primary, k)) + .freeze(), + validity, + ), + PType::F64 => PrimitiveArray::new( + primary + .into_buffer::() + .map_each_in_place(|primary| join_zero_f64(primary, k)) + .freeze(), + validity, + ), + ptype => vortex_panic!("unsupported FloatQuant ptype {ptype}"), + }); + }; + let secondary = secondary.clone().execute::(ctx)?; Ok(match PType::try_from(array.dtype())? { PType::F32 => { let secondary_values = secondary.as_slice::(); @@ -771,6 +850,45 @@ mod tests { Ok(()) } + #[test] + fn implicit_zero_secondary_roundtrip() -> VortexResult<()> { + let original = PrimitiveArray::from_option_iter([ + Some(f64::from(-10.5_f32)), + None, + Some(f64::from(-0.0_f32)), + Some(f64::from(42.25_f32)), + ]); + let encoded = FloatQuant::from_primitive_constant_secondary(original.as_view(), 29)?; + assert!(encoded.secondary().is_none()); + + let mut ctx = SESSION.create_execution_ctx(); + assert_arrays_eq!(encoded, original, &mut ctx); + assert_nth_scalar!(encoded, 3, 42.25_f64, &mut ctx); + + let sliced = encoded.into_array().slice(1..4)?; + let expected = original.into_array().slice(1..4)?; + assert_arrays_eq!(sliced, expected, &mut ctx); + + let dtype = sliced.dtype().clone(); + let len = sliced.len(); + let array_context = ArrayContext::empty(); + let serialized = + sliced.serialize(&array_context, &SESSION, &SerializeOptions::default())?; + let mut bytes = ByteBufferMut::empty(); + for buffer in serialized { + bytes.extend_from_slice(buffer.as_ref()); + } + let decoded = SerializedArray::try_from(bytes.freeze())?.decode( + &dtype, + len, + &ReadContext::new(array_context.to_ids()), + &SESSION, + )?; + assert!(decoded.as_::().secondary().is_none()); + assert_arrays_eq!(decoded, expected, &mut ctx); + Ok(()) + } + #[test] fn serialization_roundtrip() -> VortexResult<()> { let original = PrimitiveArray::from_option_iter([ diff --git a/encodings/float-quant/src/slice.rs b/encodings/float-quant/src/slice.rs index 3a83f48220d..b46d982d0d5 100644 --- a/encodings/float-quant/src/slice.rs +++ b/encodings/float-quant/src/slice.rs @@ -18,7 +18,10 @@ impl SliceReduce for FloatQuant { Ok(Some( FloatQuant::try_new( array.primary().slice(range.clone())?, - array.secondary().slice(range)?, + array + .secondary() + .map(|secondary| secondary.slice(range)) + .transpose()?, PType::try_from(array.dtype())?, array.data().k, )? diff --git a/vortex-btrblocks/Cargo.toml b/vortex-btrblocks/Cargo.toml index f517a92d942..33e55e4838c 100644 --- a/vortex-btrblocks/Cargo.toml +++ b/vortex-btrblocks/Cargo.toml @@ -78,3 +78,7 @@ test = false [[example]] name = "pco_probe" required-features = ["pco"] + +[[example]] +name = "float_compressor_bench" +required-features = ["pco", "zstd"] diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs new file mode 100644 index 00000000000..83a1adf45ab --- /dev/null +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -0,0 +1,404 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::fs::File; +use std::hint::black_box; +use std::io::BufRead; +use std::io::BufReader; +use std::path::Path; +use std::sync::Arc; +use std::time::Duration; +use std::time::Instant; + +use arrow_array::ArrayRef as ArrowArrayRef; +use arrow_schema::DataType; +use arrow_select::concat::concat; +use parquet::arrow::ProjectionMask; +use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; +use rand::RngExt; +use rand::SeedableRng; +use rand::rngs::StdRng; +use vortex_array::ArrayRef; +use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::array_session; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::assert_arrays_eq; +use vortex_arrow::ArrowSessionExt; +use vortex_btrblocks::BtrBlocksCompressor; +use vortex_btrblocks::BtrBlocksCompressorBuilder; +use vortex_btrblocks::SchemeExt; +use vortex_btrblocks::schemes::float::FloatQuantScheme; +use vortex_btrblocks::schemes::float::OrderedBlockResidualScheme; +use vortex_error::VortexResult; +use vortex_error::vortex_ensure; +use vortex_error::vortex_err; +use vortex_session::VortexSession; + +const DEFAULT_ROW_COUNT: usize = 2_000_000; +const CALIFORNIA_COLUMNS: [&str; 9] = [ + "longitude", + "latitude", + "housingMedianAge", + "totalRooms", + "totalBedrooms", + "population", + "households", + "medianIncome", + "medianHouseValue", +]; + +struct Column { + name: String, + primitive: PrimitiveArray, + array: ArrayRef, +} + +fn column(name: impl Into, primitive: PrimitiveArray) -> Column { + let array = primitive.clone().into_array(); + Column { + name: name.into(), + primitive, + array, + } +} + +fn synthetic_datasets(row_count: usize) -> Vec<(String, Vec)> { + let mut widened_rng = StdRng::seed_from_u64(1); + let widened_f32 = PrimitiveArray::from_iter((0..row_count).map(|index| { + let trend = (index % 10_000) as f32 * 0.001; + f64::from(trend + widened_rng.random_range(-1.0_f32..1.0)) + })); + + let mut walk_rng = StdRng::seed_from_u64(2); + let mut value = 1_000.0_f64; + let random_walk = PrimitiveArray::from_iter((0..row_count).map(|_| { + value += walk_rng.random_range(-0.01_f64..0.01); + value + })); + + let mut uniform_rng = StdRng::seed_from_u64(3); + let uniform = PrimitiveArray::from_iter( + (0..row_count).map(|_| uniform_rng.random_range(-1_000_000.0_f64..1_000_000.0)), + ); + + let integer_valued = PrimitiveArray::from_iter((0..row_count).map(|index| { + let permuted = (index as u64).wrapping_mul(2_654_435_761) % 1_000_000; + permuted as f64 - 500_000.0 + })); + + vec![ + ( + "synthetic-widened-f32".to_string(), + vec![column("value", widened_f32)], + ), + ( + "synthetic-random-walk".to_string(), + vec![column("value", random_walk)], + ), + ( + "synthetic-uniform".to_string(), + vec![column("value", uniform)], + ), + ( + "synthetic-integer-valued".to_string(), + vec![column("value", integer_valued)], + ), + ] +} + +fn read_california(path: &Path, row_count: usize) -> VortexResult> { + let reader = BufReader::new(File::open(path)?); + let mut values = std::array::from_fn::<_, 9, _>(|_| Vec::::new()); + for (line_index, line) in reader.lines().enumerate() { + let line = line?; + let fields = line.split(',').collect::>(); + vortex_ensure!( + fields.len() == CALIFORNIA_COLUMNS.len(), + "line {} contains {} fields instead of {}", + line_index + 1, + fields.len(), + CALIFORNIA_COLUMNS.len() + ); + for (column_index, field) in fields.into_iter().enumerate() { + values[column_index].push(field.parse::().map_err(|error| { + vortex_err!( + "cannot parse line {} column {} as f32: {}", + line_index + 1, + column_index + 1, + error + ) + })?); + } + } + for values in &mut values { + vortex_ensure!(!values.is_empty(), "California Housing input is empty"); + values.truncate(row_count); + while values.len() < row_count { + let copy_len = (row_count - values.len()).min(values.len()); + values.extend_from_within(..copy_len); + } + } + Ok(CALIFORNIA_COLUMNS + .into_iter() + .zip(values) + .map(|(name, values)| column(name, PrimitiveArray::from_iter(values))) + .collect()) +} + +fn read_parquet_numeric( + path: &Path, + row_count: usize, + session: &VortexSession, +) -> VortexResult> { + let builder = ParquetRecordBatchReaderBuilder::try_new(File::open(path)?) + .map_err(|error| vortex_err!("cannot read Parquet metadata: {error}"))?; + let schema = Arc::clone(builder.schema()); + let selected = schema + .fields() + .iter() + .enumerate() + .filter_map(|(index, field)| { + matches!( + field.data_type(), + DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::Float32 + | DataType::Float64 + ) + .then_some(index) + }) + .collect::>(); + vortex_ensure!( + !selected.is_empty(), + "Parquet file contains no numeric columns" + ); + let mask = ProjectionMask::roots(builder.parquet_schema(), selected.iter().copied()); + let reader = builder + .with_projection(mask) + .with_batch_size(65_536) + .build() + .map_err(|error| vortex_err!("cannot build Parquet reader: {error}"))?; + let mut chunks = (0..selected.len()) + .map(|_| Vec::::new()) + .collect::>(); + let mut rows_read = 0usize; + for batch in reader { + let batch = batch.map_err(|error| vortex_err!("cannot read Parquet batch: {error}"))?; + let batch_len = batch.num_rows().min(row_count - rows_read); + for (column_chunks, array) in chunks.iter_mut().zip(batch.columns()) { + column_chunks.push(array.slice(0, batch_len)); + } + rows_read += batch_len; + if rows_read == row_count { + break; + } + } + vortex_ensure!(rows_read > 0, "Parquet file contains no rows"); + + selected + .into_iter() + .zip(chunks) + .map(|(field_index, chunks)| { + let field = &schema.fields()[field_index]; + let chunk_refs = chunks + .iter() + .map(|chunk| chunk.as_ref()) + .collect::>(); + let combined = concat(&chunk_refs) + .map_err(|error| vortex_err!("cannot concatenate {}: {error}", field.name()))?; + let array = session.arrow().from_arrow_array(combined, field.as_ref())?; + let primitive = array.execute::(&mut session.create_execution_ctx())?; + Ok(column(field.name(), primitive)) + }) + .collect() +} + +fn encoding_tree(array: &ArrayRef) -> String { + let children = array.children(); + if children.is_empty() { + return array.encoding_id().to_string(); + } + let children = children + .iter() + .map(encoding_tree) + .collect::>() + .join(","); + format!("{}({children})", array.encoding_id()) +} + +fn encode_all( + compressor: &BtrBlocksCompressor, + columns: &[Column], + session: &VortexSession, +) -> VortexResult> { + columns + .iter() + .map(|column| compressor.compress(&column.array, &mut session.create_execution_ctx())) + .collect() +} + +fn decode_all(arrays: &[ArrayRef], session: &VortexSession) -> VortexResult<()> { + for array in arrays { + black_box( + array + .clone() + .execute::(&mut session.create_execution_ctx())?, + ); + } + Ok(()) +} + +fn percentile(durations: &mut [Duration], numerator: usize, denominator: usize) -> Duration { + durations.sort_unstable(); + durations[durations.len() * numerator / denominator] +} + +fn measure_dataset( + dataset: &str, + columns: &[Column], + configs: &[(&str, BtrBlocksCompressor)], + session: &VortexSession, +) -> VortexResult<()> { + let input_bytes = columns + .iter() + .map(|column| column.primitive.nbytes()) + .sum::(); + let encoded = configs + .iter() + .map(|(name, compressor)| Ok((*name, encode_all(compressor, columns, session)?))) + .collect::>>()?; + + for (config, arrays) in &encoded { + for (column, array) in columns.iter().zip(arrays) { + assert_arrays_eq!(array, column.array, &mut session.create_execution_ctx()); + println!( + "structure\t{dataset}\t{}\t{config}\t{}\t{}", + column.name, + encoding_tree(array), + array.nbytes() + ); + } + } + + let encode_iterations = (128_000_000_u64 / input_bytes).clamp(3, 10) as usize; + let mut encode_durations = (0..configs.len()) + .map(|_| Vec::with_capacity(encode_iterations)) + .collect::>(); + for iteration in 0..encode_iterations { + for offset in 0..configs.len() { + let index = (iteration + offset) % configs.len(); + let start = Instant::now(); + black_box(encode_all(&configs[index].1, columns, session)?); + encode_durations[index].push(start.elapsed()); + } + } + + let decode_iterations = (512_000_000_u64 / input_bytes).clamp(5, 30) as usize; + let mut decode_durations = (0..configs.len()) + .map(|_| Vec::with_capacity(decode_iterations)) + .collect::>(); + for iteration in 0..decode_iterations { + for offset in 0..configs.len() { + let index = (iteration + offset) % configs.len(); + let start = Instant::now(); + black_box(decode_all(&encoded[index].1, session)?); + decode_durations[index].push(start.elapsed()); + } + } + + for (index, (config, arrays)) in encoded.iter().enumerate() { + let encoded_bytes = arrays.iter().map(ArrayRef::nbytes).sum::(); + let encode_median = percentile(&mut encode_durations[index], 1, 2); + let decode_median = percentile(&mut decode_durations[index], 1, 2); + let encode_throughput = input_bytes as f64 / encode_median.as_secs_f64() / 1_000_000.0; + let decode_throughput = input_bytes as f64 / decode_median.as_secs_f64() / 1_000_000.0; + println!( + "result\t{dataset}\t{config}\t{}\t{input_bytes}\t{encoded_bytes}\t{encode_throughput:.1}\t{decode_throughput:.1}", + columns[0].primitive.len() + ); + } + Ok(()) +} + +fn compressors() -> Vec<(&'static str, BtrBlocksCompressor)> { + let new_scheme_ids = [FloatQuantScheme.id(), OrderedBlockResidualScheme.id()]; + vec![ + ( + "prior-default", + BtrBlocksCompressorBuilder::default() + .exclude_schemes(new_scheme_ids) + .build(), + ), + ( + "float-quant-only", + BtrBlocksCompressorBuilder::default() + .exclude_schemes([OrderedBlockResidualScheme.id()]) + .build(), + ), + ( + "ordered-block-residual-only", + BtrBlocksCompressorBuilder::default() + .exclude_schemes([FloatQuantScheme.id()]) + .build(), + ), + ( + "proposed-default", + BtrBlocksCompressorBuilder::default().build(), + ), + ( + "prior-compact", + BtrBlocksCompressorBuilder::default() + .exclude_schemes(new_scheme_ids) + .with_compact() + .build(), + ), + ( + "compact", + BtrBlocksCompressorBuilder::default().with_compact().build(), + ), + ] +} + +fn main() -> VortexResult<()> { + let row_count = std::env::var("VORTEX_BENCH_ROWS") + .ok() + .map(|value| { + value + .parse::() + .map_err(|error| vortex_err!("invalid VORTEX_BENCH_ROWS: {error}")) + }) + .transpose()? + .unwrap_or(DEFAULT_ROW_COUNT); + let session = array_session(); + let configs = compressors(); + + println!("structure\tdataset\tcolumn\tconfig\tencoding\tbytes"); + println!("result\tdataset\tconfig\trows\tinput-bytes\tencoded-bytes\tencode-MB/s\tdecode-MB/s"); + if std::env::var_os("VORTEX_BENCH_SKIP_SYNTHETIC").is_none() { + for (dataset, columns) in synthetic_datasets(row_count) { + measure_dataset(&dataset, &columns, &configs, &session)?; + } + } + for argument in std::env::args().skip(1) { + let path = Path::new(&argument); + let dataset = path + .file_stem() + .and_then(|name| name.to_str()) + .ok_or_else(|| vortex_err!("data path has no valid file name"))?; + let columns = if path + .extension() + .is_some_and(|extension| extension == "parquet") + { + read_parquet_numeric(path, row_count, &session)? + } else { + read_california(path, row_count)? + }; + measure_dataset(dataset, &columns, &configs, &session)?; + } + Ok(()) +} diff --git a/vortex-btrblocks/src/schemes/float/float_quant.rs b/vortex-btrblocks/src/schemes/float/float_quant.rs index 44009a09dcb..2167e8672d4 100644 --- a/vortex-btrblocks/src/schemes/float/float_quant.rs +++ b/vortex-btrblocks/src/schemes/float/float_quant.rs @@ -9,13 +9,13 @@ use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::VTable; -use vortex_array::arrays::ConstantArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::PType; use vortex_array::scalar::Scalar; use vortex_compressor::scheme::CompressionEstimate; use vortex_compressor::scheme::DeferredEstimate; use vortex_compressor::scheme::EstimateVerdict; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_fastlanes::FoR; use vortex_fastlanes::bitpack_compress::bitpack_encode_unchecked; @@ -96,14 +96,31 @@ impl Scheme for FloatQuantScheme { _ => unreachable!(), }; let compressed_primary = FoR::try_new(compressed_primary, reference)?.into_array(); - let compressed_secondary = match primitive.ptype() { - PType::F32 => ConstantArray::new(Scalar::from(0u32), primitive.len()).into_array(), - PType::F64 => ConstantArray::new(Scalar::from(0u64), primitive.len()).into_array(), - _ => unreachable!(), - }; return Ok(FloatQuant::try_new( compressed_primary, - compressed_secondary, + None, + primitive.ptype(), + analysis.k, + )? + .into_array()); + } + + if analysis.secondary_is_constant { + let encoded = FloatQuant::from_primitive_constant_secondary(primitive, analysis.k)?; + let primary = encoded + .primary() + .clone() + .execute::(exec_ctx)?; + let compressed_primary = compressor.compress_child( + &primary.into_array(), + &compress_ctx, + self.id(), + 0, + exec_ctx, + )?; + return Ok(FloatQuant::try_new( + compressed_primary, + None, primitive.ptype(), analysis.k, )? @@ -117,6 +134,7 @@ impl Scheme for FloatQuantScheme { .execute::(exec_ctx)?; let secondary = encoded .secondary() + .vortex_expect("FloatQuant::from_primitive creates a secondary child") .clone() .execute::(exec_ctx)?; let compressed_primary = compressor.compress_child( @@ -135,7 +153,7 @@ impl Scheme for FloatQuantScheme { )?; Ok(FloatQuant::try_new( compressed_primary, - compressed_secondary, + Some(compressed_secondary), primitive.ptype(), analysis.k, )? diff --git a/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs index c8470be7352..cf2ef5298cf 100644 --- a/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs @@ -21,6 +21,7 @@ use vortex_block_residual::OrderedFloat; use vortex_buffer::Buffer; use vortex_error::VortexResult; use vortex_float_quant::FloatQuant; +use vortex_float_quant::FloatQuantArraySlotsExt; use vortex_session::VortexSession; use crate::BtrBlocksCompressor; @@ -88,6 +89,7 @@ fn test_widened_f32_uses_float_quant() -> VortexResult<()> { let compressed = BtrBlocksCompressor::default().compress(&array, &mut SESSION.create_execution_ctx())?; assert!(compressed.is::()); + assert!(compressed.as_::().secondary().is_none()); Ok(()) } diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index e12e3046d77..631c1c22e58 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -18,19 +18,27 @@ use vortex::VortexSessionDefault; use vortex::array::Canonical; use vortex::array::ExecutionCtx; use vortex::array::IntoArray; +use vortex::array::arrays::Primitive; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::VarBinViewArray; use vortex::array::builders::dict::dict_encode; use vortex::array::builtins::ArrayBuiltins; use vortex::array::dtype::Nullability; +use vortex::compressor::BtrBlocksCompressorBuilder; use vortex::dtype::PType; use vortex::encodings::alp::RDEncoder; use vortex::encodings::alp::RDEncoderExt; use vortex::encodings::alp::alp_encode; +use vortex::encodings::block_residual::BlockResidual; +use vortex::encodings::block_residual::OrderedFloat; +use vortex::encodings::block_residual::OrderedFloatArraySlotsExt; use vortex::encodings::fastlanes::Delta; use vortex::encodings::fastlanes::DeltaData; use vortex::encodings::fastlanes::FoR; +use vortex::encodings::fastlanes::bitpack_compress::bitpack_encode_unchecked; use vortex::encodings::fastlanes::delta_compress; +use vortex::encodings::float_quant::FloatQuant; +use vortex::encodings::float_quant::analyze_float_quant; use vortex::encodings::fsst::fsst_compress; use vortex::encodings::fsst::fsst_train_compressor; use vortex::encodings::pco::Pco; @@ -39,7 +47,9 @@ use vortex::encodings::sequence::sequence_encode; use vortex::encodings::zigzag::zigzag_encode; use vortex::encodings::zstd::Zstd; use vortex::encodings::zstd::ZstdData; +use vortex::scalar::Scalar; use vortex_array::VortexSessionExecute; +use vortex_btrblocks::schemes::float::OrderedBlockResidualScheme; use vortex_error::VortexResult; use vortex_sequence::Sequence; use vortex_session::VortexSession; @@ -55,6 +65,9 @@ fn main() { } const NUM_VALUES: u64 = 100_000; +const FLOAT_CODEC_NUM_VALUES: u64 = 2_000_000; +const PCO_COMPRESSION_LEVEL: usize = 8; +const PCO_VALUES_PER_PAGE: usize = 8192; // Helper function to conditionally add counter based on codspeed cfg fn with_byte_counter<'a, 'b>(bencher: Bencher<'a, 'b>, bytes: u64) -> Bencher<'a, 'b> { @@ -94,6 +107,54 @@ fn setup_primitive_arrays() -> (PrimitiveArray, PrimitiveArray, PrimitiveArray) (uint_array, int_array, float_array) } +fn setup_widened_f32_array() -> PrimitiveArray { + let mut rng = StdRng::seed_from_u64(1); + PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + let trend = (index % 10_000) as f32 * 0.001; + f64::from(trend + rng.random_range(-1.0_f32..1.0)) + })) +} + +fn setup_random_walk_array() -> PrimitiveArray { + let mut rng = StdRng::seed_from_u64(2); + let mut value = 1_000.0_f64; + PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|_| { + value += rng.random_range(-0.01_f64..0.01); + value + })) +} + +fn ordered_values(array: &PrimitiveArray) -> PrimitiveArray { + let ordered = OrderedFloat::from_primitive(array.as_view()).unwrap(); + ordered + .encoded() + .clone() + .execute::(&mut SESSION.create_execution_ctx()) + .unwrap() +} + +fn encode_ordered_block_residual(array: &PrimitiveArray) -> vortex::array::ArrayRef { + let ordered = OrderedFloat::from_primitive(array.as_view()).unwrap(); + let residuals = BlockResidual::from_primitive(ordered.encoded().as_::()).unwrap(); + OrderedFloat::try_new(residuals.into_array(), PType::F64) + .unwrap() + .into_array() +} + +fn encode_float_quant_tree(array: &PrimitiveArray) -> vortex::array::ArrayRef { + let analysis = analyze_float_quant(array.as_view()).unwrap(); + assert!(analysis.secondary_is_constant); + let primary = + FloatQuant::primary_for_primitive(array.as_view(), analysis.k, analysis.primary_min) + .unwrap(); + // SAFETY: The analysis computes this width from the exact primary range. + let primary = unsafe { bitpack_encode_unchecked(primary, analysis.primary_bit_width) }.unwrap(); + let primary = FoR::try_new(primary.into_array(), Scalar::from(analysis.primary_min)).unwrap(); + FloatQuant::try_new(primary.into_array(), None, PType::F64, analysis.k) + .unwrap() + .into_array() +} + #[expect(clippy::cast_possible_truncation)] fn gen_varbin_words(len: usize, uniqueness: f64) -> Vec { let mut rng = StdRng::seed_from_u64(0); @@ -319,6 +380,128 @@ fn bench_alp_rd_decompress_f64(bencher: Bencher) { .bench_refs(|(a, ctx)| canonicalize((**a).clone(), ctx)); } +#[divan::bench(name = "ordered_float_compress_f64")] +fn bench_ordered_float_compress_f64(bencher: Bencher) { + let float_array = setup_random_walk_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| &float_array) + .bench_refs(|array| OrderedFloat::from_primitive(array.as_view()).unwrap()); +} + +#[divan::bench(name = "ordered_float_decompress_f64")] +fn bench_ordered_float_decompress_f64(bencher: Bencher) { + let float_array = setup_random_walk_array(); + let encoded = OrderedFloat::from_primitive(float_array.as_view()) + .unwrap() + .into_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "block_residual_compress_u64")] +fn bench_block_residual_compress_u64(bencher: Bencher) { + let float_array = setup_random_walk_array(); + let ordered = ordered_values(&float_array); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| &ordered) + .bench_refs(|array| BlockResidual::from_primitive(array.as_view()).unwrap()); +} + +#[divan::bench(name = "block_residual_decompress_u64")] +fn bench_block_residual_decompress_u64(bencher: Bencher) { + let float_array = setup_random_walk_array(); + let ordered = ordered_values(&float_array); + let encoded = BlockResidual::from_primitive(ordered.as_view()) + .unwrap() + .into_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "ordered_block_residual_compress_f64")] +fn bench_ordered_block_residual_compress_f64(bencher: Bencher) { + let float_array = setup_random_walk_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| &float_array) + .bench_refs(|array| encode_ordered_block_residual(array)); +} + +#[divan::bench(name = "ordered_block_residual_decompress_f64")] +fn bench_ordered_block_residual_decompress_f64(bencher: Bencher) { + let float_array = setup_random_walk_array(); + let encoded = encode_ordered_block_residual(&float_array); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "ordered_block_residual_scheme_compress_f64")] +fn bench_ordered_block_residual_scheme_compress_f64(bencher: Bencher) { + let float_array = setup_random_walk_array(); + let compressor = BtrBlocksCompressorBuilder::empty() + .with_new_scheme(&OrderedBlockResidualScheme) + .build(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| { + ( + float_array.clone().into_array(), + SESSION.create_execution_ctx(), + ) + }) + .bench_values(|(array, mut ctx)| compressor.compress(&array, &mut ctx).unwrap()); +} + +#[divan::bench(name = "float_quant_split_compress_f64")] +fn bench_float_quant_split_compress_f64(bencher: Bencher) { + let float_array = setup_widened_f32_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| &float_array) + .bench_refs(|array| { + FloatQuant::from_primitive_constant_secondary(array.as_view(), 29).unwrap() + }); +} + +#[divan::bench(name = "float_quant_split_decompress_f64")] +fn bench_float_quant_split_decompress_f64(bencher: Bencher) { + let float_array = setup_widened_f32_array(); + let encoded = FloatQuant::from_primitive_constant_secondary(float_array.as_view(), 29) + .unwrap() + .into_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "float_quant_tree_compress_f64")] +fn bench_float_quant_tree_compress_f64(bencher: Bencher) { + let float_array = setup_widened_f32_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| &float_array) + .bench_refs(|array| encode_float_quant_tree(array)); +} + +#[divan::bench(name = "float_quant_tree_decompress_f64")] +fn bench_float_quant_tree_decompress_f64(bencher: Bencher) { + let float_array = setup_widened_f32_array(); + let encoded = encode_float_quant_tree(&float_array); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + #[divan::bench(name = "pcodec_compress_f64")] fn bench_pcodec_compress_f64(bencher: Bencher) { let (_, _, float_array) = setup_primitive_arrays(); @@ -344,6 +527,72 @@ fn bench_pcodec_decompress_f64(bencher: Bencher) { .bench_refs(|(a, ctx)| canonicalize((**a).clone(), ctx)); } +#[divan::bench(name = "pcodec_compress_widened_f32_f64")] +fn bench_pcodec_compress_widened_f32_f64(bencher: Bencher) { + let float_array = setup_widened_f32_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| (&float_array, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| { + Pco::from_primitive( + array.as_view(), + PCO_COMPRESSION_LEVEL, + PCO_VALUES_PER_PAGE, + ctx, + ) + .unwrap() + }); +} + +#[divan::bench(name = "pcodec_decompress_widened_f32_f64")] +fn bench_pcodec_decompress_widened_f32_f64(bencher: Bencher) { + let float_array = setup_widened_f32_array(); + let compressed = Pco::from_primitive( + float_array.as_view(), + PCO_COMPRESSION_LEVEL, + PCO_VALUES_PER_PAGE, + &mut SESSION.create_execution_ctx(), + ) + .unwrap(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| (&compressed, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "pcodec_compress_random_walk_f64")] +fn bench_pcodec_compress_random_walk_f64(bencher: Bencher) { + let float_array = setup_random_walk_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| (&float_array, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| { + Pco::from_primitive( + array.as_view(), + PCO_COMPRESSION_LEVEL, + PCO_VALUES_PER_PAGE, + ctx, + ) + .unwrap() + }); +} + +#[divan::bench(name = "pcodec_decompress_random_walk_f64")] +fn bench_pcodec_decompress_random_walk_f64(bencher: Bencher) { + let float_array = setup_random_walk_array(); + let compressed = Pco::from_primitive( + float_array.as_view(), + PCO_COMPRESSION_LEVEL, + PCO_VALUES_PER_PAGE, + &mut SESSION.create_execution_ctx(), + ) + .unwrap(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| (&compressed, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + #[cfg(feature = "zstd")] #[divan::bench(name = "zstd_compress_u32")] fn bench_zstd_compress_u32(bencher: Bencher) { From e7216255786025a711310861502925cb1ab69bb8 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Tue, 18 Aug 2026 21:32:13 -0400 Subject: [PATCH 07/78] perf: validate native float compression schemes Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 180 +++++++++++--- .../src/block_residual_array.rs | 150 +++++++++--- encodings/block-residual/src/codec.rs | 35 ++- encodings/float-quant/src/array.rs | 26 +- .../examples/float_compressor_bench.rs | 42 +++- .../src/schemes/float/float_quant.rs | 120 ++-------- .../schemes/float/ordered_block_residual.rs | 2 +- .../schemes/float/scheme_selection_tests.rs | 7 +- vortex/benches/single_encoding_throughput.rs | 225 ++++++++++++++++++ 9 files changed, 609 insertions(+), 178 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 2765340d0cf..74e2fe484d4 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -60,9 +60,11 @@ The production codec uses one reference per block. The multi-reference prototype `FloatQuantArray` splits each float into a primary quantum and a secondary adjustment. -Bounded metadata stores the source type and split width. One or two child arrays store the integer latents. +Metadata stores only the split width. The array dtype stores the source type. -The BtrBlocks scheme compresses each present child through the normal cascade. +One or two child arrays store the integer latents. + +The current BtrBlocks scheme accepts only a constant secondary. It uses a fixed `FoR(BitPacked)` primary tree. A common path uses `FloatQuant(FoR(BitPacked))` for `f32` values stored in `f64` columns. An absent secondary child represents zero low bits. @@ -73,11 +75,11 @@ The automatic scheme accepts only `f64`. Native `f32` columns remain with ALP, A ## Selection policy -`FloatQuantScheme` uses normal sample comparison against ALP, ALP-RD, dictionary, sparse, and RLE schemes. +`FloatQuantScheme` uses the normal sample comparison against ALP, ALP-RD, dictionary, sparse, and RLE schemes. -A strong constant split can use a direct `FoR(BitPacked)` primary and an implicit-zero secondary. +The sample builds a direct `FoR(BitPacked)` primary and uses an implicit-zero secondary. -Other splits compress both children through the normal integer cascade. +The scheme rejects samples with a nonzero secondary. This avoids the recursive integer selector. `OrderedBlockResidualScheme` uses eight locality-preserving sample blocks. @@ -85,7 +87,7 @@ The ordinary BtrBlocks sample does not preserve the block-local float structure. The residual scheme requires a 1.05 compression ratio and a 1.02 win over the incumbent. -Both schemes remain eligible to displace ALP or ALP-RD when their adjusted size scores win. +Both schemes remain eligible to displace ALP or ALP-RD when their sample size scores win. ## Performance requirements @@ -107,45 +109,74 @@ Exclude source-array construction and storage input from codec throughput measur The input contains two million arbitrary `f32` values stored in an `f64` column. -| Configuration | Bytes | Compression MB/s | Decode MB/s | -| --- | ---: | ---: | ---: | -| Prior default | 14,057,966 | 550.4 | 11,310.8 | -| Default with FloatQuant | 8,753,920 | 244.9 | 14,664.3 | -| Compact | 6,051,640 | 175.4 | 4,019.0 | +| Configuration | Bytes | Compression MB/s | Decode MB/s | Scalar access ns | +| --- | ---: | ---: | ---: | ---: | +| Prior default | 14,057,966 | 553.6 | 10,944.5 | 208.7 | +| Default with FloatQuant | 8,753,920 | 504.9 | 14,795.4 | 145.5 | +| Compact | 6,051,737 | 280.0 | 4,022.0 | Not measured | FloatQuant reduced size by 37.7 percent. It remained 44.7 percent larger than Compact. -Full compressor throughput decreased by 55.5 percent. Decode throughput increased by 29.6 percent. +Full compressor throughput decreased by 8.8 percent. Decode throughput increased by 35.2 percent. + +Scalar access latency decreased by 30.3 percent. The selected tree was `FloatQuant(FoR(BitPacked))` with an implicit-zero secondary. -The isolated tree compressed at 2,503 MB/s and decoded at 15,070 MB/s. +The isolated tree compressed between 2,402 and 2,469 MB/s. + +It decoded between 14,480 and 14,810 MB/s. + +Compact Pco compressed the same input at 280 MB/s and decoded it at 4,022 MB/s. + +The tree itself exceeded Compact throughput by at least 8.6 times for compression and 3.6 times for decode. + +FloatQuant recovered 66.2 percent of the size gap between the prior default and Compact Pco. + +The direct sample tree removed the recursive integer selector from the estimate and final compression paths. + +FloatQuant meets the selected-column throughput limit on this input. + +### FloatQuant with a nonzero secondary + +The prototype changed the lowest bit for ten percent of the widened-`f32` values. + +The fixed tree was `FloatQuant(FoR(BitPacked), BitPacked)`. The secondary used one bit per value. -Pco compressed the same input at 533 MB/s and decoded it at 3,973 MB/s. +| Configuration | Bytes | Decode MB/s | Scalar access ns | +| --- | ---: | ---: | ---: | +| Prior ALP-RD default | 14,057,966 | 11,430 | 208.5 | +| Two-child FloatQuant prototype | 9,004,032 | 8,375 | 192.2 | + +The prototype reduced size by 36.0 percent. It was 2.9 percent larger than the zero-secondary tree. -The tree itself exceeded Pco throughput by 4.7 times during compression and 3.8 times during decode. +The prototype recovered 64.1 percent of the size gap between ALP-RD and Compact Pco. -The BtrBlocks analysis and candidate path caused most of the full compressor cost. +The prototype reduced decode throughput by 26.7 percent against ALP-RD. It reduced decode throughput by 43.6 percent against zero-secondary FloatQuant. -FloatQuant cannot enter the default set until its selection path meets the compression throughput limit. +Scalar access remained competitive. The direct prototype tree compressed at 3,301 MB/s. + +The default scheme rejects this form. A fused one-bit-secondary decode is the next experiment. ### OrderedFloat with BlockResidual on random walks | Configuration | Bytes | Encode MB/s | Decode MB/s | Scalar access ns | | --- | ---: | ---: | ---: | ---: | -| Prior default | 12,255,488 | 577.9 | 12,869.9 | 161.24 | -| Default with the scheme | 10,425,690 | 516.1 | 18,019.7 | 129.06 | -| Compact Pco | 9,342,749 | 316.3 | 4,611.7 | Not measured | +| Prior default | 12,255,488 | 589.8 | 12,438.1 | 207.7 | +| Default with the scheme | 10,425,690 | 529.6 | 21,728.1 | 166.7 | +| Compact Pco | 9,342,749 | 318.1 | 4,559.8 | Not measured | The residual scheme saved 14.9 percent against the prior default. It remained 11.6 percent larger than Compact. -Decode throughput increased by 40.0 percent. Random access latency decreased by 20.0 percent. +Decode throughput increased by 74.7 percent. Random access latency decreased by 19.7 percent. + +Compression throughput decreased by 10.2 percent on the selected column. -Compression throughput decreased by 10.7 percent on the selected column. +The isolated tree compressed at 2,969 MB/s and decoded at 20,870 MB/s. -The isolated tree compressed at 2,504 MB/s and decoded at 18,920 MB/s. +Compact Pco compressed the same input at 318 MB/s and decoded it at 4,560 MB/s. -Pco compressed the same input at 648 MB/s and decoded it at 4,635 MB/s. +Ordered BlockResidual recovered 62.8 percent of the size gap between ALP-RD and Compact Pco. The selector rejected the scheme on Gaussian, lognormal, decimal, widened-f32, and four-cluster inputs. @@ -157,7 +188,7 @@ It reduced the numeric subset by 6.0 percent with no isolated compressor regress ### Broad dataset revalidation -The file benchmark compared the corrected default against the new schemes. +The earlier file benchmark compared the corrected default against the new schemes. | Dataset | Prior bytes | Proposed bytes | Size change | | --- | ---: | ---: | ---: | @@ -174,9 +205,85 @@ The focused two-million-row run selected no new schemes in Taxi, Air Quality, Ar Ordered BlockResidual selected two HashTags columns. FloatQuant selected no columns in these datasets. -The FloatQuant probe reduced HashTags compression throughput by 2.4 percent without a selection. +The earlier FloatQuant probe reduced HashTags compression throughput by 2.4 percent without a selection. + +The direct sample tree removed that recursive analysis cost. + +### Pcodec paper corpus + +The focused benchmark reads the first two million numeric rows from each source. + +Timestamps use their integer representation. The California Housing size uses its original 20,640 rows. + +| Dataset | Input bytes | Prior bytes | Proposed bytes | Compact bytes | New selection | +| --- | ---: | ---: | ---: | ---: | --- | +| Air Quality | 42,834,636 | 16,645,494 | 16,645,494 | 4,298,741 | None | +| California Housing | 743,040 | 307,427 | 307,427 | 227,970 | None | +| r/place | 56,000,000 | 16,266,697 | 16,266,697 | 8,891,886 | None | +| NYC Taxi | 240,000,000 | 52,407,972 | 52,407,972 | 33,148,991 | None | +| Twitter follower graph | 32,000,000 | 7,226,752 | 7,226,752 | 3,661,121 | None | +| CMS Payments | 160,000,000 | 30,061,670 | 30,061,670 | 22,300,792 | None | + +Air Quality, California Housing, and r/place match the paper input sizes. + +The Taxi logical input matches the paper size. Vortex arrays also retain column validity. + +The Twitter source uses the first two million official edges. The paper used an unspecified ID sort. + +The Twitter result is not an exact size comparison. Independent column sort produced a 1,212,617-byte Compact result. -This analysis cost needs optimization before FloatQuant can enter the default set. +The six inputs selected no new schemes. Four inputs contain no eligible `f64` column. + +Taxi contains `f64` columns, but both new schemes lost their sample comparisons. + +CMS Payments contains one `f64` column. ALP won that column. + +The current CMS source revision differs from the paper source snapshot. + +The CMS prior and proposed compressors both encoded at approximately 1,252 MB/s. + +Their decode results differed by less than two percent. They produced identical trees. + +## General real-float follow-up + +Target float columns where Compact Pco materially beats the complete default cascade. + +The default baseline includes ALP, ALP-RD, and compression of their children. + +Use a ten-percent Pco size advantage as the initial corpus filter. This filter is not a product threshold. + +Measure gap recovery as `(default bytes - candidate bytes) / (default bytes - Pco bytes)`. + +Retain a prototype only if it recovers a material gap across unrelated real datasets. + +The retained schemes cover lower-precision values in wider floats and locally narrow ordered ranges. + +They do not replace ALP-RD for generic non-decimal floats. No measured column moved from ALP to a new scheme. + +The first general prototype will use bounded multiple references per 1,024-value block. + +Each block will test one, two, or four references. Each value will store a small reference ID and one packed residual. + +All references will share one residual width per block. Sparse high-bit patches will contain rare outliers. + +Direct access will read one reference ID, one residual, and an optional patch. The one-reference form matches `BlockResidualArray`. + +This design approximates Pco bins without entropy coding. It also avoids a rank query into separate variable-width bin streams. + +Test these secondary candidates after the multi-reference prototype: + +- Use ordered-bit XOR suffixes instead of arithmetic residuals. +- Use sign and exponent prefixes as fixed reference candidates. +- Use block-local ALP-RD split widths. +- Use independent entropy microblocks only in Compact. + +Build a gap corpus from columns where Compact Pco materially beats ALP-RD. + +Classify each gap by exponent count, prefix count, local range, low-bit entropy, and outlier rate. + +Require size, compression, decode, and scalar-access results on the same two-million-row inputs. + +Do not add a general codec to the default until it wins across several unrelated real datasets. ## Removed candidates @@ -244,13 +351,22 @@ The corrected ALP selected `ALP(FoR(BitPacked))` for the integer-valued float co The control used 5,002,240 bytes with or without the new schemes. +This round completed these steps: + +- Replaced the recursive FloatQuant child search with one fixed sample tree. +- Added scalar-access benchmarks on two million values. +- Added nonzero-secondary FloatQuant benchmarks. +- Added all six Pcodec paper datasets. +- Fixed the high-value defects from three adversarial reviews. + Complete these remaining steps: -1. Optimize the FloatQuant selection path. -2. Repeat the broad file benchmark after that change. -3. Measure random access on two million values for every retained tree. -4. Add CMS Payments, r/place, and Twitter to the paper dataset matrix. -5. Compare the geometric mean against Parquet with Zstd. +1. Run the broad Vortex compression corpus after the selector change. +2. Compare the geometric mean against Parquet with Zstd. +3. Build the Pco-versus-ALP-RD gap corpus. +4. Prototype the bounded multi-reference residual codec. +5. Test a fused nonzero-secondary decode on qualifying gap columns. +6. Update this plan after each experiment. ## Pull request structure diff --git a/encodings/block-residual/src/block_residual_array.rs b/encodings/block-residual/src/block_residual_array.rs index a662eee0c88..9a1935eecf0 100644 --- a/encodings/block-residual/src/block_residual_array.rs +++ b/encodings/block-residual/src/block_residual_array.rs @@ -262,6 +262,9 @@ impl OperationsVTable for BlockResidual { index: usize, ctx: &mut ExecutionCtx, ) -> VortexResult { + if !array.as_ref().is_valid(index, ctx)? { + return Ok(Scalar::null(array.dtype().clone())); + } let value = scalar_from_array(array, index, ctx)?; Ok(Scalar::primitive(value, array.dtype().nullability())) } @@ -461,7 +464,9 @@ fn decode_array_values( let mut values = Vec::with_capacity(logical_range.len()); let mut residuals = [0_u64; BLOCK_LEN]; - for block_index in 0..bases.len() { + let first_block = logical_range.start / BLOCK_LEN; + let last_block = logical_range.end.div_ceil(BLOCK_LEN); + for block_index in first_block..last_block { let block_start = block_index * BLOCK_LEN; let block_len = (array.data().unsliced_len - block_start).min(BLOCK_LEN); let block_stop = block_start + block_len; @@ -469,7 +474,6 @@ fn decode_array_values( continue; } - residuals.fill(0); let residual_width = residual_widths[block_index]; let high_width = high_widths[block_index]; vortex_ensure!( @@ -497,6 +501,8 @@ fn decode_array_values( &mut residuals, ); } + } else { + residuals.fill(0); } let patch_payload = @@ -504,22 +510,13 @@ fn decode_array_values( let high_payload = payload_range(high_starts, block_index, patch_highs.len(), "patch high")?; let positions = &patch_positions[patch_payload]; - vortex_ensure!( - positions - .iter() - .all(|&position| usize::from(position) < block_len) - && positions.windows(2).all(|pair| pair[0] < pair[1]), - "block residual patch positions are invalid" - ); - let expected_high_len = if positions.is_empty() { - 0 - } else { - (positions.len() * usize::from(high_width)).div_ceil(8) + 15 - }; - vortex_ensure!( - high_payload.len() == expected_high_len, - "block residual patch high payload is invalid" - ); + validate_patch_payload( + block_len, + residual_width, + high_width, + positions, + high_payload.len(), + )?; let highs = &patch_highs[high_payload]; for (patch_index, &position) in positions.iter().enumerate() { // SAFETY: The payload includes fifteen readable padding bytes. @@ -609,20 +606,24 @@ fn scalar_from_array( "patch", )?; let block_positions = &positions[patch_payload]; + let block_start = block_index * BLOCK_LEN; + let block_len = (array.data().unsliced_len - block_start).min(BLOCK_LEN); + let highs = children.patch_highs.as_slice::(); + let high_payload = payload_range( + children.high_starts.as_slice::(), + block_index, + highs.len(), + "patch high", + )?; + validate_patch_payload( + block_len, + residual_width, + high_width, + block_positions, + high_payload.len(), + )?; if let Ok(patch_index) = block_positions.binary_search(&u16::try_from(index_in_block)?) { - let highs = children.patch_highs.as_slice::(); - let high_payload = payload_range( - children.high_starts.as_slice::(), - block_index, - highs.len(), - "patch high", - )?; let high_payload = &highs[high_payload]; - vortex_ensure!( - high_payload.len() - >= (block_positions.len() * usize::from(high_width)).div_ceil(8) + 15, - "block residual patch high payload is invalid" - ); // SAFETY: The payload includes fifteen readable padding bytes. let high = unsafe { read_wide_bits( @@ -636,6 +637,36 @@ fn scalar_from_array( Ok(children.bases.as_slice::()[block_index].wrapping_add(residual)) } +fn validate_patch_payload( + block_len: usize, + residual_width: u8, + high_width: u8, + positions: &[u16], + high_payload_len: usize, +) -> VortexResult<()> { + vortex_ensure!( + positions + .iter() + .all(|&position| usize::from(position) < block_len) + && positions.windows(2).all(|pair| pair[0] < pair[1]), + "block residual patch positions are invalid" + ); + vortex_ensure!( + positions.is_empty() || (high_width > 0 && residual_width < 64), + "block residual patches require nonzero high bits" + ); + let expected_high_len = if positions.is_empty() { + 0 + } else { + (positions.len() * usize::from(high_width)).div_ceil(8) + 15 + }; + vortex_ensure!( + high_payload_len == expected_high_len, + "block residual patch high payload is invalid" + ); + Ok(()) +} + fn payload_range( starts: &[u32], block_index: usize, @@ -732,12 +763,19 @@ fn primitive_dtype(ptype: PType) -> DType { #[cfg(test)] mod tests { + use vortex_array::ArrayContext; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::array_session; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; + use vortex_array::serde::SerializeOptions; + use vortex_array::serde::SerializedArray; + use vortex_array::validity::Validity; + use vortex_buffer::Buffer; + use vortex_buffer::ByteBufferMut; use vortex_error::VortexResult; + use vortex_session::registry::ReadContext; use super::BlockResidual; @@ -761,4 +799,56 @@ mod tests { } Ok(()) } + + #[test] + fn nullable_slice_and_scalar_access() -> VortexResult<()> { + let values = (0..2_050) + .map(|index| Ok(u64::try_from(index * index)?)) + .collect::>>()?; + let validity = Validity::from_iter((0..values.len()).map(|index| index != 1_024)); + let primitive = PrimitiveArray::new(Buffer::from(values), validity); + let encoded = BlockResidual::from_primitive(primitive.as_view())?; + let session = array_session(); + crate::initialize(&session); + let mut ctx = session.create_execution_ctx(); + + assert!(encoded.execute_scalar(1_024, &mut ctx)?.is_null()); + let sliced = encoded.into_array().slice(1_023..1_026)?; + let expected = primitive.into_array().slice(1_023..1_026)?; + assert_arrays_eq!(sliced, expected, &mut ctx); + Ok(()) + } + + #[test] + fn nullable_slice_serialization_roundtrip() -> VortexResult<()> { + let values = (0..2_050) + .map(|index| Ok(u64::try_from(index * index)?)) + .collect::>>()?; + let validity = Validity::from_iter((0..values.len()).map(|index| index != 1_024)); + let primitive = PrimitiveArray::new(Buffer::from(values), validity); + let sliced = BlockResidual::from_primitive(primitive.as_view())? + .into_array() + .slice(1_023..1_026)?; + let expected = primitive.into_array().slice(1_023..1_026)?; + let dtype = sliced.dtype().clone(); + let len = sliced.len(); + let array_context = ArrayContext::empty(); + let session = array_session(); + crate::initialize(&session); + let serialized = + sliced.serialize(&array_context, &session, &SerializeOptions::default())?; + let mut bytes = ByteBufferMut::empty(); + for buffer in serialized { + bytes.extend_from_slice(buffer.as_ref()); + } + let decoded = SerializedArray::try_from(bytes.freeze())?.decode( + &dtype, + len, + &ReadContext::new(array_context.to_ids()), + &session, + )?; + assert!(decoded.is::()); + assert_arrays_eq!(decoded, expected, &mut session.create_execution_ctx()); + Ok(()) + } } diff --git a/encodings/block-residual/src/codec.rs b/encodings/block-residual/src/codec.rs index 0570030abb0..d0b9a2c5289 100644 --- a/encodings/block-residual/src/codec.rs +++ b/encodings/block-residual/src/codec.rs @@ -132,8 +132,10 @@ impl BlockResidualCodec { let residual_width = parts.residual_widths[block_index]; let high_width = parts.high_widths[block_index]; vortex_error::vortex_ensure!( - residual_width <= 64 && high_width <= 64, - "block residual bit width exceeds 64" + residual_width <= 64 + && high_width <= 64 + && u16::from(residual_width) + u16::from(high_width) <= 64, + "block residual bit widths are invalid" ); vortex_error::vortex_ensure!( residual_stop - residual_start @@ -141,6 +143,10 @@ impl BlockResidualCodec { "block residual packed word count is invalid" ); let patch_positions = &parts.patch_positions[patch_start..patch_stop]; + vortex_error::vortex_ensure!( + patch_positions.is_empty() || (high_width > 0 && residual_width < 64), + "block residual patches require nonzero high bits" + ); vortex_error::vortex_ensure!( patch_positions .iter() @@ -583,4 +589,29 @@ mod tests { assert_eq!(codec.decode()?, values); Ok(()) } + + #[test] + fn rejects_incompatible_patch_widths() -> VortexResult<()> { + let mut values = vec![0_u64; 1_024]; + values[1_023] = u64::MAX; + let mut parts = BlockResidualCodec::encode(&values)?.into_parts()?; + assert!(!parts.patch_positions.is_empty()); + + parts.residual_widths[0] = 64; + parts.high_widths[0] = 1; + assert!(BlockResidualCodec::try_from_parts(parts).is_err()); + Ok(()) + } + + #[test] + fn rejects_patches_without_high_bits() -> VortexResult<()> { + let mut values = vec![0_u64; 1_024]; + values[1_023] = u64::MAX; + let mut parts = BlockResidualCodec::encode(&values)?.into_parts()?; + assert!(!parts.patch_positions.is_empty()); + + parts.high_widths[0] = 0; + assert!(BlockResidualCodec::try_from_parts(parts).is_err()); + Ok(()) + } } diff --git a/encodings/float-quant/src/array.rs b/encodings/float-quant/src/array.rs index c50dbcabd46..92259b9eaac 100644 --- a/encodings/float-quant/src/array.rs +++ b/encodings/float-quant/src/array.rs @@ -430,7 +430,7 @@ pub fn analyze_float_quant(array: ArrayView<'_, Primitive>) -> Option, + histogram: &mut [usize], precision_bits: u8, len: usize, primary_min: u64, @@ -459,8 +459,6 @@ fn analyze_histogram( if savings > best_savings { best_k = k; best_savings = savings; - } else { - break; } } if best_savings <= 1.5 { @@ -482,7 +480,7 @@ fn analyze_histogram( fn analyze_f32(values: &[f32]) -> Option { let mut minimum = u32::MAX; let mut maximum = u32::MIN; - let mut histogram = vec![0usize; 24]; + let mut histogram = [0usize; 24]; for value in values { let bits = value.to_bits(); let ordered = ordered_u32(bits); @@ -492,7 +490,7 @@ fn analyze_f32(values: &[f32]) -> Option { histogram[zeros as usize] += 1; } analyze_histogram( - histogram, + &mut histogram, 23, values.len(), u64::from(minimum), @@ -503,7 +501,7 @@ fn analyze_f32(values: &[f32]) -> Option { fn analyze_f64(values: &[f64]) -> Option { let mut minimum = u64::MAX; let mut maximum = u64::MIN; - let mut histogram = vec![0usize; 53]; + let mut histogram = [0usize; 53]; for value in values { let bits = value.to_bits(); let ordered = ordered_u64(bits); @@ -512,7 +510,7 @@ fn analyze_f64(values: &[f64]) -> Option { let zeros = bits.trailing_zeros().min(52); histogram[zeros as usize] += 1; } - analyze_histogram(histogram, 52, values.len(), minimum, maximum) + analyze_histogram(&mut histogram, 52, values.len(), minimum, maximum) } fn category_entropy(probability: f64) -> f64 { @@ -801,6 +799,20 @@ mod tests { session }); + #[test] + fn histogram_search_continues_after_local_decline() -> VortexResult<()> { + let mut histogram = [0usize; 24]; + histogram[0] = 10; + histogram[1] = 70; + histogram[20] = 20; + + let Some(analysis) = analyze_histogram(&mut histogram, 23, 100, 0, (1 << 23) - 1) else { + vortex_bail!("a large trailing-zero group must be useful") + }; + assert_eq!(analysis.k, 20); + Ok(()) + } + #[test] fn float_bit_patterns_roundtrip() -> VortexResult<()> { let values = [ diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index 83a1adf45ab..e6b07bfc63b 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -24,6 +24,9 @@ use vortex_array::VortexSessionExecute; use vortex_array::array_session; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; +use vortex_array::builtins::ArrayBuiltins; +use vortex_array::dtype::DType; +use vortex_array::dtype::PType; use vortex_arrow::ArrowSessionExt; use vortex_btrblocks::BtrBlocksCompressor; use vortex_btrblocks::BtrBlocksCompressorBuilder; @@ -69,6 +72,16 @@ fn synthetic_datasets(row_count: usize) -> Vec<(String, Vec)> { let trend = (index % 10_000) as f32 * 0.001; f64::from(trend + widened_rng.random_range(-1.0_f32..1.0)) })); + let nonzero_secondary = + PrimitiveArray::from_iter(widened_f32.as_slice::().iter().enumerate().map( + |(index, value)| { + if index % 10 == 0 { + f64::from_bits(value.to_bits() | 1) + } else { + *value + } + }, + )); let mut walk_rng = StdRng::seed_from_u64(2); let mut value = 1_000.0_f64; @@ -92,6 +105,10 @@ fn synthetic_datasets(row_count: usize) -> Vec<(String, Vec)> { "synthetic-widened-f32".to_string(), vec![column("value", widened_f32)], ), + ( + "synthetic-nonzero-secondary".to_string(), + vec![column("value", nonzero_secondary)], + ), ( "synthetic-random-walk".to_string(), vec![column("value", random_walk)], @@ -131,12 +148,15 @@ fn read_california(path: &Path, row_count: usize) -> VortexResult> { })?); } } + let repeat_short_input = std::env::var_os("VORTEX_BENCH_REPEAT_SHORT").is_some(); for values in &mut values { vortex_ensure!(!values.is_empty(), "California Housing input is empty"); values.truncate(row_count); - while values.len() < row_count { - let copy_len = (row_count - values.len()).min(values.len()); - values.extend_from_within(..copy_len); + if repeat_short_input { + while values.len() < row_count { + let copy_len = (row_count - values.len()).min(values.len()); + values.extend_from_within(..copy_len); + } } } Ok(CALIFORNIA_COLUMNS @@ -169,6 +189,7 @@ fn read_parquet_numeric( | DataType::UInt64 | DataType::Float32 | DataType::Float64 + | DataType::Timestamp(_, _) ) .then_some(index) }) @@ -212,6 +233,11 @@ fn read_parquet_numeric( let combined = concat(&chunk_refs) .map_err(|error| vortex_err!("cannot concatenate {}: {error}", field.name()))?; let array = session.arrow().from_arrow_array(combined, field.as_ref())?; + let array = if matches!(field.data_type(), DataType::Timestamp(_, _)) { + array.cast(DType::Primitive(PType::I64, array.dtype().nullability()))? + } else { + array + }; let primitive = array.execute::(&mut session.create_execution_ctx())?; Ok(column(field.name(), primitive)) }) @@ -264,10 +290,12 @@ fn measure_dataset( configs: &[(&str, BtrBlocksCompressor)], session: &VortexSession, ) -> VortexResult<()> { - let input_bytes = columns - .iter() - .map(|column| column.primitive.nbytes()) - .sum::(); + let mut input_bytes = 0_u64; + for column in columns { + input_bytes += u64::try_from(column.primitive.len())? + * u64::try_from(column.primitive.ptype().bit_width())? + / 8; + } let encoded = configs .iter() .map(|(name, compressor)| Ok((*name, encode_all(compressor, columns, session)?))) diff --git a/vortex-btrblocks/src/schemes/float/float_quant.rs b/vortex-btrblocks/src/schemes/float/float_quant.rs index 2167e8672d4..d7bc4c6d4bb 100644 --- a/vortex-btrblocks/src/schemes/float/float_quant.rs +++ b/vortex-btrblocks/src/schemes/float/float_quant.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! Lossless float quantization with recursively compressed integer children. +//! Lossless float quantization with a fixed frame-of-reference child. use vortex_array::ArrayId; use vortex_array::ArrayRef; @@ -9,36 +9,24 @@ use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::VTable; -use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::PType; use vortex_array::scalar::Scalar; use vortex_compressor::scheme::CompressionEstimate; use vortex_compressor::scheme::DeferredEstimate; use vortex_compressor::scheme::EstimateVerdict; -use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_fastlanes::BitPacked; use vortex_fastlanes::FoR; use vortex_fastlanes::bitpack_compress::bitpack_encode_unchecked; use vortex_float_quant::FloatQuant; -use vortex_float_quant::FloatQuantAnalysis; -use vortex_float_quant::FloatQuantArraySlotsExt; use vortex_float_quant::analyze_float_quant; use crate::ArrayAndStats; use crate::CascadingCompressor; use crate::CompressorContext; use crate::Scheme; -use crate::SchemeExt; -const ALWAYS_USE_MIN_RATIO: f64 = 2.0; - -fn is_strong_constant_split(analysis: FloatQuantAnalysis, ptype: PType) -> bool { - analysis.secondary_is_constant - && analysis.primary_bit_width > 0 - && ptype.bit_width() as f64 / f64::from(analysis.primary_bit_width) >= ALWAYS_USE_MIN_RATIO -} - -/// FloatQuant split with normal BtrBlocks compression for both latent children. +/// FloatQuant split with a fixed frame-of-reference primary child. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct FloatQuantScheme; @@ -52,11 +40,7 @@ impl Scheme for FloatQuantScheme { } fn produced_encodings(&self) -> Vec { - vec![FloatQuant.id()] - } - - fn num_children(&self) -> usize { - 2 + vec![FloatQuant.id(), FoR.id(), BitPacked.id()] } fn expected_compression_ratio( @@ -74,89 +58,33 @@ impl Scheme for FloatQuantScheme { fn compress( &self, - compressor: &CascadingCompressor, + _compressor: &CascadingCompressor, data: &ArrayAndStats, - compress_ctx: CompressorContext, - exec_ctx: &mut ExecutionCtx, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> VortexResult { let primitive = data.array_as_primitive(); let Some(analysis) = analyze_float_quant(primitive) else { return Ok(primitive.array().clone()); }; - if is_strong_constant_split(analysis, primitive.ptype()) { - let biased = - FloatQuant::primary_for_primitive(primitive, analysis.k, analysis.primary_min)?; - // SAFETY: The analysis computes this width from the exact primary minimum and maximum. - let compressed_primary = - unsafe { bitpack_encode_unchecked(biased, analysis.primary_bit_width)? } - .into_array(); - let reference = match primitive.ptype() { - PType::F32 => Scalar::from(u32::try_from(analysis.primary_min)?), - PType::F64 => Scalar::from(analysis.primary_min), - _ => unreachable!(), - }; - let compressed_primary = FoR::try_new(compressed_primary, reference)?.into_array(); - return Ok(FloatQuant::try_new( - compressed_primary, - None, - primitive.ptype(), - analysis.k, - )? - .into_array()); - } - - if analysis.secondary_is_constant { - let encoded = FloatQuant::from_primitive_constant_secondary(primitive, analysis.k)?; - let primary = encoded - .primary() - .clone() - .execute::(exec_ctx)?; - let compressed_primary = compressor.compress_child( - &primary.into_array(), - &compress_ctx, - self.id(), - 0, - exec_ctx, - )?; - return Ok(FloatQuant::try_new( - compressed_primary, - None, - primitive.ptype(), - analysis.k, - )? - .into_array()); + if !analysis.secondary_is_constant || analysis.primary_bit_width == 0 { + return Ok(primitive.array().clone()); } - let encoded = FloatQuant::from_primitive(primitive, analysis.k)?; - let primary = encoded - .primary() - .clone() - .execute::(exec_ctx)?; - let secondary = encoded - .secondary() - .vortex_expect("FloatQuant::from_primitive creates a secondary child") - .clone() - .execute::(exec_ctx)?; - let compressed_primary = compressor.compress_child( - &primary.into_array(), - &compress_ctx, - self.id(), - 0, - exec_ctx, - )?; - let compressed_secondary = compressor.compress_child( - &secondary.into_array(), - &compress_ctx, - self.id(), - 1, - exec_ctx, - )?; - Ok(FloatQuant::try_new( - compressed_primary, - Some(compressed_secondary), - primitive.ptype(), - analysis.k, - )? - .into_array()) + let biased = + FloatQuant::primary_for_primitive(primitive, analysis.k, analysis.primary_min)?; + // SAFETY: The analysis computes this width from the exact primary minimum and maximum. + let compressed_primary = + unsafe { bitpack_encode_unchecked(biased, analysis.primary_bit_width)? }.into_array(); + let reference = match primitive.ptype() { + PType::F32 => Scalar::from(u32::try_from(analysis.primary_min)?), + PType::F64 => Scalar::from(analysis.primary_min), + _ => unreachable!(), + }; + let compressed_primary = FoR::try_new(compressed_primary, reference)?.into_array(); + Ok( + FloatQuant::try_new(compressed_primary, None, primitive.ptype(), analysis.k)? + .into_array(), + ) } } diff --git a/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs b/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs index c5cbedc55ed..a9bdd1bb0c9 100644 --- a/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs +++ b/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs @@ -45,7 +45,7 @@ impl Scheme for OrderedBlockResidualScheme { } fn produced_encodings(&self) -> Vec { - vec![OrderedFloat.id()] + vec![OrderedFloat.id(), BlockResidual.id()] } fn num_children(&self) -> usize { diff --git a/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs index cf2ef5298cf..cd8d7f97e5a 100644 --- a/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs @@ -12,6 +12,7 @@ use vortex_array::VortexSessionExecute; use vortex_array::arrays::Constant; use vortex_array::arrays::Dict; use vortex_array::arrays::PrimitiveArray; +use vortex_array::assert_arrays_eq; use vortex_array::builders::ArrayBuilder; use vortex_array::builders::PrimitiveBuilder; use vortex_array::dtype::Nullability; @@ -71,9 +72,9 @@ fn test_null_dominated_compressed() -> VortexResult<()> { builder.append_nulls(95); let array = builder.finish_into_primitive(); let btr = BtrBlocksCompressor::default(); - let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; - // Verify the compressed array preserves values. - assert_eq!(compressed.len(), 100); + let mut ctx = SESSION.create_execution_ctx(); + let compressed = btr.compress(&array.clone().into_array(), &mut ctx)?; + assert_arrays_eq!(compressed, array, &mut ctx); Ok(()) } diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index 631c1c22e58..b721c4ff27a 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -5,6 +5,8 @@ #![expect(clippy::cast_possible_truncation)] use std::sync::LazyLock; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; use divan::Bencher; #[cfg(not(codspeed))] @@ -38,6 +40,7 @@ use vortex::encodings::fastlanes::FoR; use vortex::encodings::fastlanes::bitpack_compress::bitpack_encode_unchecked; use vortex::encodings::fastlanes::delta_compress; use vortex::encodings::float_quant::FloatQuant; +use vortex::encodings::float_quant::FloatQuantArraySlotsExt; use vortex::encodings::float_quant::analyze_float_quant; use vortex::encodings::fsst::fsst_compress; use vortex::encodings::fsst::fsst_train_compressor; @@ -49,6 +52,8 @@ use vortex::encodings::zstd::Zstd; use vortex::encodings::zstd::ZstdData; use vortex::scalar::Scalar; use vortex_array::VortexSessionExecute; +use vortex_btrblocks::SchemeExt; +use vortex_btrblocks::schemes::float::FloatQuantScheme; use vortex_btrblocks::schemes::float::OrderedBlockResidualScheme; use vortex_error::VortexResult; use vortex_sequence::Sequence; @@ -115,6 +120,23 @@ fn setup_widened_f32_array() -> PrimitiveArray { })) } +fn setup_nonzero_secondary_array() -> PrimitiveArray { + let widened = setup_widened_f32_array(); + PrimitiveArray::from_iter( + widened + .as_slice::() + .iter() + .enumerate() + .map(|(index, value)| { + if index % 10 == 0 { + f64::from_bits(value.to_bits() | 1) + } else { + *value + } + }), + ) +} + fn setup_random_walk_array() -> PrimitiveArray { let mut rng = StdRng::seed_from_u64(2); let mut value = 1_000.0_f64; @@ -155,6 +177,54 @@ fn encode_float_quant_tree(array: &PrimitiveArray) -> vortex::array::ArrayRef { .into_array() } +fn encode_float_quant_nonzero_secondary_tree(array: &PrimitiveArray) -> vortex::array::ArrayRef { + let analysis = analyze_float_quant(array.as_view()).unwrap(); + assert!(!analysis.secondary_is_constant); + let split = FloatQuant::from_primitive(array.as_view(), analysis.k).unwrap(); + let primary = split + .primary() + .clone() + .execute::(&mut SESSION.create_execution_ctx()) + .unwrap(); + let secondary = split + .secondary() + .unwrap() + .clone() + .execute::(&mut SESSION.create_execution_ctx()) + .unwrap(); + let biased_primary = PrimitiveArray::from_iter( + primary + .as_slice::() + .iter() + .map(|value| value - analysis.primary_min), + ); + // SAFETY: The analysis computes this width from the exact primary range. + let primary = unsafe { bitpack_encode_unchecked(biased_primary, analysis.primary_bit_width) } + .unwrap() + .into_array(); + let primary = FoR::try_new(primary, Scalar::from(analysis.primary_min)) + .unwrap() + .into_array(); + // SAFETY: The input generator changes only the lowest bit. + let secondary = unsafe { bitpack_encode_unchecked(secondary, 1) } + .unwrap() + .into_array(); + FloatQuant::try_new(primary, Some(secondary), PType::F64, analysis.k) + .unwrap() + .into_array() +} + +fn encode_prior_default(array: &PrimitiveArray) -> vortex::array::ArrayRef { + BtrBlocksCompressorBuilder::default() + .exclude_schemes([FloatQuantScheme.id(), OrderedBlockResidualScheme.id()]) + .build() + .compress( + &array.clone().into_array(), + &mut SESSION.create_execution_ctx(), + ) + .unwrap() +} + #[expect(clippy::cast_possible_truncation)] fn gen_varbin_words(len: usize, uniqueness: f64) -> Vec { let mut rng = StdRng::seed_from_u64(0); @@ -443,6 +513,38 @@ fn bench_ordered_block_residual_decompress_f64(bencher: Bencher) { .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); } +#[divan::bench(name = "ordered_block_residual_scalar_at_f64")] +fn bench_ordered_block_residual_scalar_at_f64(bencher: Bencher) { + let encoded = encode_ordered_block_residual(&setup_random_walk_array()); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + +#[divan::bench(name = "ordered_block_residual_prior_default_scalar_at_f64")] +fn bench_ordered_block_residual_prior_default_scalar_at_f64(bencher: Bencher) { + let encoded = encode_prior_default(&setup_random_walk_array()); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + #[divan::bench(name = "ordered_block_residual_scheme_compress_f64")] fn bench_ordered_block_residual_scheme_compress_f64(bencher: Bencher) { let float_array = setup_random_walk_array(); @@ -502,6 +604,129 @@ fn bench_float_quant_tree_decompress_f64(bencher: Bencher) { .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); } +#[divan::bench(name = "float_quant_tree_scalar_at_f64")] +fn bench_float_quant_tree_scalar_at_f64(bencher: Bencher) { + let encoded = encode_float_quant_tree(&setup_widened_f32_array()); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + +#[divan::bench(name = "float_quant_nonzero_secondary_split_compress_f64")] +fn bench_float_quant_nonzero_secondary_split_compress_f64(bencher: Bencher) { + let float_array = setup_nonzero_secondary_array(); + let k = analyze_float_quant(float_array.as_view()).unwrap().k; + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| &float_array) + .bench_refs(|array| FloatQuant::from_primitive(array.as_view(), k).unwrap()); +} + +#[divan::bench(name = "float_quant_nonzero_secondary_split_decompress_f64")] +fn bench_float_quant_nonzero_secondary_split_decompress_f64(bencher: Bencher) { + let float_array = setup_nonzero_secondary_array(); + let k = analyze_float_quant(float_array.as_view()).unwrap().k; + let encoded = FloatQuant::from_primitive(float_array.as_view(), k) + .unwrap() + .into_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "float_quant_nonzero_secondary_tree_compress_f64")] +fn bench_float_quant_nonzero_secondary_tree_compress_f64(bencher: Bencher) { + let float_array = setup_nonzero_secondary_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| &float_array) + .bench_refs(|array| encode_float_quant_nonzero_secondary_tree(array)); +} + +#[divan::bench(name = "float_quant_nonzero_secondary_tree_decompress_f64")] +fn bench_float_quant_nonzero_secondary_tree_decompress_f64(bencher: Bencher) { + let encoded = encode_float_quant_nonzero_secondary_tree(&setup_nonzero_secondary_array()); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "float_quant_nonzero_secondary_tree_scalar_at_f64")] +fn bench_float_quant_nonzero_secondary_tree_scalar_at_f64(bencher: Bencher) { + let encoded = encode_float_quant_nonzero_secondary_tree(&setup_nonzero_secondary_array()); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + +#[divan::bench(name = "float_quant_nonzero_secondary_prior_default_compress_f64")] +fn bench_float_quant_nonzero_secondary_prior_default_compress_f64(bencher: Bencher) { + let float_array = setup_nonzero_secondary_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| &float_array) + .bench_refs(|array| encode_prior_default(array)); +} + +#[divan::bench(name = "float_quant_nonzero_secondary_prior_default_decompress_f64")] +fn bench_float_quant_nonzero_secondary_prior_default_decompress_f64(bencher: Bencher) { + let encoded = encode_prior_default(&setup_nonzero_secondary_array()); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "float_quant_nonzero_secondary_prior_default_scalar_at_f64")] +fn bench_float_quant_nonzero_secondary_prior_default_scalar_at_f64(bencher: Bencher) { + let encoded = encode_prior_default(&setup_nonzero_secondary_array()); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + +#[divan::bench(name = "float_quant_prior_default_scalar_at_f64")] +fn bench_float_quant_prior_default_scalar_at_f64(bencher: Bencher) { + let encoded = encode_prior_default(&setup_widened_f32_array()); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + #[divan::bench(name = "pcodec_compress_f64")] fn bench_pcodec_compress_f64(bencher: Bencher) { let (_, _, float_array) = setup_primitive_arrays(); From 0cf4f8b2e24e138633edfc42a86bb79d2fa64aef Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 13:23:25 +0100 Subject: [PATCH 08/78] perf: add native block residual selection Signed-off-by: Will Manning --- benchmarks/compress-bench/src/main.rs | 18 +- benchmarks/compress-bench/src/vortex.rs | 18 +- docs/plans/native-pcodec.md | 171 ++++--- .../src/block_residual_array.rs | 354 +++++++++++--- encodings/block-residual/src/codec.rs | 113 ++++- .../examples/float_compressor_bench.rs | 446 +++++++++++++++++- vortex-btrblocks/src/builder.rs | 1 + .../schemes/float/ordered_block_residual.rs | 11 +- .../src/schemes/integer/block_residual.rs | 153 ++++++ vortex-btrblocks/src/schemes/integer/mod.rs | 2 + .../schemes/integer/scheme_selection_tests.rs | 53 +++ ...lden__default__string_fsst_structured.snap | 24 +- ...n__default__temporal_timestamp_micros.snap | 38 +- vortex/benches/single_encoding_throughput.rs | 179 ++++++- 14 files changed, 1421 insertions(+), 160 deletions(-) create mode 100644 vortex-btrblocks/src/schemes/integer/block_residual.rs diff --git a/benchmarks/compress-bench/src/main.rs b/benchmarks/compress-bench/src/main.rs index 88d78733499..de3122a3a7f 100644 --- a/benchmarks/compress-bench/src/main.rs +++ b/benchmarks/compress-bench/src/main.rs @@ -81,9 +81,9 @@ struct Args { ingest_output: Option, #[arg(long)] tracing: bool, - /// Exclude FloatQuant and OrderedBlockResidual from Vortex compression. + /// Exclude FloatQuant and block-residual schemes from Vortex compression. #[arg(long)] - vortex_without_new_float: bool, + vortex_without_new_numeric: bool, /// Format for the primary stderr log sink. `text` is the default human-readable format; /// `json` emits one JSON object per event, suitable for piping into `jq`. #[arg(long, value_enum, default_value_t = LogFormat::Text)] @@ -115,7 +115,7 @@ async fn main() -> anyhow::Result<()> { args.display_format, args.output_path, args.ingest_output, - args.vortex_without_new_float, + args.vortex_without_new_numeric, ) .await } @@ -124,7 +124,7 @@ async fn main() -> anyhow::Result<()> { fn get_compressor( format: Format, gpu_decompress: bool, - vortex_without_new_float: bool, + vortex_without_new_numeric: bool, ) -> Box { if gpu_decompress { #[cfg(feature = "cuda")] @@ -136,7 +136,7 @@ fn get_compressor( } match format { - Format::OnDiskVortex => Box::new(VortexCompressor::new(!vortex_without_new_float)), + Format::OnDiskVortex => Box::new(VortexCompressor::new(!vortex_without_new_numeric)), Format::Parquet => Box::new(ParquetCompressor::new()), #[cfg(feature = "lance")] Format::Lance => Box::new(LanceCompressor), @@ -163,7 +163,7 @@ async fn run_compress( display_format: DisplayFormat, output_path: Option, ingest_output: Option, - vortex_without_new_float: bool, + vortex_without_new_numeric: bool, ) -> anyhow::Result<()> { let targets = formats .iter() @@ -242,7 +242,7 @@ async fn run_compress( iterations, dataset_handle, gpu_decompress, - vortex_without_new_float, + vortex_without_new_numeric, ) .await?; measurements.push(m); @@ -286,7 +286,7 @@ async fn run_benchmark_for_dataset( iterations: usize, dataset_handle: &dyn Dataset, gpu_decompress: bool, - vortex_without_new_float: bool, + vortex_without_new_numeric: bool, ) -> anyhow::Result<(CompressMeasurements, Vec)> { let bench_name = dataset_handle.name(); let (v3_dataset, v3_variant) = dataset_handle.v3_dataset_dims(); @@ -302,7 +302,7 @@ async fn run_benchmark_for_dataset( let mut v3_records: Vec = Vec::new(); for format in formats { - let compressor = get_compressor(*format, gpu_decompress, vortex_without_new_float); + let compressor = get_compressor(*format, gpu_decompress, vortex_without_new_numeric); for op in ops { let time = match op { diff --git a/benchmarks/compress-bench/src/vortex.rs b/benchmarks/compress-bench/src/vortex.rs index 53690f994bd..caf135165e2 100644 --- a/benchmarks/compress-bench/src/vortex.rs +++ b/benchmarks/compress-bench/src/vortex.rs @@ -30,24 +30,30 @@ use vortex_btrblocks::BtrBlocksCompressorBuilder; use vortex_btrblocks::SchemeExt; use vortex_btrblocks::schemes::float::FloatQuantScheme; use vortex_btrblocks::schemes::float::OrderedBlockResidualScheme; +use vortex_btrblocks::schemes::integer::BlockResidualScheme; /// Compressor implementation for Vortex format. pub struct VortexCompressor { - new_float_schemes: bool, + new_numeric_schemes: bool, } impl VortexCompressor { - pub fn new(new_float_schemes: bool) -> Self { - Self { new_float_schemes } + pub fn new(new_numeric_schemes: bool) -> Self { + Self { + new_numeric_schemes, + } } fn write_options(&self) -> VortexWriteOptions { let options = SESSION.write_options(); - if self.new_float_schemes { + if self.new_numeric_schemes { return options; } - let compressor = BtrBlocksCompressorBuilder::default() - .exclude_schemes([FloatQuantScheme.id(), OrderedBlockResidualScheme.id()]); + let compressor = BtrBlocksCompressorBuilder::default().exclude_schemes([ + FloatQuantScheme.id(), + OrderedBlockResidualScheme.id(), + BlockResidualScheme.id(), + ]); options.with_strategy( WriteStrategyBuilder::default() .with_btrblocks_builder(compressor) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 74e2fe484d4..ea0efb67030 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -13,10 +13,10 @@ Pco remains a compression oracle. The new arrays do not preserve Pco byte compat Focus the production work on these encodings: - `OrderedFloatArray`. -- `BlockResidualArray` with one reference per 1,024-value block. +- `BlockResidualArray` for all integer types, with one reference per 1,024-value block. - `FloatQuantArray`. -Keep `FloatQuantScheme` and `OrderedBlockResidualScheme` as BtrBlocks candidates. +Keep `FloatQuantScheme`, `OrderedBlockResidualScheme`, and `BlockResidualScheme` as BtrBlocks candidates. Remove `FloatMultArray` and `FloatMultScheme` from the focused branch. @@ -38,7 +38,9 @@ The array supports canonical decode, scalar access, slice reduction, serializati ## BlockResidualArray -`BlockResidualArray` divides unsigned integers into independent blocks of 1,024 values. +`BlockResidualArray` divides integers into independent blocks of 1,024 values. + +Unsigned values retain their bit pattern. Signed values first flip the sign bit to preserve numeric order. Each block stores one minimum value. Packed residuals store the difference from that minimum. @@ -54,6 +56,10 @@ The array supports canonical decode, scalar access, slice reduction, serializati The `OrderedFloat(BlockResidual)` execute kernel combines residual decode with the inverse float transform. +The residual payload uses the logical integer width. A 16-bit array uses 16-bit FastLanes pack and unpack operations. + +The serialized residual payload remains a `u64` child. The logical type defines the packed word interpretation. + The production codec uses one reference per block. The multi-reference prototype did not justify its encode and decode costs. ## FloatQuantArray @@ -85,7 +91,26 @@ The scheme rejects samples with a nonzero secondary. This avoids the recursive i The ordinary BtrBlocks sample does not preserve the block-local float structure. -The residual scheme requires a 1.05 compression ratio and a 1.02 win over the incumbent. +The residual scheme requires a 1.05 compression ratio. Its adjusted score includes a 1.02 decode-cost factor. + +`BlockResidualScheme` uses the same locality probe for integer arrays. + +The default scheme accepts only 32-bit and 64-bit integers. Direct 8-bit and 16-bit candidates cannot save enough absolute space. + +The scheme does not run inside trial compression for an outer scheme. Generic 64-row samples do not preserve 1,024-row locality. + +The selected outer scheme can still choose BlockResidual for its full child. + +The integer selector divides the measured compression ratio by these decode-cost factors: + +- 1.10 for 32-bit integers. +- 1.20 for 64-bit integers. + +A 1.20 factor requires about 16.7 percent fewer estimated bytes. It does not require 20 percent fewer bytes. + +The block planner adds 16 synthetic cost bits per patch. This cost favors wider packed residuals when many patches slow decode. + +The selector excludes BlockResidual from dictionary-code children. A complete BlockResidual tree can still displace a complete dictionary tree. Both schemes remain eligible to displace ALP or ALP-RD when their sample size scores win. @@ -186,63 +211,93 @@ The HashTags dataset selected this scheme for two columns. It reduced the numeric subset by 6.0 percent with no isolated compressor regression. -### Broad dataset revalidation +### Integer BlockResidual throughput -The earlier file benchmark compared the corrected default against the new schemes. +The direct benchmark uses two million block-local values. -| Dataset | Prior bytes | Proposed bytes | Size change | +| Logical type and tree | Encode GB/s | Decode GB/s | Scalar access ns | | --- | ---: | ---: | ---: | -| Taxi | 439,806,364 | 439,806,364 | 0.000 percent | -| Air Quality | 15,848,420 | 15,848,420 | 0.000 percent | -| Arade | 143,343,828 | 143,343,828 | 0.000 percent | -| Euro2016 | 164,673,900 | 164,674,236 | +0.0002 percent | -| Food | 38,894,360 | 38,922,408 | +0.072 percent | -| HashTags | 195,035,692 | 194,753,372 | -0.145 percent | +| `u64` BlockResidual | 5.00 | 36.43 | 125 | +| `u64` FoR plus BitPacked | 4.25 | 35.37 | 115 | +| `i16` BlockResidual | 1.38 | 31.12 | 125 | +| `i16` FoR plus BitPacked | 1.12 | 41.57 | 104 | -Separate benchmark processes introduced low single-digit throughput noise. +The first `i16` implementation unpacked through `u64` residuals. It decoded at 11.43 GB/s. -The focused two-million-row run selected no new schemes in Taxi, Air Quality, Arade, Euro2016, or Food. +Native-width unpack increased `i16` decode throughput by 2.78 times. -Ordered BlockResidual selected two HashTags columns. FloatQuant selected no columns in these datasets. +The synthetic `i16` BlockResidual tree uses 1,793,784 bytes. The FoR plus BitPacked tree uses 3,500,000 bytes. -The earlier FloatQuant probe reduced HashTags compression throughput by 2.4 percent without a selection. +BlockResidual is 48.7 percent smaller on that input. -The direct sample tree removed that recursive analysis cost. +Its `i16` decode throughput is 25.1 percent lower. The default selector therefore excludes direct 8-bit and 16-bit candidates. -### Pcodec paper corpus +The BlockResidual estimator measures its sampled tree exactly, with all child arrays. -The focused benchmark reads the first two million numeric rows from each source. +The incumbent and outer tree estimates remain approximate. Trial compression previously mis-ranked Dict and ALP on Taxi tips. -Timestamps use their integer representation. The California Housing size uses its original 20,640 rows. +The outer-sample exclusion removed that error from the measured tree. -| Dataset | Input bytes | Prior bytes | Proposed bytes | Compact bytes | New selection | -| --- | ---: | ---: | ---: | ---: | --- | -| Air Quality | 42,834,636 | 16,645,494 | 16,645,494 | 4,298,741 | None | -| California Housing | 743,040 | 307,427 | 307,427 | 227,970 | None | -| r/place | 56,000,000 | 16,266,697 | 16,266,697 | 8,891,886 | None | -| NYC Taxi | 240,000,000 | 52,407,972 | 52,407,972 | 33,148,991 | None | -| Twitter follower graph | 32,000,000 | 7,226,752 | 7,226,752 | 3,661,121 | None | -| CMS Payments | 160,000,000 | 30,061,670 | 30,061,670 | 22,300,792 | None | +### Broad numeric revalidation -Air Quality, California Housing, and r/place match the paper input sizes. +The focused run uses two million rows when the source contains that many rows. -The Taxi logical input matches the paper size. Vortex arrays also retain column validity. +The integer-only configuration adds `BlockResidualScheme` to the prior default. -The Twitter source uses the first two million official edges. The paper used an unspecified ID sort. +| Dataset | Prior bytes | Integer BlockResidual bytes | Complete default bytes | Integer size change | Complete size change | +| --- | ---: | ---: | ---: | ---: | ---: | +| California Housing | 307,427 | 301,125 | 301,125 | -2.0 percent | -2.0 percent | +| NYC Taxi | 52,407,972 | 52,407,972 | 52,407,972 | 0.0 percent | 0.0 percent | +| CMS Payments | 30,061,670 | 28,472,040 | 28,472,040 | -5.3 percent | -5.3 percent | +| Arade | 26,937,026 | 26,937,026 | 26,937,026 | 0.0 percent | 0.0 percent | +| Euro2016 | 44,698,131 | 39,566,892 | 39,566,892 | -11.5 percent | -11.5 percent | +| Food | 13,579,790 | 13,579,790 | 13,579,790 | 0.0 percent | 0.0 percent | +| HashTags | 22,602,522 | 22,193,089 | 21,018,949 | -1.8 percent | -7.0 percent | -The Twitter result is not an exact size comparison. Independent column sort produced a 1,212,617-byte Compact result. +Across these numeric inputs, integer BlockResidual reduced size by 3.7 percent. -The six inputs selected no new schemes. Four inputs contain no eligible `f64` column. +Its aggregate encode throughput decreased by 0.4 percent. Its aggregate decode throughput increased by 1.7 percent. -Taxi contains `f64` columns, but both new schemes lost their sample comparisons. +California Housing contains only 20,433 rows. Fixed analysis cost dominates its encode result. -CMS Payments contains one `f64` column. ALP won that column. +The complete default reduced aggregate size by 4.4 percent. Encode throughput decreased by 1.2 percent. -The current CMS source revision differs from the paper source snapshot. +Aggregate decode throughput increased by 2.3 percent. + +Euro2016 reduced size by 11.5 percent and increased decode throughput by 10.2 percent. + +CMS reduced size by 5.3 percent and increased decode throughput by 5.5 percent. -The CMS prior and proposed compressors both encoded at approximately 1,252 MB/s. +HashTags reduced size by 7.0 percent and increased decode throughput by 4.9 percent. -Their decode results differed by less than two percent. They produced identical trees. +The direct narrow-type exclusion changed no selected tree in this corpus. The earlier 1.40 factor already rejected each direct `i16` candidate. + +A FastLanes fused FoR decode trial did not improve `u64` throughput. It reduced `i16` throughput, so the implementation retains native unpack. + +The zero-width residual path now writes base values and patches directly. It skips the scratch residual block. + +### Pcodec corpus gap analysis + +The focused benchmark reads the first two million numeric rows from each source. + +Air Quality, r/place, and the Twitter graph contain no float columns in the benchmark schema. + +The float gap corpus includes California Housing, NYC Taxi, CMS Payments, and four Public BI datasets. + +Thirty float columns use at least ten percent fewer bytes with Compact Pco than with the prior default. + +Pco uses four principal mechanisms on these columns: + +- Entropy bins on ALP integer children. +- First-order Delta plus bins on raw floats or ALP integer children. +- FloatMult plus bins on Taxi tips. +- IntMult splits on selected integer children. + +No Pcodec paper gap used lookback Delta. Some Public BI columns used lookback Delta. + +The current CMS source revision differs from the paper source snapshot. + +The Twitter source uses the first two million official edges. The paper used an unspecified ID sort. ## General real-float follow-up @@ -256,19 +311,19 @@ Measure gap recovery as `(default bytes - candidate bytes) / (default bytes - Pc Retain a prototype only if it recovers a material gap across unrelated real datasets. -The retained schemes cover lower-precision values in wider floats and locally narrow ordered ranges. +The retained schemes cover lower-precision values in wider floats and locally narrow ordered or integer ranges. -They do not replace ALP-RD for generic non-decimal floats. No measured column moved from ALP to a new scheme. +The multi-reference estimator tested one, two, and four quantile references per block on thirty gap columns. -The first general prototype will use bounded multiple references per 1,024-value block. +The estimate includes reference identifiers, packed residuals, patch positions, patch highs, and block metadata. -Each block will test one, two, or four references. Each value will store a small reference ID and one packed residual. +One reference won 25 columns. Four references won five columns. Two references won no columns. -All references will share one residual width per block. Sparse high-bit patches will contain rare outliers. +The best estimate recovered 29.4 percent of the aggregate Pco size gap. -Direct access will read one reference ID, one residual, and an optional patch. The one-reference form matches `BlockResidualArray`. +It reduced default bytes by 7.9 percent across the gap columns. -This design approximates Pco bins without entropy coding. It also avoids a rank query into separate variable-width bin streams. +The extra references did not justify a new array. Generic integer BlockResidual became the next prototype. Test these secondary candidates after the multi-reference prototype: @@ -277,9 +332,7 @@ Test these secondary candidates after the multi-reference prototype: - Use block-local ALP-RD split widths. - Use independent entropy microblocks only in Compact. -Build a gap corpus from columns where Compact Pco materially beats ALP-RD. - -Classify each gap by exponent count, prefix count, local range, low-bit entropy, and outlier rate. +Classify the other gaps by exponent count, prefix count, local range, low-bit entropy, and outlier rate. Require size, compression, decode, and scalar-access results on the same two-million-row inputs. @@ -358,15 +411,25 @@ This round completed these steps: - Added nonzero-secondary FloatQuant benchmarks. - Added all six Pcodec paper datasets. - Fixed the high-value defects from three adversarial reviews. +- Extended BlockResidual to every integer type. +- Added native-width residual pack and unpack operations. +- Added integer BlockResidual to `single_encoding_throughput`. +- Measured the Pco gap across thirty float columns. +- Rejected the multi-reference residual design. +- Added the outer-sample exclusion and width-specific factors. +- Added the patch-count decode cost. +- Excluded direct 8-bit and 16-bit candidates. +- Excluded BlockResidual from dictionary-code children. Complete these remaining steps: -1. Run the broad Vortex compression corpus after the selector change. +1. Run the complete `bench-vortex` compression corpus. 2. Compare the geometric mean against Parquet with Zstd. -3. Build the Pco-versus-ALP-RD gap corpus. -4. Prototype the bounded multi-reference residual codec. -5. Test a fused nonzero-secondary decode on qualifying gap columns. -6. Update this plan after each experiment. +3. Reduce the analysis cost on short rejected columns. +4. Test a fused nonzero-secondary FloatQuant decode on selected gap columns. +5. Evaluate nonzero-secondary FloatQuant for the default compressor. +6. Investigate new schemes for real floats that Pco compresses better than ALP and ALP-RD. +7. Update this plan after each experiment. ## Pull request structure diff --git a/encodings/block-residual/src/block_residual_array.rs b/encodings/block-residual/src/block_residual_array.rs index 9a1935eecf0..e62667b78d0 100644 --- a/encodings/block-residual/src/block_residual_array.rs +++ b/encodings/block-residual/src/block_residual_array.rs @@ -7,7 +7,6 @@ use std::hash::Hash; use std::hash::Hasher; use std::ops::Range; -use fastlanes::BitPacking; use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; @@ -49,6 +48,8 @@ use vortex_session::registry::CachedId; use crate::BlockResidualCodec; use crate::BlockResidualParts; +use crate::codec::ResidualWord; +use crate::codec::packed_words_as_native; use crate::codec::read_wide_bits; const BLOCK_LEN: usize = 1024; @@ -266,7 +267,30 @@ impl OperationsVTable for BlockResidual { return Ok(Scalar::null(array.dtype().clone())); } let value = scalar_from_array(array, index, ctx)?; - Ok(Scalar::primitive(value, array.dtype().nullability())) + let nullability = array.dtype().nullability(); + Ok(match array.dtype().as_ptype() { + PType::U8 => Scalar::primitive(u8::try_from(value)?, nullability), + PType::U16 => Scalar::primitive(u16::try_from(value)?, nullability), + PType::U32 => Scalar::primitive(u32::try_from(value)?, nullability), + PType::U64 => Scalar::primitive(value, nullability), + PType::I8 => Scalar::primitive( + i8::from_le_bytes([(u8::try_from(value)? ^ (1_u8 << 7))]), + nullability, + ), + PType::I16 => Scalar::primitive( + i16::from_le_bytes((u16::try_from(value)? ^ (1_u16 << 15)).to_le_bytes()), + nullability, + ), + PType::I32 => Scalar::primitive( + i32::from_le_bytes((u32::try_from(value)? ^ (1_u32 << 31)).to_le_bytes()), + nullability, + ), + PType::I64 => Scalar::primitive( + i64::from_le_bytes((value ^ (1_u64 << 63)).to_le_bytes()), + nullability, + ), + ptype => vortex_bail!("BlockResidual scalar access does not support {ptype}"), + }) } } @@ -311,18 +335,65 @@ pub trait BlockResidualArrayExt: TypedArrayRef + BlockResidualArr impl> BlockResidualArrayExt for T {} impl BlockResidual { - /// Encode a non-negative `u64` array in independent blocks. + /// Encode an integer array in independent blocks. pub fn from_primitive(array: ArrayView<'_, Primitive>) -> VortexResult { vortex_ensure!( - array.ptype() == PType::U64, - "BlockResidual requires u64 values" + array.ptype().is_int(), + "BlockResidual requires integer values" ); let validity = array.validity()?; - let parts = BlockResidualCodec::encode(array.as_slice::())?.into_parts()?; - Self::try_new(parts, validity) + let values = match array.ptype() { + PType::U8 => array + .as_slice::() + .iter() + .map(|&value| u64::from(value)) + .collect(), + PType::U16 => array + .as_slice::() + .iter() + .map(|&value| u64::from(value)) + .collect(), + PType::U32 => array + .as_slice::() + .iter() + .map(|&value| u64::from(value)) + .collect(), + PType::U64 => array.as_slice::().to_vec(), + PType::I8 => array + .as_slice::() + .iter() + .map(|&value| u64::from((value as u8) ^ (1_u8 << 7))) + .collect(), + PType::I16 => array + .as_slice::() + .iter() + .map(|&value| u64::from((value as u16) ^ (1_u16 << 15))) + .collect(), + PType::I32 => array + .as_slice::() + .iter() + .map(|&value| u64::from((value as u32) ^ (1_u32 << 31))) + .collect(), + PType::I64 => array + .as_slice::() + .iter() + .map(|&value| (value as u64) ^ (1_u64 << 63)) + .collect(), + ptype => vortex_bail!("BlockResidual does not support {ptype}"), + }; + let parts = BlockResidualCodec::encode_with_word_width( + &values, + u8::try_from(array.ptype().bit_width())?, + )? + .into_parts()?; + Self::try_new(parts, validity, array.ptype()) } - fn try_new(parts: BlockResidualParts, validity: Validity) -> VortexResult { + fn try_new( + parts: BlockResidualParts, + validity: Validity, + ptype: PType, + ) -> VortexResult { let data = BlockResidualData { unsliced_len: parts.len, slice_start: 0, @@ -347,7 +418,7 @@ impl BlockResidual { Array::try_from_parts( ArrayParts::new( BlockResidual, - DType::Primitive(PType::U64, validity.nullability()), + DType::Primitive(ptype, validity.nullability()), data.unsliced_len, data, ) @@ -365,8 +436,8 @@ impl BlockResidualData { validity: &Validity, ) -> VortexResult<()> { vortex_ensure!( - dtype.as_ptype() == PType::U64, - "BlockResidualArray requires a u64 dtype" + dtype.is_int(), + "BlockResidualArray requires an integer dtype" ); vortex_ensure!( self.slice_start <= self.slice_stop && self.slice_stop <= self.unsliced_len, @@ -445,10 +516,10 @@ impl ExecutedBlockResidual { } } -fn decode_array_values( +fn decode_array_values( array: ArrayView<'_, BlockResidual>, ctx: &mut ExecutionCtx, - mut transform: impl FnMut(u64) -> T, + mut transform: impl FnMut(U) -> T, ) -> VortexResult { let children = ExecutedBlockResidual::new(array, ctx)?; let bases = children.bases.as_slice::(); @@ -462,7 +533,7 @@ fn decode_array_values( let patch_highs = children.patch_highs.as_slice::(); let logical_range = array.data().slice_start..array.data().slice_stop; let mut values = Vec::with_capacity(logical_range.len()); - let mut residuals = [0_u64; BLOCK_LEN]; + let mut residuals = [U::default(); BLOCK_LEN]; let first_block = logical_range.start / BLOCK_LEN; let last_block = logical_range.end.div_ceil(BLOCK_LEN); @@ -476,10 +547,11 @@ fn decode_array_values( let residual_width = residual_widths[block_index]; let high_width = high_widths[block_index]; + let base = U::from_u64(bases[block_index]); vortex_ensure!( - residual_width <= 64 - && high_width <= 64 - && u16::from(residual_width) + u16::from(high_width) <= 64, + residual_width <= U::BITS + && high_width <= U::BITS + && u16::from(residual_width) + u16::from(high_width) <= u16::from(U::BITS), "block residual bit widths are invalid" ); let residual_payload = payload_range( @@ -492,45 +564,62 @@ fn decode_array_values( residual_payload.len() == BLOCK_LEN * usize::from(residual_width) / 64, "block residual word count is invalid" ); - if residual_width > 0 { - // SAFETY: The encoder writes one complete FastLanes chunk for each block. - unsafe { - u64::unchecked_unpack( - usize::from(residual_width), - &residual_words[residual_payload], - &mut residuals, - ); - } - } else { - residuals.fill(0); - } - let patch_payload = payload_range(patch_starts, block_index, patch_positions.len(), "patch")?; let high_payload = payload_range(high_starts, block_index, patch_highs.len(), "patch high")?; let positions = &patch_positions[patch_payload]; - validate_patch_payload( - block_len, + validate_patch_header( residual_width, high_width, - positions, + positions.len(), high_payload.len(), )?; let highs = &patch_highs[high_payload]; + let local_start = logical_range.start.saturating_sub(block_start); + let local_stop = (logical_range.end - block_start).min(block_len); + + if residual_width == 0 { + let output_start = values.len(); + values + .extend(std::iter::repeat_with(|| transform(base)).take(local_stop - local_start)); + let mut previous_position = None; + for (patch_index, &position) in positions.iter().enumerate() { + validate_patch_position(block_len, previous_position, position)?; + previous_position = Some(position); + let position = usize::from(position); + if position < local_start || position >= local_stop { + continue; + } + // SAFETY: The payload includes fifteen readable padding bytes. + let high = unsafe { + read_wide_bits(highs, patch_index * usize::from(high_width), high_width) + }; + values[output_start + position - local_start] = + transform(base.wrapping_add(U::from_u64(high))); + } + continue; + } + + let packed = packed_words_as_native::(&residual_words[residual_payload]); + // SAFETY: The encoder writes one complete FastLanes chunk for each block. + unsafe { + U::unchecked_unpack(usize::from(residual_width), packed, &mut residuals); + } + let mut previous_position = None; for (patch_index, &position) in positions.iter().enumerate() { + validate_patch_position(block_len, previous_position, position)?; + previous_position = Some(position); // SAFETY: The payload includes fifteen readable padding bytes. let high = unsafe { read_wide_bits(highs, patch_index * usize::from(high_width), high_width) }; - residuals[usize::from(position)] |= high << residual_width; + residuals[usize::from(position)].apply_high(high, residual_width); } - let local_start = logical_range.start.saturating_sub(block_start); - let local_stop = (logical_range.end - block_start).min(block_len); values.extend( residuals[local_start..local_stop] .iter() - .map(|&residual| transform(bases[block_index].wrapping_add(residual))), + .map(|&residual| transform(residual.wrapping_add(base))), ); } Ok(PrimitiveArray::new(Buffer::from(values), array.validity()?)) @@ -540,14 +629,24 @@ fn decompress_array( array: ArrayView<'_, BlockResidual>, ctx: &mut ExecutionCtx, ) -> VortexResult { - decode_array_values(array, ctx, |value| value) + match array.dtype().as_ptype() { + PType::U8 => decode_array_values(array, ctx, |value: u8| value), + PType::U16 => decode_array_values(array, ctx, |value: u16| value), + PType::U32 => decode_array_values(array, ctx, |value: u32| value), + PType::U64 => decode_array_values(array, ctx, |value: u64| value), + PType::I8 => decode_array_values(array, ctx, |value: u8| (value ^ (1_u8 << 7)) as i8), + PType::I16 => decode_array_values(array, ctx, |value: u16| (value ^ (1_u16 << 15)) as i16), + PType::I32 => decode_array_values(array, ctx, |value: u32| (value ^ (1_u32 << 31)) as i32), + PType::I64 => decode_array_values(array, ctx, |value: u64| (value ^ (1_u64 << 63)) as i64), + ptype => vortex_bail!("BlockResidual decode does not support {ptype}"), + } } pub(crate) fn decompress_ordered_f64( array: ArrayView<'_, BlockResidual>, ctx: &mut ExecutionCtx, ) -> VortexResult { - decode_array_values(array, ctx, |ordered| { + decode_array_values(array, ctx, |ordered: u64| { let bits = if ordered & (1_u64 << 63) == 0 { !ordered } else { @@ -568,10 +667,11 @@ fn scalar_from_array( let children = ExecutedBlockResidual::new(array, ctx)?; let residual_width = children.residual_widths.as_slice::()[block_index]; let high_width = children.high_widths.as_slice::()[block_index]; + let logical_width = array.dtype().as_ptype().bit_width(); vortex_ensure!( - residual_width <= 64 - && high_width <= 64 - && u16::from(residual_width) + u16::from(high_width) <= 64, + usize::from(residual_width) <= logical_width + && usize::from(high_width) <= logical_width + && usize::from(residual_width) + usize::from(high_width) <= logical_width, "block residual bit widths are invalid" ); let residual_words = children.residual_words.as_slice::(); @@ -585,17 +685,28 @@ fn scalar_from_array( residual_payload.len() == BLOCK_LEN * usize::from(residual_width) / 64, "block residual word count is invalid" ); - let mut residual = if residual_width == 0 { - 0 - } else { - // SAFETY: The encoder writes one complete FastLanes chunk for each block. - unsafe { - u64::unchecked_unpack_single( - usize::from(residual_width), - &residual_words[residual_payload], - index_in_block, - ) - } + let mut residual = match logical_width { + 8 => unpack_single_residual::( + residual_width, + &residual_words[residual_payload], + index_in_block, + ), + 16 => unpack_single_residual::( + residual_width, + &residual_words[residual_payload], + index_in_block, + ), + 32 => unpack_single_residual::( + residual_width, + &residual_words[residual_payload], + index_in_block, + ), + 64 => unpack_single_residual::( + residual_width, + &residual_words[residual_payload], + index_in_block, + ), + _ => vortex_bail!("block residual logical bit width is invalid"), }; let positions = children.patch_positions.as_slice::(); @@ -637,28 +748,50 @@ fn scalar_from_array( Ok(children.bases.as_slice::()[block_index].wrapping_add(residual)) } +fn unpack_single_residual(width: u8, packed_words: &[u64], index: usize) -> u64 { + if width == 0 { + return 0; + } + let packed = packed_words_as_native::(packed_words); + // SAFETY: The encoder writes one complete FastLanes chunk for each block. + unsafe { T::unchecked_unpack_single(usize::from(width), packed, index).to_u64() } +} + fn validate_patch_payload( block_len: usize, residual_width: u8, high_width: u8, positions: &[u16], high_payload_len: usize, +) -> VortexResult<()> { + validate_patch_header( + residual_width, + high_width, + positions.len(), + high_payload_len, + )?; + let mut previous_position = None; + for &position in positions { + validate_patch_position(block_len, previous_position, position)?; + previous_position = Some(position); + } + Ok(()) +} + +fn validate_patch_header( + residual_width: u8, + high_width: u8, + patch_count: usize, + high_payload_len: usize, ) -> VortexResult<()> { vortex_ensure!( - positions - .iter() - .all(|&position| usize::from(position) < block_len) - && positions.windows(2).all(|pair| pair[0] < pair[1]), - "block residual patch positions are invalid" - ); - vortex_ensure!( - positions.is_empty() || (high_width > 0 && residual_width < 64), + patch_count == 0 || (high_width > 0 && residual_width < 64), "block residual patches require nonzero high bits" ); - let expected_high_len = if positions.is_empty() { + let expected_high_len = if patch_count == 0 { 0 } else { - (positions.len() * usize::from(high_width)).div_ceil(8) + 15 + (patch_count * usize::from(high_width)).div_ceil(8) + 15 }; vortex_ensure!( high_payload_len == expected_high_len, @@ -667,6 +800,19 @@ fn validate_patch_payload( Ok(()) } +fn validate_patch_position( + block_len: usize, + previous_position: Option, + position: u16, +) -> VortexResult<()> { + vortex_ensure!( + usize::from(position) < block_len + && previous_position.is_none_or(|previous| previous < position), + "block residual patch positions are invalid" + ); + Ok(()) +} + fn payload_range( starts: &[u32], block_index: usize, @@ -778,6 +924,7 @@ mod tests { use vortex_session::registry::ReadContext; use super::BlockResidual; + use super::BlockResidualArraySlotsExt; #[test] fn roundtrip_and_scalar_access() -> VortexResult<()> { @@ -800,6 +947,55 @@ mod tests { Ok(()) } + #[test] + fn signed_roundtrip_and_scalar_access() -> VortexResult<()> { + let values = (0..2_050) + .map(|index| match index { + 0 => i64::MIN, + 1_023 => -1, + 1_024 => 0, + 2_049 => i64::MAX, + _ => (index as i64 - 1_025) * 1_000_003, + }) + .collect::>(); + let primitive = PrimitiveArray::from_iter(values.clone()); + let encoded = BlockResidual::from_primitive(primitive.as_view())?; + let session = array_session(); + crate::initialize(&session); + let mut ctx = session.create_execution_ctx(); + + assert_arrays_eq!(encoded.clone(), primitive.into_array(), &mut ctx); + for index in [0, 1, 1_023, 1_024, 2_049] { + let scalar = encoded.execute_scalar(index, &mut ctx)?; + assert_eq!( + scalar.as_primitive().typed_value::(), + Some(values[index]) + ); + } + Ok(()) + } + + #[test] + fn narrow_integer_roundtrip() -> VortexResult<()> { + let signed = PrimitiveArray::from_iter((0..2_050).map(|index| { + let value = (index * 7919) % 65_521; + value - 32_760 + })); + let unsigned = PrimitiveArray::from_iter((0..2_050).map(|index| { + let value = (index * 7907) % 65_521; + value as u16 + })); + let signed_encoded = BlockResidual::from_primitive(signed.as_view())?; + let unsigned_encoded = BlockResidual::from_primitive(unsigned.as_view())?; + let session = array_session(); + crate::initialize(&session); + let mut ctx = session.create_execution_ctx(); + + assert_arrays_eq!(signed_encoded, signed.into_array(), &mut ctx); + assert_arrays_eq!(unsigned_encoded, unsigned.into_array(), &mut ctx); + Ok(()) + } + #[test] fn nullable_slice_and_scalar_access() -> VortexResult<()> { let values = (0..2_050) @@ -819,6 +1015,38 @@ mod tests { Ok(()) } + #[test] + fn zero_width_patched_block_roundtrip() -> VortexResult<()> { + let mut values = vec![42_u32; 2_050]; + values[1_023] = u32::MAX; + let validity = Validity::from_iter((0..values.len()).map(|index| index != 1_024)); + let primitive = PrimitiveArray::new(Buffer::from(values.clone()), validity); + let encoded = BlockResidual::from_primitive(primitive.as_view())?; + let session = array_session(); + crate::initialize(&session); + let mut ctx = session.create_execution_ctx(); + let widths = encoded + .residual_widths() + .clone() + .execute::(&mut ctx)?; + + assert_eq!(widths.as_slice::()[0], 0); + assert_eq!( + encoded + .execute_scalar(1_023, &mut ctx)? + .as_primitive() + .typed_value::(), + Some(u32::MAX) + ); + assert!(encoded.execute_scalar(1_024, &mut ctx)?.is_null()); + assert_arrays_eq!( + encoded.into_array().slice(1_022..1_025)?, + primitive.into_array().slice(1_022..1_025)?, + &mut ctx + ); + Ok(()) + } + #[test] fn nullable_slice_serialization_roundtrip() -> VortexResult<()> { let values = (0..2_050) diff --git a/encodings/block-residual/src/codec.rs b/encodings/block-residual/src/codec.rs index d0b9a2c5289..1c0aa74dede 100644 --- a/encodings/block-residual/src/codec.rs +++ b/encodings/block-residual/src/codec.rs @@ -6,6 +6,7 @@ use vortex_error::VortexResult; const CHUNK_LEN: usize = 1024; const HIGH_PADDING: usize = 15; +const PATCH_DECODE_PENALTY_BITS: usize = 16; const SERIALIZED_BLOCK_METADATA_BYTES: usize = 12; /// Block-local residual codec for ordered unsigned latents. @@ -44,9 +45,17 @@ struct BlockResidualBlock { impl BlockResidualCodec { /// Encode ordered unsigned latents in independent 1024-value blocks. pub fn encode(values: &[u64]) -> VortexResult { + Self::encode_with_word_width(values, 64) + } + + pub(crate) fn encode_with_word_width(values: &[u64], word_width: u8) -> VortexResult { + vortex_error::vortex_ensure!( + matches!(word_width, 8 | 16 | 32 | 64), + "block residual word width is invalid" + ); let blocks = values .chunks(CHUNK_LEN) - .map(encode_block) + .map(|block| encode_block(block, word_width)) .collect::>>()?; Ok(Self { len: values.len(), @@ -342,7 +351,7 @@ impl BlockResidualCodec { } } -fn encode_block(values: &[u64]) -> VortexResult { +fn encode_block(values: &[u64], word_width: u8) -> VortexResult { let base = values.iter().copied().min().unwrap_or(0); let mut residuals = Vec::with_capacity(CHUNK_LEN); let mut width_counts = [0usize; 65]; @@ -367,6 +376,7 @@ fn encode_block(values: &[u64]) -> VortexResult { }; let cost_bits = usize::from(residual_width) * CHUNK_LEN + patch_count * (u16::BITS as usize + usize::from(high_width)) + + patch_count * PATCH_DECODE_PENALTY_BITS + u64::BITS as usize + SERIALIZED_BLOCK_METADATA_BYTES * 8 + usize::from(patch_count > 0) * HIGH_PADDING * 8; @@ -384,6 +394,7 @@ fn encode_block(values: &[u64]) -> VortexResult { residuals, patch_count: best.3, }, + word_width, ) } @@ -416,14 +427,18 @@ struct BlockPlan { patch_count: usize, } -fn materialize_block(values: &[u64], plan: BlockPlan) -> VortexResult { +fn materialize_block( + values: &[u64], + plan: BlockPlan, + word_width: u8, +) -> VortexResult { let residual_mask = low_mask(plan.residual_width); let low_residuals = plan .residuals .iter() .map(|&residual| residual & residual_mask) .collect::>(); - let residuals = fast_pack(&low_residuals, plan.residual_width); + let residuals = fast_pack(&low_residuals, plan.residual_width, word_width); let mut patch_positions = Vec::with_capacity(plan.patch_count); let mut patch_highs = BitWriter::with_capacity(plan.patch_count * 8); if plan.high_width > 0 { @@ -454,14 +469,85 @@ fn materialize_block(values: &[u64], plan: BlockPlan) -> VortexResult Vec { +fn fast_pack(values: &[u64], width: u8, word_width: u8) -> Vec { if width == 0 { return Vec::new(); } - let mut packed = vec![0u64; CHUNK_LEN * usize::from(width) / u64::BITS as usize]; + match word_width { + 8 => fast_pack_native::(values, width), + 16 => fast_pack_native::(values, width), + 32 => fast_pack_native::(values, width), + 64 => fast_pack_native::(values, width), + _ => unreachable!("validated block residual word width"), + } +} + +fn fast_pack_native(values: &[u64], width: u8) -> Vec { + let mut packed_words = vec![0u64; CHUNK_LEN * usize::from(width) / u64::BITS as usize]; + let unpacked = values.iter().copied().map(T::from_u64).collect::>(); + let packed_native = packed_words_as_native_mut::(&mut packed_words); // SAFETY: Both slices have the exact lengths required for one FastLanes chunk. - unsafe { u64::unchecked_pack(usize::from(width), values, &mut packed) }; - packed + unsafe { T::unchecked_pack(usize::from(width), &unpacked, packed_native) }; + packed_words +} + +pub(crate) trait ResidualWord: BitPacking + Copy + Default { + const BITS: u8; + + fn from_u64(value: u64) -> Self; + + fn to_u64(self) -> u64; + + fn wrapping_add(self, other: Self) -> Self; + + fn apply_high(&mut self, high: u64, shift: u8); +} + +macro_rules! impl_residual_word { + ($T:ty, $bits:literal) => { + impl ResidualWord for $T { + const BITS: u8 = $bits; + + #[allow(clippy::cast_possible_truncation)] + fn from_u64(value: u64) -> Self { + value as $T + } + + fn to_u64(self) -> u64 { + u64::from(self) + } + + fn wrapping_add(self, other: Self) -> Self { + self.wrapping_add(other) + } + + #[allow(clippy::cast_possible_truncation)] + fn apply_high(&mut self, high: u64, shift: u8) { + *self |= (high as $T) << shift; + } + } + }; +} + +impl_residual_word!(u8, 8); +impl_residual_word!(u16, 16); +impl_residual_word!(u32, 32); +impl_residual_word!(u64, 64); + +pub(crate) fn packed_words_as_native(words: &[u64]) -> &[T] { + // SAFETY: Unsigned integer types permit every bit pattern. A `u64` slice has sufficient + // alignment, and packed FastLanes payloads always contain a whole number of bytes. + let (prefix, native, suffix) = unsafe { words.align_to::() }; + debug_assert!(prefix.is_empty() && suffix.is_empty()); + native +} + +fn packed_words_as_native_mut(words: &mut [u64]) -> &mut [T] { + // SAFETY: Unsigned integer types permit every bit pattern. A `u64` slice has sufficient + // alignment, and packed FastLanes payloads always contain a whole number of bytes. + let (prefix, native, suffix) = unsafe { words.align_to_mut::() }; + debug_assert!(prefix.is_empty() && suffix.is_empty()); + native } fn bit_width(value: u64) -> u8 { @@ -614,4 +700,15 @@ mod tests { assert!(BlockResidualCodec::try_from_parts(parts).is_err()); Ok(()) } + + #[test] + fn patch_penalty_prefers_dense_residuals() -> VortexResult<()> { + let mut values = vec![0_u64; 1_024]; + values[..307].fill(4_095); + let parts = BlockResidualCodec::encode(&values)?.into_parts()?; + + assert_eq!(parts.residual_widths, [12]); + assert!(parts.patch_positions.is_empty()); + Ok(()) + } } diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index e6b07bfc63b..1594225cdf3 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -15,6 +15,13 @@ use arrow_schema::DataType; use arrow_select::concat::concat; use parquet::arrow::ProjectionMask; use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; +use pco::ChunkConfig; +use pco::data_types::Number; +use pco::metadata::ChunkLatentVarMeta; +use pco::metadata::DeltaEncoding; +use pco::metadata::DynBins; +use pco::metadata::Mode; +use pco::wrapped::FileCompressor; use rand::RngExt; use rand::SeedableRng; use rand::rngs::StdRng; @@ -28,11 +35,14 @@ use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::PType; use vortex_arrow::ArrowSessionExt; +use vortex_block_residual::BlockResidual; +use vortex_block_residual::BlockResidualArraySlotsExt; use vortex_btrblocks::BtrBlocksCompressor; use vortex_btrblocks::BtrBlocksCompressorBuilder; use vortex_btrblocks::SchemeExt; use vortex_btrblocks::schemes::float::FloatQuantScheme; use vortex_btrblocks::schemes::float::OrderedBlockResidualScheme; +use vortex_btrblocks::schemes::integer::BlockResidualScheme; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_err; @@ -257,6 +267,58 @@ fn encoding_tree(array: &ArrayRef) -> String { format!("{}({children})", array.encoding_id()) } +fn profile_block_residual_array( + dataset: &str, + column: &str, + config: &str, + path: &str, + array: &ArrayRef, + session: &VortexSession, +) -> VortexResult<()> { + if let Some(residuals) = array.as_typed::() { + let mut ctx = session.create_execution_ctx(); + let residual_widths = residuals + .residual_widths() + .clone() + .execute::(&mut ctx)?; + let high_widths = residuals + .high_widths() + .clone() + .execute::(&mut ctx)?; + let blocks = residual_widths.len(); + let average_residual_width = residual_widths + .as_slice::() + .iter() + .map(|&width| f64::from(width)) + .sum::() + / blocks as f64; + let average_high_width = high_widths + .as_slice::() + .iter() + .map(|&width| f64::from(width)) + .sum::() + / blocks as f64; + println!( + "block-residual-profile\t{dataset}\t{column}\t{config}\t{path}\t{}\t{}\t{}\t{blocks}\t{average_residual_width:.3}\t{average_high_width:.3}\t{}", + array.dtype().as_ptype(), + array.len(), + residuals.patch_positions().len(), + array.nbytes(), + ); + } + for (child_index, child) in array.children().iter().enumerate() { + profile_block_residual_array( + dataset, + column, + config, + &format!("{path}/{child_index}"), + child, + session, + )?; + } + Ok(()) +} + fn encode_all( compressor: &BtrBlocksCompressor, columns: &[Column], @@ -284,6 +346,312 @@ fn percentile(durations: &mut [Duration], numerator: usize, denominator: usize) durations[durations.len() * numerator / denominator] } +struct BinProfile { + count: usize, + ans_size_log: u32, + max_offset_bits: u32, + average_bits: f64, +} + +fn bin_profile(meta: &ChunkLatentVarMeta) -> BinProfile { + pco::match_latent_enum!(&meta.bins, DynBins(bins) => { + let total_weight = (1_u64 << meta.ans_size_log) as f64; + let average_bits = bins + .iter() + .map(|bin| { + let ans_bits = f64::from(meta.ans_size_log) - f64::from(bin.weight).log2(); + (ans_bits + f64::from(bin.offset_bits)) * f64::from(bin.weight) / total_weight + }) + .sum(); + BinProfile { + count: bins.len(), + ans_size_log: meta.ans_size_log, + max_offset_bits: bins + .iter() + .map(|bin| bin.offset_bits) + .max() + .unwrap_or_default(), + average_bits, + } + }) +} + +fn optional_bin_profile(meta: Option<&ChunkLatentVarMeta>) -> BinProfile { + meta.map(bin_profile).unwrap_or(BinProfile { + count: 0, + ans_size_log: 0, + max_offset_bits: 0, + average_bits: 0.0, + }) +} + +fn mode_name(mode: &Mode) -> String { + match mode { + Mode::Classic => "classic".to_string(), + Mode::IntMult(_) => "int-mult".to_string(), + Mode::FloatMult(_) => "float-mult".to_string(), + Mode::FloatQuant(k) => format!("float-quant-{k}"), + Mode::Dict(_) => "dict".to_string(), + _ => "unknown".to_string(), + } +} + +fn delta_name(delta: &DeltaEncoding) -> String { + match delta { + DeltaEncoding::NoOp => "none".to_string(), + DeltaEncoding::Consecutive { + order, + secondary_uses_delta, + } => format!("consecutive-{order}-secondary-{secondary_uses_delta}"), + DeltaEncoding::Lookback { + config, + secondary_uses_delta, + } => format!( + "lookback-state-{}-window-{}-secondary-{secondary_uses_delta}", + config.state_n_log, config.window_n_log + ), + DeltaEncoding::Conv1(config) => format!("conv1-quantization-{}", config.quantization), + _ => "unknown".to_string(), + } +} + +fn profile_pco_values( + dataset: &str, + column: &str, + path: &str, + ptype: PType, + values: &[T], +) -> VortexResult<()> { + let file_compressor = FileCompressor::default(); + for (chunk_index, chunk) in values.chunks(pco::DEFAULT_MAX_PAGE_N).enumerate() { + let compressor = file_compressor + .chunk_compressor(chunk, &ChunkConfig::default()) + .map_err(|error| vortex_err!("cannot profile Pco chunk: {error}"))?; + let meta = compressor.meta(); + let delta = optional_bin_profile(meta.per_latent_var.delta.as_ref()); + let primary = bin_profile(&meta.per_latent_var.primary); + let secondary = optional_bin_profile(meta.per_latent_var.secondary.as_ref()); + println!( + "pco-profile\t{dataset}\t{column}\t{path}\t{ptype}\t{chunk_index}\t{}\t{}\t{}\t{}\t{}\t{:.3}\t{}\t{}\t{}\t{:.3}\t{}\t{}\t{}\t{:.3}", + chunk.len(), + mode_name(&meta.mode), + delta_name(&meta.delta_encoding), + delta.count, + delta.max_offset_bits, + delta.average_bits, + primary.count, + primary.ans_size_log, + primary.max_offset_bits, + primary.average_bits, + secondary.count, + secondary.ans_size_log, + secondary.max_offset_bits, + secondary.average_bits, + ); + } + Ok(()) +} + +fn reference_id_bits(reference_count: usize) -> usize { + match reference_count { + 0 | 1 => 0, + 2 => 1, + _ => 2, + } +} + +fn estimate_multi_reference_block( + values: &[u64], + requested_references: usize, + bits: usize, +) -> usize { + const BLOCK_LEN: usize = 1024; + const METADATA_BYTES: usize = 12; + const HIGH_PADDING_BYTES: usize = 15; + + let mut sorted = values.to_vec(); + sorted.sort_unstable(); + let mut references = (0..requested_references) + .map(|index| sorted[index * sorted.len() / requested_references]) + .collect::>(); + references.dedup(); + + let mut width_counts = [0_usize; 65]; + let mut maximum_width = 0_usize; + for &value in values { + let reference_index = references + .partition_point(|&reference| reference <= value) + .saturating_sub(1); + let residual = value - references[reference_index]; + let width = u64::BITS as usize - residual.leading_zeros() as usize; + width_counts[width] += 1; + maximum_width = maximum_width.max(width); + } + + let mut patch_count = values.len(); + let mut best_bits = usize::MAX; + for residual_width in 0..=maximum_width { + patch_count -= width_counts[residual_width]; + let high_width = if patch_count == 0 { + 0 + } else { + maximum_width - residual_width + }; + let cost_bits = residual_width * BLOCK_LEN + + reference_id_bits(references.len()) * BLOCK_LEN + + references.len() * bits + + patch_count * (u16::BITS as usize + high_width) + + METADATA_BYTES * 8 + + usize::from(patch_count > 0) * HIGH_PADDING_BYTES * 8; + best_bits = best_bits.min(cost_bits); + } + best_bits.div_ceil(8) +} + +fn estimate_multi_reference(values: &[u64], requested_references: usize, bits: usize) -> usize { + values + .chunks(1024) + .map(|block| estimate_multi_reference_block(block, requested_references, bits)) + .sum() +} + +fn ordered_f32(value: f32) -> u64 { + let bits = value.to_bits(); + u64::from(if bits & (1_u32 << 31) == 0 { + bits ^ (1_u32 << 31) + } else { + !bits + }) +} + +fn ordered_f64(value: f64) -> u64 { + let bits = value.to_bits(); + if bits & (1_u64 << 63) == 0 { + bits ^ (1_u64 << 63) + } else { + !bits + } +} + +fn ordered_values(values: &PrimitiveArray) -> Vec { + match values.ptype() { + PType::F32 => values + .as_slice::() + .iter() + .copied() + .map(ordered_f32) + .collect(), + PType::F64 => values + .as_slice::() + .iter() + .copied() + .map(ordered_f64) + .collect(), + PType::I16 => values + .as_slice::() + .iter() + .map(|&value| u64::from((value as u16) ^ (1_u16 << 15))) + .collect(), + PType::I32 => values + .as_slice::() + .iter() + .map(|&value| u64::from((value as u32) ^ (1_u32 << 31))) + .collect(), + PType::I64 => values + .as_slice::() + .iter() + .map(|&value| (value as u64) ^ (1_u64 << 63)) + .collect(), + PType::U16 => values + .as_slice::() + .iter() + .map(|&value| u64::from(value)) + .collect(), + PType::U32 => values + .as_slice::() + .iter() + .map(|&value| u64::from(value)) + .collect(), + PType::U64 => values.as_slice::().to_vec(), + ptype => unreachable!("Pco does not support {ptype}"), + } +} + +fn profile_pco_array( + dataset: &str, + column: &str, + path: &str, + array: &ArrayRef, + session: &VortexSession, +) -> VortexResult<()> { + if array.encoding_id().as_ref() == "vortex.pco" { + let mut ctx = session.create_execution_ctx(); + let primitive = array.clone().execute::(&mut ctx)?; + let primitive_array = primitive.clone().into_array(); + let mask = primitive_array + .validity()? + .execute_mask(primitive.len(), &mut ctx)?; + let values = primitive_array + .filter(mask)? + .execute::(&mut ctx)?; + let ordered = ordered_values(&values); + let validity_bytes = array.children().iter().map(ArrayRef::nbytes).sum::(); + let bits = values.ptype().bit_width(); + let one_reference = + u64::try_from(estimate_multi_reference(&ordered, 1, bits))? + validity_bytes; + let two_references = + u64::try_from(estimate_multi_reference(&ordered, 2, bits))? + validity_bytes; + let four_references = + u64::try_from(estimate_multi_reference(&ordered, 4, bits))? + validity_bytes; + println!( + "multi-ref-estimate\t{dataset}\t{column}\t{path}\t{}\t{}\t{}\t{}\t{}\t{}", + values.ptype(), + array.nbytes(), + one_reference, + two_references, + four_references, + one_reference.min(two_references).min(four_references), + ); + match values.ptype() { + PType::F32 => { + profile_pco_values(dataset, column, path, PType::F32, values.as_slice::())? + } + PType::F64 => { + profile_pco_values(dataset, column, path, PType::F64, values.as_slice::())? + } + PType::I16 => { + profile_pco_values(dataset, column, path, PType::I16, values.as_slice::())? + } + PType::I32 => { + profile_pco_values(dataset, column, path, PType::I32, values.as_slice::())? + } + PType::I64 => { + profile_pco_values(dataset, column, path, PType::I64, values.as_slice::())? + } + PType::U16 => { + profile_pco_values(dataset, column, path, PType::U16, values.as_slice::())? + } + PType::U32 => { + profile_pco_values(dataset, column, path, PType::U32, values.as_slice::())? + } + PType::U64 => { + profile_pco_values(dataset, column, path, PType::U64, values.as_slice::())? + } + ptype => return Err(vortex_err!("cannot profile Pco ptype {ptype}")), + } + } + for (child_index, child) in array.children().iter().enumerate() { + profile_pco_array( + dataset, + column, + &format!("{path}/{child_index}"), + child, + session, + )?; + } + Ok(()) +} + fn measure_dataset( dataset: &str, columns: &[Column], @@ -305,11 +673,57 @@ fn measure_dataset( for (column, array) in columns.iter().zip(arrays) { assert_arrays_eq!(array, column.array, &mut session.create_execution_ctx()); println!( - "structure\t{dataset}\t{}\t{config}\t{}\t{}", + "structure\t{dataset}\t{}\t{}\t{config}\t{}\t{}", column.name, + column.primitive.ptype(), encoding_tree(array), array.nbytes() ); + if *config == "integer-block-residual-only" { + profile_block_residual_array( + dataset, + &column.name, + config, + "root", + array, + session, + )?; + } + } + } + + let prior_default = encoded + .iter() + .find(|(config, _)| *config == "prior-default") + .ok_or_else(|| vortex_err!("prior-default configuration is missing"))?; + let prior_compact = encoded + .iter() + .find(|(config, _)| *config == "prior-compact") + .ok_or_else(|| vortex_err!("prior-compact configuration is missing"))?; + for (column_index, column) in columns.iter().enumerate() { + if !column.primitive.ptype().is_float() { + continue; + } + let default_array = &prior_default.1[column_index]; + let compact_array = &prior_compact.1[column_index]; + let input_bytes = column.primitive.len() * column.primitive.ptype().byte_width(); + let default_bytes = default_array.nbytes(); + let compact_bytes = compact_array.nbytes(); + let compact_savings = if default_bytes == 0 { + 0.0 + } else { + 100.0 * (1.0 - compact_bytes as f64 / default_bytes as f64) + }; + println!( + "float-column\t{dataset}\t{}\t{}\t{}\t{input_bytes}\t{default_bytes}\t{compact_bytes}\t{compact_savings:.3}\t{}\t{}", + column.name, + column.primitive.ptype(), + column.primitive.len(), + encoding_tree(default_array), + encoding_tree(compact_array), + ); + if compact_bytes * 10 <= default_bytes * 9 { + profile_pco_array(dataset, &column.name, "root", compact_array, session)?; } } @@ -354,7 +768,11 @@ fn measure_dataset( } fn compressors() -> Vec<(&'static str, BtrBlocksCompressor)> { - let new_scheme_ids = [FloatQuantScheme.id(), OrderedBlockResidualScheme.id()]; + let new_scheme_ids = [ + FloatQuantScheme.id(), + OrderedBlockResidualScheme.id(), + BlockResidualScheme.id(), + ]; vec![ ( "prior-default", @@ -365,13 +783,19 @@ fn compressors() -> Vec<(&'static str, BtrBlocksCompressor)> { ( "float-quant-only", BtrBlocksCompressorBuilder::default() - .exclude_schemes([OrderedBlockResidualScheme.id()]) + .exclude_schemes([OrderedBlockResidualScheme.id(), BlockResidualScheme.id()]) .build(), ), ( "ordered-block-residual-only", BtrBlocksCompressorBuilder::default() - .exclude_schemes([FloatQuantScheme.id()]) + .exclude_schemes([FloatQuantScheme.id(), BlockResidualScheme.id()]) + .build(), + ), + ( + "integer-block-residual-only", + BtrBlocksCompressorBuilder::default() + .exclude_schemes([FloatQuantScheme.id(), OrderedBlockResidualScheme.id()]) .build(), ), ( @@ -405,8 +829,20 @@ fn main() -> VortexResult<()> { let session = array_session(); let configs = compressors(); - println!("structure\tdataset\tcolumn\tconfig\tencoding\tbytes"); + println!("structure\tdataset\tcolumn\tptype\tconfig\tencoding\tbytes"); + println!( + "float-column\tdataset\tcolumn\tptype\trows\tinput-bytes\tdefault-bytes\tcompact-bytes\tcompact-savings-pct\tdefault-encoding\tcompact-encoding" + ); + println!( + "pco-profile\tdataset\tcolumn\tpath\tptype\tchunk\trows\tmode\tdelta\tdelta-bins\tdelta-max-offset-bits\tdelta-average-bits\tprimary-bins\tprimary-ans-log\tprimary-max-offset-bits\tprimary-average-bits\tsecondary-bins\tsecondary-ans-log\tsecondary-max-offset-bits\tsecondary-average-bits" + ); + println!( + "multi-ref-estimate\tdataset\tcolumn\tpath\tptype\tpco-child-bytes\tone-reference-bytes\ttwo-reference-bytes\tfour-reference-bytes\tbest-bytes" + ); println!("result\tdataset\tconfig\trows\tinput-bytes\tencoded-bytes\tencode-MB/s\tdecode-MB/s"); + println!( + "block-residual-profile\tdataset\tcolumn\tconfig\tpath\tptype\trows\tpatches\tblocks\taverage-residual-width\taverage-high-width\tbytes" + ); if std::env::var_os("VORTEX_BENCH_SKIP_SYNTHETIC").is_none() { for (dataset, columns) in synthetic_datasets(row_count) { measure_dataset(&dataset, &columns, &configs, &session)?; diff --git a/vortex-btrblocks/src/builder.rs b/vortex-btrblocks/src/builder.rs index 3020d9ae348..90f8fb00adc 100644 --- a/vortex-btrblocks/src/builder.rs +++ b/vortex-btrblocks/src/builder.rs @@ -31,6 +31,7 @@ pub const ALL_SCHEMES: &[&dyn Scheme] = &[ // NOTE: ZigZag should precede BitPacking because we don't want negative numbers. &integer::ZigZagScheme, &integer::BitPackingScheme, + &integer::BlockResidualScheme, &integer::SparseScheme, &integer::IntDictScheme, &integer::RunEndScheme, diff --git a/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs b/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs index a9bdd1bb0c9..33200c3d65d 100644 --- a/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs +++ b/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs @@ -17,7 +17,6 @@ use vortex_block_residual::OrderedFloat; use vortex_block_residual::OrderedFloatArraySlotsExt; use vortex_compressor::scheme::CompressionEstimate; use vortex_compressor::scheme::DeferredEstimate; -use vortex_compressor::scheme::EstimateScore; use vortex_compressor::scheme::EstimateVerdict; use vortex_error::VortexResult; @@ -29,7 +28,7 @@ use crate::Scheme; const BLOCK_LEN: usize = 1024; const ESTIMATE_BLOCKS: usize = 8; const MIN_COMPRESSION_RATIO: f64 = 1.05; -const MIN_WIN_OVER_INCUMBENT: f64 = 1.02; +const DECODE_COST_FACTOR: f64 = 1.02; /// Compress `f64` values as block-local residuals of ordered IEEE bits. #[derive(Debug, Copy, Clone, PartialEq, Eq)] @@ -62,7 +61,7 @@ impl Scheme for OrderedBlockResidualScheme { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |_compressor, data, best_so_far, _compress_ctx, exec_ctx| { + |_compressor, data, _best_so_far, _compress_ctx, exec_ctx| { let sample = locality_sample(data.array_as_primitive(), exec_ctx)?; let before_nbytes = sample.nbytes(); let ordered = OrderedFloat::from_primitive(sample.as_view())?; @@ -77,11 +76,11 @@ impl Scheme for OrderedBlockResidualScheme { if ratio < MIN_COMPRESSION_RATIO { return Ok(EstimateVerdict::Skip); } - let incumbent = best_so_far.and_then(EstimateScore::finite_ratio); - if incumbent.is_some_and(|best| ratio < best * MIN_WIN_OVER_INCUMBENT) { + let adjusted_ratio = ratio / DECODE_COST_FACTOR; + if adjusted_ratio < MIN_COMPRESSION_RATIO { return Ok(EstimateVerdict::Skip); } - Ok(EstimateVerdict::Ratio(ratio)) + Ok(EstimateVerdict::Ratio(adjusted_ratio)) }, ))) } diff --git a/vortex-btrblocks/src/schemes/integer/block_residual.rs b/vortex-btrblocks/src/schemes/integer/block_residual.rs new file mode 100644 index 00000000000..61689c974e9 --- /dev/null +++ b/vortex-btrblocks/src/schemes/integer/block_residual.rs @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Integer compression with one block-local reference and packed residuals. + +use vortex_array::ArrayId; +use vortex_array::ArrayRef; +use vortex_array::Canonical; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::VTable; +use vortex_array::arrays::Primitive; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::match_each_integer_ptype; +use vortex_block_residual::BlockResidual; +use vortex_compressor::builtins::BinaryDictScheme; +use vortex_compressor::builtins::FloatDictScheme; +use vortex_compressor::builtins::IntDictScheme; +use vortex_compressor::builtins::StringDictScheme; +use vortex_compressor::scheme::AncestorExclusion; +use vortex_compressor::scheme::ChildSelection; +use vortex_compressor::scheme::CompressionEstimate; +use vortex_compressor::scheme::DeferredEstimate; +use vortex_compressor::scheme::EstimateVerdict; +use vortex_error::VortexResult; + +use crate::ArrayAndStats; +use crate::CascadingCompressor; +use crate::CompressorContext; +use crate::Scheme; +use crate::SchemeExt; + +const BLOCK_LEN: usize = 1024; +const ESTIMATE_BLOCKS: usize = 8; +const MIN_COMPRESSION_RATIO: f64 = 1.05; + +/// Compress integers with one reference and packed residuals per 1,024-value block. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct BlockResidualScheme; + +impl Scheme for BlockResidualScheme { + fn scheme_name(&self) -> &'static str { + "vortex.int.block_residual" + } + + fn matches(&self, canonical: &Canonical) -> bool { + canonical.dtype().is_int() && canonical.dtype().as_ptype().bit_width() >= 32 + } + + fn produced_encodings(&self) -> Vec { + vec![BlockResidual.id()] + } + + fn ancestor_exclusions(&self) -> Vec { + vec![ + AncestorExclusion { + ancestor: IntDictScheme.id(), + children: ChildSelection::One(1), + }, + AncestorExclusion { + ancestor: FloatDictScheme.id(), + children: ChildSelection::One(1), + }, + AncestorExclusion { + ancestor: StringDictScheme.id(), + children: ChildSelection::One(1), + }, + AncestorExclusion { + ancestor: BinaryDictScheme.id(), + children: ChildSelection::One(1), + }, + ] + } + + fn expected_compression_ratio( + &self, + _data: &ArrayAndStats, + compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> CompressionEstimate { + if compress_ctx.finished_cascading() || compress_ctx.is_sample() { + return CompressionEstimate::Verdict(EstimateVerdict::Skip); + } + CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( + |_compressor, data, _best_so_far, _compress_ctx, exec_ctx| { + let sample = locality_sample(data.array_as_primitive(), exec_ctx)?; + let before_nbytes = sample.nbytes(); + let residuals = BlockResidual::from_primitive(sample.as_view())?; + let after_nbytes = residuals.nbytes(); + if after_nbytes == 0 { + return Ok(EstimateVerdict::Skip); + } + + let ratio = before_nbytes as f64 / after_nbytes as f64; + if ratio < MIN_COMPRESSION_RATIO { + return Ok(EstimateVerdict::Skip); + } + let speed_penalty = match sample.ptype().bit_width() { + 32 => 1.10, + 64 => 1.20, + _ => unreachable!("BlockResidual only matches 32-bit and 64-bit integers"), + }; + let adjusted_ratio = ratio / speed_penalty; + if adjusted_ratio < MIN_COMPRESSION_RATIO { + return Ok(EstimateVerdict::Skip); + } + Ok(EstimateVerdict::Ratio(adjusted_ratio)) + }, + ))) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> VortexResult { + Ok(BlockResidual::from_primitive(data.array_as_primitive())?.into_array()) + } +} + +fn locality_sample( + primitive: vortex_array::ArrayView<'_, Primitive>, + exec_ctx: &mut ExecutionCtx, +) -> VortexResult { + let validity = primitive + .validity()? + .execute_mask(primitive.len(), exec_ctx)?; + let full_blocks = primitive.len() / BLOCK_LEN; + + if full_blocks <= ESTIMATE_BLOCKS { + return primitive + .array() + .clone() + .execute::(exec_ctx); + } + + let sample_blocks = ESTIMATE_BLOCKS.min(full_blocks); + Ok(match_each_integer_ptype!(primitive.ptype(), |T| { + let values = primitive.as_slice::(); + let mut sample = Vec::with_capacity(sample_blocks * BLOCK_LEN); + for sample_index in 0..sample_blocks { + let block_index = sample_index * full_blocks / sample_blocks; + let start = block_index * BLOCK_LEN; + sample.extend( + (start..start + BLOCK_LEN) + .map(|index| validity.value(index).then_some(values[index])), + ); + } + PrimitiveArray::from_option_iter(sample) + })) +} diff --git a/vortex-btrblocks/src/schemes/integer/mod.rs b/vortex-btrblocks/src/schemes/integer/mod.rs index 3aae2ae5601..d0e4386fa73 100644 --- a/vortex-btrblocks/src/schemes/integer/mod.rs +++ b/vortex-btrblocks/src/schemes/integer/mod.rs @@ -4,6 +4,7 @@ //! Integer compression schemes. mod bitpacking; +mod block_residual; #[cfg(feature = "unstable_encodings")] mod delta; mod for_; @@ -17,6 +18,7 @@ mod zigzag; mod pco; pub use bitpacking::BitPackingScheme; +pub use block_residual::BlockResidualScheme; #[cfg(feature = "unstable_encodings")] pub use delta::DeltaScheme; pub use for_::FoRScheme; diff --git a/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs index e4227a472ec..0e8083bef4f 100644 --- a/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs @@ -14,10 +14,12 @@ use vortex_array::VortexSessionExecute; use vortex_array::arrays::Constant; use vortex_array::arrays::Dict; use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::dict::DictArraySlotsExt; use vortex_array::expr::stats::Precision; use vortex_array::expr::stats::Stat; use vortex_array::expr::stats::StatsProviderExt; use vortex_array::validity::Validity; +use vortex_block_residual::BlockResidual; use vortex_buffer::Buffer; use vortex_error::VortexResult; use vortex_fastlanes::BitPacked; @@ -50,6 +52,52 @@ fn test_for_compressed() -> VortexResult<()> { Ok(()) } +#[test] +fn test_block_residual_compressed() -> VortexResult<()> { + let values = (0..8_192) + .map(|index| { + let block = index / 1_024; + let residual = (index * 2_654_435_761_usize) % 1_024; + (block as i64 - 4) * 1_000_000_000_000 + residual as i64 + }) + .collect::>(); + let array = PrimitiveArray::from_iter(values); + let compressed = BtrBlocksCompressor::default() + .compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; + + assert!( + compressed.is::(), + "expected BlockResidual, got tree:\n{}", + compressed.display_tree() + ); + Ok(()) +} + +#[test] +fn test_block_residual_skips_narrow_integers() -> VortexResult<()> { + let values = (0..8_192) + .map(|index| { + let block = index / 1_024; + let residual = (index * 2_654_435_761_usize) % 32; + Ok(i16::try_from(block * 1_000 + residual)?) + }) + .collect::>>()?; + let array = PrimitiveArray::from_iter(values); + let compressed = BtrBlocksCompressor::default() + .compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; + + assert!( + !contains_block_residual(&compressed), + "BlockResidual must not encode narrow integers:\n{}", + compressed.display_tree() + ); + Ok(()) +} + +fn contains_block_residual(array: &vortex_array::ArrayRef) -> bool { + array.is::() || array.children().iter().any(contains_block_residual) +} + #[test] fn test_bitpacking_compressed() -> VortexResult<()> { let values: Vec = (0..1000).map(|i| i % 16).collect(); @@ -110,6 +158,11 @@ fn test_dict_compressed() -> VortexResult<()> { let btr = BtrBlocksCompressor::default(); let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; assert!(compressed.is::()); + assert!( + !contains_block_residual(compressed.as_::().codes()), + "BlockResidual must not encode dictionary codes:\n{}", + compressed.display_tree() + ); Ok(()) } diff --git a/vortex-btrblocks/tests/snapshots/golden__default__string_fsst_structured.snap b/vortex-btrblocks/tests/snapshots/golden__default__string_fsst_structured.snap index 327f050b0d7..85e3422cb1b 100644 --- a/vortex-btrblocks/tests/snapshots/golden__default__string_fsst_structured.snap +++ b/vortex-btrblocks/tests/snapshots/golden__default__string_fsst_structured.snap @@ -3,7 +3,7 @@ source: vortex-btrblocks/tests/golden.rs expression: rendered --- input: utf8, len=16384, nbytes=653785 -root: vortex.fsst(utf8, len=16384) nbytes=151382 +root: vortex.fsst(utf8, len=16384) nbytes=141400 metadata: len: 16384, nsymbols: 223 uncompressed_lengths: vortex.sparse(u8, len=16384) nbytes=3154 metadata: fill_value: 24u8 @@ -11,5 +11,23 @@ root: vortex.fsst(utf8, len=16384) nbytes=151382 metadata: ptype: u16 patch_values: vortex.constant(u8, len=1575) nbytes=2 metadata: scalar: 23u8 - codes_offsets: fastlanes.bitpacked(u32, len=16385) nbytes=36992 - metadata: bit_width: 17, offset: 0 + codes_offsets: vortex.block_residual(u32, len=16385) nbytes=27010 + metadata: blocks: 17, slice: 0..16385 + bases: vortex.primitive(u64, len=17) nbytes=136 + metadata: ptype: u64 + residual_widths: vortex.primitive(u8, len=17) nbytes=17 + metadata: ptype: u8 + high_widths: vortex.primitive(u8, len=17) nbytes=17 + metadata: ptype: u8 + residual_starts: vortex.primitive(u32, len=18) nbytes=72 + metadata: ptype: u32 + patch_starts: vortex.primitive(u32, len=18) nbytes=72 + metadata: ptype: u32 + high_starts: vortex.primitive(u32, len=18) nbytes=72 + metadata: ptype: u32 + residual_words: vortex.primitive(u64, len=3328) nbytes=26624 + metadata: ptype: u64 + patch_positions: vortex.primitive(u16, len=0) nbytes=0 + metadata: ptype: u16 + patch_highs: vortex.primitive(u8, len=0) nbytes=0 + metadata: ptype: u8 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__temporal_timestamp_micros.snap b/vortex-btrblocks/tests/snapshots/golden__default__temporal_timestamp_micros.snap index 6f38e8e6f5d..caa6e265676 100644 --- a/vortex-btrblocks/tests/snapshots/golden__default__temporal_timestamp_micros.snap +++ b/vortex-btrblocks/tests/snapshots/golden__default__temporal_timestamp_micros.snap @@ -3,9 +3,37 @@ source: vortex-btrblocks/tests/golden.rs expression: rendered --- input: vortex.timestamp[µs, tz=UTC](i64), len=16384, nbytes=131072 -root: vortex.ext(vortex.timestamp[µs, tz=UTC](i64), len=16384) nbytes=67584 +root: vortex.datetimeparts(vortex.timestamp[µs, tz=UTC](i64), len=16384) nbytes=60983 metadata: - storage: fastlanes.for(i64, len=16384) nbytes=67584 - metadata: reference: 1700000000891673i64 - encoded: fastlanes.bitpacked(i64, len=16384) nbytes=67584 - metadata: bit_width: 33, offset: 0 + days: vortex.dict(u16, len=16384) nbytes=10 + metadata: all_values_referenced: true + codes: vortex.runend(u8, len=16384) nbytes=6 + metadata: offset: 0 + ends: vortex.primitive(u16, len=2) nbytes=4 + metadata: ptype: u16 + values: vortex.primitive(u8, len=2) nbytes=2 + metadata: ptype: u8 + values: vortex.primitive(u16, len=2) nbytes=4 + metadata: ptype: u16 + seconds: vortex.block_residual(u32, len=16384) nbytes=20013 + metadata: blocks: 16, slice: 0..16384 + bases: vortex.primitive(u64, len=16) nbytes=128 + metadata: ptype: u64 + residual_widths: vortex.primitive(u8, len=16) nbytes=16 + metadata: ptype: u8 + high_widths: vortex.primitive(u8, len=16) nbytes=16 + metadata: ptype: u8 + residual_starts: vortex.primitive(u32, len=17) nbytes=68 + metadata: ptype: u32 + patch_starts: vortex.primitive(u32, len=17) nbytes=68 + metadata: ptype: u32 + high_starts: vortex.primitive(u32, len=17) nbytes=68 + metadata: ptype: u32 + residual_words: vortex.primitive(u64, len=2432) nbytes=19456 + metadata: ptype: u64 + patch_positions: vortex.primitive(u16, len=47) nbytes=94 + metadata: ptype: u16 + patch_highs: vortex.primitive(u8, len=99) nbytes=99 + metadata: ptype: u8 + subseconds: fastlanes.bitpacked(u32, len=16384) nbytes=40960 + metadata: bit_width: 20, offset: 0 diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index b721c4ff27a..6fd33bf4747 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -34,9 +34,12 @@ use vortex::encodings::alp::alp_encode; use vortex::encodings::block_residual::BlockResidual; use vortex::encodings::block_residual::OrderedFloat; use vortex::encodings::block_residual::OrderedFloatArraySlotsExt; +use vortex::encodings::fastlanes::BitPacked; use vortex::encodings::fastlanes::Delta; use vortex::encodings::fastlanes::DeltaData; use vortex::encodings::fastlanes::FoR; +use vortex::encodings::fastlanes::FoRArrayExt; +use vortex::encodings::fastlanes::FoRArraySlotsExt; use vortex::encodings::fastlanes::bitpack_compress::bitpack_encode_unchecked; use vortex::encodings::fastlanes::delta_compress; use vortex::encodings::float_quant::FloatQuant; @@ -55,6 +58,7 @@ use vortex_array::VortexSessionExecute; use vortex_btrblocks::SchemeExt; use vortex_btrblocks::schemes::float::FloatQuantScheme; use vortex_btrblocks::schemes::float::OrderedBlockResidualScheme; +use vortex_btrblocks::schemes::integer::BlockResidualScheme; use vortex_error::VortexResult; use vortex_sequence::Sequence; use vortex_session::VortexSession; @@ -146,6 +150,31 @@ fn setup_random_walk_array() -> PrimitiveArray { })) } +fn setup_block_local_u64_array() -> PrimitiveArray { + PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + let block = index / 1_024; + let residual = index.wrapping_mul(2_654_435_761) % 1_024; + block * 1_000_000 + residual + })) +} + +fn setup_block_local_i16_array() -> PrimitiveArray { + PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + let block = (index / 1_024) % 128; + let residual = index.wrapping_mul(2_654_435_761) % 128; + (block * 128 + residual) as i16 + })) +} + +fn encode_for_bitpacked_tree(array: &PrimitiveArray, bit_width: u8) -> vortex::array::ArrayRef { + let mut ctx = SESSION.create_execution_ctx(); + let encoded = FoR::encode(array.clone(), &mut ctx).unwrap(); + let bitpacked = BitPacked::encode(encoded.encoded(), bit_width, &mut ctx).unwrap(); + FoR::try_new(bitpacked.into_array(), encoded.reference_scalar().clone()) + .unwrap() + .into_array() +} + fn ordered_values(array: &PrimitiveArray) -> PrimitiveArray { let ordered = OrderedFloat::from_primitive(array.as_view()).unwrap(); ordered @@ -216,7 +245,11 @@ fn encode_float_quant_nonzero_secondary_tree(array: &PrimitiveArray) -> vortex:: fn encode_prior_default(array: &PrimitiveArray) -> vortex::array::ArrayRef { BtrBlocksCompressorBuilder::default() - .exclude_schemes([FloatQuantScheme.id(), OrderedBlockResidualScheme.id()]) + .exclude_schemes([ + FloatQuantScheme.id(), + OrderedBlockResidualScheme.id(), + BlockResidualScheme.id(), + ]) .build() .compress( &array.clone().into_array(), @@ -494,6 +527,150 @@ fn bench_block_residual_decompress_u64(bencher: Bencher) { .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); } +#[divan::bench(name = "block_local_block_residual_compress_u64")] +fn bench_block_local_block_residual_compress_u64(bencher: Bencher) { + let array = setup_block_local_u64_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| &array) + .bench_refs(|array| BlockResidual::from_primitive(array.as_view()).unwrap()); +} + +#[divan::bench(name = "block_local_block_residual_decompress_u64")] +fn bench_block_local_block_residual_decompress_u64(bencher: Bencher) { + let encoded = BlockResidual::from_primitive(setup_block_local_u64_array().as_view()) + .unwrap() + .into_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "block_local_block_residual_scalar_at_u64")] +fn bench_block_local_block_residual_scalar_at_u64(bencher: Bencher) { + let encoded = BlockResidual::from_primitive(setup_block_local_u64_array().as_view()) + .unwrap() + .into_array(); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + +#[divan::bench(name = "block_local_for_bitpacked_compress_u64")] +fn bench_block_local_for_bitpacked_compress_u64(bencher: Bencher) { + let array = setup_block_local_u64_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| &array) + .bench_refs(|array| encode_for_bitpacked_tree(array, 31)); +} + +#[divan::bench(name = "block_local_for_bitpacked_decompress_u64")] +fn bench_block_local_for_bitpacked_decompress_u64(bencher: Bencher) { + let encoded = encode_for_bitpacked_tree(&setup_block_local_u64_array(), 31); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "block_local_for_bitpacked_scalar_at_u64")] +fn bench_block_local_for_bitpacked_scalar_at_u64(bencher: Bencher) { + let encoded = encode_for_bitpacked_tree(&setup_block_local_u64_array(), 31); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + +#[divan::bench(name = "block_local_block_residual_compress_i16")] +fn bench_block_local_block_residual_compress_i16(bencher: Bencher) { + let array = setup_block_local_i16_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 2) + .with_inputs(|| &array) + .bench_refs(|array| BlockResidual::from_primitive(array.as_view()).unwrap()); +} + +#[divan::bench(name = "block_local_block_residual_decompress_i16")] +fn bench_block_local_block_residual_decompress_i16(bencher: Bencher) { + let encoded = BlockResidual::from_primitive(setup_block_local_i16_array().as_view()) + .unwrap() + .into_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 2) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "block_local_block_residual_scalar_at_i16")] +fn bench_block_local_block_residual_scalar_at_i16(bencher: Bencher) { + let encoded = BlockResidual::from_primitive(setup_block_local_i16_array().as_view()) + .unwrap() + .into_array(); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + +#[divan::bench(name = "block_local_for_bitpacked_compress_i16")] +fn bench_block_local_for_bitpacked_compress_i16(bencher: Bencher) { + let array = setup_block_local_i16_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 2) + .with_inputs(|| &array) + .bench_refs(|array| encode_for_bitpacked_tree(array, 14)); +} + +#[divan::bench(name = "block_local_for_bitpacked_decompress_i16")] +fn bench_block_local_for_bitpacked_decompress_i16(bencher: Bencher) { + let encoded = encode_for_bitpacked_tree(&setup_block_local_i16_array(), 14); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 2) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "block_local_for_bitpacked_scalar_at_i16")] +fn bench_block_local_for_bitpacked_scalar_at_i16(bencher: Bencher) { + let encoded = encode_for_bitpacked_tree(&setup_block_local_i16_array(), 14); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + #[divan::bench(name = "ordered_block_residual_compress_f64")] fn bench_ordered_block_residual_compress_f64(bencher: Bencher) { let float_array = setup_random_walk_array(); From ed9efb1c885bee7ed1620807faf4c126cdc85444 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 14:09:02 +0100 Subject: [PATCH 09/78] perf: expand native numeric compression coverage Signed-off-by: Will Manning --- benchmarks/compress-bench/src/main.rs | 2 + docs/plans/native-pcodec.md | 137 +++++++- .../src/block_residual_array.rs | 42 +-- .../block-residual/src/ordered_float_array.rs | 47 ++- vortex-bench/src/datasets/feature_vectors.rs | 32 ++ vortex-btrblocks/src/builder.rs | 12 + vortex-btrblocks/src/lib.rs | 26 ++ .../src/schemes/float/float_quant.rs | 19 +- .../schemes/float/ordered_block_residual.rs | 53 ++-- .../schemes/float/scheme_selection_tests.rs | 49 ++- .../src/schemes/integer/block_residual.rs | 7 +- .../schemes/integer/scheme_selection_tests.rs | 26 ++ vortex/benches/single_encoding_throughput.rs | 294 +++++++++++++++++- 13 files changed, 665 insertions(+), 81 deletions(-) diff --git a/benchmarks/compress-bench/src/main.rs b/benchmarks/compress-bench/src/main.rs index de3122a3a7f..6e81a8a4376 100644 --- a/benchmarks/compress-bench/src/main.rs +++ b/benchmarks/compress-bench/src/main.rs @@ -27,6 +27,7 @@ use vortex_bench::compress::benchmark_decompress; use vortex_bench::compress::calculate_ratios; use vortex_bench::create_output_writer; use vortex_bench::datasets::Dataset; +use vortex_bench::datasets::feature_vectors::GloveEmbeddingsData; use vortex_bench::datasets::struct_list_of_ints::StructListOfInts; use vortex_bench::datasets::taxi_data::TaxiData; use vortex_bench::datasets::tpch_l_comment::TPCHLCommentCanonical; @@ -197,6 +198,7 @@ async fn run_compress( let datasets: Vec<&dyn Dataset> = [ &TaxiData as &dyn Dataset, + &GloveEmbeddingsData, PBI_DATASETS.get(Arade), PBI_DATASETS.get(Bimbo), PBI_DATASETS.get(CMSprovider), diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index ea0efb67030..40ae1e10317 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -26,6 +26,22 @@ The `wm/pcodec-entropy-experiments` branch preserves the complete entropy and bi Do not add adjacent Delta, Delta-of-delta, Delta with lookback, or convolution Delta. +## Current state + +The branch implements `OrderedFloatArray`, `BlockResidualArray`, and `FloatQuantArray`. + +The default candidate set includes their three BtrBlocks schemes. + +`OrderedFloat(BlockResidual)` now supports `f32` and `f64` inputs. + +Direct integer BlockResidual supports every integer type. The default selector accepts only 32-bit and 64-bit inputs. + +The retained schemes win on specific structures. They do not replace ALP or ALP-RD across general float data. + +The GloVe result identifies a separate gap inside the ALP integer child. + +The current selector factors and patch cost remain provisional. Final calibration requires the complete corpus and selected-tree evidence. + ## OrderedFloatArray `OrderedFloatArray` maps IEEE float bits to unsigned integers with the same order. @@ -219,8 +235,8 @@ The direct benchmark uses two million block-local values. | --- | ---: | ---: | ---: | | `u64` BlockResidual | 5.00 | 36.43 | 125 | | `u64` FoR plus BitPacked | 4.25 | 35.37 | 115 | -| `i16` BlockResidual | 1.38 | 31.12 | 125 | -| `i16` FoR plus BitPacked | 1.12 | 41.57 | 104 | +| `i16` BlockResidual | 1.39 | 31.86 | 125 | +| `i16` FoR plus BitPacked | 1.15 | 42.14 | 125 | The first `i16` implementation unpacked through `u64` residuals. It decoded at 11.43 GB/s. @@ -230,7 +246,7 @@ The synthetic `i16` BlockResidual tree uses 1,793,784 bytes. The FoR plus BitPac BlockResidual is 48.7 percent smaller on that input. -Its `i16` decode throughput is 25.1 percent lower. The default selector therefore excludes direct 8-bit and 16-bit candidates. +Its `i16` decode throughput is 24.4 percent lower. The default selector therefore excludes direct 8-bit and 16-bit candidates. The BlockResidual estimator measures its sampled tree exactly, with all child arrays. @@ -270,12 +286,93 @@ CMS reduced size by 5.3 percent and increased decode throughput by 5.5 percent. HashTags reduced size by 7.0 percent and increased decode throughput by 4.9 percent. -The direct narrow-type exclusion changed no selected tree in this corpus. The earlier 1.40 factor already rejected each direct `i16` candidate. +The direct narrow-type exclusion changed no selected tree in this corpus. + +Before that exclusion, an earlier 1.40 trial factor rejected each direct `i16` candidate. A FastLanes fused FoR decode trial did not improve `u64` throughput. It reduced `i16` throughput, so the implementation retains native unpack. The zero-width residual path now writes base values and patches directly. It skips the scratch residual block. +### Native f32 OrderedFloat with BlockResidual + +The input contains two million f32 values with narrow ordered-bit ranges inside each 1,024-value block. + +| Operation | Result | +| --- | ---: | +| Encode | 2.45 GB/s | +| Decode | 30.30 GB/s | +| Scalar access | 167 ns | + +The fused decode now handles `OrderedFloat(BlockResidual)` trees with u32 latents. + +The BtrBlocks candidate now accepts f32 and f64 inputs. + +These results remove the prior f32 type exclusion. Corpus comparisons against ALP and ALP-RD remain necessary. + +### Direct 32-bit comparison + +The direct benchmark compares two million block-local values against a whole-column FoR plus BitPacked tree. + +| Type and tree | Encode GB/s | Decode GB/s | Scalar access ns | +| --- | ---: | ---: | ---: | +| i32 BlockResidual | 2.69 | 33.66 | 124 | +| i32 FoR plus BitPacked | 2.59 | 26.93 | 119 | +| u32 BlockResidual | 2.74 | 33.55 | 98 | +| u32 FoR plus BitPacked | 2.62 | 39.70 | 87 | + +One width factor does not predict both signed and unsigned results. + +Retain the current factor until the final corpus calibration uses selected-tree evidence. + +### Patch-density sweep + +The u32 input uses one large outlier at each configured stride. + +| Outlier stride | Approximate patch share | Decode GB/s | Scalar access ns | +| ---: | ---: | ---: | ---: | +| 256 | 0.4 percent | 57.88 | 84 | +| 64 | 1.6 percent | 49.31 | 86 | +| 16 | 6.3 percent | 26.32 | 104 | +| 4 | 25.0 percent | 9.57 | 118 | +| 1 | Packed residuals | 33.02 | 99 | + +Scalar access remains bounded across the sweep. + +Bulk decode has a severe patch-density cliff before the planner changes to packed residuals. + +The final selector calibration must compare alternate patch costs and separate zero-width blocks from packed blocks. + +### GloVe embeddings + +The corpus now includes the real GloVe dataset with 100,000 rows and 200 f32 values per row. + +The complete proposed and previous Vortex defaults both use 68,375,768 bytes. + +The Vortex file uses 0.92 times the bytes of Parquet with Zstd. + +The first two million embedding values use `ALP(ZigZag(BitPacked))` under both default configurations. + +That tree uses 6,274,834 bytes for 8,000,000 raw bytes. + +Compact uses `ALP(Pco)` and 5,287,510 bytes. Compact is 15.7 percent smaller than the default tree. + +FloatQuant and OrderedFloat with BlockResidual both lose to ALP on this dataset. + +Pco receives the `i32` primary child from ALP. Each 262,144-value Pco chunk selects `IntMult(10)` with no Delta encoding. + +`IntMult(10)` splits each integer into a quotient and a remainder modulo ten. + +The quotient uses 23 to 31 entropy bins. Its average encoded cost is 16.95 to 16.99 bits per value. + +The remainder uses ten entropy bins and no offset bits. Its average encoded cost is about 1.08 bits per value. + +The complete Pco child uses 4,518,870 bytes for two million values. This result includes model metadata and page overhead. + +The size gap comes from a decimal quotient and remainder split plus entropy coding. Pco does not use a Delta or float mode here. + +GloVe therefore identifies an ALP-child compression gap. It does not identify a failure in the ALP float transform. + ### Pcodec corpus gap analysis The focused benchmark reads the first two million numeric rows from each source. @@ -325,7 +422,7 @@ It reduced default bytes by 7.9 percent across the gap columns. The extra references did not justify a new array. Generic integer BlockResidual became the next prototype. -Test these secondary candidates after the multi-reference prototype: +Test these secondary candidates after the quotient and remainder prototype: - Use ordered-bit XOR suffixes instead of arithmetic residuals. - Use sign and exponent prefixes as fixed reference candidates. @@ -420,16 +517,32 @@ This round completed these steps: - Added the patch-count decode cost. - Excluded direct 8-bit and 16-bit candidates. - Excluded BlockResidual from dictionary-code children. +- Added fused f32 OrderedFloat with BlockResidual decode and selection. +- Added u32, i32, f32, and patch-density benchmarks. +- Removed the full patch scan from scalar access. +- Made null payload bits neutral for the new default candidates. +- Excluded integer BlockResidual from the CUDA-compatible preset. +- Added the real GloVe embeddings dataset to the compression corpus. Complete these remaining steps: -1. Run the complete `bench-vortex` compression corpus. -2. Compare the geometric mean against Parquet with Zstd. -3. Reduce the analysis cost on short rejected columns. -4. Test a fused nonzero-secondary FloatQuant decode on selected gap columns. -5. Evaluate nonzero-secondary FloatQuant for the default compressor. -6. Investigate new schemes for real floats that Pco compresses better than ALP and ALP-RD. -7. Update this plan after each experiment. +1. Profile the ALP integer child on GloVe and unrelated no-Delta Pco wins. +2. Record each selected `IntMult` base, quotient distribution, remainder distribution, and Pco bit cost. +3. Prototype a bounded-access quotient and remainder array with a constant base. +4. Test existing integer trees for both children before a new entropy backend. +5. Compare that prototype against `ALP(ZigZag(BitPacked))` and `ALP(Pco)` on the same inputs. +6. If existing children lose materially, test a small-alphabet entropy child for the remainder. +7. Test a fused nonzero-secondary FloatQuant decode on selected gap columns. +8. Evaluate nonzero-secondary FloatQuant for the default compressor. +9. Compare native f32 FloatQuant against ALP and ALP-RD on the same inputs. +10. Compare alternate patch costs across the patch-density sweep. +11. Evaluate narrow BlockResidual as a Compact-only candidate. +12. Reduce the analysis cost on short rejected columns. +13. Measure BlockResidual under temporal, FSST, sparse, run-end, and list parents. +14. Run the complete `bench-vortex` compression corpus after the candidate set stabilizes. +15. Compare the geometric mean against Parquet with Zstd. +16. Calibrate all selector thresholds from the complete corpus and selected-tree evidence. +17. Update this plan after each experiment. ## Pull request structure diff --git a/encodings/block-residual/src/block_residual_array.rs b/encodings/block-residual/src/block_residual_array.rs index e62667b78d0..d47f194727d 100644 --- a/encodings/block-residual/src/block_residual_array.rs +++ b/encodings/block-residual/src/block_residual_array.rs @@ -642,6 +642,20 @@ fn decompress_array( } } +pub(crate) fn decompress_ordered_f32( + array: ArrayView<'_, BlockResidual>, + ctx: &mut ExecutionCtx, +) -> VortexResult { + decode_array_values(array, ctx, |ordered: u32| { + let bits = if ordered & (1_u32 << 31) == 0 { + !ordered + } else { + ordered ^ (1_u32 << 31) + }; + f32::from_bits(bits) + }) +} + pub(crate) fn decompress_ordered_f64( array: ArrayView<'_, BlockResidual>, ctx: &mut ExecutionCtx, @@ -717,8 +731,6 @@ fn scalar_from_array( "patch", )?; let block_positions = &positions[patch_payload]; - let block_start = block_index * BLOCK_LEN; - let block_len = (array.data().unsliced_len - block_start).min(BLOCK_LEN); let highs = children.patch_highs.as_slice::(); let high_payload = payload_range( children.high_starts.as_slice::(), @@ -726,11 +738,10 @@ fn scalar_from_array( highs.len(), "patch high", )?; - validate_patch_payload( - block_len, + validate_patch_header( residual_width, high_width, - block_positions, + block_positions.len(), high_payload.len(), )?; if let Ok(patch_index) = block_positions.binary_search(&u16::try_from(index_in_block)?) { @@ -757,27 +768,6 @@ fn unpack_single_residual(width: u8, packed_words: &[u64], inde unsafe { T::unchecked_unpack_single(usize::from(width), packed, index).to_u64() } } -fn validate_patch_payload( - block_len: usize, - residual_width: u8, - high_width: u8, - positions: &[u16], - high_payload_len: usize, -) -> VortexResult<()> { - validate_patch_header( - residual_width, - high_width, - positions.len(), - high_payload_len, - )?; - let mut previous_position = None; - for &position in positions { - validate_patch_position(block_len, previous_position, position)?; - previous_position = Some(position); - } - Ok(()) -} - fn validate_patch_header( residual_width: u8, high_width: u8, diff --git a/encodings/block-residual/src/ordered_float_array.rs b/encodings/block-residual/src/ordered_float_array.rs index 690ed357175..a33f11cc31b 100644 --- a/encodings/block-residual/src/ordered_float_array.rs +++ b/encodings/block-residual/src/ordered_float_array.rs @@ -43,6 +43,7 @@ use vortex_session::VortexSession; use vortex_session::registry::CachedId; use crate::BlockResidual; +use crate::block_residual_array::decompress_ordered_f32; use crate::block_residual_array::decompress_ordered_f64; /// IEEE floats mapped to unsigned integers that preserve numeric order. @@ -169,11 +170,11 @@ impl VTable for OrderedFloat { fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { let decoded = if let Some(block_residual) = array.encoded().as_typed::() { - vortex_ensure!( - array.dtype().as_ptype() == PType::F64, - "fused BlockResidual decode requires f64" - ); - decompress_ordered_f64(block_residual, ctx)? + match array.dtype().as_ptype() { + PType::F32 => decompress_ordered_f32(block_residual, ctx)?, + PType::F64 => decompress_ordered_f64(block_residual, ctx)?, + ptype => vortex_bail!("unsupported OrderedFloat ptype {ptype}"), + } } else { decode_primitive(array.as_view(), ctx)? }; @@ -372,11 +373,15 @@ mod tests { use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::array_session; + use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; + use vortex_array::dtype::PType; use vortex_error::VortexResult; use super::OrderedFloat; + use super::OrderedFloatArraySlotsExt; + use crate::BlockResidual; #[test] fn roundtrip_special_values() -> VortexResult<()> { @@ -396,4 +401,36 @@ mod tests { assert_arrays_eq!(encoded, primitive.into_array(), &mut ctx); Ok(()) } + + #[test] + fn roundtrip_f32_block_residual() -> VortexResult<()> { + let values = (0..2_050) + .scan(1_000.0_f32, |value, index| { + *value += ((index * 7_919 % 101) as f32 - 50.0) * 0.0001; + Some(*value) + }) + .collect::>(); + let primitive = PrimitiveArray::from_iter(values.clone()); + let ordered = OrderedFloat::from_primitive(primitive.as_view())?; + let residuals = BlockResidual::from_primitive(ordered.encoded().as_::())?; + let encoded = OrderedFloat::try_new(residuals.into_array(), PType::F32)?; + let session = array_session(); + crate::initialize(&session); + let mut ctx = session.create_execution_ctx(); + + assert_arrays_eq!(encoded, primitive, &mut ctx); + assert_eq!( + encoded + .execute_scalar(1_024, &mut ctx)? + .as_primitive() + .typed_value::(), + Some(values[1_024]) + ); + assert_arrays_eq!( + encoded.into_array().slice(1_023..1_026)?, + primitive.into_array().slice(1_023..1_026)?, + &mut ctx + ); + Ok(()) + } } diff --git a/vortex-bench/src/datasets/feature_vectors.rs b/vortex-bench/src/datasets/feature_vectors.rs index e09f29f49ec..87e91a0573c 100644 --- a/vortex-bench/src/datasets/feature_vectors.rs +++ b/vortex-bench/src/datasets/feature_vectors.rs @@ -18,13 +18,20 @@ use parquet::arrow::ArrowWriter; use rand::RngExt; use rand::SeedableRng; use rand::rngs::StdRng; +use vortex::array::ArrayRef; +use vortex::array::ExecutionCtx; use crate::CompactionStrategy; use crate::Format; +use crate::conversions::parquet_to_vortex_chunks; use crate::conversions::write_parquet_as_vortex; +use crate::datasets::Dataset; use crate::idempotent_async; use crate::random_access::BenchDataset; use crate::random_access::data_path; +use crate::vector_dataset::TrainLayout; +use crate::vector_dataset::VectorDataset; +use crate::vector_dataset::download; /// Dataset identifier used for data path generation. pub const DATASET: &str = "feature_vectors"; @@ -34,6 +41,31 @@ pub const ROW_COUNT: usize = 1_000_000; pub struct FeatureVectorsData; +/// A real f32 embedding dataset from the GloVe vector benchmark corpus. +pub struct GloveEmbeddingsData; + +#[async_trait] +impl Dataset for GloveEmbeddingsData { + fn name(&self) -> &str { + "glove-embeddings-100k" + } + + async fn to_vortex_array(&self, _ctx: &mut ExecutionCtx) -> Result { + Ok(parquet_to_vortex_chunks(self.to_parquet_path().await?) + .await? + .into()) + } + + async fn to_parquet_path(&self) -> Result { + let paths = download(VectorDataset::GloveSmall100k, TrainLayout::Single).await?; + paths + .train_files + .into_iter() + .next() + .ok_or_else(|| anyhow::anyhow!("GloVe embedding train file is missing")) + } +} + #[async_trait] impl BenchDataset for FeatureVectorsData { fn name(&self) -> &str { diff --git a/vortex-btrblocks/src/builder.rs b/vortex-btrblocks/src/builder.rs index 90f8fb00adc..bd6fcdb430d 100644 --- a/vortex-btrblocks/src/builder.rs +++ b/vortex-btrblocks/src/builder.rs @@ -173,6 +173,7 @@ impl BtrBlocksCompressorBuilder { allow(unused_mut) )] let mut excluded: Vec = vec![ + integer::BlockResidualScheme.id(), integer::SparseScheme.id(), integer::IntRLEScheme.id(), float::ALPRDScheme.id(), @@ -304,6 +305,17 @@ mod tests { ); } + #[test] + fn cuda_compatible_excludes_block_residual() { + let builder = BtrBlocksCompressorBuilder::default().only_cuda_compatible(); + assert!( + !builder + .schemes + .iter() + .any(|scheme| scheme.id() == integer::BlockResidualScheme.id()) + ); + } + #[test] fn cuda_compatible_uses_fsst_for_strings() { let builder = BtrBlocksCompressorBuilder::default().only_cuda_compatible(); diff --git a/vortex-btrblocks/src/lib.rs b/vortex-btrblocks/src/lib.rs index 1ca05c86b4e..59064ab3432 100644 --- a/vortex-btrblocks/src/lib.rs +++ b/vortex-btrblocks/src/lib.rs @@ -80,6 +80,11 @@ pub use builder::ALL_SCHEMES; pub use builder::BtrBlocksCompressorBuilder; pub use canonical_compressor::BtrBlocksCompressor; pub use schemes::patches::compress_patches; +use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; +use vortex_array::arrays::Primitive; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::match_each_native_ptype; pub use vortex_compressor::CascadingCompressor; pub use vortex_compressor::scheme::CompressorContext; pub use vortex_compressor::scheme::MAX_CASCADE; @@ -92,3 +97,24 @@ pub use vortex_compressor::stats::FloatStats; pub use vortex_compressor::stats::GenerateStatsOptions; pub use vortex_compressor::stats::IntegerStats; pub use vortex_compressor::stats::StringStats; +use vortex_error::VortexResult; + +fn normalize_null_values( + array: ArrayView<'_, Primitive>, + ctx: &mut ExecutionCtx, +) -> VortexResult { + let validity = array.validity()?; + let primitive = array.into_owned(); + if validity.definitely_no_nulls() { + return Ok(primitive); + } + + match_each_native_ptype!(primitive.ptype(), |T| { + primitive.map_each_with_validity::( + ctx, + |(value, valid)| { + if valid { value } else { T::default() } + }, + ) + }) +} diff --git a/vortex-btrblocks/src/schemes/float/float_quant.rs b/vortex-btrblocks/src/schemes/float/float_quant.rs index d7bc4c6d4bb..4ad0dc9fee1 100644 --- a/vortex-btrblocks/src/schemes/float/float_quant.rs +++ b/vortex-btrblocks/src/schemes/float/float_quant.rs @@ -25,6 +25,7 @@ use crate::ArrayAndStats; use crate::CascadingCompressor; use crate::CompressorContext; use crate::Scheme; +use crate::normalize_null_values; /// FloatQuant split with a fixed frame-of-reference primary child. #[derive(Debug, Copy, Clone, PartialEq, Eq)] @@ -61,18 +62,22 @@ impl Scheme for FloatQuantScheme { _compressor: &CascadingCompressor, data: &ArrayAndStats, _compress_ctx: CompressorContext, - _exec_ctx: &mut ExecutionCtx, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - let primitive = data.array_as_primitive(); - let Some(analysis) = analyze_float_quant(primitive) else { - return Ok(primitive.array().clone()); + let source = data.array_as_primitive(); + let primitive = normalize_null_values(source, exec_ctx)?; + let Some(analysis) = analyze_float_quant(primitive.as_view()) else { + return Ok(source.array().clone()); }; if !analysis.secondary_is_constant || analysis.primary_bit_width == 0 { - return Ok(primitive.array().clone()); + return Ok(source.array().clone()); } - let biased = - FloatQuant::primary_for_primitive(primitive, analysis.k, analysis.primary_min)?; + let biased = FloatQuant::primary_for_primitive( + primitive.as_view(), + analysis.k, + analysis.primary_min, + )?; // SAFETY: The analysis computes this width from the exact primary minimum and maximum. let compressed_primary = unsafe { bitpack_encode_unchecked(biased, analysis.primary_bit_width)? }.into_array(); diff --git a/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs b/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs index 33200c3d65d..85990fff4b9 100644 --- a/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs +++ b/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs @@ -11,7 +11,7 @@ use vortex_array::IntoArray; use vortex_array::VTable; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; -use vortex_array::dtype::PType; +use vortex_array::match_each_float_ptype; use vortex_block_residual::BlockResidual; use vortex_block_residual::OrderedFloat; use vortex_block_residual::OrderedFloatArraySlotsExt; @@ -24,13 +24,14 @@ use crate::ArrayAndStats; use crate::CascadingCompressor; use crate::CompressorContext; use crate::Scheme; +use crate::normalize_null_values; const BLOCK_LEN: usize = 1024; const ESTIMATE_BLOCKS: usize = 8; const MIN_COMPRESSION_RATIO: f64 = 1.05; const DECODE_COST_FACTOR: f64 = 1.02; -/// Compress `f64` values as block-local residuals of ordered IEEE bits. +/// Compress floats as block-local residuals of ordered IEEE bits. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct OrderedBlockResidualScheme; @@ -40,7 +41,7 @@ impl Scheme for OrderedBlockResidualScheme { } fn matches(&self, canonical: &Canonical) -> bool { - canonical.dtype().is_float() && canonical.dtype().as_ptype() == PType::F64 + canonical.dtype().is_float() } fn produced_encodings(&self) -> Vec { @@ -53,16 +54,17 @@ impl Scheme for OrderedBlockResidualScheme { fn expected_compression_ratio( &self, - data: &ArrayAndStats, + _data: &ArrayAndStats, compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { - if compress_ctx.finished_cascading() || data.array_as_primitive().ptype() != PType::F64 { + if compress_ctx.finished_cascading() { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( |_compressor, data, _best_so_far, _compress_ctx, exec_ctx| { let sample = locality_sample(data.array_as_primitive(), exec_ctx)?; + let sample = normalize_null_values(sample.as_view(), exec_ctx)?; let before_nbytes = sample.nbytes(); let ordered = OrderedFloat::from_primitive(sample.as_view())?; let residuals = @@ -90,12 +92,13 @@ impl Scheme for OrderedBlockResidualScheme { _compressor: &CascadingCompressor, data: &ArrayAndStats, _compress_ctx: CompressorContext, - _exec_ctx: &mut ExecutionCtx, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - let ordered = OrderedFloat::from_primitive(data.array_as_primitive())?; + let primitive = normalize_null_values(data.array_as_primitive(), exec_ctx)?; + let ordered = OrderedFloat::from_primitive(primitive.as_view())?; let ordered_values = ordered.encoded().as_::(); let residuals = BlockResidual::from_primitive(ordered_values)?; - Ok(OrderedFloat::try_new(residuals.into_array(), PType::F64)?.into_array()) + Ok(OrderedFloat::try_new(residuals.into_array(), primitive.ptype())?.into_array()) } } @@ -103,30 +106,30 @@ fn locality_sample( primitive: vortex_array::ArrayView<'_, Primitive>, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - let values = primitive.as_slice::(); let validity = primitive .validity()? .execute_mask(primitive.len(), exec_ctx)?; let full_blocks = primitive.len() / BLOCK_LEN; if full_blocks <= ESTIMATE_BLOCKS { - return Ok(PrimitiveArray::from_option_iter( - values - .iter() - .copied() - .enumerate() - .map(|(index, value)| validity.value(index).then_some(value)), - )); + return primitive + .array() + .clone() + .execute::(exec_ctx); } let sample_blocks = ESTIMATE_BLOCKS.min(full_blocks); - let mut sample = Vec::with_capacity(sample_blocks * BLOCK_LEN); - for sample_index in 0..sample_blocks { - let block_index = sample_index * full_blocks / sample_blocks; - let start = block_index * BLOCK_LEN; - sample.extend( - (start..start + BLOCK_LEN).map(|index| validity.value(index).then_some(values[index])), - ); - } - Ok(PrimitiveArray::from_option_iter(sample)) + Ok(match_each_float_ptype!(primitive.ptype(), |T| { + let values = primitive.as_slice::(); + let mut sample = Vec::with_capacity(sample_blocks * BLOCK_LEN); + for sample_index in 0..sample_blocks { + let block_index = sample_index * full_blocks / sample_blocks; + let start = block_index * BLOCK_LEN; + sample.extend( + (start..start + BLOCK_LEN) + .map(|index| validity.value(index).then_some(values[index])), + ); + } + PrimitiveArray::from_option_iter(sample) + })) } diff --git a/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs index cd8d7f97e5a..2765c79c092 100644 --- a/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs @@ -94,6 +94,31 @@ fn test_widened_f32_uses_float_quant() -> VortexResult<()> { Ok(()) } +#[test] +fn test_float_quant_ignores_null_payloads() -> VortexResult<()> { + let values = (0u32..16_384) + .map(|index| { + let mantissa = index.wrapping_mul(7_919) & 0x007f_ffff; + f64::from(f32::from_bits(0x3f80_0000 | mantissa)) + }) + .collect::>(); + let validity = Validity::from_iter((0..values.len()).map(|index| index % 17 != 0)); + let mut alternate = values.clone(); + for index in (0..alternate.len()).step_by(17) { + alternate[index] = f64::from_bits(0x3ff0_0000_0000_0001 + index as u64); + } + let first = PrimitiveArray::new(Buffer::copy_from(&values), validity.clone()).into_array(); + let second = PrimitiveArray::new(Buffer::copy_from(&alternate), validity).into_array(); + let compressor = BtrBlocksCompressor::default(); + let first = compressor.compress(&first, &mut SESSION.create_execution_ctx())?; + let second = compressor.compress(&second, &mut SESSION.create_execution_ctx())?; + + assert!(first.is::()); + assert!(second.is::()); + assert_eq!(first.nbytes(), second.nbytes()); + Ok(()) +} + #[test] fn test_f32_does_not_use_float_quant() -> VortexResult<()> { let values = (0u32..16_384) @@ -132,7 +157,7 @@ fn test_random_walk_uses_ordered_block_residual() -> VortexResult<()> { let mut state = 0x4d59_5df4_d0f3_3173_u64; let mut value = 0.0_f64; - let values = (0..65_536) + let values = (0usize..65_536) .map(|_| { let radius = (-2.0 * uniform(&mut state).ln()).sqrt(); let normal = radius * (TAU * uniform(&mut state)).cos(); @@ -147,3 +172,25 @@ fn test_random_walk_uses_ordered_block_residual() -> VortexResult<()> { assert!(compressed.children()[0].is::()); Ok(()) } + +#[test] +fn test_f32_random_walk_uses_ordered_block_residual() -> VortexResult<()> { + let values = (0_u32..65_536) + .map(|index| { + let block = index / 1_024; + let residual = index.wrapping_mul(7_919) % 1_024; + f32::from_bits(0x3f80_0000 + (block * 0x1_0000) + residual) + }) + .collect::>(); + let array = PrimitiveArray::from_iter(values).into_array(); + let compressed = + BtrBlocksCompressor::default().compress(&array, &mut SESSION.create_execution_ctx())?; + + assert!( + compressed.is::(), + "expected OrderedFloat, got tree:\n{}", + compressed.display_tree() + ); + assert!(compressed.children()[0].is::()); + Ok(()) +} diff --git a/vortex-btrblocks/src/schemes/integer/block_residual.rs b/vortex-btrblocks/src/schemes/integer/block_residual.rs index 61689c974e9..b430ecf00da 100644 --- a/vortex-btrblocks/src/schemes/integer/block_residual.rs +++ b/vortex-btrblocks/src/schemes/integer/block_residual.rs @@ -29,6 +29,7 @@ use crate::CascadingCompressor; use crate::CompressorContext; use crate::Scheme; use crate::SchemeExt; +use crate::normalize_null_values; const BLOCK_LEN: usize = 1024; const ESTIMATE_BLOCKS: usize = 8; @@ -84,6 +85,7 @@ impl Scheme for BlockResidualScheme { CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( |_compressor, data, _best_so_far, _compress_ctx, exec_ctx| { let sample = locality_sample(data.array_as_primitive(), exec_ctx)?; + let sample = normalize_null_values(sample.as_view(), exec_ctx)?; let before_nbytes = sample.nbytes(); let residuals = BlockResidual::from_primitive(sample.as_view())?; let after_nbytes = residuals.nbytes(); @@ -114,9 +116,10 @@ impl Scheme for BlockResidualScheme { _compressor: &CascadingCompressor, data: &ArrayAndStats, _compress_ctx: CompressorContext, - _exec_ctx: &mut ExecutionCtx, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - Ok(BlockResidual::from_primitive(data.array_as_primitive())?.into_array()) + let primitive = normalize_null_values(data.array_as_primitive(), exec_ctx)?; + Ok(BlockResidual::from_primitive(primitive.as_view())?.into_array()) } } diff --git a/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs index 0e8083bef4f..e859f34735c 100644 --- a/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs @@ -73,6 +73,32 @@ fn test_block_residual_compressed() -> VortexResult<()> { Ok(()) } +#[test] +fn test_block_residual_ignores_null_payloads() -> VortexResult<()> { + let values = (0usize..8_192) + .map(|index| { + let block = index / 1_024; + let residual = index.wrapping_mul(2_654_435_761) % 1_024; + (block as i64 - 4) * 1_000_000_000_000 + residual as i64 + }) + .collect::>(); + let validity = Validity::from_iter((0..values.len()).map(|index| index % 17 != 0)); + let mut alternate = values.clone(); + for index in (0..alternate.len()).step_by(17) { + alternate[index] = i64::MAX - index as i64; + } + let first = PrimitiveArray::new(Buffer::copy_from(&values), validity.clone()).into_array(); + let second = PrimitiveArray::new(Buffer::copy_from(&alternate), validity).into_array(); + let compressor = BtrBlocksCompressor::default(); + let first = compressor.compress(&first, &mut SESSION.create_execution_ctx())?; + let second = compressor.compress(&second, &mut SESSION.create_execution_ctx())?; + + assert!(first.is::()); + assert!(second.is::()); + assert_eq!(first.nbytes(), second.nbytes()); + Ok(()) +} + #[test] fn test_block_residual_skips_narrow_integers() -> VortexResult<()> { let values = (0..8_192) diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index 6fd33bf4747..227215d1da2 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -124,6 +124,13 @@ fn setup_widened_f32_array() -> PrimitiveArray { })) } +fn setup_quantized_f32_array() -> PrimitiveArray { + PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + let mantissa = (index.wrapping_mul(7_919) as u32 & 0x7fff) << 8; + f32::from_bits(0x3f80_0000 | mantissa) + })) +} + fn setup_nonzero_secondary_array() -> PrimitiveArray { let widened = setup_widened_f32_array(); PrimitiveArray::from_iter( @@ -158,6 +165,40 @@ fn setup_block_local_u64_array() -> PrimitiveArray { })) } +fn setup_block_local_u32_array() -> PrimitiveArray { + PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + let block = index / 1_024; + let residual = index.wrapping_mul(2_654_435_761) % 1_024; + (block * 1_000_000 + residual) as u32 + })) +} + +fn setup_block_local_i32_array() -> PrimitiveArray { + PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + let block = index / 1_024; + let residual = index.wrapping_mul(2_654_435_761) % 1_024; + (block as i32 - 1_000) * 1_000_000 + residual as i32 + })) +} + +fn setup_patch_density_u32_array(stride: u64) -> PrimitiveArray { + PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + if index % stride == 0 { + u32::MAX - index as u32 + } else { + 42 + } + })) +} + +fn setup_ordered_f32_array() -> PrimitiveArray { + PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + let block = index / 1_024; + let residual = index.wrapping_mul(7_919) % 1_024; + f32::from_bits(0x3f80_0000 + (block as u32 * 0x1_0000) + residual as u32) + })) +} + fn setup_block_local_i16_array() -> PrimitiveArray { PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { let block = (index / 1_024) % 128; @@ -187,7 +228,7 @@ fn ordered_values(array: &PrimitiveArray) -> PrimitiveArray { fn encode_ordered_block_residual(array: &PrimitiveArray) -> vortex::array::ArrayRef { let ordered = OrderedFloat::from_primitive(array.as_view()).unwrap(); let residuals = BlockResidual::from_primitive(ordered.encoded().as_::()).unwrap(); - OrderedFloat::try_new(residuals.into_array(), PType::F64) + OrderedFloat::try_new(residuals.into_array(), array.ptype()) .unwrap() .into_array() } @@ -200,8 +241,14 @@ fn encode_float_quant_tree(array: &PrimitiveArray) -> vortex::array::ArrayRef { .unwrap(); // SAFETY: The analysis computes this width from the exact primary range. let primary = unsafe { bitpack_encode_unchecked(primary, analysis.primary_bit_width) }.unwrap(); - let primary = FoR::try_new(primary.into_array(), Scalar::from(analysis.primary_min)).unwrap(); - FloatQuant::try_new(primary.into_array(), None, PType::F64, analysis.k) + let reference = if array.ptype() == PType::F32 { + Scalar::from(u32::try_from(analysis.primary_min).unwrap()) + } else { + debug_assert_eq!(array.ptype(), PType::F64); + Scalar::from(analysis.primary_min) + }; + let primary = FoR::try_new(primary.into_array(), reference).unwrap(); + FloatQuant::try_new(primary.into_array(), None, array.ptype(), analysis.k) .unwrap() .into_array() } @@ -599,6 +646,179 @@ fn bench_block_local_for_bitpacked_scalar_at_u64(bencher: Bencher) { .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); } +#[divan::bench(name = "block_local_block_residual_compress_u32")] +fn bench_block_local_block_residual_compress_u32(bencher: Bencher) { + let array = setup_block_local_u32_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) + .with_inputs(|| &array) + .bench_refs(|array| BlockResidual::from_primitive(array.as_view()).unwrap()); +} + +#[divan::bench(name = "block_local_block_residual_decompress_u32")] +fn bench_block_local_block_residual_decompress_u32(bencher: Bencher) { + let encoded = BlockResidual::from_primitive(setup_block_local_u32_array().as_view()) + .unwrap() + .into_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "block_local_block_residual_scalar_at_u32")] +fn bench_block_local_block_residual_scalar_at_u32(bencher: Bencher) { + let encoded = BlockResidual::from_primitive(setup_block_local_u32_array().as_view()) + .unwrap() + .into_array(); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + +#[divan::bench(name = "block_local_for_bitpacked_compress_u32")] +fn bench_block_local_for_bitpacked_compress_u32(bencher: Bencher) { + let array = setup_block_local_u32_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) + .with_inputs(|| &array) + .bench_refs(|array| encode_for_bitpacked_tree(array, 31)); +} + +#[divan::bench(name = "block_local_for_bitpacked_decompress_u32")] +fn bench_block_local_for_bitpacked_decompress_u32(bencher: Bencher) { + let encoded = encode_for_bitpacked_tree(&setup_block_local_u32_array(), 31); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "block_local_for_bitpacked_scalar_at_u32")] +fn bench_block_local_for_bitpacked_scalar_at_u32(bencher: Bencher) { + let encoded = encode_for_bitpacked_tree(&setup_block_local_u32_array(), 31); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + +#[divan::bench(name = "block_local_block_residual_compress_i32")] +fn bench_block_local_block_residual_compress_i32(bencher: Bencher) { + let array = setup_block_local_i32_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) + .with_inputs(|| &array) + .bench_refs(|array| BlockResidual::from_primitive(array.as_view()).unwrap()); +} + +#[divan::bench(name = "block_local_block_residual_decompress_i32")] +fn bench_block_local_block_residual_decompress_i32(bencher: Bencher) { + let encoded = BlockResidual::from_primitive(setup_block_local_i32_array().as_view()) + .unwrap() + .into_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "block_local_block_residual_scalar_at_i32")] +fn bench_block_local_block_residual_scalar_at_i32(bencher: Bencher) { + let encoded = BlockResidual::from_primitive(setup_block_local_i32_array().as_view()) + .unwrap() + .into_array(); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + +#[divan::bench(name = "block_local_for_bitpacked_compress_i32")] +fn bench_block_local_for_bitpacked_compress_i32(bencher: Bencher) { + let array = setup_block_local_i32_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) + .with_inputs(|| &array) + .bench_refs(|array| encode_for_bitpacked_tree(array, 31)); +} + +#[divan::bench(name = "block_local_for_bitpacked_decompress_i32")] +fn bench_block_local_for_bitpacked_decompress_i32(bencher: Bencher) { + let encoded = encode_for_bitpacked_tree(&setup_block_local_i32_array(), 31); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "block_local_for_bitpacked_scalar_at_i32")] +fn bench_block_local_for_bitpacked_scalar_at_i32(bencher: Bencher) { + let encoded = encode_for_bitpacked_tree(&setup_block_local_i32_array(), 31); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + +#[divan::bench(args = [256, 64, 16, 4, 1])] +fn patch_density_block_residual_decompress_u32(bencher: Bencher, stride: u64) { + let encoded = BlockResidual::from_primitive(setup_patch_density_u32_array(stride).as_view()) + .unwrap() + .into_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(args = [256, 64, 16, 4, 1])] +fn patch_density_block_residual_scalar_at_u32(bencher: Bencher, stride: u64) { + let encoded = BlockResidual::from_primitive(setup_patch_density_u32_array(stride).as_view()) + .unwrap() + .into_array(); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + #[divan::bench(name = "block_local_block_residual_compress_i16")] fn bench_block_local_block_residual_compress_i16(bencher: Bencher) { let array = setup_block_local_i16_array(); @@ -722,6 +942,40 @@ fn bench_ordered_block_residual_prior_default_scalar_at_f64(bencher: Bencher) { .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); } +#[divan::bench(name = "ordered_block_residual_compress_f32")] +fn bench_ordered_block_residual_compress_f32(bencher: Bencher) { + let float_array = setup_ordered_f32_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) + .with_inputs(|| &float_array) + .bench_refs(|array| encode_ordered_block_residual(array)); +} + +#[divan::bench(name = "ordered_block_residual_decompress_f32")] +fn bench_ordered_block_residual_decompress_f32(bencher: Bencher) { + let encoded = encode_ordered_block_residual(&setup_ordered_f32_array()); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "ordered_block_residual_scalar_at_f32")] +fn bench_ordered_block_residual_scalar_at_f32(bencher: Bencher) { + let encoded = encode_ordered_block_residual(&setup_ordered_f32_array()); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + #[divan::bench(name = "ordered_block_residual_scheme_compress_f64")] fn bench_ordered_block_residual_scheme_compress_f64(bencher: Bencher) { let float_array = setup_random_walk_array(); @@ -797,6 +1051,40 @@ fn bench_float_quant_tree_scalar_at_f64(bencher: Bencher) { .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); } +#[divan::bench(name = "float_quant_tree_compress_f32")] +fn bench_float_quant_tree_compress_f32(bencher: Bencher) { + let float_array = setup_quantized_f32_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) + .with_inputs(|| &float_array) + .bench_refs(|array| encode_float_quant_tree(array)); +} + +#[divan::bench(name = "float_quant_tree_decompress_f32")] +fn bench_float_quant_tree_decompress_f32(bencher: Bencher) { + let encoded = encode_float_quant_tree(&setup_quantized_f32_array()); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "float_quant_tree_scalar_at_f32")] +fn bench_float_quant_tree_scalar_at_f32(bencher: Bencher) { + let encoded = encode_float_quant_tree(&setup_quantized_f32_array()); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + #[divan::bench(name = "float_quant_nonzero_secondary_split_compress_f64")] fn bench_float_quant_nonzero_secondary_split_compress_f64(bencher: Bencher) { let float_array = setup_nonzero_secondary_array(); From c351b9a15aa69f26f2631601fd3d2e3d4414aa70 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 14:22:22 +0100 Subject: [PATCH 10/78] bench: profile quotient remainder candidates Signed-off-by: Will Manning --- .../examples/float_compressor_bench.rs | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index 1594225cdf3..e6cf36feca8 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -47,6 +47,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_session::VortexSession; +use vortex_utils::aliases::hash_map::HashMap; const DEFAULT_ROW_COUNT: usize = 2_000_000; const CALIFORNIA_COLUMNS: [&str; 9] = [ @@ -577,6 +578,53 @@ fn ordered_values(values: &PrimitiveArray) -> Vec { } } +fn profile_quotient_remainder( + dataset: &str, + column: &str, + path: &str, + ptype: PType, + pco_bytes: u64, + ordered: &[u64], + session: &VortexSession, +) -> VortexResult<()> { + const BASES: [u64; 9] = [2, 4, 5, 8, 10, 16, 32, 100, 1_000]; + + let compressor = BtrBlocksCompressor::default(); + for base in BASES { + let mut remainder_counts = HashMap::::new(); + for value in ordered { + *remainder_counts.entry(value % base).or_default() += 1; + } + let remainder_entropy = remainder_counts + .values() + .map(|&count| { + let probability = count as f64 / ordered.len() as f64; + -probability * probability.log2() + }) + .sum::(); + let most_common_remainder_share = + remainder_counts.values().copied().max().unwrap_or_default() as f64 + / ordered.len() as f64; + let quotients = PrimitiveArray::from_iter(ordered.iter().map(|value| value / base)); + let remainders = PrimitiveArray::from_iter(ordered.iter().map(|value| value % base)); + let quotient = + compressor.compress("ients.into_array(), &mut session.create_execution_ctx())?; + let remainder = compressor.compress( + &remainders.into_array(), + &mut session.create_execution_ctx(), + )?; + println!( + "quotient-remainder-estimate\t{dataset}\t{column}\t{path}\t{ptype}\t{base}\t{pco_bytes}\t{remainder_entropy:.3}\t{most_common_remainder_share:.3}\t{}\t{}\t{}\t{}\t{}", + quotient.nbytes(), + remainder.nbytes(), + quotient.nbytes() + remainder.nbytes(), + encoding_tree("ient), + encoding_tree(&remainder), + ); + } + Ok(()) +} + fn profile_pco_array( dataset: &str, column: &str, @@ -595,6 +643,15 @@ fn profile_pco_array( .filter(mask)? .execute::(&mut ctx)?; let ordered = ordered_values(&values); + profile_quotient_remainder( + dataset, + column, + path, + values.ptype(), + array.nbytes(), + &ordered, + session, + )?; let validity_bytes = array.children().iter().map(ArrayRef::nbytes).sum::(); let bits = values.ptype().bit_width(); let one_reference = @@ -727,6 +784,10 @@ fn measure_dataset( } } + if std::env::var_os("VORTEX_BENCH_PROFILE_ONLY").is_some() { + return Ok(()); + } + let encode_iterations = (128_000_000_u64 / input_bytes).clamp(3, 10) as usize; let mut encode_durations = (0..configs.len()) .map(|_| Vec::with_capacity(encode_iterations)) @@ -839,6 +900,9 @@ fn main() -> VortexResult<()> { println!( "multi-ref-estimate\tdataset\tcolumn\tpath\tptype\tpco-child-bytes\tone-reference-bytes\ttwo-reference-bytes\tfour-reference-bytes\tbest-bytes" ); + println!( + "quotient-remainder-estimate\tdataset\tcolumn\tpath\tptype\tbase\tpco-child-bytes\tremainder-entropy\tmost-common-remainder-share\tquotient-bytes\tremainder-bytes\ttotal-bytes\tquotient-encoding\tremainder-encoding" + ); println!("result\tdataset\tconfig\trows\tinput-bytes\tencoded-bytes\tencode-MB/s\tdecode-MB/s"); println!( "block-residual-profile\tdataset\tcolumn\tconfig\tpath\tptype\trows\tpatches\tblocks\taverage-residual-width\taverage-high-width\tbytes" From 37b7547b536bf07f0b55eabe972b890567e7b03f Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 14:25:28 +0100 Subject: [PATCH 11/78] bench: estimate bitmap patch alternatives Signed-off-by: Will Manning --- .../examples/float_compressor_bench.rs | 90 ++++++++++++++++++- 1 file changed, 86 insertions(+), 4 deletions(-) diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index e6cf36feca8..a240c1948ee 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -516,6 +516,80 @@ fn estimate_multi_reference(values: &[u64], requested_references: usize, bits: u .sum() } +fn estimate_bitmap_patches(values: &[u64], bits: usize) -> usize { + const BLOCK_LEN: usize = 1_024; + const METADATA_BYTES: usize = 12; + const HIGH_PADDING_BYTES: usize = 15; + + values + .chunks(BLOCK_LEN) + .map(|block| { + let base = block.iter().copied().min().unwrap_or_default(); + let mut width_counts = [0_usize; 65]; + let mut maximum_width = 0_usize; + for value in block { + let residual = value - base; + let width = u64::BITS as usize - residual.leading_zeros() as usize; + width_counts[width] += 1; + maximum_width = maximum_width.max(width); + } + + let mut patch_count = block.len(); + let mut best_bits = usize::MAX; + for residual_width in 0..=maximum_width { + patch_count -= width_counts[residual_width]; + let high_width = if patch_count == 0 { + 0 + } else { + maximum_width - residual_width + }; + let cost_bits = residual_width * BLOCK_LEN + + bits + + usize::from(patch_count > 0) * BLOCK_LEN + + patch_count * high_width + + METADATA_BYTES * 8 + + usize::from(patch_count > 0) * HIGH_PADDING_BYTES * 8; + best_bits = best_bits.min(cost_bits); + } + best_bits.div_ceil(8) + }) + .sum() +} + +fn estimate_mode_bitmap(values: &[u64], bits: usize) -> usize { + const BLOCK_LEN: usize = 1_024; + const METADATA_BYTES: usize = 8; + const VALUE_PADDING_BYTES: usize = 15; + + values + .chunks(BLOCK_LEN) + .map(|block| { + let mut counts = HashMap::::new(); + for value in block { + *counts.entry(*value).or_default() += 1; + } + let mode = counts + .into_iter() + .max_by_key(|(_, count)| *count) + .map(|(value, _)| value) + .unwrap_or_default(); + let exceptions = block.iter().filter(|&&value| value != mode).count(); + let exception_width = block + .iter() + .filter(|&&value| value != mode) + .map(|&value| u64::BITS as usize - value.leading_zeros() as usize) + .max() + .unwrap_or_default(); + let cost_bits = bits + + BLOCK_LEN + + exceptions * exception_width + + METADATA_BYTES * 8 + + usize::from(exceptions > 0) * VALUE_PADDING_BYTES * 8; + cost_bits.div_ceil(8) + }) + .sum() +} + fn ordered_f32(value: f32) -> u64 { let bits = value.to_bits(); u64::from(if bits & (1_u32 << 31) == 0 { @@ -607,6 +681,10 @@ fn profile_quotient_remainder( / ordered.len() as f64; let quotients = PrimitiveArray::from_iter(ordered.iter().map(|value| value / base)); let remainders = PrimitiveArray::from_iter(ordered.iter().map(|value| value % base)); + let quotient_bitmap_bytes = + estimate_bitmap_patches(quotients.as_slice::(), ptype.bit_width()); + let remainder_mode_bitmap_bytes = + estimate_mode_bitmap(remainders.as_slice::(), ptype.bit_width()); let quotient = compressor.compress("ients.into_array(), &mut session.create_execution_ctx())?; let remainder = compressor.compress( @@ -614,10 +692,11 @@ fn profile_quotient_remainder( &mut session.create_execution_ctx(), )?; println!( - "quotient-remainder-estimate\t{dataset}\t{column}\t{path}\t{ptype}\t{base}\t{pco_bytes}\t{remainder_entropy:.3}\t{most_common_remainder_share:.3}\t{}\t{}\t{}\t{}\t{}", + "quotient-remainder-estimate\t{dataset}\t{column}\t{path}\t{ptype}\t{base}\t{pco_bytes}\t{remainder_entropy:.3}\t{most_common_remainder_share:.3}\t{}\t{}\t{}\t{quotient_bitmap_bytes}\t{remainder_mode_bitmap_bytes}\t{}\t{}\t{}", quotient.nbytes(), remainder.nbytes(), quotient.nbytes() + remainder.nbytes(), + quotient_bitmap_bytes + remainder_mode_bitmap_bytes, encoding_tree("ient), encoding_tree(&remainder), ); @@ -660,14 +739,17 @@ fn profile_pco_array( u64::try_from(estimate_multi_reference(&ordered, 2, bits))? + validity_bytes; let four_references = u64::try_from(estimate_multi_reference(&ordered, 4, bits))? + validity_bytes; + let bitmap_patches = + u64::try_from(estimate_bitmap_patches(&ordered, bits))? + validity_bytes; println!( - "multi-ref-estimate\t{dataset}\t{column}\t{path}\t{}\t{}\t{}\t{}\t{}\t{}", + "multi-ref-estimate\t{dataset}\t{column}\t{path}\t{}\t{}\t{}\t{}\t{}\t{}\t{}", values.ptype(), array.nbytes(), one_reference, two_references, four_references, one_reference.min(two_references).min(four_references), + bitmap_patches, ); match values.ptype() { PType::F32 => { @@ -898,10 +980,10 @@ fn main() -> VortexResult<()> { "pco-profile\tdataset\tcolumn\tpath\tptype\tchunk\trows\tmode\tdelta\tdelta-bins\tdelta-max-offset-bits\tdelta-average-bits\tprimary-bins\tprimary-ans-log\tprimary-max-offset-bits\tprimary-average-bits\tsecondary-bins\tsecondary-ans-log\tsecondary-max-offset-bits\tsecondary-average-bits" ); println!( - "multi-ref-estimate\tdataset\tcolumn\tpath\tptype\tpco-child-bytes\tone-reference-bytes\ttwo-reference-bytes\tfour-reference-bytes\tbest-bytes" + "multi-ref-estimate\tdataset\tcolumn\tpath\tptype\tpco-child-bytes\tone-reference-bytes\ttwo-reference-bytes\tfour-reference-bytes\tbest-reference-bytes\tbitmap-patch-bytes" ); println!( - "quotient-remainder-estimate\tdataset\tcolumn\tpath\tptype\tbase\tpco-child-bytes\tremainder-entropy\tmost-common-remainder-share\tquotient-bytes\tremainder-bytes\ttotal-bytes\tquotient-encoding\tremainder-encoding" + "quotient-remainder-estimate\tdataset\tcolumn\tpath\tptype\tbase\tpco-child-bytes\tremainder-entropy\tmost-common-remainder-share\tquotient-bytes\tremainder-bytes\ttotal-bytes\tquotient-bitmap-bytes\tremainder-mode-bitmap-bytes\tbitmap-total-bytes\tquotient-encoding\tremainder-encoding" ); println!("result\tdataset\tconfig\trows\tinput-bytes\tencoded-bytes\tencode-MB/s\tdecode-MB/s"); println!( From 27aca12ff052850f91e2dd6a2e80a0b580552873 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 14:53:45 +0100 Subject: [PATCH 12/78] perf: evaluate native f32 float quant Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 130 +++++++++++++++--- .../examples/float_compressor_bench.rs | 15 ++ .../src/schemes/float/float_quant.rs | 5 +- .../schemes/float/scheme_selection_tests.rs | 17 +++ 4 files changed, 145 insertions(+), 22 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 40ae1e10317..f131b43a1de 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -24,6 +24,8 @@ Remove `RangeEntropyArray`, `RangeEntropyScheme`, and `BitSplitCodec` from the f The `wm/pcodec-entropy-experiments` branch preserves the complete entropy and bit-split prototypes. +Keep fixed-bin range packing as an experimental Compact candidate. It is not a default candidate. + Do not add adjacent Delta, Delta-of-delta, Delta with lookback, or convolution Delta. ## Current state @@ -34,11 +36,15 @@ The default candidate set includes their three BtrBlocks schemes. `OrderedFloat(BlockResidual)` now supports `f32` and `f64` inputs. +The current FloatQuant prototype also accepts native `f32` and `f64` inputs. + Direct integer BlockResidual supports every integer type. The default selector accepts only 32-bit and 64-bit inputs. The retained schemes win on specific structures. They do not replace ALP or ALP-RD across general float data. -The GloVe result identifies a separate gap inside the ALP integer child. +The GloVe result identifies a separate entropy gap inside the ALP integer child. + +Generic quotient and remainder trees do not close that gap. The current selector factors and patch cost remain provisional. Final calibration requires the complete corpus and selected-tree evidence. @@ -93,7 +99,9 @@ An absent secondary child represents zero low bits. The array supports exact IEEE bit-pattern round trips, nulls, slices, scalar access, and serialization. -The automatic scheme accepts only `f64`. Native `f32` columns remain with ALP, ALP-RD, and existing schemes. +The automatic scheme accepts `f32` and `f64` inputs. + +Native `f32` selection remains provisional because the full compressor misses the compression throughput limit. ## Selection policy @@ -103,6 +111,10 @@ The sample builds a direct `FoR(BitPacked)` primary and uses an implicit-zero se The scheme rejects samples with a nonzero secondary. This avoids the recursive integer selector. +The current native `f32` path uses the generic deferred sample. + +The next selector uses a cheap prefilter and an exact fixed-tree estimate on the same one-percent sample. + `OrderedBlockResidualScheme` uses eight locality-preserving sample blocks. The ordinary BtrBlocks sample does not preserve the block-local float structure. @@ -310,6 +322,29 @@ The BtrBlocks candidate now accepts f32 and f64 inputs. These results remove the prior f32 type exclusion. Corpus comparisons against ALP and ALP-RD remain necessary. +### Native f32 FloatQuant + +The input contains two million native `f32` values with eight zero low mantissa bits. + +| Configuration | Selected tree | Bytes | Encode MB/s | Decode MB/s | +| --- | --- | ---: | ---: | ---: | +| Prior default | `ALP-RD(BitPacked, BitPacked)` | 5,752,576 | 750.4 | 10,634.2 | +| Default with FloatQuant | `FloatQuant(FoR(BitPacked))` | 3,751,680 | 542.1 | 18,970.4 | +| FloatQuant only | `FloatQuant(FoR(BitPacked))` | 3,751,680 | 554.3 | 18,427.9 | +| Compact Pco | `Pco` | 201,374 | About 304 | About 2,933 | + +FloatQuant reduced size by 34.8 percent against ALP-RD. + +Decode throughput increased by 78.4 percent. + +Full compression throughput decreased by 27.8 percent. This result exceeds the 20-percent limit. + +The isolated FloatQuant tree compressed at about 1.29 GB/s and decoded at about 19.6 GB/s. + +The large difference between isolated and full compression identifies selector and sample cost as the next target. + +The Compact size is specific to this synthetic pattern. It does not establish a general real-float result. + ### Direct 32-bit comparison The direct benchmark compares two million block-local values against a whole-column FoR plus BitPacked tree. @@ -373,6 +408,56 @@ The size gap comes from a decimal quotient and remainder split plus entropy codi GloVe therefore identifies an ALP-child compression gap. It does not identify a failure in the ALP float transform. +### GloVe quotient and remainder prototypes + +The first prototype splits the exact ALP integer child with a constant base. + +| Candidate | Bytes | +| --- | ---: | +| Pco `IntMult(10)` child | 4,518,870 | +| Base ten with ordinary integer children | 6,002,688 | +| Best tested ordinary base, base 100 | 5,572,096 | +| Base ten with bitmap-patched quotient and mode bitmap remainder | 5,221,476 | +| Direct bitmap patches on the unsplit child | 5,541,496 | +| Direct position patches on the unsplit child | 5,507,862 | + +Ordinary integer trees lose materially to Pco on the quotient and remainder. + +Bitmap patches improve size, but they do not match the Pco child. + +The results reject a general quotient and remainder array with ordinary children. + +They support a small-alphabet entropy experiment and a fixed-bin Compact experiment. + +### Fixed-bin range backend experiments + +The experimental branch compares ANS symbols with fixed-width bin identifiers. + +The packed variants retain variable-width offsets inside each bin. + +| Input and backend | Bytes | Encode MB/s | Decode MB/s | +| --- | ---: | ---: | ---: | +| GloVe ALP child, ANS | 5,264,329 | 414.2 | 3,476.5 | +| GloVe ALP child, packed bins | 5,573,968 | 765.8 | 7,700.0 | +| GloVe `IntMult(10)`, ANS | 4,716,002 | 457.8 | 1,944.6 | +| GloVe `IntMult(10)`, packed bins | 5,562,181 | 821.1 | 3,661.7 | +| CMS ALP child, ANS | 3,293,477 | 406.6 | 3,823.0 | +| CMS ALP child, packed bins | 3,647,593 | 642.7 | 8,358.2 | +| Food ALP child, ANS | 5,405,851 | 392.5 | 3,936.6 | +| Food ALP child, packed bins | 5,525,544 | 607.4 | 8,525.6 | + +ANS approaches Pco size, but it retains an expensive decode path. + +Packed bins decode faster, but they lose part of the size benefit. + +Packed bins remain plausible for Compact on selected columns. + +Fast scalar access requires checkpoints for the variable offset stream. + +A checkpoint interval of 32 values bounds scalar work to 31 offset widths. + +This design adds about 0.5 bits per value with 16-bit checkpoints. + ### Pcodec corpus gap analysis The focused benchmark reads the first two million numeric rows from each source. @@ -392,6 +477,12 @@ Pco uses four principal mechanisms on these columns: No Pcodec paper gap used lookback Delta. Some Public BI columns used lookback Delta. +GloVe, CMS Payments, and Food include important no-Delta wins. + +GloVe uses `IntMult(10)` plus entropy bins. CMS Payments and Food use classic entropy bins. + +Arade relies mainly on first-order consecutive Delta. Two columns combine `IntMult` with Delta. + The current CMS source revision differs from the paper source snapshot. The Twitter source uses the first two million official edges. The paper used an unspecified ID sort. @@ -524,25 +615,26 @@ This round completed these steps: - Excluded integer BlockResidual from the CUDA-compatible preset. - Added the real GloVe embeddings dataset to the compression corpus. +The Pco mode profile and quotient and remainder experiments are complete. + Complete these remaining steps: -1. Profile the ALP integer child on GloVe and unrelated no-Delta Pco wins. -2. Record each selected `IntMult` base, quotient distribution, remainder distribution, and Pco bit cost. -3. Prototype a bounded-access quotient and remainder array with a constant base. -4. Test existing integer trees for both children before a new entropy backend. -5. Compare that prototype against `ALP(ZigZag(BitPacked))` and `ALP(Pco)` on the same inputs. -6. If existing children lose materially, test a small-alphabet entropy child for the remainder. -7. Test a fused nonzero-secondary FloatQuant decode on selected gap columns. -8. Evaluate nonzero-secondary FloatQuant for the default compressor. -9. Compare native f32 FloatQuant against ALP and ALP-RD on the same inputs. -10. Compare alternate patch costs across the patch-density sweep. -11. Evaluate narrow BlockResidual as a Compact-only candidate. -12. Reduce the analysis cost on short rejected columns. -13. Measure BlockResidual under temporal, FSST, sparse, run-end, and list parents. -14. Run the complete `bench-vortex` compression corpus after the candidate set stabilizes. -15. Compare the geometric mean against Parquet with Zstd. -16. Calibrate all selector thresholds from the complete corpus and selected-tree evidence. -17. Update this plan after each experiment. +1. Replace the native `f32` FloatQuant deferred sample with a cheap prefilter and fixed-tree estimator. +2. Add FloatQuant scheme analysis to `single_encoding_throughput`. +3. Re-run the native `f32` size and throughput comparison. +4. Test a fused nonzero-secondary FloatQuant decode on selected gap columns. +5. Decide whether nonzero-secondary FloatQuant is a default candidate. +6. Compare alternate BlockResidual patch costs across the patch-density sweep. +7. Evaluate narrow BlockResidual as a Compact-only candidate. +8. Reduce analysis cost on short rejected columns. +9. Measure BlockResidual under temporal, FSST, sparse, run-end, and list parents. +10. Prototype bounded scalar checkpoints for fixed-bin range packing. +11. Compare fixed-bin packing against default and Pco on unrelated no-Delta wins. +12. Add real embedding datasets beyond GloVe when licenses and loaders permit them. +13. Run the complete `bench-vortex` compression corpus after the candidate set stabilizes. +14. Compare the geometric mean against Parquet with Zstd. +15. Calibrate all selector thresholds from complete corpus and selected-tree evidence. +16. Update this plan after each experiment. ## Pull request structure diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index a240c1948ee..c20332513cf 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -93,6 +93,10 @@ fn synthetic_datasets(row_count: usize) -> Vec<(String, Vec)> { } }, )); + let quantized_f32 = PrimitiveArray::from_iter((0_u32..).take(row_count).map(|index| { + let mantissa = (index.wrapping_mul(7_919) & 0x7fff) << 8; + f32::from_bits(0x3f80_0000 | mantissa) + })); let mut walk_rng = StdRng::seed_from_u64(2); let mut value = 1_000.0_f64; @@ -120,6 +124,10 @@ fn synthetic_datasets(row_count: usize) -> Vec<(String, Vec)> { "synthetic-nonzero-secondary".to_string(), vec![column("value", nonzero_secondary)], ), + ( + "synthetic-quantized-f32".to_string(), + vec![column("value", quantized_f32)], + ), ( "synthetic-random-walk".to_string(), vec![column("value", random_walk)], @@ -990,7 +998,14 @@ fn main() -> VortexResult<()> { "block-residual-profile\tdataset\tcolumn\tconfig\tpath\tptype\trows\tpatches\tblocks\taverage-residual-width\taverage-high-width\tbytes" ); if std::env::var_os("VORTEX_BENCH_SKIP_SYNTHETIC").is_none() { + let synthetic_filter = std::env::var("VORTEX_BENCH_SYNTHETIC").ok(); for (dataset, columns) in synthetic_datasets(row_count) { + if synthetic_filter + .as_deref() + .is_some_and(|filter| filter != dataset) + { + continue; + } measure_dataset(&dataset, &columns, &configs, &session)?; } } diff --git a/vortex-btrblocks/src/schemes/float/float_quant.rs b/vortex-btrblocks/src/schemes/float/float_quant.rs index 4ad0dc9fee1..8c1a027fc81 100644 --- a/vortex-btrblocks/src/schemes/float/float_quant.rs +++ b/vortex-btrblocks/src/schemes/float/float_quant.rs @@ -46,12 +46,11 @@ impl Scheme for FloatQuantScheme { fn expected_compression_ratio( &self, - data: &ArrayAndStats, + _data: &ArrayAndStats, compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { - let primitive = data.array_as_primitive(); - if compress_ctx.finished_cascading() || primitive.ptype() != PType::F64 { + if compress_ctx.finished_cascading() { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } CompressionEstimate::Deferred(DeferredEstimate::Sample) diff --git a/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs index 2765c79c092..197e44858cd 100644 --- a/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs @@ -134,6 +134,23 @@ fn test_f32_does_not_use_float_quant() -> VortexResult<()> { Ok(()) } +#[test] +fn test_quantized_f32_uses_float_quant() -> VortexResult<()> { + let values = (0_u32..65_536) + .map(|index| { + let mantissa = (index.wrapping_mul(7_919) & 0x7fff) << 8; + f32::from_bits(0x3f80_0000 | mantissa) + }) + .collect::>(); + let array = PrimitiveArray::from_iter(values).into_array(); + let compressed = + BtrBlocksCompressor::default().compress(&array, &mut SESSION.create_execution_ctx())?; + + assert!(compressed.is::()); + assert!(compressed.as_::().secondary().is_none()); + Ok(()) +} + #[test] fn test_repeated_f64_prefers_existing_scheme() -> VortexResult<()> { let values = (0u32..16_384) From 4465ba2977b8f2722261b9266f0b50d146e3155c Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 15:12:35 +0100 Subject: [PATCH 13/78] perf: fuse float quant with bitpacking Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 84 +++++---- .../src/bitpacking/array/bitpack_compress.rs | 68 ++++++++ .../src/schemes/float/float_quant.rs | 165 +++++++++++++++--- vortex/benches/single_encoding_throughput.rs | 107 +++++++++++- 4 files changed, 367 insertions(+), 57 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index f131b43a1de..1b947fce351 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -36,7 +36,7 @@ The default candidate set includes their three BtrBlocks schemes. `OrderedFloat(BlockResidual)` now supports `f32` and `f64` inputs. -The current FloatQuant prototype also accepts native `f32` and `f64` inputs. +The FloatQuant candidate accepts native `f32` and `f64` inputs. Direct integer BlockResidual supports every integer type. The default selector accepts only 32-bit and 64-bit inputs. @@ -101,7 +101,9 @@ The array supports exact IEEE bit-pattern round trips, nulls, slices, scalar acc The automatic scheme accepts `f32` and `f64` inputs. -Native `f32` selection remains provisional because the full compressor misses the compression throughput limit. +Native `f32` selection now meets the selected-column and rejected-column throughput limits. + +Final default inclusion still requires broad corpus validation. ## Selection policy @@ -111,9 +113,13 @@ The sample builds a direct `FoR(BitPacked)` primary and uses an implicit-zero se The scheme rejects samples with a nonzero secondary. This avoids the recursive integer selector. -The current native `f32` path uses the generic deferred sample. +The scheme uses a cheap bit-pattern prefilter over a stratified one-percent sample. + +If the sample qualifies, the estimator builds the exact fixed tree on that sample. -The next selector uses a cheap prefilter and an exact fixed-tree estimate on the same one-percent sample. +The final encoder maps float blocks directly into the FastLanes packer. + +This fused path does not allocate a full primary integer buffer. `OrderedBlockResidualScheme` uses eight locality-preserving sample blocks. @@ -164,25 +170,25 @@ The input contains two million arbitrary `f32` values stored in an `f64` column. | Configuration | Bytes | Compression MB/s | Decode MB/s | Scalar access ns | | --- | ---: | ---: | ---: | ---: | -| Prior default | 14,057,966 | 553.6 | 10,944.5 | 208.7 | -| Default with FloatQuant | 8,753,920 | 504.9 | 14,795.4 | 145.5 | -| Compact | 6,051,737 | 280.0 | 4,022.0 | Not measured | +| Prior default | 14,057,966 | 534.4 | 10,072.7 | 208.7 | +| Default with FloatQuant | 8,753,920 | 565.4 | 14,032.5 | 145.5 | +| Compact | 6,051,737 | 272.2 | 3,912.7 | Not measured | FloatQuant reduced size by 37.7 percent. It remained 44.7 percent larger than Compact. -Full compressor throughput decreased by 8.8 percent. Decode throughput increased by 35.2 percent. +Full compressor throughput increased by 5.8 percent. Decode throughput increased by 39.3 percent. Scalar access latency decreased by 30.3 percent. The selected tree was `FloatQuant(FoR(BitPacked))` with an implicit-zero secondary. -The isolated tree compressed between 2,402 and 2,469 MB/s. +The fused FloatQuant scheme compressed at 4,799 MB/s. It decoded between 14,480 and 14,810 MB/s. -Compact Pco compressed the same input at 280 MB/s and decoded it at 4,022 MB/s. +Compact Pco compressed the same input at 272 MB/s and decoded it at 3,913 MB/s. -The tree itself exceeded Compact throughput by at least 8.6 times for compression and 3.6 times for decode. +The scheme exceeded Compact throughput by 17.6 times for compression and 3.6 times for decode. FloatQuant recovered 66.2 percent of the size gap between the prior default and Compact Pco. @@ -328,20 +334,28 @@ The input contains two million native `f32` values with eight zero low mantissa | Configuration | Selected tree | Bytes | Encode MB/s | Decode MB/s | | --- | --- | ---: | ---: | ---: | -| Prior default | `ALP-RD(BitPacked, BitPacked)` | 5,752,576 | 750.4 | 10,634.2 | -| Default with FloatQuant | `FloatQuant(FoR(BitPacked))` | 3,751,680 | 542.1 | 18,970.4 | -| FloatQuant only | `FloatQuant(FoR(BitPacked))` | 3,751,680 | 554.3 | 18,427.9 | -| Compact Pco | `Pco` | 201,374 | About 304 | About 2,933 | +| Prior default | `ALP-RD(BitPacked, BitPacked)` | 5,752,576 | 664.7 | 10,284.4 | +| Default with FloatQuant | `FloatQuant(FoR(BitPacked))` | 3,751,680 | 745.4 | 18,461.6 | +| FloatQuant only | `FloatQuant(FoR(BitPacked))` | 3,751,680 | 746.6 | 18,169.8 | +| Compact Pco | `Pco` | 201,374 | 294.2 | 2,885.0 | FloatQuant reduced size by 34.8 percent against ALP-RD. -Decode throughput increased by 78.4 percent. +Decode throughput increased by 79.5 percent. + +Full compression throughput increased by 12.1 percent in the interleaved compressor benchmark. + +The fused FloatQuant scheme compressed at 2.33 GB/s. + +The prior materialized tree compressed at 1.25 GB/s in the same direct benchmark. + +The proposed default compressed at 753 MB/s in the Divan benchmark. -Full compression throughput decreased by 27.8 percent. This result exceeds the 20-percent limit. +The prior default compressed at 776 MB/s in that benchmark. The difference is 2.9 percent. -The isolated FloatQuant tree compressed at about 1.29 GB/s and decoded at about 19.6 GB/s. +On rejected general `f32` data, the proposed default compressed at 390 MB/s. -The large difference between isolated and full compression identifies selector and sample cost as the next target. +The prior default compressed at 394 MB/s. The difference is 1.2 percent. The Compact size is specific to this synthetic pattern. It does not establish a general real-float result. @@ -614,27 +628,27 @@ This round completed these steps: - Made null payload bits neutral for the new default candidates. - Excluded integer BlockResidual from the CUDA-compatible preset. - Added the real GloVe embeddings dataset to the compression corpus. +- Added native `f32` FloatQuant selection and scheme benchmarks. +- Replaced the full primary buffer with fused FloatQuant and FastLanes packing. +- Replaced the generic FloatQuant sample with a direct fixed-tree estimator. The Pco mode profile and quotient and remainder experiments are complete. Complete these remaining steps: -1. Replace the native `f32` FloatQuant deferred sample with a cheap prefilter and fixed-tree estimator. -2. Add FloatQuant scheme analysis to `single_encoding_throughput`. -3. Re-run the native `f32` size and throughput comparison. -4. Test a fused nonzero-secondary FloatQuant decode on selected gap columns. -5. Decide whether nonzero-secondary FloatQuant is a default candidate. -6. Compare alternate BlockResidual patch costs across the patch-density sweep. -7. Evaluate narrow BlockResidual as a Compact-only candidate. -8. Reduce analysis cost on short rejected columns. -9. Measure BlockResidual under temporal, FSST, sparse, run-end, and list parents. -10. Prototype bounded scalar checkpoints for fixed-bin range packing. -11. Compare fixed-bin packing against default and Pco on unrelated no-Delta wins. -12. Add real embedding datasets beyond GloVe when licenses and loaders permit them. -13. Run the complete `bench-vortex` compression corpus after the candidate set stabilizes. -14. Compare the geometric mean against Parquet with Zstd. -15. Calibrate all selector thresholds from complete corpus and selected-tree evidence. -16. Update this plan after each experiment. +1. Test a fused nonzero-secondary FloatQuant decode on selected gap columns. +2. Decide whether nonzero-secondary FloatQuant is a default candidate. +3. Compare alternate BlockResidual patch costs across the patch-density sweep. +4. Evaluate narrow BlockResidual as a Compact-only candidate. +5. Reduce analysis cost on short rejected columns. +6. Measure BlockResidual under temporal, FSST, sparse, run-end, and list parents. +7. Prototype bounded scalar checkpoints for fixed-bin range packing. +8. Compare fixed-bin packing against default and Pco on unrelated no-Delta wins. +9. Add real embedding datasets beyond GloVe when licenses and loaders permit them. +10. Run the complete `bench-vortex` compression corpus after the candidate set stabilizes. +11. Compare the geometric mean against Parquet with Zstd. +12. Calibrate all selector thresholds from complete corpus and selected-tree evidence. +13. Update this plan after each experiment. ## Pull request structure diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs index a393db6ecc8..006fae2a1dc 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs @@ -194,6 +194,59 @@ pub fn bitpack_primitive(array: &[T], bit_width: u8 output.freeze() } +/// Maps and bit-packs primitive values without a full intermediate buffer. +/// +/// The mapped values must fit in `bit_width` bits. This function does not create patches. +pub fn bitpack_primitive_map(array: &[S], bit_width: u8, mut map: F) -> Buffer +where + T: NativePType + BitPacking, + F: FnMut(&S) -> T, +{ + if bit_width == 0 { + return Buffer::::empty(); + } + + let bit_width = usize::from(bit_width); + let num_chunks = array.len().div_ceil(1024); + let num_full_chunks = array.len() / 1024; + let packed_len = 128 * bit_width / size_of::(); + let mut output = BufferMut::::with_capacity(num_chunks * packed_len); + let mut mapped = [T::zero(); 1024]; + + for chunk_index in 0..num_full_chunks { + let start = chunk_index * 1024; + mapped + .iter_mut() + .zip(&array[start..start + 1024]) + .for_each(|(output, input)| *output = map(input)); + let output_len = output.len(); + // SAFETY: The output has capacity for one packed vector and both slices have exact sizes. + unsafe { + output.set_len(output_len + packed_len); + BitPacking::unchecked_pack(bit_width, &mapped, &mut output[output_len..][..packed_len]); + } + } + + if num_chunks != num_full_chunks { + let start = num_full_chunks * 1024; + let last_chunk_len = array.len() - start; + mapped[..last_chunk_len] + .iter_mut() + .zip(&array[start..]) + .for_each(|(output, input)| *output = map(input)); + mapped[last_chunk_len..].fill(T::zero()); + + let output_len = output.len(); + // SAFETY: The output has capacity for one packed vector and both slices have exact sizes. + unsafe { + output.set_len(output_len + packed_len); + BitPacking::unchecked_pack(bit_width, &mapped, &mut output[output_len..][..packed_len]); + } + } + + output.freeze() +} + pub fn gather_patches( parray: &PrimitiveArray, bit_width: u8, @@ -464,6 +517,21 @@ mod test { ); } + #[test] + fn bitpack_primitive_map_matches_materialized_input() { + for len in [0, 1, 1023, 1024, 1025, 4097] { + let input = (0..len).map(|value| value as u32).collect::>(); + let mapped = input + .iter() + .map(|value| value.wrapping_mul(31) & 0x3ff) + .collect::>(); + assert_eq!( + bitpack_primitive_map(&input, 10, |value| value.wrapping_mul(31) & 0x3ff), + bitpack_primitive(&mapped, 10), + ); + } + } + #[test] fn null_patches() { let mut ctx = SESSION.create_execution_ctx(); diff --git a/vortex-btrblocks/src/schemes/float/float_quant.rs b/vortex-btrblocks/src/schemes/float/float_quant.rs index 8c1a027fc81..decd9e4b307 100644 --- a/vortex-btrblocks/src/schemes/float/float_quant.rs +++ b/vortex-btrblocks/src/schemes/float/float_quant.rs @@ -3,13 +3,19 @@ //! Lossless float quantization with a fixed frame-of-reference child. +use std::ops::Range; + use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::VTable; +use vortex_array::arrays::Primitive; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::buffer::BufferHandle; use vortex_array::dtype::PType; +use vortex_array::match_each_float_ptype; use vortex_array::scalar::Scalar; use vortex_compressor::scheme::CompressionEstimate; use vortex_compressor::scheme::DeferredEstimate; @@ -17,8 +23,9 @@ use vortex_compressor::scheme::EstimateVerdict; use vortex_error::VortexResult; use vortex_fastlanes::BitPacked; use vortex_fastlanes::FoR; -use vortex_fastlanes::bitpack_compress::bitpack_encode_unchecked; +use vortex_fastlanes::bitpack_compress::bitpack_primitive_map; use vortex_float_quant::FloatQuant; +use vortex_float_quant::FloatQuantAnalysis; use vortex_float_quant::analyze_float_quant; use crate::ArrayAndStats; @@ -27,6 +34,10 @@ use crate::CompressorContext; use crate::Scheme; use crate::normalize_null_values; +const SAMPLE_BLOCK_LEN: usize = 64; +const MIN_SAMPLE_BLOCKS: usize = 16; +const SAMPLE_BLOCK_MULTIPLE: usize = 16; + /// FloatQuant split with a fixed frame-of-reference primary child. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct FloatQuantScheme; @@ -53,7 +64,28 @@ impl Scheme for FloatQuantScheme { if compress_ctx.finished_cascading() { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } - CompressionEstimate::Deferred(DeferredEstimate::Sample) + CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( + |_compressor, data, _best_so_far, _compress_ctx, exec_ctx| { + let sample = float_quant_sample(data.array_as_primitive(), exec_ctx)?; + let Some(analysis) = analyze_float_quant(sample.as_view()) else { + return Ok(EstimateVerdict::Skip); + }; + if !analysis.secondary_is_constant || analysis.primary_bit_width == 0 { + return Ok(EstimateVerdict::Skip); + } + + let before_nbytes = sample.nbytes(); + let compressed = encode_constant_secondary(sample.as_view(), analysis)?; + let after_nbytes = compressed.nbytes(); + if after_nbytes == 0 || after_nbytes >= before_nbytes { + return Ok(EstimateVerdict::Skip); + } + + Ok(EstimateVerdict::Ratio( + before_nbytes as f64 / after_nbytes as f64, + )) + }, + ))) } fn compress( @@ -72,23 +104,116 @@ impl Scheme for FloatQuantScheme { return Ok(source.array().clone()); } - let biased = FloatQuant::primary_for_primitive( - primitive.as_view(), - analysis.k, - analysis.primary_min, - )?; - // SAFETY: The analysis computes this width from the exact primary minimum and maximum. - let compressed_primary = - unsafe { bitpack_encode_unchecked(biased, analysis.primary_bit_width)? }.into_array(); - let reference = match primitive.ptype() { - PType::F32 => Scalar::from(u32::try_from(analysis.primary_min)?), - PType::F64 => Scalar::from(analysis.primary_min), - _ => unreachable!(), - }; - let compressed_primary = FoR::try_new(compressed_primary, reference)?.into_array(); - Ok( - FloatQuant::try_new(compressed_primary, None, primitive.ptype(), analysis.k)? - .into_array(), - ) + encode_constant_secondary(primitive.as_view(), analysis) + } +} + +fn encode_constant_secondary( + primitive: vortex_array::ArrayView<'_, Primitive>, + analysis: FloatQuantAnalysis, +) -> VortexResult { + let (packed, latent_ptype, reference) = match primitive.ptype() { + PType::F32 => { + let primary_min = u32::try_from(analysis.primary_min)?; + let packed = bitpack_primitive_map( + primitive.as_slice::(), + analysis.primary_bit_width, + |value| (ordered_u32(value.to_bits()) >> analysis.k) - primary_min, + ) + .into_byte_buffer(); + (packed, PType::U32, Scalar::from(primary_min)) + } + PType::F64 => { + let packed = bitpack_primitive_map( + primitive.as_slice::(), + analysis.primary_bit_width, + |value| (ordered_u64(value.to_bits()) >> analysis.k) - analysis.primary_min, + ) + .into_byte_buffer(); + (packed, PType::U64, Scalar::from(analysis.primary_min)) + } + _ => unreachable!(), + }; + let compressed_primary = BitPacked::try_new( + BufferHandle::new_host(packed), + latent_ptype, + primitive.validity()?, + None, + analysis.primary_bit_width, + primitive.len(), + 0, + )? + .into_array(); + let compressed_primary = FoR::try_new(compressed_primary, reference)?.into_array(); + Ok(FloatQuant::try_new(compressed_primary, None, primitive.ptype(), analysis.k)?.into_array()) +} + +#[inline] +fn ordered_u32(bits: u32) -> u32 { + if bits & (1_u32 << 31) == 0 { + bits ^ (1_u32 << 31) + } else { + !bits } } + +#[inline] +fn ordered_u64(bits: u64) -> u64 { + if bits & (1_u64 << 63) == 0 { + bits ^ (1_u64 << 63) + } else { + !bits + } +} + +fn float_quant_sample( + primitive: vortex_array::ArrayView<'_, Primitive>, + exec_ctx: &mut ExecutionCtx, +) -> VortexResult { + let sample_blocks = (primitive.len() / 100 / SAMPLE_BLOCK_LEN) + .next_multiple_of(SAMPLE_BLOCK_MULTIPLE) + .max(MIN_SAMPLE_BLOCKS); + let sample_len = sample_blocks * SAMPLE_BLOCK_LEN; + if primitive.len() <= sample_len { + return normalize_null_values(primitive, exec_ctx); + } + + let ranges = float_quant_sample_ranges(primitive.len(), sample_blocks); + let validity = primitive.validity()?; + if validity.definitely_no_nulls() { + return Ok(match_each_float_ptype!(primitive.ptype(), |T| { + let values = primitive.as_slice::(); + let mut sample = Vec::with_capacity(sample_len); + for range in ranges { + sample.extend_from_slice(&values[range]); + } + PrimitiveArray::from_iter(sample) + })); + } + + let validity = validity.execute_mask(primitive.len(), exec_ctx)?; + Ok(match_each_float_ptype!(primitive.ptype(), |T| { + let values = primitive.as_slice::(); + PrimitiveArray::from_option_iter( + ranges + .into_iter() + .flatten() + .map(|index| validity.value(index).then_some(values[index])), + ) + })) +} + +fn float_quant_sample_ranges(len: usize, sample_blocks: usize) -> Vec> { + let partition_len = len / sample_blocks; + let long_partitions = len % sample_blocks; + let mut partition_start = 0; + (0..sample_blocks) + .map(|partition_index| { + let current_partition_len = + partition_len + usize::from(partition_index < long_partitions); + let start = partition_start + (current_partition_len - SAMPLE_BLOCK_LEN) / 2; + partition_start += current_partition_len; + start..start + SAMPLE_BLOCK_LEN + }) + .collect() +} diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index 227215d1da2..fd182a17c2b 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -55,6 +55,7 @@ use vortex::encodings::zstd::Zstd; use vortex::encodings::zstd::ZstdData; use vortex::scalar::Scalar; use vortex_array::VortexSessionExecute; +use vortex_btrblocks::BtrBlocksCompressor; use vortex_btrblocks::SchemeExt; use vortex_btrblocks::schemes::float::FloatQuantScheme; use vortex_btrblocks::schemes::float::OrderedBlockResidualScheme; @@ -93,6 +94,12 @@ fn canonicalize(array: impl IntoArray, ctx: &mut ExecutionCtx) -> VortexResult(ctx) } +fn bench_compressor(bencher: Bencher, array: PrimitiveArray, compressor: BtrBlocksCompressor) { + with_byte_counter(bencher, array.nbytes()) + .with_inputs(|| (array.clone().into_array(), SESSION.create_execution_ctx())) + .bench_values(|(array, mut ctx)| compressor.compress(&array, &mut ctx).unwrap()); +} + // Setup functions fn setup_primitive_arrays() -> (PrimitiveArray, PrimitiveArray, PrimitiveArray) { let mut ctx = SESSION.create_execution_ctx(); @@ -131,6 +138,13 @@ fn setup_quantized_f32_array() -> PrimitiveArray { })) } +fn setup_general_f32_array() -> PrimitiveArray { + PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + let mantissa = index.wrapping_mul(7_919) as u32 & 0x007f_ffff; + f32::from_bits(0x3f80_0000 | mantissa) + })) +} + fn setup_nonzero_secondary_array() -> PrimitiveArray { let widened = setup_widened_f32_array(); PrimitiveArray::from_iter( @@ -1016,7 +1030,7 @@ fn bench_float_quant_split_decompress_f64(bencher: Bencher) { .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); } -#[divan::bench(name = "float_quant_tree_compress_f64")] +#[divan::bench(name = "float_quant_materialized_tree_compress_f64")] fn bench_float_quant_tree_compress_f64(bencher: Bencher) { let float_array = setup_widened_f32_array(); @@ -1025,6 +1039,14 @@ fn bench_float_quant_tree_compress_f64(bencher: Bencher) { .bench_refs(|array| encode_float_quant_tree(array)); } +#[divan::bench(name = "float_quant_scheme_compress_f64")] +fn bench_float_quant_scheme_compress_f64(bencher: Bencher) { + let compressor = BtrBlocksCompressorBuilder::empty() + .with_new_scheme(&FloatQuantScheme) + .build(); + bench_compressor(bencher, setup_widened_f32_array(), compressor); +} + #[divan::bench(name = "float_quant_tree_decompress_f64")] fn bench_float_quant_tree_decompress_f64(bencher: Bencher) { let float_array = setup_widened_f32_array(); @@ -1051,7 +1073,7 @@ fn bench_float_quant_tree_scalar_at_f64(bencher: Bencher) { .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); } -#[divan::bench(name = "float_quant_tree_compress_f32")] +#[divan::bench(name = "float_quant_materialized_tree_compress_f32")] fn bench_float_quant_tree_compress_f32(bencher: Bencher) { let float_array = setup_quantized_f32_array(); @@ -1060,6 +1082,18 @@ fn bench_float_quant_tree_compress_f32(bencher: Bencher) { .bench_refs(|array| encode_float_quant_tree(array)); } +#[divan::bench(name = "float_quant_alp_rd_compress_f32")] +fn bench_float_quant_alp_rd_compress_f32(bencher: Bencher) { + let float_array = setup_quantized_f32_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) + .with_inputs(|| &float_array) + .bench_refs(|array| { + let encoder = RDEncoder::new(array.as_slice::()); + encoder.encode(array.as_view()) + }); +} + #[divan::bench(name = "float_quant_tree_decompress_f32")] fn bench_float_quant_tree_decompress_f32(bencher: Bencher) { let encoded = encode_float_quant_tree(&setup_quantized_f32_array()); @@ -1085,6 +1119,75 @@ fn bench_float_quant_tree_scalar_at_f32(bencher: Bencher) { .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); } +#[divan::bench(name = "float_quant_analyze_f32")] +fn bench_float_quant_analyze_f32(bencher: Bencher) { + let float_array = setup_quantized_f32_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) + .with_inputs(|| &float_array) + .bench_refs(|array| analyze_float_quant(array.as_view()).unwrap()); +} + +#[divan::bench(name = "float_quant_scheme_compress_f32")] +fn bench_float_quant_scheme_compress_f32(bencher: Bencher) { + let compressor = BtrBlocksCompressorBuilder::empty() + .with_new_scheme(&FloatQuantScheme) + .build(); + bench_compressor(bencher, setup_quantized_f32_array(), compressor); +} + +#[divan::bench(name = "float_quant_prior_default_compress_f32")] +fn bench_float_quant_prior_default_compress_f32(bencher: Bencher) { + let compressor = BtrBlocksCompressorBuilder::default() + .exclude_schemes([ + FloatQuantScheme.id(), + OrderedBlockResidualScheme.id(), + BlockResidualScheme.id(), + ]) + .build(); + bench_compressor(bencher, setup_quantized_f32_array(), compressor); +} + +#[divan::bench(name = "float_quant_default_compress_f32")] +fn bench_float_quant_default_compress_f32(bencher: Bencher) { + let compressor = BtrBlocksCompressorBuilder::default() + .exclude_schemes([OrderedBlockResidualScheme.id(), BlockResidualScheme.id()]) + .build(); + bench_compressor(bencher, setup_quantized_f32_array(), compressor); +} + +#[divan::bench(name = "float_quant_proposed_default_compress_f32")] +fn bench_float_quant_proposed_default_compress_f32(bencher: Bencher) { + let compressor = BtrBlocksCompressorBuilder::default().build(); + bench_compressor(bencher, setup_quantized_f32_array(), compressor); +} + +#[divan::bench(name = "float_quant_scheme_reject_f32")] +fn bench_float_quant_scheme_reject_f32(bencher: Bencher) { + let compressor = BtrBlocksCompressorBuilder::empty() + .with_new_scheme(&FloatQuantScheme) + .build(); + bench_compressor(bencher, setup_general_f32_array(), compressor); +} + +#[divan::bench(name = "float_quant_prior_default_reject_f32")] +fn bench_float_quant_prior_default_reject_f32(bencher: Bencher) { + let compressor = BtrBlocksCompressorBuilder::default() + .exclude_schemes([ + FloatQuantScheme.id(), + OrderedBlockResidualScheme.id(), + BlockResidualScheme.id(), + ]) + .build(); + bench_compressor(bencher, setup_general_f32_array(), compressor); +} + +#[divan::bench(name = "float_quant_proposed_default_reject_f32")] +fn bench_float_quant_proposed_default_reject_f32(bencher: Bencher) { + let compressor = BtrBlocksCompressorBuilder::default().build(); + bench_compressor(bencher, setup_general_f32_array(), compressor); +} + #[divan::bench(name = "float_quant_nonzero_secondary_split_compress_f64")] fn bench_float_quant_nonzero_secondary_split_compress_f64(bencher: Bencher) { let float_array = setup_nonzero_secondary_array(); From 47daa71b8aa52c865d6e644bcc426341d53c9831 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 15:22:49 +0100 Subject: [PATCH 14/78] perf: fuse float quant pair decode Signed-off-by: Will Manning --- Cargo.lock | 1 + docs/plans/native-pcodec.md | 50 ++++--- .../bitpacking/array/bitpack_decompress.rs | 131 ++++++++++++++++++ encodings/float-quant/Cargo.toml | 1 + encodings/float-quant/src/array.rs | 121 ++++++++++++++++ 5 files changed, 288 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 39dd8297e02..ed8f3a51940 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10216,6 +10216,7 @@ dependencies = [ "vortex-array", "vortex-buffer", "vortex-error", + "vortex-fastlanes", "vortex-session", "vortex-utils", ] diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 1b947fce351..dc818e019d9 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -97,6 +97,10 @@ The current BtrBlocks scheme accepts only a constant secondary. It uses a fixed A common path uses `FloatQuant(FoR(BitPacked))` for `f32` values stored in `f64` columns. An absent secondary child represents zero low bits. +The array decode kernel also supports `FloatQuant(FoR(BitPacked), BitPacked)`. + +The kernel unpacks both aligned children and reconstructs each float in one pass. + The array supports exact IEEE bit-pattern round trips, nulls, slices, scalar access, and serialization. The automatic scheme accepts `f32` and `f64` inputs. @@ -205,17 +209,27 @@ The fixed tree was `FloatQuant(FoR(BitPacked), BitPacked)`. The secondary used o | Configuration | Bytes | Decode MB/s | Scalar access ns | | --- | ---: | ---: | ---: | | Prior ALP-RD default | 14,057,966 | 11,430 | 208.5 | -| Two-child FloatQuant prototype | 9,004,032 | 8,375 | 192.2 | +| Two-child FloatQuant prototype | 9,004,032 | 13,350 | 192.2 | The prototype reduced size by 36.0 percent. It was 2.9 percent larger than the zero-secondary tree. The prototype recovered 64.1 percent of the size gap between ALP-RD and Compact Pco. -The prototype reduced decode throughput by 26.7 percent against ALP-RD. It reduced decode throughput by 43.6 percent against zero-secondary FloatQuant. +The first generic decode path reached 8,375 MB/s. + +The fused pair kernel increased decode throughput by 59.4 percent. + +The fused tree decoded 9.3 percent faster than ALP-RD. It decoded 13.5 percent slower than zero-secondary FloatQuant. Scalar access remained competitive. The direct prototype tree compressed at 3,301 MB/s. -The default scheme rejects this form. A fused one-bit-secondary decode is the next experiment. +The fused kernel supports aligned, patch-free BitPacked secondary children of any width. + +The measured input used a one-bit secondary. Real gap columns can require wider secondary values or patches. + +The result clears the isolated decode requirement. The default scheme still rejects this form. + +Default selection requires a fixed-tree estimator, a direct final encoder, and full-writer validation on real gap columns. ### OrderedFloat with BlockResidual on random walks @@ -634,21 +648,25 @@ This round completed these steps: The Pco mode profile and quotient and remainder experiments are complete. +The fused nonzero-secondary FloatQuant decode experiment is complete. + Complete these remaining steps: -1. Test a fused nonzero-secondary FloatQuant decode on selected gap columns. -2. Decide whether nonzero-secondary FloatQuant is a default candidate. -3. Compare alternate BlockResidual patch costs across the patch-density sweep. -4. Evaluate narrow BlockResidual as a Compact-only candidate. -5. Reduce analysis cost on short rejected columns. -6. Measure BlockResidual under temporal, FSST, sparse, run-end, and list parents. -7. Prototype bounded scalar checkpoints for fixed-bin range packing. -8. Compare fixed-bin packing against default and Pco on unrelated no-Delta wins. -9. Add real embedding datasets beyond GloVe when licenses and loaders permit them. -10. Run the complete `bench-vortex` compression corpus after the candidate set stabilizes. -11. Compare the geometric mean against Parquet with Zstd. -12. Calibrate all selector thresholds from complete corpus and selected-tree evidence. -13. Update this plan after each experiment. +1. Extend the FloatQuant estimator to small nonzero secondary widths. +2. Add a direct final encoder for both FloatQuant children. +3. Measure selected and rejected FloatQuant columns from the real gap corpus. +4. Decide whether nonzero-secondary FloatQuant enters the default candidate set. +5. Compare alternate BlockResidual patch costs across the patch-density sweep. +6. Evaluate narrow BlockResidual as a Compact-only candidate. +7. Reduce analysis cost on short rejected columns. +8. Measure BlockResidual under temporal, FSST, sparse, run-end, and list parents. +9. Prototype bounded scalar checkpoints for fixed-bin range packing. +10. Compare fixed-bin packing against default and Pco on unrelated no-Delta wins. +11. Add real embedding datasets beyond GloVe when licenses and loaders permit them. +12. Run the complete `bench-vortex` compression corpus after the candidate set stabilizes. +13. Compare the geometric mean against Parquet with Zstd. +14. Calibrate all selector thresholds from complete corpus and selected-tree evidence. +15. Update this plan after each experiment. ## Pull request structure diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs index 692e7dcdd7f..24f0eb029e7 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs @@ -17,8 +17,11 @@ use vortex_array::match_each_integer_ptype; use vortex_array::match_each_unsigned_integer_ptype; use vortex_array::patches::Patches; use vortex_array::scalar::Scalar; +use vortex_buffer::Buffer; +use vortex_buffer::BufferMut; use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_error::vortex_ensure; use crate::BitPacked; use crate::BitPackedArrayExt; @@ -45,6 +48,98 @@ pub fn unpack_primitive_array( Ok(builder.finish_into_primitive()) } +/// Unpacks two aligned bit-packed arrays and maps each value pair into one output buffer. +/// +/// This path requires equal lengths and offsets. Neither input can contain patches. +pub fn unpack_pair_map( + left: ArrayView<'_, BitPacked>, + right: ArrayView<'_, BitPacked>, + mut map: F, +) -> VortexResult> +where + T: BitPackedUnpack + BitPacking, + U: NativePType, + F: FnMut(T, T) -> U, +{ + vortex_ensure!(left.len() == right.len(), "bit-packed pair length differs"); + vortex_ensure!( + left.offset() == right.offset(), + "bit-packed pair offset differs" + ); + vortex_ensure!( + left.dtype().as_ptype() == T::PTYPE && right.dtype().as_ptype() == T::PTYPE, + "bit-packed pair requires {} inputs", + T::PTYPE + ); + vortex_ensure!( + left.patches().is_none() && right.patches().is_none(), + "bit-packed pair mapping does not support patches" + ); + if left.is_empty() { + return Ok(Buffer::empty()); + } + + let len = left.len(); + let offset = usize::from(left.offset()); + let num_chunks = (offset + len).div_ceil(1024); + let left_width = usize::from(left.bit_width()); + let right_width = usize::from(right.bit_width()); + let left_packed_len = 128 * left_width / size_of::(); + let right_packed_len = 128 * right_width / size_of::(); + let left_packed = left.packed_slice::(); + let right_packed = right.packed_slice::(); + let mut left_values = [T::default(); 1024]; + let mut right_values = [T::default(); 1024]; + let mut output = BufferMut::with_capacity(len); + + for chunk_index in 0..num_chunks { + if left_width == 0 { + left_values.fill(T::default()); + } else { + let start = chunk_index * left_packed_len; + // SAFETY: BitPacked validation guarantees one packed FastLanes block here. + unsafe { + BitPacking::unchecked_unpack( + left_width, + &left_packed[start..start + left_packed_len], + &mut left_values, + ); + } + } + if right_width == 0 { + right_values.fill(T::default()); + } else { + let start = chunk_index * right_packed_len; + // SAFETY: BitPacked validation guarantees one packed FastLanes block here. + unsafe { + BitPacking::unchecked_unpack( + right_width, + &right_packed[start..start + right_packed_len], + &mut right_values, + ); + } + } + + let start = if chunk_index == 0 { offset } else { 0 }; + let logical_start = chunk_index * 1024; + let end = (offset + len - logical_start).min(1024); + let output_len = output.len(); + let mapped_len = end - start; + for (output, (&left, &right)) in output.spare_capacity_mut()[..mapped_len].iter_mut().zip( + left_values[start..end] + .iter() + .zip(&right_values[start..end]), + ) { + output.write(map(left, right)); + } + // SAFETY: The loop initialized each new output value. + unsafe { output.set_len(output_len + mapped_len) }; + } + + debug_assert_eq!(output.len(), len); + Ok(output.freeze()) +} + /// Unpack a bit-packed array directly into a same-typed `PrimitiveBuilder`. /// /// This is the fast path for ordinary decompression: full FastLanes chunks are unpacked straight @@ -203,6 +298,7 @@ pub fn count_exceptions(bit_width: u8, bit_width_freq: &[usize]) -> usize { #[cfg(test)] mod tests { + use std::ops::Range; use std::sync::Arc; use std::sync::LazyLock; @@ -228,6 +324,41 @@ mod tests { use crate::BitPackedData; use crate::bitpack_compress::bitpack_encode; + #[test] + fn unpack_pair_map_matches_sliced_inputs() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let left_values = PrimitiveArray::from_iter((0_u32..3073).map(|value| value & 0x3ff)); + let right_values = + PrimitiveArray::from_iter((0_u32..3073).map(|value| value.wrapping_mul(7) & 0x7f)); + let left = bitpack_encode(&left_values, 10, None, &mut ctx)?.into_array(); + let right = bitpack_encode(&right_values, 7, None, &mut ctx)?.into_array(); + + for range in [ + Range { + start: 0, + end: 3073, + }, + 3..2051, + 1023..2050, + ] { + let left = left.slice(range.clone())?; + let right = right.slice(range.clone())?; + let actual = unpack_pair_map::( + left.as_::(), + right.as_::(), + |left, right| u64::from(left) * 17 + u64::from(right), + )?; + let expected = left_values.as_slice::()[range.clone()] + .iter() + .copied() + .zip(right_values.as_slice::()[range].iter().copied()) + .map(|(left, right)| u64::from(left) * 17 + u64::from(right)) + .collect::>(); + assert_eq!(actual, expected); + } + Ok(()) + } + fn encode(array: &PrimitiveArray, bit_width: u8) -> BitPackedArray { bitpack_encode(array, bit_width, None, &mut SESSION.create_execution_ctx()).unwrap() } diff --git a/encodings/float-quant/Cargo.toml b/encodings/float-quant/Cargo.toml index 88cb361b555..c3592f20840 100644 --- a/encodings/float-quant/Cargo.toml +++ b/encodings/float-quant/Cargo.toml @@ -17,6 +17,7 @@ version = { workspace = true } vortex-array = { workspace = true } vortex-buffer = { workspace = true } vortex-error = { workspace = true } +vortex-fastlanes = { workspace = true } vortex-session = { workspace = true } vortex-utils = { workspace = true } diff --git a/encodings/float-quant/src/array.rs b/encodings/float-quant/src/array.rs index 92259b9eaac..2fce7464db3 100644 --- a/encodings/float-quant/src/array.rs +++ b/encodings/float-quant/src/array.rs @@ -37,6 +37,12 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; +use vortex_fastlanes::BitPacked; +use vortex_fastlanes::BitPackedArrayExt; +use vortex_fastlanes::FoR; +use vortex_fastlanes::FoRArrayExt; +use vortex_fastlanes::FoRArraySlotsExt; +use vortex_fastlanes::bitpack_decompress::unpack_pair_map; use vortex_session::VortexSession; use vortex_session::registry::CachedId; @@ -719,6 +725,10 @@ fn decode( array: ArrayView<'_, FloatQuant>, ctx: &mut ExecutionCtx, ) -> VortexResult { + if let Some(decoded) = decode_fastlanes_pair(array)? { + return Ok(decoded); + } + let primary = array.primary().clone().execute::(ctx)?; let validity = primary.validity()?; let k = array.data().k; @@ -773,6 +783,59 @@ fn decode( }) } +fn decode_fastlanes_pair(array: ArrayView<'_, FloatQuant>) -> VortexResult> { + let Some(secondary) = array + .secondary() + .and_then(|child| child.as_opt::()) + else { + return Ok(None); + }; + let Some(primary_for) = array.primary().as_opt::() else { + return Ok(None); + }; + let Some(primary) = primary_for.encoded().as_opt::() else { + return Ok(None); + }; + if primary.patches().is_some() + || secondary.patches().is_some() + || primary.offset() != secondary.offset() + { + return Ok(None); + } + + let validity = primary.validity()?; + let k = array.data().k; + Ok(Some(match PType::try_from(array.dtype())? { + PType::F32 => { + let reference = primary_for + .reference_scalar() + .as_primitive() + .typed_value::() + .vortex_expect("validated f32 primary reference"); + PrimitiveArray::new( + unpack_pair_map::(primary, secondary, |primary, secondary| { + join_f32(primary.wrapping_add(reference), secondary, k) + })?, + validity, + ) + } + PType::F64 => { + let reference = primary_for + .reference_scalar() + .as_primitive() + .typed_value::() + .vortex_expect("validated f64 primary reference"); + PrimitiveArray::new( + unpack_pair_map::(primary, secondary, |primary, secondary| { + join_f64(primary.wrapping_add(reference), secondary, k) + })?, + validity, + ) + } + ptype => vortex_panic!("unsupported FloatQuant ptype {ptype}"), + })) +} + #[cfg(test)] mod tests { use std::sync::LazyLock; @@ -901,6 +964,64 @@ mod tests { Ok(()) } + #[test] + fn fastlanes_pair_decode_roundtrip_and_slice() -> VortexResult<()> { + let original = PrimitiveArray::from_iter((0_u32..4097).map(|index| { + let mantissa = index.wrapping_mul(7_919) & 0x007f_ffff; + let value = f64::from(f32::from_bits(0x3f80_0000 | mantissa)); + if index % 10 == 0 { + f64::from_bits(value.to_bits() | 1) + } else { + value + } + })); + let analysis = analyze_float_quant(original.as_view()).vortex_expect("FloatQuant input"); + assert!(!analysis.secondary_is_constant); + let split = FloatQuant::from_primitive(original.as_view(), analysis.k)?; + let primary = split + .primary() + .clone() + .execute::(&mut SESSION.create_execution_ctx())?; + let secondary = split + .secondary() + .vortex_expect("nonzero secondary") + .clone() + .execute::(&mut SESSION.create_execution_ctx())?; + let biased_primary = PrimitiveArray::from_iter( + primary + .as_slice::() + .iter() + .map(|value| value - analysis.primary_min), + ); + // SAFETY: The analysis computes the exact primary width. + let primary = unsafe { + vortex_fastlanes::bitpack_compress::bitpack_encode_unchecked( + biased_primary, + analysis.primary_bit_width, + )? + }; + let primary = FoR::try_new(primary.into_array(), Scalar::from(analysis.primary_min))?; + // SAFETY: The test changes only one low bit. + let secondary = + unsafe { vortex_fastlanes::bitpack_compress::bitpack_encode_unchecked(secondary, 1)? }; + let encoded = FloatQuant::try_new( + primary.into_array(), + Some(secondary.into_array()), + PType::F64, + analysis.k, + )? + .into_array(); + + let mut ctx = SESSION.create_execution_ctx(); + assert_arrays_eq!(encoded, original, &mut ctx); + assert_arrays_eq!( + encoded.slice(3..2051)?, + original.into_array().slice(3..2051)?, + &mut ctx + ); + Ok(()) + } + #[test] fn serialization_roundtrip() -> VortexResult<()> { let original = PrimitiveArray::from_option_iter([ From 097f9f5c74ea08d7a3f1dc8898fc4188f4fb1698 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 15:39:06 +0100 Subject: [PATCH 15/78] perf: select two-child float quant Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 93 +++++++------ .../src/bitpacking/array/bitpack_compress.rs | 94 +++++++++++++ encodings/float-quant/src/array.rs | 123 +++++++++--------- .../src/schemes/float/float_quant.rs | 112 ++++++++++++---- .../schemes/float/scheme_selection_tests.rs | 30 +++++ vortex/benches/single_encoding_throughput.rs | 63 ++++++++- 6 files changed, 387 insertions(+), 128 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index dc818e019d9..e344e0d0c14 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -92,12 +92,12 @@ Metadata stores only the split width. The array dtype stores the source type. One or two child arrays store the integer latents. -The current BtrBlocks scheme accepts only a constant secondary. It uses a fixed `FoR(BitPacked)` primary tree. +The BtrBlocks scheme uses a fixed `FoR(BitPacked)` primary tree. A common path uses `FloatQuant(FoR(BitPacked))` for `f32` values stored in `f64` columns. An absent secondary child represents zero low bits. -The array decode kernel also supports `FloatQuant(FoR(BitPacked), BitPacked)`. +Nonzero low bits use `FloatQuant(FoR(BitPacked), BitPacked)`. The kernel unpacks both aligned children and reconstructs each float in one pass. @@ -105,25 +105,31 @@ The array supports exact IEEE bit-pattern round trips, nulls, slices, scalar acc The automatic scheme accepts `f32` and `f64` inputs. -Native `f32` selection now meets the selected-column and rejected-column throughput limits. +Native `f32` and two-child selection meet the selected-column and rejected-column throughput limits. -Final default inclusion still requires broad corpus validation. +Final selector factors still require broad corpus validation. ## Selection policy `FloatQuantScheme` uses the normal sample comparison against ALP, ALP-RD, dictionary, sparse, and RLE schemes. -The sample builds a direct `FoR(BitPacked)` primary and uses an implicit-zero secondary. +The sample builds the exact one-child or two-child fixed tree. -The scheme rejects samples with a nonzero secondary. This avoids the recursive integer selector. +The prefilter tracks the bitwise union of all low float bits. -The scheme uses a cheap bit-pattern prefilter over a stratified one-percent sample. +For each split, it compares the removed bit count with the required secondary width. + +A candidate must save at least two fixed bits through the split. + +The prefilter uses a stratified one-percent sample. If the sample qualifies, the estimator builds the exact fixed tree on that sample. -The final encoder maps float blocks directly into the FastLanes packer. +The final encoder maps float blocks directly into one or two FastLanes packers. + +This fused path does not allocate full primary or secondary integer buffers. -This fused path does not allocate a full primary integer buffer. +The scheme does not call the recursive integer selector. `OrderedBlockResidualScheme` uses eight locality-preserving sample blocks. @@ -202,34 +208,47 @@ FloatQuant meets the selected-column throughput limit on this input. ### FloatQuant with a nonzero secondary -The prototype changed the lowest bit for ten percent of the widened-`f32` values. +The input changed the lowest bit for ten percent of the widened-`f32` values. The fixed tree was `FloatQuant(FoR(BitPacked), BitPacked)`. The secondary used one bit per value. -| Configuration | Bytes | Decode MB/s | Scalar access ns | -| --- | ---: | ---: | ---: | -| Prior ALP-RD default | 14,057,966 | 11,430 | 208.5 | -| Two-child FloatQuant prototype | 9,004,032 | 13,350 | 192.2 | +| Configuration | Bytes | Encode MB/s | Decode MB/s | Scalar access ns | +| --- | ---: | ---: | ---: | ---: | +| Prior ALP-RD default | 14,057,966 | 564.5 | 12,234.8 | 187 | +| Default with FloatQuant | 9,004,032 | 633.5 | 13,462.8 | 174 | +| Compact Pco | 6,171,139 | 268.4 | 3,054.5 | Not measured | -The prototype reduced size by 36.0 percent. It was 2.9 percent larger than the zero-secondary tree. +FloatQuant reduced size by 36.0 percent. It was 2.9 percent larger than the zero-secondary tree. -The prototype recovered 64.1 percent of the size gap between ALP-RD and Compact Pco. +FloatQuant recovered 64.1 percent of the size gap between ALP-RD and Compact Pco. The first generic decode path reached 8,375 MB/s. The fused pair kernel increased decode throughput by 59.4 percent. -The fused tree decoded 9.3 percent faster than ALP-RD. It decoded 13.5 percent slower than zero-secondary FloatQuant. +The complete default encoded 12.2 percent faster and decoded 10.0 percent faster than the prior default. -Scalar access remained competitive. The direct prototype tree compressed at 3,301 MB/s. +Scalar access latency decreased by 7.0 percent. + +The direct two-child scheme compressed at 6,469 MB/s. The fused kernel supports aligned, patch-free BitPacked secondary children of any width. -The measured input used a one-bit secondary. Real gap columns can require wider secondary values or patches. +Decode throughput was 12,900 MB/s with a one-bit secondary. + +It was 12,410 MB/s with a 16-bit secondary. + +The real profile selected this tree only for `HashTags_1.twitter#id`. + +It reduced that column by 6.1 percent against ALP-RD. + +OrderedFloat with BlockResidual reduced the column by 14.5 percent and won the final comparison. + +No other tested Pcodec or Public BI column selected the two-child tree. -The result clears the isolated decode requirement. The default scheme still rejects this form. +Rejected analysis changed encode throughput by less than one percent on most measured datasets. -Default selection requires a fixed-tree estimator, a direct final encoder, and full-writer validation on real gap columns. +Retain the two-child form in the default candidate set for final corpus calibration. ### OrderedFloat with BlockResidual on random walks @@ -645,28 +664,28 @@ This round completed these steps: - Added native `f32` FloatQuant selection and scheme benchmarks. - Replaced the full primary buffer with fused FloatQuant and FastLanes packing. - Replaced the generic FloatQuant sample with a direct fixed-tree estimator. +- Added fixed-tree analysis for nonzero FloatQuant secondary values. +- Added direct paired FastLanes packing for both FloatQuant children. +- Added fused two-child FloatQuant decode and width-sweep benchmarks. +- Validated selected and rejected two-child FloatQuant paths. The Pco mode profile and quotient and remainder experiments are complete. -The fused nonzero-secondary FloatQuant decode experiment is complete. +The nonzero-secondary FloatQuant implementation and focused validation are complete. Complete these remaining steps: -1. Extend the FloatQuant estimator to small nonzero secondary widths. -2. Add a direct final encoder for both FloatQuant children. -3. Measure selected and rejected FloatQuant columns from the real gap corpus. -4. Decide whether nonzero-secondary FloatQuant enters the default candidate set. -5. Compare alternate BlockResidual patch costs across the patch-density sweep. -6. Evaluate narrow BlockResidual as a Compact-only candidate. -7. Reduce analysis cost on short rejected columns. -8. Measure BlockResidual under temporal, FSST, sparse, run-end, and list parents. -9. Prototype bounded scalar checkpoints for fixed-bin range packing. -10. Compare fixed-bin packing against default and Pco on unrelated no-Delta wins. -11. Add real embedding datasets beyond GloVe when licenses and loaders permit them. -12. Run the complete `bench-vortex` compression corpus after the candidate set stabilizes. -13. Compare the geometric mean against Parquet with Zstd. -14. Calibrate all selector thresholds from complete corpus and selected-tree evidence. -15. Update this plan after each experiment. +1. Compare alternate BlockResidual patch costs across the patch-density sweep. +2. Evaluate narrow BlockResidual as a Compact-only candidate. +3. Reduce analysis cost on short rejected columns. +4. Measure BlockResidual under temporal, FSST, sparse, run-end, and list parents. +5. Prototype bounded scalar checkpoints for fixed-bin range packing. +6. Compare fixed-bin packing against default and Pco on unrelated no-Delta wins. +7. Add real embedding datasets beyond GloVe when licenses and loaders permit them. +8. Run the complete `bench-vortex` compression corpus after the candidate set stabilizes. +9. Compare the geometric mean against Parquet with Zstd. +10. Calibrate all selector thresholds from complete corpus and selected-tree evidence. +11. Update this plan after each experiment. ## Pull request structure diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs index 006fae2a1dc..cb85537f303 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs @@ -247,6 +247,80 @@ where output.freeze() } +/// Maps each primitive value to two values and bit-packs both outputs. +/// +/// Each mapped value must fit in its corresponding bit width. This function does not create +/// patches. +pub fn bitpack_primitive_map_pair( + array: &[S], + left_bit_width: u8, + right_bit_width: u8, + mut map: F, +) -> (Buffer, Buffer) +where + T: NativePType + BitPacking, + F: FnMut(&S) -> (T, T), +{ + if left_bit_width == 0 && right_bit_width == 0 { + return (Buffer::empty(), Buffer::empty()); + } + + let left_bit_width = usize::from(left_bit_width); + let right_bit_width = usize::from(right_bit_width); + let num_chunks = array.len().div_ceil(1024); + let num_full_chunks = array.len() / 1024; + let left_packed_len = 128 * left_bit_width / size_of::(); + let right_packed_len = 128 * right_bit_width / size_of::(); + let mut left_output = BufferMut::::with_capacity(num_chunks * left_packed_len); + let mut right_output = BufferMut::::with_capacity(num_chunks * right_packed_len); + let mut left_mapped = [T::zero(); 1024]; + let mut right_mapped = [T::zero(); 1024]; + + for chunk_index in 0..num_full_chunks { + let start = chunk_index * 1024; + left_mapped + .iter_mut() + .zip(&mut right_mapped) + .zip(&array[start..start + 1024]) + .for_each(|((left, right), input)| (*left, *right) = map(input)); + append_packed_chunk(&mut left_output, &left_mapped, left_bit_width); + append_packed_chunk(&mut right_output, &right_mapped, right_bit_width); + } + + if num_chunks != num_full_chunks { + let start = num_full_chunks * 1024; + let last_chunk_len = array.len() - start; + left_mapped[..last_chunk_len] + .iter_mut() + .zip(&mut right_mapped[..last_chunk_len]) + .zip(&array[start..]) + .for_each(|((left, right), input)| (*left, *right) = map(input)); + left_mapped[last_chunk_len..].fill(T::zero()); + right_mapped[last_chunk_len..].fill(T::zero()); + append_packed_chunk(&mut left_output, &left_mapped, left_bit_width); + append_packed_chunk(&mut right_output, &right_mapped, right_bit_width); + } + + (left_output.freeze(), right_output.freeze()) +} + +fn append_packed_chunk( + output: &mut BufferMut, + values: &[T; 1024], + bit_width: usize, +) { + if bit_width == 0 { + return; + } + let packed_len = 128 * bit_width / size_of::(); + let output_len = output.len(); + // SAFETY: The output has capacity for one packed vector and both slices have exact sizes. + unsafe { + output.set_len(output_len + packed_len); + BitPacking::unchecked_pack(bit_width, values, &mut output[output_len..][..packed_len]); + } +} + pub fn gather_patches( parray: &PrimitiveArray, bit_width: u8, @@ -532,6 +606,26 @@ mod test { } } + #[test] + fn bitpack_primitive_map_pair_matches_materialized_inputs() { + for len in [0, 1, 1023, 1024, 1025, 4097] { + let input = (0..len).map(|value| value as u32).collect::>(); + let left = input + .iter() + .map(|value| value.wrapping_mul(31) & 0x3ff) + .collect::>(); + let right = input + .iter() + .map(|value| value.wrapping_mul(7) & 0x7) + .collect::>(); + let actual = bitpack_primitive_map_pair(&input, 10, 3, |value| { + (value.wrapping_mul(31) & 0x3ff, value.wrapping_mul(7) & 0x7) + }); + assert_eq!(actual.0, bitpack_primitive(&left, 10)); + assert_eq!(actual.1, bitpack_primitive(&right, 3)); + } + } + #[test] fn null_patches() { let mut ctx = SESSION.create_execution_ctx(); diff --git a/encodings/float-quant/src/array.rs b/encodings/float-quant/src/array.rs index 2fce7464db3..7724adf2d76 100644 --- a/encodings/float-quant/src/array.rs +++ b/encodings/float-quant/src/array.rs @@ -417,8 +417,8 @@ pub struct FloatQuantAnalysis { pub primary_bit_width: u8, /// Minimum primary value before frame-of-reference subtraction. pub primary_min: u64, - /// True when every secondary value is zero. - pub secondary_is_constant: bool, + /// Bit width of the secondary values. Zero identifies an implicit-zero child. + pub secondary_bit_width: u8, } /// Estimate a useful low-bit split width for a canonical float array. @@ -435,8 +435,8 @@ pub fn analyze_float_quant(array: ArrayView<'_, Primitive>) -> Option best_savings { - best_k = k; - best_savings = savings; + + let shifted_min = primary_min >> k; + let shifted_max = primary_max >> k; + let primary_bit_width = + u8::try_from(u64::BITS - (shifted_max - shifted_min).leading_zeros()) + .unwrap_or(u8::MAX); + let total_bit_width = primary_bit_width + secondary_bit_width; + let candidate = ( + total_bit_width, + secondary_bit_width, + k, + primary_bit_width, + shifted_min, + ); + if best.is_none_or(|current| candidate < current) { + best = Some(candidate); } } - if best_savings <= 1.5 { - return None; - } - let secondary_is_constant = histogram[usize::from(best_k)] == len; - let primary_min = primary_min >> best_k; - let primary_max = primary_max >> best_k; - let primary_bit_width = - u8::try_from(u64::BITS - (primary_max - primary_min).leading_zeros()).unwrap_or(u8::MAX); + + let (_, secondary_bit_width, k, primary_bit_width, primary_min) = best?; Some(FloatQuantAnalysis { - k: best_k, + k, primary_bit_width, primary_min, - secondary_is_constant, + secondary_bit_width, }) } fn analyze_f32(values: &[f32]) -> Option { let mut minimum = u32::MAX; let mut maximum = u32::MIN; - let mut histogram = [0usize; 24]; + let mut low_bits_or = 0_u32; for value in values { let bits = value.to_bits(); let ordered = ordered_u32(bits); minimum = minimum.min(ordered); maximum = maximum.max(ordered); - let zeros = bits.trailing_zeros().min(23); - histogram[zeros as usize] += 1; + low_bits_or |= bits; } - analyze_histogram( - &mut histogram, + analyze_bits( + u64::from(low_bits_or), 23, values.len(), u64::from(minimum), @@ -507,24 +505,15 @@ fn analyze_f32(values: &[f32]) -> Option { fn analyze_f64(values: &[f64]) -> Option { let mut minimum = u64::MAX; let mut maximum = u64::MIN; - let mut histogram = [0usize; 53]; + let mut low_bits_or = 0_u64; for value in values { let bits = value.to_bits(); let ordered = ordered_u64(bits); minimum = minimum.min(ordered); maximum = maximum.max(ordered); - let zeros = bits.trailing_zeros().min(52); - histogram[zeros as usize] += 1; - } - analyze_histogram(&mut histogram, 52, values.len(), minimum, maximum) -} - -fn category_entropy(probability: f64) -> f64 { - if probability == 0.0 || probability == 1.0 { - 0.0 - } else { - -probability * probability.log2() + low_bits_or |= bits; } + analyze_bits(low_bits_or, 52, values.len(), minimum, maximum) } fn latent_ptype(ptype: PType) -> VortexResult { @@ -863,16 +852,24 @@ mod tests { }); #[test] - fn histogram_search_continues_after_local_decline() -> VortexResult<()> { - let mut histogram = [0usize; 24]; - histogram[0] = 10; - histogram[1] = 70; - histogram[20] = 20; - - let Some(analysis) = analyze_histogram(&mut histogram, 23, 100, 0, (1 << 23) - 1) else { - vortex_bail!("a large trailing-zero group must be useful") - }; - assert_eq!(analysis.k, 20); + fn fixed_tree_analysis_accounts_for_secondary_width() -> VortexResult<()> { + let values = PrimitiveArray::from_iter((0_u32..4096).map(|index| { + let value = f64::from(f32::from_bits(0x3f80_0000 | index.wrapping_mul(7_919))); + if index % 10 == 0 { + f64::from_bits(value.to_bits() | 1) + } else { + value + } + })); + let analysis = analyze_float_quant(values.as_view()).vortex_expect("FloatQuant input"); + assert_eq!(analysis.k, 29); + assert_eq!(analysis.secondary_bit_width, 1); + + let general = + PrimitiveArray::from_iter((0_u32..4096).map(|index| { + f32::from_bits(0x3f80_0000 | (index.wrapping_mul(7_919) & 0x007f_ffff)) + })); + assert_eq!(analyze_float_quant(general.as_view()), None); Ok(()) } @@ -976,7 +973,7 @@ mod tests { } })); let analysis = analyze_float_quant(original.as_view()).vortex_expect("FloatQuant input"); - assert!(!analysis.secondary_is_constant); + assert_eq!(analysis.secondary_bit_width, 1); let split = FloatQuant::from_primitive(original.as_view(), analysis.k)?; let primary = split .primary() @@ -1001,9 +998,13 @@ mod tests { )? }; let primary = FoR::try_new(primary.into_array(), Scalar::from(analysis.primary_min))?; - // SAFETY: The test changes only one low bit. - let secondary = - unsafe { vortex_fastlanes::bitpack_compress::bitpack_encode_unchecked(secondary, 1)? }; + // SAFETY: The analysis computes the exact secondary width. + let secondary = unsafe { + vortex_fastlanes::bitpack_compress::bitpack_encode_unchecked( + secondary, + analysis.secondary_bit_width, + )? + }; let encoded = FloatQuant::try_new( primary.into_array(), Some(secondary.into_array()), diff --git a/vortex-btrblocks/src/schemes/float/float_quant.rs b/vortex-btrblocks/src/schemes/float/float_quant.rs index decd9e4b307..0bfe5c6ee11 100644 --- a/vortex-btrblocks/src/schemes/float/float_quant.rs +++ b/vortex-btrblocks/src/schemes/float/float_quant.rs @@ -17,6 +17,7 @@ use vortex_array::buffer::BufferHandle; use vortex_array::dtype::PType; use vortex_array::match_each_float_ptype; use vortex_array::scalar::Scalar; +use vortex_array::validity::Validity; use vortex_compressor::scheme::CompressionEstimate; use vortex_compressor::scheme::DeferredEstimate; use vortex_compressor::scheme::EstimateVerdict; @@ -24,6 +25,7 @@ use vortex_error::VortexResult; use vortex_fastlanes::BitPacked; use vortex_fastlanes::FoR; use vortex_fastlanes::bitpack_compress::bitpack_primitive_map; +use vortex_fastlanes::bitpack_compress::bitpack_primitive_map_pair; use vortex_float_quant::FloatQuant; use vortex_float_quant::FloatQuantAnalysis; use vortex_float_quant::analyze_float_quant; @@ -70,12 +72,8 @@ impl Scheme for FloatQuantScheme { let Some(analysis) = analyze_float_quant(sample.as_view()) else { return Ok(EstimateVerdict::Skip); }; - if !analysis.secondary_is_constant || analysis.primary_bit_width == 0 { - return Ok(EstimateVerdict::Skip); - } - let before_nbytes = sample.nbytes(); - let compressed = encode_constant_secondary(sample.as_view(), analysis)?; + let compressed = encode_float_quant(sample.as_view(), analysis)?; let after_nbytes = compressed.nbytes(); if after_nbytes == 0 || after_nbytes >= before_nbytes { return Ok(EstimateVerdict::Skip); @@ -100,42 +98,84 @@ impl Scheme for FloatQuantScheme { let Some(analysis) = analyze_float_quant(primitive.as_view()) else { return Ok(source.array().clone()); }; - if !analysis.secondary_is_constant || analysis.primary_bit_width == 0 { - return Ok(source.array().clone()); - } - - encode_constant_secondary(primitive.as_view(), analysis) + encode_float_quant(primitive.as_view(), analysis) } } -fn encode_constant_secondary( +fn encode_float_quant( primitive: vortex_array::ArrayView<'_, Primitive>, analysis: FloatQuantAnalysis, ) -> VortexResult { - let (packed, latent_ptype, reference) = match primitive.ptype() { + let (primary_packed, secondary_packed, latent_ptype, reference) = match primitive.ptype() { PType::F32 => { let primary_min = u32::try_from(analysis.primary_min)?; - let packed = bitpack_primitive_map( - primitive.as_slice::(), - analysis.primary_bit_width, - |value| (ordered_u32(value.to_bits()) >> analysis.k) - primary_min, + let values = primitive.as_slice::(); + let (primary, secondary) = if analysis.secondary_bit_width == 0 { + ( + bitpack_primitive_map(values, analysis.primary_bit_width, |value| { + (ordered_u32(value.to_bits()) >> analysis.k) - primary_min + }), + None, + ) + } else { + let low_mask = (1_u32 << analysis.k) - 1; + let (primary, secondary) = bitpack_primitive_map_pair( + values, + analysis.primary_bit_width, + analysis.secondary_bit_width, + |value| { + let bits = value.to_bits(); + ( + (ordered_u32(bits) >> analysis.k) - primary_min, + bits & low_mask, + ) + }, + ); + (primary, Some(secondary)) + }; + ( + primary.into_byte_buffer(), + secondary.map(|packed| packed.into_byte_buffer()), + PType::U32, + Scalar::from(primary_min), ) - .into_byte_buffer(); - (packed, PType::U32, Scalar::from(primary_min)) } PType::F64 => { - let packed = bitpack_primitive_map( - primitive.as_slice::(), - analysis.primary_bit_width, - |value| (ordered_u64(value.to_bits()) >> analysis.k) - analysis.primary_min, + let values = primitive.as_slice::(); + let (primary, secondary) = if analysis.secondary_bit_width == 0 { + ( + bitpack_primitive_map(values, analysis.primary_bit_width, |value| { + (ordered_u64(value.to_bits()) >> analysis.k) - analysis.primary_min + }), + None, + ) + } else { + let low_mask = (1_u64 << analysis.k) - 1; + let (primary, secondary) = bitpack_primitive_map_pair( + values, + analysis.primary_bit_width, + analysis.secondary_bit_width, + |value| { + let bits = value.to_bits(); + ( + (ordered_u64(bits) >> analysis.k) - analysis.primary_min, + bits & low_mask, + ) + }, + ); + (primary, Some(secondary)) + }; + ( + primary.into_byte_buffer(), + secondary.map(|packed| packed.into_byte_buffer()), + PType::U64, + Scalar::from(analysis.primary_min), ) - .into_byte_buffer(); - (packed, PType::U64, Scalar::from(analysis.primary_min)) } _ => unreachable!(), }; let compressed_primary = BitPacked::try_new( - BufferHandle::new_host(packed), + BufferHandle::new_host(primary_packed), latent_ptype, primitive.validity()?, None, @@ -145,7 +185,27 @@ fn encode_constant_secondary( )? .into_array(); let compressed_primary = FoR::try_new(compressed_primary, reference)?.into_array(); - Ok(FloatQuant::try_new(compressed_primary, None, primitive.ptype(), analysis.k)?.into_array()) + let compressed_secondary = secondary_packed + .map(|packed| { + BitPacked::try_new( + BufferHandle::new_host(packed), + latent_ptype, + Validity::NonNullable, + None, + analysis.secondary_bit_width, + primitive.len(), + 0, + ) + .map(IntoArray::into_array) + }) + .transpose()?; + Ok(FloatQuant::try_new( + compressed_primary, + compressed_secondary, + primitive.ptype(), + analysis.k, + )? + .into_array()) } #[inline] diff --git a/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs index 197e44858cd..4fc35157b32 100644 --- a/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs @@ -21,6 +21,8 @@ use vortex_block_residual::BlockResidual; use vortex_block_residual::OrderedFloat; use vortex_buffer::Buffer; use vortex_error::VortexResult; +use vortex_fastlanes::BitPacked; +use vortex_fastlanes::BitPackedArrayExt; use vortex_float_quant::FloatQuant; use vortex_float_quant::FloatQuantArraySlotsExt; use vortex_session::VortexSession; @@ -94,6 +96,34 @@ fn test_widened_f32_uses_float_quant() -> VortexResult<()> { Ok(()) } +#[test] +fn test_nonzero_secondary_uses_float_quant() -> VortexResult<()> { + let values = (0u32..65_536) + .map(|index| { + let mantissa = index.wrapping_mul(7_919) & 0x007f_ffff; + let value = f64::from(f32::from_bits(0x3f80_0000 | mantissa)); + if index % 10 == 0 { + f64::from_bits(value.to_bits() | 1) + } else { + value + } + }) + .collect::>(); + let array = PrimitiveArray::from_iter(values).into_array(); + let mut ctx = SESSION.create_execution_ctx(); + let compressed = BtrBlocksCompressor::default().compress(&array, &mut ctx)?; + + assert!(compressed.is::()); + let float_quant = compressed.as_::(); + let secondary = float_quant + .secondary() + .ok_or_else(|| vortex_error::vortex_err!("missing nonzero FloatQuant secondary"))? + .as_::(); + assert_eq!(secondary.bit_width(), 1); + assert_arrays_eq!(compressed, array, &mut ctx); + Ok(()) +} + #[test] fn test_float_quant_ignores_null_payloads() -> VortexResult<()> { let values = (0u32..16_384) diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index fd182a17c2b..67ea3818763 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -162,6 +162,21 @@ fn setup_nonzero_secondary_array() -> PrimitiveArray { ) } +fn setup_secondary_width_array(width: u8) -> PrimitiveArray { + let widened = setup_widened_f32_array(); + let low_mask = (1_u64 << width) - 1; + PrimitiveArray::from_iter( + widened + .as_slice::() + .iter() + .enumerate() + .map(|(index, value)| { + let low = (index as u64).wrapping_mul(2_654_435_761) & low_mask; + f64::from_bits(value.to_bits() | low) + }), + ) +} + fn setup_random_walk_array() -> PrimitiveArray { let mut rng = StdRng::seed_from_u64(2); let mut value = 1_000.0_f64; @@ -249,7 +264,7 @@ fn encode_ordered_block_residual(array: &PrimitiveArray) -> vortex::array::Array fn encode_float_quant_tree(array: &PrimitiveArray) -> vortex::array::ArrayRef { let analysis = analyze_float_quant(array.as_view()).unwrap(); - assert!(analysis.secondary_is_constant); + assert_eq!(analysis.secondary_bit_width, 0); let primary = FloatQuant::primary_for_primitive(array.as_view(), analysis.k, analysis.primary_min) .unwrap(); @@ -269,7 +284,7 @@ fn encode_float_quant_tree(array: &PrimitiveArray) -> vortex::array::ArrayRef { fn encode_float_quant_nonzero_secondary_tree(array: &PrimitiveArray) -> vortex::array::ArrayRef { let analysis = analyze_float_quant(array.as_view()).unwrap(); - assert!(!analysis.secondary_is_constant); + assert_ne!(analysis.secondary_bit_width, 0); let split = FloatQuant::from_primitive(array.as_view(), analysis.k).unwrap(); let primary = split .primary() @@ -295,8 +310,8 @@ fn encode_float_quant_nonzero_secondary_tree(array: &PrimitiveArray) -> vortex:: let primary = FoR::try_new(primary, Scalar::from(analysis.primary_min)) .unwrap() .into_array(); - // SAFETY: The input generator changes only the lowest bit. - let secondary = unsafe { bitpack_encode_unchecked(secondary, 1) } + // SAFETY: The analysis computes the exact secondary width. + let secondary = unsafe { bitpack_encode_unchecked(secondary, analysis.secondary_bit_width) } .unwrap() .into_array(); FloatQuant::try_new(primary, Some(secondary), PType::F64, analysis.k) @@ -1220,6 +1235,23 @@ fn bench_float_quant_nonzero_secondary_tree_compress_f64(bencher: Bencher) { .bench_refs(|array| encode_float_quant_nonzero_secondary_tree(array)); } +#[divan::bench(name = "float_quant_nonzero_secondary_scheme_compress_f64")] +fn bench_float_quant_nonzero_secondary_scheme_compress_f64(bencher: Bencher) { + let compressor = BtrBlocksCompressorBuilder::empty() + .with_new_scheme(&FloatQuantScheme) + .build(); + bench_compressor(bencher, setup_nonzero_secondary_array(), compressor); +} + +#[divan::bench(name = "float_quant_nonzero_secondary_default_compress_f64")] +fn bench_float_quant_nonzero_secondary_default_compress_f64(bencher: Bencher) { + bench_compressor( + bencher, + setup_nonzero_secondary_array(), + BtrBlocksCompressorBuilder::default().build(), + ); +} + #[divan::bench(name = "float_quant_nonzero_secondary_tree_decompress_f64")] fn bench_float_quant_nonzero_secondary_tree_decompress_f64(bencher: Bencher) { let encoded = encode_float_quant_nonzero_secondary_tree(&setup_nonzero_secondary_array()); @@ -1229,6 +1261,29 @@ fn bench_float_quant_nonzero_secondary_tree_decompress_f64(bencher: Bencher) { .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); } +#[divan::bench(name = "float_quant_nonzero_secondary_default_decompress_f64")] +fn bench_float_quant_nonzero_secondary_default_decompress_f64(bencher: Bencher) { + let input = setup_nonzero_secondary_array().into_array(); + let encoded = BtrBlocksCompressorBuilder::default() + .build() + .compress(&input, &mut SESSION.create_execution_ctx()) + .unwrap(); + assert!(encoded.is::()); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(args = [1, 4, 8, 16])] +fn float_quant_secondary_width_decompress_f64(bencher: Bencher, width: u8) { + let encoded = encode_float_quant_nonzero_secondary_tree(&setup_secondary_width_array(width)); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + #[divan::bench(name = "float_quant_nonzero_secondary_tree_scalar_at_f64")] fn bench_float_quant_nonzero_secondary_tree_scalar_at_f64(bencher: Bencher) { let encoded = encode_float_quant_nonzero_secondary_tree(&setup_nonzero_secondary_array()); From be658d235ff1dce2cac2e1bd93b53475c24a07cd Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 16:00:01 +0100 Subject: [PATCH 16/78] perf: specialize block residual decode Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 106 +++++-- .../src/block_residual_array.rs | 288 ++++++++++++++++-- encodings/block-residual/src/codec.rs | 15 +- vortex/benches/single_encoding_throughput.rs | 46 +++ 4 files changed, 408 insertions(+), 47 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index e344e0d0c14..824e1c82563 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -46,7 +46,9 @@ The GloVe result identifies a separate entropy gap inside the ALP integer child. Generic quotient and remainder trees do not close that gap. -The current selector factors and patch cost remain provisional. Final calibration requires the complete corpus and selected-tree evidence. +The current selector factors and dense-patch cost remain provisional. + +Final calibration requires the complete corpus and selected-tree evidence. ## OrderedFloatArray @@ -152,7 +154,13 @@ The integer selector divides the measured compression ratio by these decode-cost A 1.20 factor requires about 16.7 percent fewer estimated bytes. It does not require 20 percent fewer bytes. -The block planner adds 16 synthetic cost bits per patch. This cost favors wider packed residuals when many patches slow decode. +The block planner adds 16 synthetic cost bits per patch. + +This cost preserves useful sparse-patch trees. It does not prevent every dense-patch decode regression. + +A global 96-bit patch cost prevented the synthetic dense-patch regressions. It also removed useful sparse-patch compression on HashTags. + +The next planner experiment will add cost only after a block enters the measured slow patch-density region. The selector excludes BlockResidual from dictionary-code children. A complete BlockResidual tree can still displace a complete dictionary tree. @@ -392,16 +400,28 @@ The prior default compressed at 394 MB/s. The difference is 1.2 percent. The Compact size is specific to this synthetic pattern. It does not establish a general real-float result. -### Direct 32-bit comparison +### Direct integer comparison after decode specialization The direct benchmark compares two million block-local values against a whole-column FoR plus BitPacked tree. -| Type and tree | Encode GB/s | Decode GB/s | Scalar access ns | -| --- | ---: | ---: | ---: | -| i32 BlockResidual | 2.69 | 33.66 | 124 | -| i32 FoR plus BitPacked | 2.59 | 26.93 | 119 | -| u32 BlockResidual | 2.74 | 33.55 | 98 | -| u32 FoR plus BitPacked | 2.62 | 39.70 | 87 | +The `u32` decode path now unpacks residuals and adds the block base directly into the output buffer. + +Signed integers and ordered floats still require a transform after residual decode. + +| Type and tree | Decode GB/s | Difference | +| --- | ---: | ---: | +| `u32` BlockResidual | 46.27 | +16.1 percent | +| `u32` FoR plus BitPacked | 39.85 | Baseline | +| `i32` BlockResidual | 33.86 | +31.8 percent | +| `i32` FoR plus BitPacked | 25.69 | Baseline | +| `u64` BlockResidual | 36.88 | +0.6 percent | +| `u64` FoR plus BitPacked | 36.65 | Baseline | +| `i16` BlockResidual | 32.34 | -22.2 percent | +| `i16` FoR plus BitPacked | 41.59 | Baseline | + +The specialization removes the prior `u32` decode disadvantage. + +The narrow integer exclusion remains valid. Native-width unpack does not close the `i16` gap. One width factor does not predict both signed and unsigned results. @@ -423,7 +443,42 @@ Scalar access remains bounded across the sweep. Bulk decode has a severe patch-density cliff before the planner changes to packed residuals. -The final selector calibration must compare alternate patch costs and separate zero-width blocks from packed blocks. +Before the decode specialization, the complete selector exposed two failures with the 16-bit patch cost. + +| Outlier stride | Prior bytes | BlockResidual bytes | Prior decode GB/s | BlockResidual decode GB/s | +| ---: | ---: | ---: | ---: | ---: | +| 4 | 5,508,488 | 3,072,310 | 18.11 | 9.01 | +| 1 | 5,252,352 | 2,543,221 | 44.73 | 33.39 | + +The stride-4 tree remains a bad selection because one quarter of the values use patches. + +A 96-bit cost selected a near-full residual width for both cases. + +The complete selector then retained BitPacked at stride 4. It retained patch-free BlockResidual at stride 1. + +With the direct-output path, decode reached 18.06 GB/s at stride 4 and 45.94 GB/s at stride 1. + +These results combine the 96-bit planner with the decode specialization. The next sweep must isolate both effects. + +The global 96-bit cost also increased proposed-default size on real data: + +| Dataset | Size change from 16-bit cost | +| --- | ---: | +| Euro2016 | +0.1 percent | +| HashTags | +3.2 percent | +| Air Quality | +0.04 percent | + +On HashTags, size increased from 21,306,418 bytes to 21,989,476 bytes. + +Decode throughput changed from 22.54 GB/s to 22.77 GB/s. Encode throughput did not change materially. + +The global 96-bit cost is rejected. + +The next planner must preserve low-density patches and penalize the dense patch region. + +BlockResidual also composes with outer encodings. Sparse and RunEnd children selected BlockResidual in HashTags and the synthetic low-density sweep. + +This composition reduced size, but it requires parent-specific throughput validation. ### GloVe embeddings @@ -455,6 +510,12 @@ The size gap comes from a decimal quotient and remainder split plus entropy codi GloVe therefore identifies an ALP-child compression gap. It does not identify a failure in the ALP float transform. +The current Vortex tree does not leave the embedding values uncompressed. + +It uses ALP for the float transform, then ZigZag and BitPacked for the integer child. + +The Pco advantage comes from a different encoding of the same ALP integer child. + ### GloVe quotient and remainder prototypes The first prototype splits the exact ALP integer child with a constant base. @@ -675,17 +736,20 @@ The nonzero-secondary FloatQuant implementation and focused validation are compl Complete these remaining steps: -1. Compare alternate BlockResidual patch costs across the patch-density sweep. -2. Evaluate narrow BlockResidual as a Compact-only candidate. -3. Reduce analysis cost on short rejected columns. -4. Measure BlockResidual under temporal, FSST, sparse, run-end, and list parents. -5. Prototype bounded scalar checkpoints for fixed-bin range packing. -6. Compare fixed-bin packing against default and Pco on unrelated no-Delta wins. -7. Add real embedding datasets beyond GloVe when licenses and loaders permit them. -8. Run the complete `bench-vortex` compression corpus after the candidate set stabilizes. -9. Compare the geometric mean against Parquet with Zstd. -10. Calibrate all selector thresholds from complete corpus and selected-tree evidence. -11. Update this plan after each experiment. +1. Prototype a density-aware BlockResidual patch cost. +2. Re-run the patch sweep and the affected real columns after each planner change. +3. Measure BlockResidual under temporal, FSST, Sparse, RunEnd, and list parents. +4. Reduce analysis cost on short rejected columns. +5. Evaluate narrow BlockResidual as a Compact-only candidate. +6. Prototype bounded scalar checkpoints for fixed-bin range packing. +7. Compare fixed-bin packing against default and Pco on unrelated no-Delta wins. +8. Add real embedding datasets beyond GloVe when licenses and loaders permit them. +9. Classify more float columns where Pco beats ALP, ALP-RD, and the new schemes. +10. Prototype one lightweight scheme for the repeated real-float gap classes. +11. Run the complete `bench-vortex` compression corpus after the candidate set stabilizes. +12. Compare the geometric mean against Parquet with Zstd. +13. Calibrate all selector thresholds from complete corpus and selected-tree evidence. +14. Update this plan after each experiment. ## Pull request structure diff --git a/encodings/block-residual/src/block_residual_array.rs b/encodings/block-residual/src/block_residual_array.rs index d47f194727d..298fa8952b4 100644 --- a/encodings/block-residual/src/block_residual_array.rs +++ b/encodings/block-residual/src/block_residual_array.rs @@ -38,7 +38,8 @@ use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityVTable; use vortex_array::vtable::child_to_validity; use vortex_array::vtable::validity_to_child; -use vortex_buffer::Buffer; +use vortex_buffer::BufferMut; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -516,7 +517,7 @@ impl ExecutedBlockResidual { } } -fn decode_array_values( +fn decode_array_values( array: ArrayView<'_, BlockResidual>, ctx: &mut ExecutionCtx, mut transform: impl FnMut(U) -> T, @@ -532,7 +533,10 @@ fn decode_array_values( let patch_positions = children.patch_positions.as_slice::(); let patch_highs = children.patch_highs.as_slice::(); let logical_range = array.data().slice_start..array.data().slice_stop; - let mut values = Vec::with_capacity(logical_range.len()); + let mut direct_values = + DIRECT_OUTPUT.then(|| BufferMut::::with_capacity(logical_range.len())); + let mut transformed_values = + (!DIRECT_OUTPUT).then(|| BufferMut::::with_capacity(logical_range.len())); let mut residuals = [U::default(); BLOCK_LEN]; let first_block = logical_range.start / BLOCK_LEN; @@ -547,7 +551,7 @@ fn decode_array_values( let residual_width = residual_widths[block_index]; let high_width = high_widths[block_index]; - let base = U::from_u64(bases[block_index]); + let base = ::from_u64(bases[block_index]); vortex_ensure!( residual_width <= U::BITS && high_width <= U::BITS @@ -580,9 +584,14 @@ fn decode_array_values( let local_stop = (logical_range.end - block_start).min(block_len); if residual_width == 0 { - let output_start = values.len(); - values - .extend(std::iter::repeat_with(|| transform(base)).take(local_stop - local_start)); + let output_start = decoded_len(direct_values.as_ref(), transformed_values.as_ref()); + append_repeated_decoded::( + &mut direct_values, + &mut transformed_values, + base, + local_stop - local_start, + &mut transform, + ); let mut previous_position = None; for (patch_index, &position) in positions.iter().enumerate() { validate_patch_position(block_len, previous_position, position)?; @@ -595,16 +604,48 @@ fn decode_array_values( let high = unsafe { read_wide_bits(highs, patch_index * usize::from(high_width), high_width) }; - values[output_start + position - local_start] = - transform(base.wrapping_add(U::from_u64(high))); + set_decoded::( + &mut direct_values, + &mut transformed_values, + output_start + position - local_start, + base.wrapping_add(::from_u64(high)), + &mut transform, + ); } continue; } let packed = packed_words_as_native::(&residual_words[residual_payload]); + if positions.is_empty() && DIRECT_OUTPUT && local_start == 0 && local_stop == BLOCK_LEN { + // SAFETY: The encoder writes one complete FastLanes chunk, and the output has capacity. + unsafe { + append_unpacked_add( + direct_values + .as_mut() + .vortex_expect("direct BlockResidual output is present"), + usize::from(residual_width), + packed, + base, + ); + } + continue; + } // SAFETY: The encoder writes one complete FastLanes chunk for each block. unsafe { - U::unchecked_unpack(usize::from(residual_width), packed, &mut residuals); + if positions.is_empty() && DIRECT_OUTPUT { + U::unpack_add(usize::from(residual_width), packed, base, &mut residuals); + } else { + U::unchecked_unpack(usize::from(residual_width), packed, &mut residuals); + } + } + if positions.is_empty() && DIRECT_OUTPUT { + append_decoded::( + &mut direct_values, + &mut transformed_values, + &residuals[local_start..local_stop], + &mut transform, + ); + continue; } let mut previous_position = None; for (patch_index, &position) in positions.iter().enumerate() { @@ -616,13 +657,182 @@ fn decode_array_values( residuals[usize::from(position)].apply_high(high, residual_width); } - values.extend( - residuals[local_start..local_stop] - .iter() - .map(|&residual| transform(residual.wrapping_add(base))), + append_residuals::( + &mut direct_values, + &mut transformed_values, + &mut residuals[local_start..local_stop], + base, + &mut transform, ); } - Ok(PrimitiveArray::new(Buffer::from(values), array.validity()?)) + let validity = array.validity()?; + if DIRECT_OUTPUT { + Ok(PrimitiveArray::new( + direct_values + .vortex_expect("direct BlockResidual output is present") + .freeze(), + validity, + )) + } else { + Ok(PrimitiveArray::new( + transformed_values + .vortex_expect("transformed BlockResidual output is present") + .freeze(), + validity, + )) + } +} + +fn decoded_len( + direct: Option<&BufferMut>, + transformed: Option<&BufferMut>, +) -> usize { + direct.map_or_else(|| transformed.map_or(0, BufferMut::len), BufferMut::len) +} + +fn append_repeated_decoded( + direct: &mut Option>, + transformed: &mut Option>, + value: U, + count: usize, + transform: &mut impl FnMut(U) -> T, +) { + if DIRECT_OUTPUT { + append_repeated( + direct + .as_mut() + .vortex_expect("direct BlockResidual output is present"), + value, + count, + ); + } else { + let output = transformed + .as_mut() + .vortex_expect("transformed BlockResidual output is present"); + let output_len = output.len(); + for destination in &mut output.spare_capacity_mut()[..count] { + destination.write(transform(value)); + } + // SAFETY: The loop initialized each new output value. + unsafe { output.set_len(output_len + count) }; + } +} + +fn append_decoded( + direct: &mut Option>, + transformed: &mut Option>, + values: &[U], + transform: &mut impl FnMut(U) -> T, +) { + if DIRECT_OUTPUT { + append_values( + direct + .as_mut() + .vortex_expect("direct BlockResidual output is present"), + values, + ); + } else { + let output = transformed + .as_mut() + .vortex_expect("transformed BlockResidual output is present"); + let output_len = output.len(); + for (destination, &value) in output.spare_capacity_mut()[..values.len()] + .iter_mut() + .zip(values) + { + destination.write(transform(value)); + } + // SAFETY: The loop initialized each new output value. + unsafe { output.set_len(output_len + values.len()) }; + } +} + +fn append_residuals( + direct: &mut Option>, + transformed: &mut Option>, + residuals: &mut [U], + base: U, + transform: &mut impl FnMut(U) -> T, +) { + if DIRECT_OUTPUT { + for residual in residuals.iter_mut() { + *residual = residual.wrapping_add(base); + } + append_values( + direct + .as_mut() + .vortex_expect("direct BlockResidual output is present"), + residuals, + ); + } else { + let output = transformed + .as_mut() + .vortex_expect("transformed BlockResidual output is present"); + let output_len = output.len(); + for (destination, &residual) in output.spare_capacity_mut()[..residuals.len()] + .iter_mut() + .zip(residuals.iter()) + { + destination.write(transform(residual.wrapping_add(base))); + } + // SAFETY: The loop initialized each new output value. + unsafe { output.set_len(output_len + residuals.len()) }; + } +} + +fn set_decoded( + direct: &mut Option>, + transformed: &mut Option>, + index: usize, + value: U, + transform: &mut impl FnMut(U) -> T, +) { + if DIRECT_OUTPUT { + direct + .as_mut() + .vortex_expect("direct BlockResidual output is present")[index] = value; + } else { + transformed + .as_mut() + .vortex_expect("transformed BlockResidual output is present")[index] = transform(value); + } +} + +fn append_repeated(output: &mut BufferMut, value: T, count: usize) { + let output_len = output.len(); + for destination in &mut output.spare_capacity_mut()[..count] { + destination.write(value); + } + // SAFETY: The loop initialized each new output value. + unsafe { output.set_len(output_len + count) }; +} + +fn append_values(output: &mut BufferMut, values: &[T]) { + let output_len = output.len(); + for (destination, &value) in output.spare_capacity_mut()[..values.len()] + .iter_mut() + .zip(values) + { + destination.write(value); + } + // SAFETY: The loop initialized each new output value. + unsafe { output.set_len(output_len + values.len()) }; +} + +unsafe fn append_unpacked_add( + output: &mut BufferMut, + bit_width: usize, + packed: &[T], + base: T, +) { + let output_len = output.len(); + let destination = output.spare_capacity_mut()[..BLOCK_LEN].as_mut_ptr(); + // SAFETY: The caller guarantees capacity. FastLanes initializes one complete output chunk. + let destination = unsafe { std::slice::from_raw_parts_mut(destination.cast::(), BLOCK_LEN) }; + // SAFETY: The caller provides one complete FastLanes input and output chunk. + unsafe { T::unpack_add(bit_width, packed, base, destination) }; + // SAFETY: FastLanes initialized each new output value. + unsafe { output.set_len(output_len + BLOCK_LEN) }; } fn decompress_array( @@ -630,14 +840,22 @@ fn decompress_array( ctx: &mut ExecutionCtx, ) -> VortexResult { match array.dtype().as_ptype() { - PType::U8 => decode_array_values(array, ctx, |value: u8| value), - PType::U16 => decode_array_values(array, ctx, |value: u16| value), - PType::U32 => decode_array_values(array, ctx, |value: u32| value), - PType::U64 => decode_array_values(array, ctx, |value: u64| value), - PType::I8 => decode_array_values(array, ctx, |value: u8| (value ^ (1_u8 << 7)) as i8), - PType::I16 => decode_array_values(array, ctx, |value: u16| (value ^ (1_u16 << 15)) as i16), - PType::I32 => decode_array_values(array, ctx, |value: u32| (value ^ (1_u32 << 31)) as i32), - PType::I64 => decode_array_values(array, ctx, |value: u64| (value ^ (1_u64 << 63)) as i64), + PType::U8 => decode_array_values::(array, ctx, |value| value), + PType::U16 => decode_array_values::(array, ctx, |value| value), + PType::U32 => decode_array_values::(array, ctx, |value| value), + PType::U64 => decode_array_values::(array, ctx, |value| value), + PType::I8 => { + decode_array_values::(array, ctx, |value| (value ^ (1_u8 << 7)) as i8) + } + PType::I16 => decode_array_values::(array, ctx, |value| { + (value ^ (1_u16 << 15)) as i16 + }), + PType::I32 => decode_array_values::(array, ctx, |value| { + (value ^ (1_u32 << 31)) as i32 + }), + PType::I64 => decode_array_values::(array, ctx, |value| { + (value ^ (1_u64 << 63)) as i64 + }), ptype => vortex_bail!("BlockResidual decode does not support {ptype}"), } } @@ -646,7 +864,7 @@ pub(crate) fn decompress_ordered_f32( array: ArrayView<'_, BlockResidual>, ctx: &mut ExecutionCtx, ) -> VortexResult { - decode_array_values(array, ctx, |ordered: u32| { + decode_array_values::(array, ctx, |ordered| { let bits = if ordered & (1_u32 << 31) == 0 { !ordered } else { @@ -660,7 +878,7 @@ pub(crate) fn decompress_ordered_f64( array: ArrayView<'_, BlockResidual>, ctx: &mut ExecutionCtx, ) -> VortexResult { - decode_array_values(array, ctx, |ordered: u64| { + decode_array_values::(array, ctx, |ordered| { let bits = if ordered & (1_u64 << 63) == 0 { !ordered } else { @@ -986,6 +1204,26 @@ mod tests { Ok(()) } + #[test] + fn u32_direct_decode_roundtrip() -> VortexResult<()> { + let primitive = PrimitiveArray::from_iter((0..2_050_u32).map(|index| { + let block = index / 1_024; + block * 1_000_000 + (index * 7_919) % 1_024 + })); + let encoded = BlockResidual::from_primitive(primitive.as_view())?; + let session = array_session(); + crate::initialize(&session); + let mut ctx = session.create_execution_ctx(); + + assert_arrays_eq!(encoded, primitive.clone().into_array(), &mut ctx); + assert_arrays_eq!( + encoded.into_array().slice(1_023..1_026)?, + primitive.into_array().slice(1_023..1_026)?, + &mut ctx + ); + Ok(()) + } + #[test] fn nullable_slice_and_scalar_access() -> VortexResult<()> { let values = (0..2_050) diff --git a/encodings/block-residual/src/codec.rs b/encodings/block-residual/src/codec.rs index 1c0aa74dede..4f665c2466c 100644 --- a/encodings/block-residual/src/codec.rs +++ b/encodings/block-residual/src/codec.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use fastlanes::BitPacking; +use fastlanes::FoR as FastLanesFoR; use vortex_error::VortexResult; const CHUNK_LEN: usize = 1024; @@ -491,7 +492,7 @@ fn fast_pack_native(values: &[u64], width: u8) -> Vec { packed_words } -pub(crate) trait ResidualWord: BitPacking + Copy + Default { +pub(crate) trait ResidualWord: BitPacking + FastLanesFoR + Copy + Default { const BITS: u8; fn from_u64(value: u64) -> Self; @@ -501,6 +502,8 @@ pub(crate) trait ResidualWord: BitPacking + Copy + Default { fn wrapping_add(self, other: Self) -> Self; fn apply_high(&mut self, high: u64, shift: u8); + + unsafe fn unpack_add(bit_width: usize, packed: &[Self], base: Self, output: &mut [Self]); } macro_rules! impl_residual_word { @@ -525,6 +528,16 @@ macro_rules! impl_residual_word { fn apply_high(&mut self, high: u64, shift: u8) { *self |= (high as $T) << shift; } + + unsafe fn unpack_add( + bit_width: usize, + packed: &[Self], + base: Self, + output: &mut [Self], + ) { + // SAFETY: The caller provides one complete FastLanes input and output chunk. + unsafe { FastLanesFoR::unchecked_unfor_pack(bit_width, packed, base, output) }; + } } }; } diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index 67ea3818763..c72fd3cd099 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -334,6 +334,16 @@ fn encode_prior_default(array: &PrimitiveArray) -> vortex::array::ArrayRef { .unwrap() } +fn encode_proposed_default(array: &PrimitiveArray) -> vortex::array::ArrayRef { + BtrBlocksCompressorBuilder::default() + .build() + .compress( + &array.clone().into_array(), + &mut SESSION.create_execution_ctx(), + ) + .unwrap() +} + #[expect(clippy::cast_possible_truncation)] fn gen_varbin_words(len: usize, uniqueness: f64) -> Vec { let mut rng = StdRng::seed_from_u64(0); @@ -848,6 +858,42 @@ fn patch_density_block_residual_scalar_at_u32(bencher: Bencher, stride: u64) { .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); } +#[divan::bench(args = [256, 64, 16, 4, 1])] +fn patch_density_prior_default_compress_u32(bencher: Bencher, stride: u64) { + let input = setup_patch_density_u32_array(stride); + let compressor = BtrBlocksCompressorBuilder::default() + .exclude_schemes([BlockResidualScheme.id()]) + .build(); + bench_compressor(bencher, input, compressor); +} + +#[divan::bench(args = [256, 64, 16, 4, 1])] +fn patch_density_default_compress_u32(bencher: Bencher, stride: u64) { + bench_compressor( + bencher, + setup_patch_density_u32_array(stride), + BtrBlocksCompressorBuilder::default().build(), + ); +} + +#[divan::bench(args = [256, 64, 16, 4, 1])] +fn patch_density_prior_default_decompress_u32(bencher: Bencher, stride: u64) { + let encoded = encode_prior_default(&setup_patch_density_u32_array(stride)); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(args = [256, 64, 16, 4, 1])] +fn patch_density_default_decompress_u32(bencher: Bencher, stride: u64) { + let encoded = encode_proposed_default(&setup_patch_density_u32_array(stride)); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + #[divan::bench(name = "block_local_block_residual_compress_i16")] fn bench_block_local_block_residual_compress_i16(bencher: Bencher) { let array = setup_block_local_i16_array(); From be0b43b5148a0e95b631a60f02c134cf967c8533 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 16:10:47 +0100 Subject: [PATCH 17/78] perf: penalize dense block residual patches Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 74 ++++++++++++++----- .../examples/float_compressor_bench.rs | 54 +++++++++++++- .../schemes/float/ordered_block_residual.rs | 3 +- .../src/schemes/integer/block_residual.rs | 37 +++++++++- vortex-btrblocks/src/schemes/integer/mod.rs | 1 + .../schemes/integer/scheme_selection_tests.rs | 15 ++++ 6 files changed, 160 insertions(+), 24 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 824e1c82563..9245c9cf257 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -46,7 +46,7 @@ The GloVe result identifies a separate entropy gap inside the ALP integer child. Generic quotient and remainder trees do not close that gap. -The current selector factors and dense-patch cost remain provisional. +The current selector factors and nonlinear patch cost remain provisional. Final calibration requires the complete corpus and selected-tree evidence. @@ -156,11 +156,17 @@ A 1.20 factor requires about 16.7 percent fewer estimated bytes. It does not req The block planner adds 16 synthetic cost bits per patch. -This cost preserves useful sparse-patch trees. It does not prevent every dense-patch decode regression. +This cost selects the physical residual width. It preserves useful sparse-patch trees. A global 96-bit patch cost prevented the synthetic dense-patch regressions. It also removed useful sparse-patch compression on HashTags. -The next planner experiment will add cost only after a block enters the measured slow patch-density region. +The BtrBlocks estimator adds a separate nonlinear decode cost. + +For `p` patches and `n` values, it adds `320 * p * p / n` synthetic bits. + +This formula adds 80 bits per patch at 25 percent density. It adds 32 bits per patch at 10 percent density. + +The physical encoding remains size-optimal. The adjusted size only affects selection. The selector excludes BlockResidual from dictionary-code children. A complete BlockResidual tree can still displace a complete dictionary tree. @@ -474,7 +480,37 @@ Decode throughput changed from 22.54 GB/s to 22.77 GB/s. Encode throughput did n The global 96-bit cost is rejected. -The next planner must preserve low-density patches and penalize the dense patch region. +The first per-block density gate also rejected the two useful HashTags trees. + +The selector-level cost uses total sample density instead. It preserves the physical encoding and compares the complete tree against its incumbent. + +The dense synthetic tree now retains BitPacked. + +| Synthetic input | Prior tree | Proposed tree | Proposed bytes | Result | +| --- | --- | --- | ---: | --- | +| 25 percent patches | BitPacked | BitPacked | 5,508,488 | Reject BlockResidual | +| Packed residuals | FoR with BitPacked | BlockResidual | 2,543,221 | Select BlockResidual | + +The packed-residual prior tree uses 5,252,352 bytes. + +BlockResidual reduced its size by 51.6 percent. Encode throughput decreased by 9.4 percent, and decode throughput increased by 8.0 percent. + +Two HashTags trees explain the real-data tradeoff: + +| Column and candidate | Prior bytes | Candidate bytes | Encode change | Decode change | Decision | +| --- | ---: | ---: | ---: | ---: | --- | +| `twitter#in_reply_to_user_id` BlockResidual | 701,203 | 399,111 | -23.4 percent | -3.0 percent | Reject | +| `interaction#received_at` Ordered BlockResidual | 3,263,939 | 3,018,443 | -38.3 percent | -28.5 percent | Reject | + +The first tree fails the selected-column encode gate. The second tree fails both throughput gates. + +With the nonlinear cost, complete HashTags size is 21,874,126 bytes. + +The prior default uses 22,602,522 bytes. The unpenalized new default used 21,306,418 bytes but selected both failing trees. + +The gated default reduces size by 3.2 percent against the prior default. + +Its measured aggregate encode throughput increased by 0.9 percent. Decode throughput increased by 5.6 percent. BlockResidual also composes with outer encodings. Sparse and RunEnd children selected BlockResidual in HashTags and the synthetic low-density sweep. @@ -718,6 +754,9 @@ This round completed these steps: - Excluded BlockResidual from dictionary-code children. - Added fused f32 OrderedFloat with BlockResidual decode and selection. - Added u32, i32, f32, and patch-density benchmarks. +- Added complete-compressor patch-density datasets and a column filter to the profiling benchmark. +- Added patch-density statistics to the BlockResidual profile. +- Replaced the global patch cost experiment with a nonlinear selector cost. - Removed the full patch scan from scalar access. - Made null payload bits neutral for the new default candidates. - Excluded integer BlockResidual from the CUDA-compatible preset. @@ -736,20 +775,19 @@ The nonzero-secondary FloatQuant implementation and focused validation are compl Complete these remaining steps: -1. Prototype a density-aware BlockResidual patch cost. -2. Re-run the patch sweep and the affected real columns after each planner change. -3. Measure BlockResidual under temporal, FSST, Sparse, RunEnd, and list parents. -4. Reduce analysis cost on short rejected columns. -5. Evaluate narrow BlockResidual as a Compact-only candidate. -6. Prototype bounded scalar checkpoints for fixed-bin range packing. -7. Compare fixed-bin packing against default and Pco on unrelated no-Delta wins. -8. Add real embedding datasets beyond GloVe when licenses and loaders permit them. -9. Classify more float columns where Pco beats ALP, ALP-RD, and the new schemes. -10. Prototype one lightweight scheme for the repeated real-float gap classes. -11. Run the complete `bench-vortex` compression corpus after the candidate set stabilizes. -12. Compare the geometric mean against Parquet with Zstd. -13. Calibrate all selector thresholds from complete corpus and selected-tree evidence. -14. Update this plan after each experiment. +1. Validate the nonlinear patch cost on the complete corpus. +2. Measure BlockResidual under temporal, FSST, Sparse, RunEnd, and list parents. +3. Reduce analysis cost on short rejected columns. +4. Evaluate narrow BlockResidual as a Compact-only candidate. +5. Prototype bounded scalar checkpoints for fixed-bin range packing. +6. Compare fixed-bin packing against default and Pco on unrelated no-Delta wins. +7. Add real embedding datasets beyond GloVe when licenses and loaders permit them. +8. Classify more float columns where Pco beats ALP, ALP-RD, and the new schemes. +9. Prototype one lightweight scheme for the repeated real-float gap classes. +10. Run the complete `bench-vortex` compression corpus after the candidate set stabilizes. +11. Compare the geometric mean against Parquet with Zstd. +12. Calibrate all selector thresholds from complete corpus and selected-tree evidence. +13. Update this plan after each experiment. ## Pull request structure diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index c20332513cf..cfd1a241007 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -115,6 +115,17 @@ fn synthetic_datasets(row_count: usize) -> Vec<(String, Vec)> { permuted as f64 - 500_000.0 })); + let patch_density_4 = PrimitiveArray::from_iter((0..row_count).map(|index| { + if index % 4 == 0 { + u32::MAX - u32::try_from(index).unwrap_or(u32::MAX) + } else { + 42 + } + })); + let patch_density_1 = PrimitiveArray::from_iter( + (0..row_count).map(|index| u32::MAX - u32::try_from(index).unwrap_or(u32::MAX)), + ); + vec![ ( "synthetic-widened-f32".to_string(), @@ -140,6 +151,14 @@ fn synthetic_datasets(row_count: usize) -> Vec<(String, Vec)> { "synthetic-integer-valued".to_string(), vec![column("value", integer_valued)], ), + ( + "synthetic-patch-density-4".to_string(), + vec![column("value", patch_density_4)], + ), + ( + "synthetic-patch-density-1".to_string(), + vec![column("value", patch_density_1)], + ), ] } @@ -307,8 +326,25 @@ fn profile_block_residual_array( .map(|&width| f64::from(width)) .sum::() / blocks as f64; + let patch_starts = residuals + .patch_starts() + .clone() + .execute::(&mut ctx)?; + let patch_starts = patch_starts.as_slice::(); + let mut maximum_patch_density = 0.0_f64; + let mut blocks_above_one_eighth = 0usize; + let mut blocks_at_one_quarter = 0usize; + for (block_index, starts) in patch_starts.windows(2).enumerate() { + let patch_count = usize::try_from(starts[1] - starts[0])?; + let block_start = block_index * 1_024; + let block_len = (array.len() - block_start).min(1_024); + maximum_patch_density = + maximum_patch_density.max(patch_count as f64 / block_len as f64); + blocks_above_one_eighth += usize::from(patch_count * 8 > block_len); + blocks_at_one_quarter += usize::from(patch_count * 4 >= block_len); + } println!( - "block-residual-profile\t{dataset}\t{column}\t{config}\t{path}\t{}\t{}\t{}\t{blocks}\t{average_residual_width:.3}\t{average_high_width:.3}\t{}", + "block-residual-profile\t{dataset}\t{column}\t{config}\t{path}\t{}\t{}\t{}\t{blocks}\t{average_residual_width:.3}\t{average_high_width:.3}\t{maximum_patch_density:.3}\t{blocks_above_one_eighth}\t{blocks_at_one_quarter}\t{}", array.dtype().as_ptype(), array.len(), residuals.patch_positions().len(), @@ -826,7 +862,10 @@ fn measure_dataset( encoding_tree(array), array.nbytes() ); - if *config == "integer-block-residual-only" { + if matches!( + *config, + "integer-block-residual-only" | "ordered-block-residual-only" + ) { profile_block_residual_array( dataset, &column.name, @@ -995,7 +1034,7 @@ fn main() -> VortexResult<()> { ); println!("result\tdataset\tconfig\trows\tinput-bytes\tencoded-bytes\tencode-MB/s\tdecode-MB/s"); println!( - "block-residual-profile\tdataset\tcolumn\tconfig\tpath\tptype\trows\tpatches\tblocks\taverage-residual-width\taverage-high-width\tbytes" + "block-residual-profile\tdataset\tcolumn\tconfig\tpath\tptype\trows\tpatches\tblocks\taverage-residual-width\taverage-high-width\tmaximum-patch-density\tblocks-above-one-eighth\tblocks-at-one-quarter\tbytes" ); if std::env::var_os("VORTEX_BENCH_SKIP_SYNTHETIC").is_none() { let synthetic_filter = std::env::var("VORTEX_BENCH_SYNTHETIC").ok(); @@ -1015,7 +1054,7 @@ fn main() -> VortexResult<()> { .file_stem() .and_then(|name| name.to_str()) .ok_or_else(|| vortex_err!("data path has no valid file name"))?; - let columns = if path + let mut columns = if path .extension() .is_some_and(|extension| extension == "parquet") { @@ -1023,6 +1062,13 @@ fn main() -> VortexResult<()> { } else { read_california(path, row_count)? }; + if let Ok(column_filter) = std::env::var("VORTEX_BENCH_COLUMN") { + columns.retain(|column| column.name == column_filter); + vortex_ensure!( + !columns.is_empty(), + "dataset does not contain the requested column" + ); + } measure_dataset(dataset, &columns, &configs, &session)?; } Ok(()) diff --git a/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs b/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs index 85990fff4b9..95dc49d5d4c 100644 --- a/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs +++ b/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs @@ -25,6 +25,7 @@ use crate::CascadingCompressor; use crate::CompressorContext; use crate::Scheme; use crate::normalize_null_values; +use crate::schemes::integer::patch_adjusted_nbytes; const BLOCK_LEN: usize = 1024; const ESTIMATE_BLOCKS: usize = 8; @@ -69,7 +70,7 @@ impl Scheme for OrderedBlockResidualScheme { let ordered = OrderedFloat::from_primitive(sample.as_view())?; let residuals = BlockResidual::from_primitive(ordered.encoded().as_::())?; - let after_nbytes = residuals.nbytes(); + let after_nbytes = patch_adjusted_nbytes(&residuals); if after_nbytes == 0 { return Ok(EstimateVerdict::Skip); } diff --git a/vortex-btrblocks/src/schemes/integer/block_residual.rs b/vortex-btrblocks/src/schemes/integer/block_residual.rs index b430ecf00da..e8080cadc85 100644 --- a/vortex-btrblocks/src/schemes/integer/block_residual.rs +++ b/vortex-btrblocks/src/schemes/integer/block_residual.rs @@ -13,6 +13,8 @@ use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::match_each_integer_ptype; use vortex_block_residual::BlockResidual; +use vortex_block_residual::BlockResidualArray; +use vortex_block_residual::BlockResidualArraySlotsExt; use vortex_compressor::builtins::BinaryDictScheme; use vortex_compressor::builtins::FloatDictScheme; use vortex_compressor::builtins::IntDictScheme; @@ -34,6 +36,9 @@ use crate::normalize_null_values; const BLOCK_LEN: usize = 1024; const ESTIMATE_BLOCKS: usize = 8; const MIN_COMPRESSION_RATIO: f64 = 1.05; +// The 25-percent patch sweep needs about 80 cost bits per patch to reject its slow tree. +// This quadratic slope reaches that cost at 25 percent while it keeps sparse patches cheap. +const PATCH_DENSITY_COST_BITS: u64 = 320; /// Compress integers with one reference and packed residuals per 1,024-value block. #[derive(Debug, Copy, Clone, PartialEq, Eq)] @@ -88,7 +93,7 @@ impl Scheme for BlockResidualScheme { let sample = normalize_null_values(sample.as_view(), exec_ctx)?; let before_nbytes = sample.nbytes(); let residuals = BlockResidual::from_primitive(sample.as_view())?; - let after_nbytes = residuals.nbytes(); + let after_nbytes = patch_adjusted_nbytes(&residuals); if after_nbytes == 0 { return Ok(EstimateVerdict::Skip); } @@ -123,6 +128,24 @@ impl Scheme for BlockResidualScheme { } } +pub(crate) fn patch_adjusted_nbytes(residuals: &BlockResidualArray) -> u64 { + residuals.nbytes().saturating_add(patch_density_cost_bytes( + residuals.len(), + residuals.patch_positions().len(), + )) +} + +fn patch_density_cost_bytes(len: usize, patch_count: usize) -> u64 { + if len == 0 { + return 0; + } + let patch_count = u64::try_from(patch_count).unwrap_or(u64::MAX); + patch_count + .saturating_mul(patch_count) + .saturating_mul(PATCH_DENSITY_COST_BITS) + .div_ceil(u64::try_from(len).unwrap_or(u64::MAX).saturating_mul(8)) +} + fn locality_sample( primitive: vortex_array::ArrayView<'_, Primitive>, exec_ctx: &mut ExecutionCtx, @@ -154,3 +177,15 @@ fn locality_sample( PrimitiveArray::from_option_iter(sample) })) } + +#[cfg(test)] +mod tests { + use super::patch_density_cost_bytes; + + #[test] + fn patch_density_cost_is_nonlinear() { + assert_eq!(patch_density_cost_bytes(1_024, 0), 0); + assert_eq!(patch_density_cost_bytes(1_024, 102), 407); + assert_eq!(patch_density_cost_bytes(1_024, 256), 2_560); + } +} diff --git a/vortex-btrblocks/src/schemes/integer/mod.rs b/vortex-btrblocks/src/schemes/integer/mod.rs index d0e4386fa73..f597384537e 100644 --- a/vortex-btrblocks/src/schemes/integer/mod.rs +++ b/vortex-btrblocks/src/schemes/integer/mod.rs @@ -19,6 +19,7 @@ mod pco; pub use bitpacking::BitPackingScheme; pub use block_residual::BlockResidualScheme; +pub(crate) use block_residual::patch_adjusted_nbytes; #[cfg(feature = "unstable_encodings")] pub use delta::DeltaScheme; pub use for_::FoRScheme; diff --git a/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs index e859f34735c..101f0806dd1 100644 --- a/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs @@ -120,6 +120,21 @@ fn test_block_residual_skips_narrow_integers() -> VortexResult<()> { Ok(()) } +#[test] +fn test_block_residual_rejects_dense_patches() -> VortexResult<()> { + let values = (0..8_192_u32).map(|index| if index % 4 == 0 { u32::MAX - index } else { 42 }); + let array = PrimitiveArray::from_iter(values); + let compressed = BtrBlocksCompressor::default() + .compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; + + assert!( + !contains_block_residual(&compressed), + "dense patches must not select BlockResidual:\n{}", + compressed.display_tree() + ); + Ok(()) +} + fn contains_block_residual(array: &vortex_array::ArrayRef) -> bool { array.is::() || array.children().iter().any(contains_block_residual) } From a14aa0cd843a72dfbba5efc32eeee16d468fe46b Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 16:22:02 +0100 Subject: [PATCH 18/78] bench: validate block residual compositions Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 45 ++++- .../examples/float_compressor_bench.rs | 160 ++++++++++++++++-- .../schemes/float/scheme_selection_tests.rs | 27 +++ .../schemes/integer/scheme_selection_tests.rs | 46 +++++ 4 files changed, 259 insertions(+), 19 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 9245c9cf257..271e045d9c2 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -36,6 +36,10 @@ The default candidate set includes their three BtrBlocks schemes. `OrderedFloat(BlockResidual)` now supports `f32` and `f64` inputs. +The float scheme applies `OrderedFloat` first, then `BlockResidual` to the unsigned child. + +The serialized tree uses `OrderedFloat(BlockResidual(...))` because the outer array restores the float dtype. + The FloatQuant candidate accepts native `f32` and `f64` inputs. Direct integer BlockResidual supports every integer type. The default selector accepts only 32-bit and 64-bit inputs. @@ -319,7 +323,9 @@ The incumbent and outer tree estimates remain approximate. Trial compression pre The outer-sample exclusion removed that error from the measured tree. -### Broad numeric revalidation +### Broad numeric revalidation before patch-density calibration + +This table predates the nonlinear patch cost. The later patch-density section contains the current HashTags result. The focused run uses two million rows when the source contains that many rows. @@ -514,7 +520,38 @@ Its measured aggregate encode throughput increased by 0.9 percent. Decode throug BlockResidual also composes with outer encodings. Sparse and RunEnd children selected BlockResidual in HashTags and the synthetic low-density sweep. -This composition reduced size, but it requires parent-specific throughput validation. +### Complete parent compositions + +The finalized benchmark decodes every nested child to recursive canonical form. + +Each numeric case contains two million source values. The FSST case contains 500,000 strings. + +| Complete tree | Prior bytes | Proposed bytes | Encode change | Decode change | +| --- | ---: | ---: | ---: | ---: | +| Timestamp storage with direct BlockResidual | 9,254,235 | 7,293,370 | -5.1 percent | +348.9 percent | +| `List(BlockResidual)` | 12,755,712 | 2,543,268 | +1.0 percent | +24.8 percent | +| FSST with two BlockResidual children | 4,675,014 | 4,118,592 | -0.4 percent | +8.1 percent | +| `ALP(BlockResidual)` | 7,753,472 | 2,543,268 | -0.2 percent | +1.0 percent | +| `Sparse(BlockResidual, BlockResidual)` | 1,070,594 | 383,297 | +1.9 percent | +6.3 percent | +| `RunEnd(Sequence, BlockResidual)` | 739,968 | 159,124 | +0.5 percent | +1.6 percent | + +Each composition reduced size and passed both throughput gates. + +The timestamp candidate displaced `DateTimeParts`. It reduced size by 21.2 percent and decoded 4.5 times faster. + +The list, ALP, Sparse, and RunEnd cases reduced size by 64.2 to 80.1 percent. + +The FSST case reduced size by 11.9 percent with no material encode cost. + +A separate float case selected direct `OrderedFloat(BlockResidual)`. + +It reduced size by 69.9 percent, increased encode throughput by 128.7 percent, and increased decode throughput by 56.4 percent. + +The attempted ALP-RD case did not select ALP-RD. Evidence for BlockResidual under an ALP-RD child remains missing. + +Focused selection tests now cover ALP, Sparse, and RunEnd child composition. + +Golden snapshots cover temporal and FSST parent trees. ### GloVe embeddings @@ -755,6 +792,8 @@ This round completed these steps: - Added fused f32 OrderedFloat with BlockResidual decode and selection. - Added u32, i32, f32, and patch-density benchmarks. - Added complete-compressor patch-density datasets and a column filter to the profiling benchmark. +- Added complete temporal, FSST, Sparse, RunEnd, list, and ALP composition benchmarks. +- Added selection tests for BlockResidual under ALP, Sparse, and RunEnd. - Added patch-density statistics to the BlockResidual profile. - Replaced the global patch cost experiment with a nonlinear selector cost. - Removed the full patch scan from scalar access. @@ -776,7 +815,7 @@ The nonzero-secondary FloatQuant implementation and focused validation are compl Complete these remaining steps: 1. Validate the nonlinear patch cost on the complete corpus. -2. Measure BlockResidual under temporal, FSST, Sparse, RunEnd, and list parents. +2. Add a true ALP-RD child composition case if a real selected tree exposes one. 3. Reduce analysis cost on short rejected columns. 4. Evaluate narrow BlockResidual as a Compact-only candidate. 5. Prototype bounded scalar checkpoints for fixed-bin range packing. diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index cfd1a241007..af567b76961 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -27,13 +27,19 @@ use rand::SeedableRng; use rand::rngs::StdRng; use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::RecursiveCanonical; use vortex_array::VortexSessionExecute; use vortex_array::array_session; +use vortex_array::arrays::ListArray; use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::TemporalArray; +use vortex_array::arrays::VarBinViewArray; use vortex_array::assert_arrays_eq; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::PType; +use vortex_array::extension::datetime::TimeUnit; +use vortex_array::validity::Validity; use vortex_arrow::ArrowSessionExt; use vortex_block_residual::BlockResidual; use vortex_block_residual::BlockResidualArraySlotsExt; @@ -64,20 +70,36 @@ const CALIFORNIA_COLUMNS: [&str; 9] = [ struct Column { name: String, - primitive: PrimitiveArray, + primitive: Option, array: ArrayRef, + input_bytes: u64, + dtype_label: String, } fn column(name: impl Into, primitive: PrimitiveArray) -> Column { let array = primitive.clone().into_array(); + let input_bytes = array.nbytes(); + let dtype_label = primitive.ptype().to_string(); Column { name: name.into(), - primitive, + primitive: Some(primitive), array, + input_bytes, + dtype_label, } } -fn synthetic_datasets(row_count: usize) -> Vec<(String, Vec)> { +fn array_column(name: impl Into, array: ArrayRef) -> Column { + Column { + name: name.into(), + primitive: None, + input_bytes: array.nbytes(), + dtype_label: array.dtype().to_string(), + array, + } +} + +fn synthetic_datasets(row_count: usize) -> VortexResult)>> { let mut widened_rng = StdRng::seed_from_u64(1); let widened_f32 = PrimitiveArray::from_iter((0..row_count).map(|index| { let trend = (index % 10_000) as f32 * 0.001; @@ -114,6 +136,19 @@ fn synthetic_datasets(row_count: usize) -> Vec<(String, Vec)> { let permuted = (index as u64).wrapping_mul(2_654_435_761) % 1_000_000; permuted as f64 - 500_000.0 })); + let block_local_integer_valued = PrimitiveArray::from_iter((0..row_count).map(|index| { + let block = index / 1_024; + let residual = index.wrapping_mul(2_654_435_761) % 1_024; + (block * 1_000_000 + residual) as f64 + })); + let block_local_ordered_float = PrimitiveArray::from_iter((0..row_count).map(|index| { + let block = index / 1_024; + let residual = index.wrapping_mul(2_654_435_761) % 1_024; + let bits = 0x3ff0_0000_0000_0000_u64 + + u64::try_from(block).unwrap_or(u64::MAX) * 0x1_0000 + + u64::try_from(residual).unwrap_or(u64::MAX); + f64::from_bits(bits) + })); let patch_density_4 = PrimitiveArray::from_iter((0..row_count).map(|index| { if index % 4 == 0 { @@ -125,8 +160,72 @@ fn synthetic_datasets(row_count: usize) -> Vec<(String, Vec)> { let patch_density_1 = PrimitiveArray::from_iter( (0..row_count).map(|index| u32::MAX - u32::try_from(index).unwrap_or(u32::MAX)), ); + let sparse_block_local = PrimitiveArray::from_iter((0..row_count).map(|index| { + if index % 16 == 0 { + let value_index = index / 16; + let block = value_index / 1_024; + let residual = value_index.wrapping_mul(2_654_435_761) % 1_024; + u64::try_from(block).unwrap_or(u64::MAX) * 1_000_000_000_000 + + u64::try_from(residual).unwrap_or(u64::MAX) + } else { + 42 + } + })); + let runend_block_local = PrimitiveArray::from_iter((0..row_count).map(|index| { + let value_index = index / 16; + let block = value_index / 1_024; + let residual = value_index.wrapping_mul(2_654_435_761) % 1_024; + u64::try_from(block).unwrap_or(u64::MAX) * 1_000_000_000_000 + + u64::try_from(residual).unwrap_or(u64::MAX) + })); - vec![ + let mut temporal_rng = StdRng::seed_from_u64(4); + let mut timestamp = 1_700_000_000_000_000_i64; + let timestamp_values = PrimitiveArray::from_iter((0..row_count).map(|_| { + timestamp += temporal_rng.random_range(1_000_i64..1_000_000); + timestamp + })); + let temporal = TemporalArray::new_timestamp( + timestamp_values.into_array(), + TimeUnit::Microseconds, + Some(Arc::from("UTC")), + ) + .into_array(); + + let list_elements = PrimitiveArray::from_iter((0..row_count).map(|index| { + let block = index / 1_024; + let residual = index.wrapping_mul(2_654_435_761) % 1_024; + u64::try_from(block).unwrap_or(u64::MAX) * 1_000_000_000_000 + + u64::try_from(residual).unwrap_or(u64::MAX) + })); + let mut list_offsets = Vec::with_capacity(row_count.div_ceil(8) + 1); + list_offsets.push(0_u32); + let mut list_offset = 0usize; + while list_offset < row_count { + list_offset = (list_offset + 8).min(row_count); + list_offsets.push(u32::try_from(list_offset).unwrap_or(u32::MAX)); + } + let list = ListArray::try_new( + list_elements.into_array(), + PrimitiveArray::from_iter(list_offsets).into_array(), + Validity::NonNullable, + )? + .into_array(); + + let string_count = row_count.min(500_000); + let strings = (0..string_count) + .map(|index| { + format!( + "user{:06}@example{}.com", + index.wrapping_mul(2_654_435_761) % 1_000_000, + index % 100 + ) + }) + .collect::>(); + let fsst_strings = + VarBinViewArray::from_iter_str(strings.iter().map(String::as_str)).into_array(); + + Ok(vec![ ( "synthetic-widened-f32".to_string(), vec![column("value", widened_f32)], @@ -151,6 +250,14 @@ fn synthetic_datasets(row_count: usize) -> Vec<(String, Vec)> { "synthetic-integer-valued".to_string(), vec![column("value", integer_valued)], ), + ( + "synthetic-alp-block-local".to_string(), + vec![column("value", block_local_integer_valued)], + ), + ( + "synthetic-ordered-float-block-local".to_string(), + vec![column("value", block_local_ordered_float)], + ), ( "synthetic-patch-density-4".to_string(), vec![column("value", patch_density_4)], @@ -159,7 +266,27 @@ fn synthetic_datasets(row_count: usize) -> Vec<(String, Vec)> { "synthetic-patch-density-1".to_string(), vec![column("value", patch_density_1)], ), - ] + ( + "synthetic-sparse-block-local".to_string(), + vec![column("value", sparse_block_local)], + ), + ( + "synthetic-runend-block-local".to_string(), + vec![column("value", runend_block_local)], + ), + ( + "synthetic-temporal-parent".to_string(), + vec![array_column("value", temporal)], + ), + ( + "synthetic-list-parent".to_string(), + vec![array_column("value", list)], + ), + ( + "synthetic-fsst-parent".to_string(), + vec![array_column("value", fsst_strings)], + ), + ]) } fn read_california(path: &Path, row_count: usize) -> VortexResult> { @@ -380,7 +507,7 @@ fn decode_all(arrays: &[ArrayRef], session: &VortexSession) -> VortexResult<()> black_box( array .clone() - .execute::(&mut session.create_execution_ctx())?, + .execute::(&mut session.create_execution_ctx())?, ); } Ok(()) @@ -843,9 +970,7 @@ fn measure_dataset( ) -> VortexResult<()> { let mut input_bytes = 0_u64; for column in columns { - input_bytes += u64::try_from(column.primitive.len())? - * u64::try_from(column.primitive.ptype().bit_width())? - / 8; + input_bytes += column.input_bytes; } let encoded = configs .iter() @@ -858,7 +983,7 @@ fn measure_dataset( println!( "structure\t{dataset}\t{}\t{}\t{config}\t{}\t{}", column.name, - column.primitive.ptype(), + column.dtype_label, encoding_tree(array), array.nbytes() ); @@ -887,12 +1012,15 @@ fn measure_dataset( .find(|(config, _)| *config == "prior-compact") .ok_or_else(|| vortex_err!("prior-compact configuration is missing"))?; for (column_index, column) in columns.iter().enumerate() { - if !column.primitive.ptype().is_float() { + let Some(primitive) = &column.primitive else { + continue; + }; + if !primitive.ptype().is_float() { continue; } let default_array = &prior_default.1[column_index]; let compact_array = &prior_compact.1[column_index]; - let input_bytes = column.primitive.len() * column.primitive.ptype().byte_width(); + let input_bytes = primitive.len() * primitive.ptype().byte_width(); let default_bytes = default_array.nbytes(); let compact_bytes = compact_array.nbytes(); let compact_savings = if default_bytes == 0 { @@ -903,8 +1031,8 @@ fn measure_dataset( println!( "float-column\t{dataset}\t{}\t{}\t{}\t{input_bytes}\t{default_bytes}\t{compact_bytes}\t{compact_savings:.3}\t{}\t{}", column.name, - column.primitive.ptype(), - column.primitive.len(), + primitive.ptype(), + primitive.len(), encoding_tree(default_array), encoding_tree(compact_array), ); @@ -951,7 +1079,7 @@ fn measure_dataset( let decode_throughput = input_bytes as f64 / decode_median.as_secs_f64() / 1_000_000.0; println!( "result\t{dataset}\t{config}\t{}\t{input_bytes}\t{encoded_bytes}\t{encode_throughput:.1}\t{decode_throughput:.1}", - columns[0].primitive.len() + columns[0].array.len() ); } Ok(()) @@ -1038,7 +1166,7 @@ fn main() -> VortexResult<()> { ); if std::env::var_os("VORTEX_BENCH_SKIP_SYNTHETIC").is_none() { let synthetic_filter = std::env::var("VORTEX_BENCH_SYNTHETIC").ok(); - for (dataset, columns) in synthetic_datasets(row_count) { + for (dataset, columns) in synthetic_datasets(row_count)? { if synthetic_filter .as_deref() .is_some_and(|filter| filter != dataset) diff --git a/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs index 4fc35157b32..4615a6cf54f 100644 --- a/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs @@ -241,3 +241,30 @@ fn test_f32_random_walk_uses_ordered_block_residual() -> VortexResult<()> { assert!(compressed.children()[0].is::()); Ok(()) } + +#[test] +fn test_block_residual_composes_with_alp() -> VortexResult<()> { + let values = (0..65_536_usize).map(|index| { + let block = index / 1_024; + let residual = index.wrapping_mul(2_654_435_761) % 1_024; + (block * 1_000_000 + residual) as f64 + }); + let array = PrimitiveArray::from_iter(values).into_array(); + let compressed = + BtrBlocksCompressor::default().compress(&array, &mut SESSION.create_execution_ctx())?; + + assert!( + compressed.is::(), + "expected ALP, got tree:\n{}", + compressed.display_tree() + ); + assert!( + compressed + .children() + .iter() + .any(|child| child.is::()), + "expected a BlockResidual child:\n{}", + compressed.display_tree() + ); + Ok(()) +} diff --git a/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs index 101f0806dd1..c8999b6acab 100644 --- a/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs @@ -135,6 +135,52 @@ fn test_block_residual_rejects_dense_patches() -> VortexResult<()> { Ok(()) } +#[test] +fn test_block_residual_composes_with_sparse() -> VortexResult<()> { + let values = (0..65_536_usize).map(|index| { + if index % 16 == 0 { + let value_index = index / 16; + let block = value_index / 1_024; + let residual = value_index.wrapping_mul(2_654_435_761) % 1_024; + block as u64 * 1_000_000_000_000 + residual as u64 + } else { + 42 + } + }); + let array = PrimitiveArray::from_iter(values); + let compressed = BtrBlocksCompressor::default() + .compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; + + assert!(compressed.is::()); + assert!( + contains_block_residual(&compressed), + "expected a BlockResidual child:\n{}", + compressed.display_tree() + ); + Ok(()) +} + +#[test] +fn test_block_residual_composes_with_runend() -> VortexResult<()> { + let values = (0..65_536_usize).map(|index| { + let value_index = index / 16; + let block = value_index / 1_024; + let residual = value_index.wrapping_mul(2_654_435_761) % 1_024; + block as u64 * 1_000_000_000_000 + residual as u64 + }); + let array = PrimitiveArray::from_iter(values); + let compressed = BtrBlocksCompressor::default() + .compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; + + assert!(compressed.is::()); + assert!( + contains_block_residual(&compressed), + "expected a BlockResidual child:\n{}", + compressed.display_tree() + ); + Ok(()) +} + fn contains_block_residual(array: &vortex_array::ArrayRef) -> bool { array.is::() || array.children().iter().any(contains_block_residual) } From e117cee6d356aa043dca747426e228b2785461bb Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 16:37:36 +0100 Subject: [PATCH 19/78] perf: estimate block residual without payloads Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 43 +++- .../src/block_residual_array.rs | 183 ++++++++++++++---- encodings/block-residual/src/codec.rs | 90 +++++++-- .../block-residual/src/ordered_float_array.rs | 38 ++++ .../schemes/float/ordered_block_residual.rs | 28 ++- .../src/schemes/integer/block_residual.rs | 36 ++-- vortex-btrblocks/src/schemes/integer/mod.rs | 2 +- vortex-btrblocks/src/schemes/mod.rs | 31 +++ 8 files changed, 361 insertions(+), 90 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 271e045d9c2..a94d43aba36 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -141,6 +141,12 @@ The scheme does not call the recursive integer selector. The ordinary BtrBlocks sample does not preserve the block-local float structure. +The estimator runs the exact production width planner on native integer or float slices. + +It does not create packed payloads, patch payloads, temporary ordered integers, or child arrays. + +All-valid samples use direct block copies. Nullable samples retain the validity mask. + The residual scheme requires a 1.05 compression ratio. Its adjusted score includes a 1.02 decode-cost factor. `BlockResidualScheme` uses the same locality probe for integer arrays. @@ -520,6 +526,38 @@ Its measured aggregate encode throughput increased by 0.9 percent. Decode throug BlockResidual also composes with outer encodings. Sparse and RunEnd children selected BlockResidual in HashTags and the synthetic low-density sweep. +### BlockResidual analysis cost + +California Housing contains 20,433 rows and nine nonnullable `f32` columns. + +Every new scheme rejects every column. The output remains 290,737 bytes. + +The first eight-block estimator reduced complete encode throughput by 14 to 15 percent. + +The exact width planner and direct all-valid copies reduced that cost to 6.3 percent. + +| Configuration | Encode MB/s | +| --- | ---: | +| Prior default | 214.4 | +| FloatQuant only | 216.4 | +| Ordered BlockResidual only | 208.1 | +| Integer BlockResidual only | 213.2 | +| Complete proposed default | 200.8 | + +A four-block trial reduced analysis cost further. It mis-ranked the California longitude column. + +The selected `ALP(BlockResidual)` tree saved only 5.7 percent against the prior tree. + +That result did not meet the intended 32-bit speed-adjusted margin. The production candidate retains eight sample blocks. + +The two-million-value random walk retained `OrderedFloat(BlockResidual)`. + +It used 10,428,451 bytes, encoded at 526.7 MB/s, and decoded at 22.29 GB/s. + +The prior default used 12,255,488 bytes, encoded at 595.8 MB/s, and decoded at 12.55 GB/s. + +The selected candidate remained inside both throughput gates. + ### Complete parent compositions The finalized benchmark decodes every nested child to recursive canonical form. @@ -794,6 +832,9 @@ This round completed these steps: - Added complete-compressor patch-density datasets and a column filter to the profiling benchmark. - Added complete temporal, FSST, Sparse, RunEnd, list, and ALP composition benchmarks. - Added selection tests for BlockResidual under ALP, Sparse, and RunEnd. +- Added exact allocation-free BlockResidual size estimates. +- Added an all-valid fast path for locality sample copies. +- Rejected a four-block short-column estimate after a real mis-ranking. - Added patch-density statistics to the BlockResidual profile. - Replaced the global patch cost experiment with a nonlinear selector cost. - Removed the full patch scan from scalar access. @@ -816,7 +857,7 @@ Complete these remaining steps: 1. Validate the nonlinear patch cost on the complete corpus. 2. Add a true ALP-RD child composition case if a real selected tree exposes one. -3. Reduce analysis cost on short rejected columns. +3. Validate the remaining rejected analysis cost in the complete corpus. 4. Evaluate narrow BlockResidual as a Compact-only candidate. 5. Prototype bounded scalar checkpoints for fixed-bin range packing. 6. Compare fixed-bin packing against default and Pco on unrelated no-Delta wins. diff --git a/encodings/block-residual/src/block_residual_array.rs b/encodings/block-residual/src/block_residual_array.rs index 298fa8952b4..1bc17d0a157 100644 --- a/encodings/block-residual/src/block_residual_array.rs +++ b/encodings/block-residual/src/block_residual_array.rs @@ -49,6 +49,7 @@ use vortex_session::registry::CachedId; use crate::BlockResidualCodec; use crate::BlockResidualParts; +use crate::codec::BlockResidualCodecEstimate; use crate::codec::ResidualWord; use crate::codec::packed_words_as_native; use crate::codec::read_wide_bits; @@ -131,6 +132,39 @@ impl ArrayEq for BlockResidualData { #[derive(Clone, Debug)] pub struct BlockResidual; +/// Exact encoded size and patch count without materialized payloads. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct BlockResidualEstimate { + nbytes: u64, + patch_count: usize, +} + +impl BlockResidualEstimate { + pub(crate) fn try_new( + encoded_nbytes: usize, + validity_nbytes: u64, + patch_count: usize, + ) -> VortexResult { + let nbytes = u64::try_from(encoded_nbytes)? + .checked_add(validity_nbytes) + .ok_or_else(|| vortex_error::vortex_err!("BlockResidual estimate size overflow"))?; + Ok(Self { + nbytes, + patch_count, + }) + } + + /// Return the estimated physical bytes. + pub fn nbytes(self) -> u64 { + self.nbytes + } + + /// Return the estimated patch count. + pub fn patch_count(self) -> usize { + self.patch_count + } +} + impl VTable for BlockResidual { type TypedArrayData = BlockResidualData; type OperationsVTable = Self; @@ -336,6 +370,58 @@ pub trait BlockResidualArrayExt: TypedArrayRef + BlockResidualArr impl> BlockResidualArrayExt for T {} impl BlockResidual { + /// Estimate the exact encoded size without materializing packed payloads. + pub fn estimate_primitive( + array: ArrayView<'_, Primitive>, + ) -> VortexResult { + vortex_ensure!( + array.ptype().is_int(), + "BlockResidual requires integer values" + ); + let BlockResidualCodecEstimate { + encoded_nbytes, + patch_count, + } = match array.ptype() { + PType::U8 => { + BlockResidualCodec::estimate_transformed(array.as_slice::(), u64::from) + } + PType::U16 => { + BlockResidualCodec::estimate_transformed(array.as_slice::(), u64::from) + } + PType::U32 => { + BlockResidualCodec::estimate_transformed(array.as_slice::(), u64::from) + } + PType::U64 => { + BlockResidualCodec::estimate_transformed(array.as_slice::(), |value| value) + } + PType::I8 => { + BlockResidualCodec::estimate_transformed(array.as_slice::(), |value| { + u64::from((value as u8) ^ (1_u8 << 7)) + }) + } + PType::I16 => { + BlockResidualCodec::estimate_transformed(array.as_slice::(), |value| { + u64::from((value as u16) ^ (1_u16 << 15)) + }) + } + PType::I32 => { + BlockResidualCodec::estimate_transformed(array.as_slice::(), |value| { + u64::from((value as u32) ^ (1_u32 << 31)) + }) + } + PType::I64 => { + BlockResidualCodec::estimate_transformed(array.as_slice::(), |value| { + (value as u64) ^ (1_u64 << 63) + }) + } + ptype => vortex_bail!("BlockResidual does not support {ptype}"), + }; + let validity_nbytes = validity_to_child(&array.validity()?, array.len()) + .map(|validity| validity.nbytes()) + .unwrap_or(0); + BlockResidualEstimate::try_new(encoded_nbytes, validity_nbytes, patch_count) + } + /// Encode an integer array in independent blocks. pub fn from_primitive(array: ArrayView<'_, Primitive>) -> VortexResult { vortex_ensure!( @@ -343,45 +429,7 @@ impl BlockResidual { "BlockResidual requires integer values" ); let validity = array.validity()?; - let values = match array.ptype() { - PType::U8 => array - .as_slice::() - .iter() - .map(|&value| u64::from(value)) - .collect(), - PType::U16 => array - .as_slice::() - .iter() - .map(|&value| u64::from(value)) - .collect(), - PType::U32 => array - .as_slice::() - .iter() - .map(|&value| u64::from(value)) - .collect(), - PType::U64 => array.as_slice::().to_vec(), - PType::I8 => array - .as_slice::() - .iter() - .map(|&value| u64::from((value as u8) ^ (1_u8 << 7))) - .collect(), - PType::I16 => array - .as_slice::() - .iter() - .map(|&value| u64::from((value as u16) ^ (1_u16 << 15))) - .collect(), - PType::I32 => array - .as_slice::() - .iter() - .map(|&value| u64::from((value as u32) ^ (1_u32 << 31))) - .collect(), - PType::I64 => array - .as_slice::() - .iter() - .map(|&value| (value as u64) ^ (1_u64 << 63)) - .collect(), - ptype => vortex_bail!("BlockResidual does not support {ptype}"), - }; + let values = ordered_values(array)?; let parts = BlockResidualCodec::encode_with_word_width( &values, u8::try_from(array.ptype().bit_width())?, @@ -428,6 +476,48 @@ impl BlockResidual { } } +fn ordered_values(array: ArrayView<'_, Primitive>) -> VortexResult> { + Ok(match array.ptype() { + PType::U8 => array + .as_slice::() + .iter() + .map(|&value| u64::from(value)) + .collect(), + PType::U16 => array + .as_slice::() + .iter() + .map(|&value| u64::from(value)) + .collect(), + PType::U32 => array + .as_slice::() + .iter() + .map(|&value| u64::from(value)) + .collect(), + PType::U64 => array.as_slice::().to_vec(), + PType::I8 => array + .as_slice::() + .iter() + .map(|&value| u64::from((value as u8) ^ (1_u8 << 7))) + .collect(), + PType::I16 => array + .as_slice::() + .iter() + .map(|&value| u64::from((value as u16) ^ (1_u16 << 15))) + .collect(), + PType::I32 => array + .as_slice::() + .iter() + .map(|&value| u64::from((value as u32) ^ (1_u32 << 31))) + .collect(), + PType::I64 => array + .as_slice::() + .iter() + .map(|&value| (value as u64) ^ (1_u64 << 63)) + .collect(), + ptype => vortex_bail!("BlockResidual does not support {ptype}"), + }) +} + impl BlockResidualData { fn validate( &self, @@ -1275,6 +1365,21 @@ mod tests { Ok(()) } + #[test] + fn estimate_matches_materialized_size() -> VortexResult<()> { + let mut values = vec![42_u32; 2_050]; + values[1_023] = u32::MAX; + values[2_049] = u32::MAX - 1; + let validity = Validity::from_iter((0..values.len()).map(|index| index != 1_024)); + let primitive = PrimitiveArray::new(Buffer::from(values), validity); + let estimate = BlockResidual::estimate_primitive(primitive.as_view())?; + let encoded = BlockResidual::from_primitive(primitive.as_view())?; + + assert_eq!(estimate.nbytes(), encoded.nbytes()); + assert_eq!(estimate.patch_count(), encoded.patch_positions().len()); + Ok(()) + } + #[test] fn nullable_slice_serialization_roundtrip() -> VortexResult<()> { let values = (0..2_050) diff --git a/encodings/block-residual/src/codec.rs b/encodings/block-residual/src/codec.rs index 4f665c2466c..8c91b14ccff 100644 --- a/encodings/block-residual/src/codec.rs +++ b/encodings/block-residual/src/codec.rs @@ -32,6 +32,11 @@ pub struct BlockResidualParts { pub patch_highs: Vec, } +pub(crate) struct BlockResidualCodecEstimate { + pub encoded_nbytes: usize, + pub patch_count: usize, +} + #[derive(Clone, Debug, PartialEq, Eq)] struct BlockResidualBlock { len: u16, @@ -64,6 +69,34 @@ impl BlockResidualCodec { }) } + pub(crate) fn estimate_transformed( + values: &[T], + transform: impl Fn(T) -> u64 + Copy, + ) -> BlockResidualCodecEstimate { + let mut encoded_nbytes = 3 * size_of::(); + let mut total_patch_count = 0; + for values in values.chunks(CHUNK_LEN) { + let plan = estimate_block(values, transform); + let residual_nbytes = CHUNK_LEN * usize::from(plan.residual_width) / 8; + let patch_high_nbytes = if plan.patch_count == 0 { + 0 + } else { + (plan.patch_count * usize::from(plan.high_width)).div_ceil(8) + HIGH_PADDING + }; + encoded_nbytes += size_of::() + + 2 * size_of::() + + 3 * size_of::() + + residual_nbytes + + plan.patch_count * size_of::() + + patch_high_nbytes; + total_patch_count += plan.patch_count; + } + BlockResidualCodecEstimate { + encoded_nbytes, + patch_count: total_patch_count, + } + } + /// Convert the codec into serialized array children. pub fn into_parts(self) -> VortexResult { let mut parts = BlockResidualParts { @@ -366,7 +399,45 @@ fn encode_block(values: &[u64], word_width: u8) -> VortexResult(values: &[T], transform: impl Fn(T) -> u64) -> BlockWidthPlan { + let base = values.iter().copied().map(&transform).min().unwrap_or(0); + let mut width_counts = [0usize; 65]; + let mut maximum_width = 0u8; + for &value in values { + let width = bit_width(transform(value) - base); + width_counts[usize::from(width)] += 1; + maximum_width = maximum_width.max(width); + } + choose_width(&width_counts, maximum_width, values.len()) +} + +fn choose_width( + width_counts: &[usize; 65], + maximum_width: u8, + value_count: usize, +) -> BlockWidthPlan { + let mut patch_count = value_count; let mut best = (usize::MAX, maximum_width, 0u8, 0usize); for residual_width in 0..=maximum_width { patch_count -= width_counts[usize::from(residual_width)]; @@ -385,18 +456,11 @@ fn encode_block(values: &[u64], word_width: u8) -> VortexResult; @@ -247,6 +251,32 @@ pub trait OrderedFloatArrayExt: TypedArrayRef + OrderedFloatArrayS impl> OrderedFloatArrayExt for T {} impl OrderedFloat { + /// Estimate BlockResidual bytes for ordered float bits without materialized integer values. + pub fn estimate_block_residual( + array: ArrayView<'_, Primitive>, + ) -> VortexResult { + let BlockResidualCodecEstimate { + encoded_nbytes, + patch_count, + } = match array.ptype() { + PType::F32 => { + BlockResidualCodec::estimate_transformed(array.as_slice::(), |value| { + u64::from(ordered_u32(value.to_bits())) + }) + } + PType::F64 => { + BlockResidualCodec::estimate_transformed(array.as_slice::(), |value| { + ordered_u64(value.to_bits()) + }) + } + ptype => vortex_bail!("OrderedFloat requires f32 or f64, got {ptype}"), + }; + let validity_nbytes = validity_to_child(&array.validity()?, array.len()) + .map(|validity| validity.nbytes()) + .unwrap_or(0); + BlockResidualEstimate::try_new(encoded_nbytes, validity_nbytes, patch_count) + } + /// Construct an ordered float array from an unsigned child. pub fn try_new(encoded: ArrayRef, float_ptype: PType) -> VortexResult { vortex_ensure!( @@ -382,6 +412,7 @@ mod tests { use super::OrderedFloat; use super::OrderedFloatArraySlotsExt; use crate::BlockResidual; + use crate::BlockResidualArraySlotsExt; #[test] fn roundtrip_special_values() -> VortexResult<()> { @@ -394,7 +425,11 @@ mod tests { f64::INFINITY, f64::from_bits(0x7ff8_0000_0000_0042), ]); + let estimate = OrderedFloat::estimate_block_residual(primitive.as_view())?; let encoded = OrderedFloat::from_primitive(primitive.as_view())?; + let residuals = BlockResidual::from_primitive(encoded.encoded().as_::())?; + assert_eq!(estimate.nbytes(), residuals.nbytes()); + assert_eq!(estimate.patch_count(), residuals.patch_positions().len()); let session = array_session(); crate::initialize(&session); let mut ctx = session.create_execution_ctx(); @@ -411,8 +446,11 @@ mod tests { }) .collect::>(); let primitive = PrimitiveArray::from_iter(values.clone()); + let estimate = OrderedFloat::estimate_block_residual(primitive.as_view())?; let ordered = OrderedFloat::from_primitive(primitive.as_view())?; let residuals = BlockResidual::from_primitive(ordered.encoded().as_::())?; + assert_eq!(estimate.nbytes(), residuals.nbytes()); + assert_eq!(estimate.patch_count(), residuals.patch_positions().len()); let encoded = OrderedFloat::try_new(residuals.into_array(), PType::F32)?; let session = array_session(); crate::initialize(&session); diff --git a/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs b/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs index 95dc49d5d4c..4fbfe07aed8 100644 --- a/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs +++ b/vortex-btrblocks/src/schemes/float/ordered_block_residual.rs @@ -25,7 +25,8 @@ use crate::CascadingCompressor; use crate::CompressorContext; use crate::Scheme; use crate::normalize_null_values; -use crate::schemes::integer::patch_adjusted_nbytes; +use crate::schemes::integer::patch_adjusted_estimate_nbytes; +use crate::schemes::sample_primitive_blocks; const BLOCK_LEN: usize = 1024; const ESTIMATE_BLOCKS: usize = 8; @@ -67,10 +68,8 @@ impl Scheme for OrderedBlockResidualScheme { let sample = locality_sample(data.array_as_primitive(), exec_ctx)?; let sample = normalize_null_values(sample.as_view(), exec_ctx)?; let before_nbytes = sample.nbytes(); - let ordered = OrderedFloat::from_primitive(sample.as_view())?; - let residuals = - BlockResidual::from_primitive(ordered.encoded().as_::())?; - let after_nbytes = patch_adjusted_nbytes(&residuals); + let estimate = OrderedFloat::estimate_block_residual(sample.as_view())?; + let after_nbytes = patch_adjusted_estimate_nbytes(estimate, sample.len()); if after_nbytes == 0 { return Ok(EstimateVerdict::Skip); } @@ -121,16 +120,13 @@ fn locality_sample( let sample_blocks = ESTIMATE_BLOCKS.min(full_blocks); Ok(match_each_float_ptype!(primitive.ptype(), |T| { - let values = primitive.as_slice::(); - let mut sample = Vec::with_capacity(sample_blocks * BLOCK_LEN); - for sample_index in 0..sample_blocks { - let block_index = sample_index * full_blocks / sample_blocks; - let start = block_index * BLOCK_LEN; - sample.extend( - (start..start + BLOCK_LEN) - .map(|index| validity.value(index).then_some(values[index])), - ); - } - PrimitiveArray::from_option_iter(sample) + sample_primitive_blocks( + primitive.as_slice::(), + validity.all_true(), + |index| validity.value(index), + full_blocks, + sample_blocks, + BLOCK_LEN, + ) })) } diff --git a/vortex-btrblocks/src/schemes/integer/block_residual.rs b/vortex-btrblocks/src/schemes/integer/block_residual.rs index e8080cadc85..a45d5a7bffd 100644 --- a/vortex-btrblocks/src/schemes/integer/block_residual.rs +++ b/vortex-btrblocks/src/schemes/integer/block_residual.rs @@ -13,8 +13,7 @@ use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::match_each_integer_ptype; use vortex_block_residual::BlockResidual; -use vortex_block_residual::BlockResidualArray; -use vortex_block_residual::BlockResidualArraySlotsExt; +use vortex_block_residual::BlockResidualEstimate; use vortex_compressor::builtins::BinaryDictScheme; use vortex_compressor::builtins::FloatDictScheme; use vortex_compressor::builtins::IntDictScheme; @@ -32,6 +31,7 @@ use crate::CompressorContext; use crate::Scheme; use crate::SchemeExt; use crate::normalize_null_values; +use crate::schemes::sample_primitive_blocks; const BLOCK_LEN: usize = 1024; const ESTIMATE_BLOCKS: usize = 8; @@ -92,8 +92,8 @@ impl Scheme for BlockResidualScheme { let sample = locality_sample(data.array_as_primitive(), exec_ctx)?; let sample = normalize_null_values(sample.as_view(), exec_ctx)?; let before_nbytes = sample.nbytes(); - let residuals = BlockResidual::from_primitive(sample.as_view())?; - let after_nbytes = patch_adjusted_nbytes(&residuals); + let estimate = BlockResidual::estimate_primitive(sample.as_view())?; + let after_nbytes = patch_adjusted_estimate_nbytes(estimate, sample.len()); if after_nbytes == 0 { return Ok(EstimateVerdict::Skip); } @@ -128,11 +128,10 @@ impl Scheme for BlockResidualScheme { } } -pub(crate) fn patch_adjusted_nbytes(residuals: &BlockResidualArray) -> u64 { - residuals.nbytes().saturating_add(patch_density_cost_bytes( - residuals.len(), - residuals.patch_positions().len(), - )) +pub(crate) fn patch_adjusted_estimate_nbytes(estimate: BlockResidualEstimate, len: usize) -> u64 { + estimate + .nbytes() + .saturating_add(patch_density_cost_bytes(len, estimate.patch_count())) } fn patch_density_cost_bytes(len: usize, patch_count: usize) -> u64 { @@ -164,17 +163,14 @@ fn locality_sample( let sample_blocks = ESTIMATE_BLOCKS.min(full_blocks); Ok(match_each_integer_ptype!(primitive.ptype(), |T| { - let values = primitive.as_slice::(); - let mut sample = Vec::with_capacity(sample_blocks * BLOCK_LEN); - for sample_index in 0..sample_blocks { - let block_index = sample_index * full_blocks / sample_blocks; - let start = block_index * BLOCK_LEN; - sample.extend( - (start..start + BLOCK_LEN) - .map(|index| validity.value(index).then_some(values[index])), - ); - } - PrimitiveArray::from_option_iter(sample) + sample_primitive_blocks( + primitive.as_slice::(), + validity.all_true(), + |index| validity.value(index), + full_blocks, + sample_blocks, + BLOCK_LEN, + ) })) } diff --git a/vortex-btrblocks/src/schemes/integer/mod.rs b/vortex-btrblocks/src/schemes/integer/mod.rs index f597384537e..43ff8bbb6cd 100644 --- a/vortex-btrblocks/src/schemes/integer/mod.rs +++ b/vortex-btrblocks/src/schemes/integer/mod.rs @@ -19,7 +19,7 @@ mod pco; pub use bitpacking::BitPackingScheme; pub use block_residual::BlockResidualScheme; -pub(crate) use block_residual::patch_adjusted_nbytes; +pub(crate) use block_residual::patch_adjusted_estimate_nbytes; #[cfg(feature = "unstable_encodings")] pub use delta::DeltaScheme; pub use for_::FoRScheme; diff --git a/vortex-btrblocks/src/schemes/mod.rs b/vortex-btrblocks/src/schemes/mod.rs index a0e9b042a66..765cc6804e0 100644 --- a/vortex-btrblocks/src/schemes/mod.rs +++ b/vortex-btrblocks/src/schemes/mod.rs @@ -13,6 +13,8 @@ pub mod temporal; pub(crate) mod patches; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::dtype::NativePType; use vortex_compressor::builtins::BinaryDictScheme; use vortex_compressor::builtins::FloatDictScheme; use vortex_compressor::builtins::IntDictScheme; @@ -24,6 +26,35 @@ use vortex_compressor::scheme::SchemeExt; use crate::schemes::integer::SparseScheme; +fn sample_primitive_blocks( + values: &[T], + all_valid: bool, + is_valid: impl Fn(usize) -> bool, + full_blocks: usize, + sample_blocks: usize, + block_len: usize, +) -> PrimitiveArray { + if all_valid { + let mut sample = Vec::with_capacity(sample_blocks * block_len); + for sample_index in 0..sample_blocks { + let block_index = sample_index * full_blocks / sample_blocks; + let start = block_index * block_len; + sample.extend_from_slice(&values[start..start + block_len]); + } + PrimitiveArray::from_iter(sample) + } else { + let mut sample = Vec::with_capacity(sample_blocks * block_len); + for sample_index in 0..sample_blocks { + let block_index = sample_index * full_blocks / sample_blocks; + let start = block_index * block_len; + sample.extend( + (start..start + block_len).map(|index| is_valid(index).then_some(values[index])), + ); + } + PrimitiveArray::from_option_iter(sample) + } +} + /// Shared descendant exclusion rules for RLE schemes. /// /// RLE indices (child 1) and offsets (child 2) are monotonically increasing positions with all From ace5630dd1f944e3a808567cce52dc6e05e1ced9 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 16:42:55 +0100 Subject: [PATCH 20/78] bench: compare narrow residuals with pco Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 20 ++++++++++- .../examples/float_compressor_bench.rs | 9 +++++ vortex/benches/single_encoding_throughput.rs | 33 +++++++++++++++++++ 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index a94d43aba36..7385d2ec2b3 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -323,6 +323,24 @@ BlockResidual is 48.7 percent smaller on that input. Its `i16` decode throughput is 24.4 percent lower. The default selector therefore excludes direct 8-bit and 16-bit candidates. +### Narrow BlockResidual in Compact + +The Compact comparison uses the same two million block-local `i16` values. + +| Tree | Bytes | Encode MB/s | Decode MB/s | +| --- | ---: | ---: | ---: | +| FoR plus BitPacked | 3,501,568 | 1,129 | 40,970 | +| BlockResidual | 1,793,784 | 1,361 | 31,330 | +| Compact Pco | 241,832 | 391 | 1,680 | + +BlockResidual uses 7.4 times as many bytes as Pco. + +It encodes 3.5 times faster and decodes 18.6 times faster than Pco. + +Compact gives priority to the Pco size point. Narrow BlockResidual therefore has no Compact-only role. + +The 48.8 percent saving against FoR plus BitPacked supports one more narrow decode optimization pass. + The BlockResidual estimator measures its sampled tree exactly, with all child arrays. The incumbent and outer tree estimates remain approximate. Trial compression previously mis-ranked Dict and ALP on Taxi tips. @@ -858,7 +876,7 @@ Complete these remaining steps: 1. Validate the nonlinear patch cost on the complete corpus. 2. Add a true ALP-RD child composition case if a real selected tree exposes one. 3. Validate the remaining rejected analysis cost in the complete corpus. -4. Evaluate narrow BlockResidual as a Compact-only candidate. +4. Optimize narrow BlockResidual decode and recheck the default throughput gate. 5. Prototype bounded scalar checkpoints for fixed-bin range packing. 6. Compare fixed-bin packing against default and Pco on unrelated no-Delta wins. 7. Add real embedding datasets beyond GloVe when licenses and loaders permit them. diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index af567b76961..0c680aa0c70 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -160,6 +160,11 @@ fn synthetic_datasets(row_count: usize) -> VortexResult let patch_density_1 = PrimitiveArray::from_iter( (0..row_count).map(|index| u32::MAX - u32::try_from(index).unwrap_or(u32::MAX)), ); + let block_local_i16 = PrimitiveArray::from_iter((0..row_count).map(|index| { + let block = (index / 1_024) % 128; + let residual = index.wrapping_mul(2_654_435_761) % 128; + i16::try_from(block * 128 + residual).unwrap_or(i16::MAX) + })); let sparse_block_local = PrimitiveArray::from_iter((0..row_count).map(|index| { if index % 16 == 0 { let value_index = index / 16; @@ -266,6 +271,10 @@ fn synthetic_datasets(row_count: usize) -> VortexResult "synthetic-patch-density-1".to_string(), vec![column("value", patch_density_1)], ), + ( + "synthetic-block-local-i16".to_string(), + vec![column("value", block_local_i16)], + ), ( "synthetic-sparse-block-local".to_string(), vec![column("value", sparse_block_local)], diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index c72fd3cd099..60084b1a770 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -966,6 +966,39 @@ fn bench_block_local_for_bitpacked_scalar_at_i16(bencher: Bencher) { .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); } +#[divan::bench(name = "block_local_pcodec_compress_i16")] +fn bench_block_local_pcodec_compress_i16(bencher: Bencher) { + let array = setup_block_local_i16_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 2) + .with_inputs(|| (&array, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| { + Pco::from_primitive( + array.as_view(), + PCO_COMPRESSION_LEVEL, + PCO_VALUES_PER_PAGE, + ctx, + ) + .unwrap() + }); +} + +#[divan::bench(name = "block_local_pcodec_decompress_i16")] +fn bench_block_local_pcodec_decompress_i16(bencher: Bencher) { + let array = setup_block_local_i16_array(); + let compressed = Pco::from_primitive( + array.as_view(), + PCO_COMPRESSION_LEVEL, + PCO_VALUES_PER_PAGE, + &mut SESSION.create_execution_ctx(), + ) + .unwrap(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 2) + .with_inputs(|| (&compressed, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + #[divan::bench(name = "ordered_block_residual_compress_f64")] fn bench_ordered_block_residual_compress_f64(bencher: Bencher) { let float_array = setup_random_walk_array(); From 1c9abba26129b3e364c079d7244e22a65766def6 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 16:57:50 +0100 Subject: [PATCH 21/78] bench: prototype checkpointed range packing Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 42 +- .../examples/float_compressor_bench.rs | 80 ++- .../examples/support/range_packed.rs | 605 ++++++++++++++++++ 3 files changed, 715 insertions(+), 12 deletions(-) create mode 100644 vortex-btrblocks/examples/support/range_packed.rs diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 7385d2ec2b3..2d46cf9f908 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -695,6 +695,30 @@ A checkpoint interval of 32 values bounds scalar work to 31 offset widths. This design adds about 0.5 bits per value with 16-bit checkpoints. +### Fixed-bin checkpoint prototype + +The checkpoint prototype uses 1,024-value blocks and one 16-bit offset checkpoint per 32 values. + +The bin optimizer scores fixed-width identifiers. It restricts bin counts to powers of two. + +The bulk decoder uses one narrow-offset specialization for bins with widths of at most 57 bits. + +| Input | Default bytes | Pco bytes | Fixed-bin bytes | Encode MB/s | Decode MB/s | Scalar ns | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Synthetic uniform f64 | 14,104,090 | 13,768,605 | 14,037,682 | 1,194 | 10,534 | 48 | +| GloVe ALP child | 6,274,834 | 5,287,510 | 6,177,168 | 539 | 6,600 | 25 | +| CMS ALP child | 4,146,124 | 3,261,563 | 3,515,002 | 1,280 | 13,554 | 23 | + +The GloVe fixed-bin tree saves only 1.6 percent against default. + +The CMS fixed-bin tree saves 15.2 percent against default. It remains 7.8 percent larger than Compact Pco. + +The first CMS decoder reached 8,057 MB/s. The specialized decoder increased throughput by 68.2 percent. + +The complete CMS default tree decodes at 14,649 MB/s. The fixed-bin child is 7.5 percent slower. + +An integrated Vortex array is required for the complete-tree throughput comparison. + ### Pcodec corpus gap analysis The focused benchmark reads the first two million numeric rows from each source. @@ -876,16 +900,14 @@ Complete these remaining steps: 1. Validate the nonlinear patch cost on the complete corpus. 2. Add a true ALP-RD child composition case if a real selected tree exposes one. 3. Validate the remaining rejected analysis cost in the complete corpus. -4. Optimize narrow BlockResidual decode and recheck the default throughput gate. -5. Prototype bounded scalar checkpoints for fixed-bin range packing. -6. Compare fixed-bin packing against default and Pco on unrelated no-Delta wins. -7. Add real embedding datasets beyond GloVe when licenses and loaders permit them. -8. Classify more float columns where Pco beats ALP, ALP-RD, and the new schemes. -9. Prototype one lightweight scheme for the repeated real-float gap classes. -10. Run the complete `bench-vortex` compression corpus after the candidate set stabilizes. -11. Compare the geometric mean against Parquet with Zstd. -12. Calibrate all selector thresholds from complete corpus and selected-tree evidence. -13. Update this plan after each experiment. +4. Integrate fixed-bin packing as an experimental array and measure complete CMS and GloVe trees. +5. Add real embedding datasets beyond GloVe when licenses and loaders permit them. +6. Classify more float columns where Pco beats ALP, ALP-RD, and the new schemes. +7. Prototype one lightweight scheme for the repeated real-float gap classes. +8. Run the complete `bench-vortex` compression corpus after the candidate set stabilizes. +9. Compare the geometric mean against Parquet with Zstd. +10. Calibrate all selector thresholds from complete corpus and selected-tree evidence. +11. Update this plan after each experiment. ## Pull request structure diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index 0c680aa0c70..6c1d8ce9797 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -55,6 +55,11 @@ use vortex_error::vortex_err; use vortex_session::VortexSession; use vortex_utils::aliases::hash_map::HashMap; +#[path = "support/range_packed.rs"] +mod range_packed; + +use range_packed::RangePackedCodec; + const DEFAULT_ROW_COUNT: usize = 2_000_000; const CALIFORNIA_COLUMNS: [&str; 9] = [ "longitude", @@ -884,6 +889,61 @@ fn profile_quotient_remainder( Ok(()) } +fn profile_range_packed( + dataset: &str, + column: &str, + path: &str, + logical_bytes: usize, + pco_bytes: u64, + validity_bytes: u64, + ordered: &[u64], +) -> VortexResult<()> { + if ordered.is_empty() { + return Ok(()); + } + + let mut encode_durations = Vec::with_capacity(5); + let mut codec = RangePackedCodec::encode(ordered, 1_024)?; + for _ in 0..5 { + let start = Instant::now(); + codec = RangePackedCodec::encode(black_box(ordered), 1_024)?; + encode_durations.push(start.elapsed()); + } + let encode_median = percentile(&mut encode_durations, 1, 2); + + vortex_ensure!(codec.decode()? == ordered, "range packed decode differs"); + let mut decode_durations = Vec::with_capacity(20); + for _ in 0..20 { + let start = Instant::now(); + black_box(codec.decode()?); + decode_durations.push(start.elapsed()); + } + let decode_median = percentile(&mut decode_durations, 1, 2); + + let scalar_iterations = 200_000; + let mut scalar_index = 0usize; + let mut scalar_checksum = 0u64; + let scalar_start = Instant::now(); + for _ in 0..scalar_iterations { + scalar_index = scalar_index.wrapping_add(2_654_435_761) % ordered.len(); + scalar_checksum ^= black_box(codec.scalar_at(scalar_index)?); + } + black_box(scalar_checksum); + let scalar_duration = scalar_start.elapsed(); + let input_bytes = ordered.len() * logical_bytes; + let encode_throughput = input_bytes as f64 / encode_median.as_secs_f64() / 1_000_000.0; + let decode_throughput = input_bytes as f64 / decode_median.as_secs_f64() / 1_000_000.0; + let scalar_nanoseconds = + scalar_duration.as_secs_f64() * 1_000_000_000.0 / scalar_iterations as f64; + let encoded_bytes = u64::try_from(codec.encoded_size())? + validity_bytes; + println!( + "fixed-bin-checkpoint\t{dataset}\t{column}\t{path}\t{}\t{pco_bytes}\t{encoded_bytes}\t{}\t{encode_throughput:.1}\t{decode_throughput:.1}\t{scalar_nanoseconds:.1}", + ordered.len(), + codec.bin_count(), + ); + Ok(()) +} + fn profile_pco_array( dataset: &str, column: &str, @@ -902,6 +962,18 @@ fn profile_pco_array( .filter(mask)? .execute::(&mut ctx)?; let ordered = ordered_values(&values); + let validity_bytes = array.children().iter().map(ArrayRef::nbytes).sum::(); + if std::env::var_os("VORTEX_BENCH_FIXED_BIN").is_some() { + profile_range_packed( + dataset, + column, + path, + values.ptype().byte_width(), + array.nbytes(), + validity_bytes, + &ordered, + )?; + } profile_quotient_remainder( dataset, column, @@ -911,7 +983,6 @@ fn profile_pco_array( &ordered, session, )?; - let validity_bytes = array.children().iter().map(ArrayRef::nbytes).sum::(); let bits = values.ptype().bit_width(); let one_reference = u64::try_from(estimate_multi_reference(&ordered, 1, bits))? + validity_bytes; @@ -1045,7 +1116,9 @@ fn measure_dataset( encoding_tree(default_array), encoding_tree(compact_array), ); - if compact_bytes * 10 <= default_bytes * 9 { + if compact_bytes * 10 <= default_bytes * 9 + || std::env::var_os("VORTEX_BENCH_FIXED_BIN").is_some() + { profile_pco_array(dataset, &column.name, "root", compact_array, session)?; } } @@ -1169,6 +1242,9 @@ fn main() -> VortexResult<()> { println!( "quotient-remainder-estimate\tdataset\tcolumn\tpath\tptype\tbase\tpco-child-bytes\tremainder-entropy\tmost-common-remainder-share\tquotient-bytes\tremainder-bytes\ttotal-bytes\tquotient-bitmap-bytes\tremainder-mode-bitmap-bytes\tbitmap-total-bytes\tquotient-encoding\tremainder-encoding" ); + println!( + "fixed-bin-checkpoint\tdataset\tcolumn\tpath\trows\tpco-bytes\tfixed-bin-bytes\tbins\tencode-MB/s\tdecode-MB/s\tscalar-ns" + ); println!("result\tdataset\tconfig\trows\tinput-bytes\tencoded-bytes\tencode-MB/s\tdecode-MB/s"); println!( "block-residual-profile\tdataset\tcolumn\tconfig\tpath\tptype\trows\tpatches\tblocks\taverage-residual-width\taverage-high-width\tmaximum-patch-density\tblocks-above-one-eighth\tblocks-at-one-quarter\tbytes" diff --git a/vortex-btrblocks/examples/support/range_packed.rs b/vortex-btrblocks/examples/support/range_packed.rs new file mode 100644 index 00000000000..3a6945bbbc0 --- /dev/null +++ b/vortex-btrblocks/examples/support/range_packed.rs @@ -0,0 +1,605 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::cmp::Ordering; +use std::mem::MaybeUninit; +use std::ops::Range; + +use vortex_error::VortexResult; +use vortex_error::vortex_ensure; + +const MAX_BINS: usize = 64; +const SPLIT_CANDIDATES: usize = 64; +const TRAINING_SAMPLE_SIZE: usize = 8_192; +const BIN_TABLE_COST_BITS: f64 = 128.0; +const SYMBOL_PADDING: usize = 8; +const OFFSET_PADDING: usize = 15; +const CHECKPOINT_INTERVAL: usize = 32; +const MAX_BLOCK_LEN: usize = 1_024; + +/// Fixed-width range identifiers with variable-width offsets and scalar checkpoints. +#[derive(Clone, Debug)] +pub struct RangePackedCodec { + len: usize, + block_len: usize, + symbol_width: u8, + bins: Vec, + block_offsets: Vec, + payload: Vec, +} + +#[derive(Clone, Copy, Debug)] +struct Bin { + lower: u64, + offset_bits: u8, +} + +impl RangePackedCodec { + pub fn encode(values: &[u64], block_len: usize) -> VortexResult { + vortex_ensure!( + block_len > 0 && block_len <= MAX_BLOCK_LEN, + "range packed block length must be in 1..={MAX_BLOCK_LEN}" + ); + if values.is_empty() { + return Ok(Self { + len: 0, + block_len, + symbol_width: 0, + bins: Vec::new(), + block_offsets: vec![0], + payload: Vec::new(), + }); + } + + let (sample, minimum, maximum) = training_sample(values); + let bins = cover_domain(optimize_bins(&sample), minimum, maximum); + vortex_ensure!( + bins.len().is_power_of_two(), + "range packed bin count must be a power of two" + ); + let symbols = assign_bins(values, &bins)?; + let symbol_width = bit_width(u64::try_from(bins.len() - 1)?); + let block_count = values.len().div_ceil(block_len); + let mut block_offsets = Vec::with_capacity(block_count + 1); + let mut payload = Vec::new(); + block_offsets.push(0); + for block_index in 0..block_count { + let start = block_index * block_len; + let stop = (start + block_len).min(values.len()); + encode_block( + &values[start..stop], + &symbols[start..stop], + &bins, + symbol_width, + &mut payload, + )?; + block_offsets.push(u32::try_from(payload.len())?); + } + + Ok(Self { + len: values.len(), + block_len, + symbol_width, + bins, + block_offsets, + payload, + }) + } + + pub fn decode(&self) -> VortexResult> { + if self.bins.iter().all(|bin| bin.offset_bits <= 57) { + self.decode_with_width::() + } else { + self.decode_with_width::() + } + } + + fn decode_with_width(&self) -> VortexResult> { + let mut values = Vec::with_capacity(self.len); + let table = DecodeTable::new(&self.bins); + for block_index in 0..self.block_count() { + let payload = self.block_payload(block_index)?; + let value_count = self.block_value_count(block_index); + let output_start = values.len(); + decode_block::( + payload, + value_count, + self.symbol_width, + &table, + &mut values.spare_capacity_mut()[..value_count], + )?; + // SAFETY: decode_block initialized each output slot. + unsafe { values.set_len(output_start + value_count) }; + } + Ok(values) + } + + pub fn scalar_at(&self, index: usize) -> VortexResult { + vortex_ensure!(index < self.len, "range packed index exceeds array length"); + let block_index = index / self.block_len; + let index_in_block = index % self.block_len; + let value_count = self.block_value_count(block_index); + let payload = self.block_payload(block_index)?; + let layout = BlockLayout::new(value_count, self.symbol_width, payload.len())?; + let symbol = read_symbol(payload, index_in_block, self.symbol_width)?; + let node = *self + .bins + .get(symbol) + .ok_or_else(|| vortex_error::vortex_err!("range packed symbol exceeds bin table"))?; + let checkpoint_index = index_in_block / CHECKPOINT_INTERVAL; + let checkpoint_start = layout.checkpoints_start + checkpoint_index * size_of::(); + let checkpoint = + u16::from_le_bytes([payload[checkpoint_start], payload[checkpoint_start + 1]]); + let mut offset_position = usize::from(checkpoint); + let scan_start = checkpoint_index * CHECKPOINT_INTERVAL; + for preceding in scan_start..index_in_block { + let preceding_symbol = read_symbol(payload, preceding, self.symbol_width)?; + offset_position += usize::from( + self.bins + .get(preceding_symbol) + .ok_or_else(|| { + vortex_error::vortex_err!("range packed symbol exceeds bin table") + })? + .offset_bits, + ); + } + let offset = read_bits_padded( + &payload[layout.offsets_start..], + offset_position, + node.offset_bits, + ); + Ok(node.lower.wrapping_add(offset)) + } + + pub fn encoded_size(&self) -> usize { + self.bins.len() * (size_of::() + size_of::()) + + self.block_offsets.len() * size_of::() + + self.payload.len() + } + + pub fn bin_count(&self) -> usize { + self.bins.len() + } + + fn block_count(&self) -> usize { + self.len.div_ceil(self.block_len) + } + + fn block_value_count(&self, block_index: usize) -> usize { + (self.len - block_index * self.block_len).min(self.block_len) + } + + fn block_payload(&self, block_index: usize) -> VortexResult<&[u8]> { + let start = usize::try_from(self.block_offsets[block_index])?; + let stop = usize::try_from(self.block_offsets[block_index + 1])?; + Ok(&self.payload[start..stop]) + } +} + +struct DecodeTable { + lowers: [u64; MAX_BINS], + widths: [u8; MAX_BINS], + bin_count: usize, +} + +impl DecodeTable { + fn new(bins: &[Bin]) -> Self { + let mut lowers = [0_u64; MAX_BINS]; + let mut widths = [0_u8; MAX_BINS]; + for (index, bin) in bins.iter().enumerate() { + lowers[index] = bin.lower; + widths[index] = bin.offset_bits; + } + Self { + lowers, + widths, + bin_count: bins.len(), + } + } +} + +struct BlockLayout { + checkpoints_start: usize, + offsets_start: usize, +} + +impl BlockLayout { + fn new(value_count: usize, symbol_width: u8, payload_len: usize) -> VortexResult { + let symbol_bytes = (value_count * usize::from(symbol_width)).div_ceil(8); + let checkpoints_start = symbol_bytes + SYMBOL_PADDING; + let checkpoint_bytes = value_count.div_ceil(CHECKPOINT_INTERVAL) * size_of::(); + let offsets_start = checkpoints_start + checkpoint_bytes; + vortex_ensure!( + payload_len >= offsets_start + OFFSET_PADDING, + "range packed block is too short" + ); + Ok(Self { + checkpoints_start, + offsets_start, + }) + } +} + +fn encode_block( + values: &[u64], + symbols: &[u8], + bins: &[Bin], + symbol_width: u8, + payload: &mut Vec, +) -> VortexResult<()> { + let mut symbol_writer = BitWriter::with_capacity(values.len()); + let mut offset_writer = BitWriter::with_capacity(size_of_val(values)); + let mut checkpoints = Vec::with_capacity(values.len().div_ceil(CHECKPOINT_INTERVAL)); + for (index, (&value, &symbol)) in values.iter().zip(symbols).enumerate() { + if index.is_multiple_of(CHECKPOINT_INTERVAL) { + checkpoints.push(u16::try_from(offset_writer.bit_len())?); + } + symbol_writer.write(u64::from(symbol), symbol_width); + let bin = bins[usize::from(symbol)]; + offset_writer.write(value - bin.lower, bin.offset_bits); + } + payload.extend_from_slice(&symbol_writer.finish()); + payload.extend_from_slice(&[0; SYMBOL_PADDING]); + for checkpoint in checkpoints { + payload.extend_from_slice(&checkpoint.to_le_bytes()); + } + payload.extend_from_slice(&offset_writer.finish()); + payload.extend_from_slice(&[0; OFFSET_PADDING]); + Ok(()) +} + +#[expect( + clippy::cast_possible_truncation, + reason = "the symbol mask never exceeds 63" +)] +fn decode_block( + payload: &[u8], + value_count: usize, + symbol_width: u8, + table: &DecodeTable, + values: &mut [MaybeUninit], +) -> VortexResult<()> { + let layout = BlockLayout::new(value_count, symbol_width, payload.len())?; + let offsets = &payload[layout.offsets_start..]; + let mut offset_position = 0usize; + let full_len = value_count / 8 * 8; + let symbol_mask = low_mask(symbol_width); + for base in (0..full_len).step_by(8) { + let symbol_position = base * usize::from(symbol_width); + let packed = read_bits_padded(payload, symbol_position, symbol_width * 8); + for lane in 0..8 { + let symbol = ((packed >> (lane * usize::from(symbol_width))) & symbol_mask) as usize; + debug_assert!(symbol < table.bin_count); + // SAFETY: A power-of-two bin count makes every symbol code valid. + let width = unsafe { *table.widths.get_unchecked(symbol) }; + let offset = read_offset::(offsets, offset_position, width); + offset_position += usize::from(width); + // SAFETY: A power-of-two bin count makes every symbol code valid. + let lower = unsafe { *table.lowers.get_unchecked(symbol) }; + // SAFETY: This lane is within a complete eight-value output batch. + unsafe { + values + .get_unchecked_mut(base + lane) + .write(lower.wrapping_add(offset)) + }; + } + } + for index in full_len..value_count { + let symbol = read_symbol(payload, index, symbol_width)?; + debug_assert!(symbol < table.bin_count); + // SAFETY: A power-of-two bin count makes every symbol code valid. + let width = unsafe { *table.widths.get_unchecked(symbol) }; + let offset = read_offset::(offsets, offset_position, width); + offset_position += usize::from(width); + // SAFETY: A power-of-two bin count makes every symbol code valid. + let lower = unsafe { *table.lowers.get_unchecked(symbol) }; + // SAFETY: The trailing index is less than value_count. + unsafe { + values + .get_unchecked_mut(index) + .write(lower.wrapping_add(offset)) + }; + } + let offset_bytes = offset_position.div_ceil(8); + vortex_ensure!( + offsets.len() == offset_bytes + OFFSET_PADDING, + "range packed offset stream has unused bytes" + ); + Ok(()) +} + +fn read_symbol(payload: &[u8], index: usize, symbol_width: u8) -> VortexResult { + usize::try_from(read_bits_padded( + payload, + index * usize::from(symbol_width), + symbol_width, + )) + .map_err(Into::into) +} + +#[inline(always)] +fn read_offset(offsets: &[u8], bit_position: usize, width: u8) -> u64 { + if WIDE { + read_bits_padded(offsets, bit_position, width) + } else if width == 0 { + 0 + } else { + let byte_position = bit_position / 8; + let bits_past_byte = bit_position % 8; + // SAFETY: Offset streams include fifteen readable padding bytes. + let packed = unsafe { read_u64_unaligned(offsets.as_ptr().add(byte_position)) }; + (packed >> bits_past_byte) & low_mask(width) + } +} + +fn read_bits_padded(bytes: &[u8], bit_position: usize, width: u8) -> u64 { + if width == 0 { + return 0; + } + let byte_position = bit_position / 8; + let bits_past_byte = bit_position % 8; + // SAFETY: Every encoded stream includes at least eight readable padding bytes. + let first = unsafe { read_u64_unaligned(bytes.as_ptr().add(byte_position)) }; + if usize::from(width) <= 64 - bits_past_byte { + (first >> bits_past_byte) & low_mask(width) + } else { + // SAFETY: Offset streams include fifteen readable padding bytes. + let second = unsafe { read_u64_unaligned(bytes.as_ptr().add(byte_position + 7)) }; + let processed = 56 - bits_past_byte; + ((first >> bits_past_byte) | (second << processed)) & low_mask(width) + } +} + +unsafe fn read_u64_unaligned(pointer: *const u8) -> u64 { + // SAFETY: The caller provides eight readable bytes at the pointer. + u64::from_le(unsafe { pointer.cast::().read_unaligned() }) +} + +fn assign_bins(values: &[u64], bins: &[Bin]) -> VortexResult> { + values + .iter() + .map(|&value| { + let symbol = bins + .partition_point(|bin| bin.lower <= value) + .saturating_sub(1); + let bin = bins[symbol]; + vortex_ensure!( + value - bin.lower <= low_mask(bin.offset_bits), + "range packed value exceeds its bin" + ); + Ok(u8::try_from(symbol)?) + }) + .collect() +} + +fn optimize_bins(sorted: &[u64]) -> Vec { + let mut segments = Vec::with_capacity(MAX_BINS); + segments.push(0..sorted.len()); + let mut best_segments = segments.clone(); + let mut best_cost = partition_cost(sorted, &segments); + + while segments.len() < MAX_BINS { + let best = segments + .iter() + .enumerate() + .filter_map(|(segment_index, segment)| { + best_split(sorted, segment).map(|split| (segment_index, split)) + }) + .max_by(|(_, left), (_, right)| { + left.gain + .partial_cmp(&right.gain) + .unwrap_or(Ordering::Equal) + }); + let Some((segment_index, split)) = best else { + break; + }; + if split.gain <= 0.0 { + break; + } + let segment = segments.remove(segment_index); + segments.push(segment.start..split.at); + segments.push(split.at..segment.end); + segments.sort_unstable_by_key(|segment| segment.start); + let cost = partition_cost(sorted, &segments); + if segments.len().is_power_of_two() && cost < best_cost { + best_cost = cost; + best_segments.clone_from(&segments); + } + } + + best_segments + .into_iter() + .map(|segment| Bin::from_values(sorted[segment.start], sorted[segment.end - 1])) + .collect() +} + +fn partition_cost(sorted: &[u64], segments: &[Range]) -> f64 { + let residual_cost = segments + .iter() + .map(|segment| { + segment.len() as f64 + * f64::from(bit_width(sorted[segment.end - 1] - sorted[segment.start])) + }) + .sum::(); + let symbol_cost = sorted.len() as f64 * f64::from(bit_width((segments.len() - 1) as u64)); + residual_cost + symbol_cost + segments.len() as f64 * BIN_TABLE_COST_BITS +} + +fn training_sample(values: &[u64]) -> (Vec, u64, u64) { + let mut minimum = u64::MAX; + let mut maximum = 0; + for &value in values { + minimum = minimum.min(value); + maximum = maximum.max(value); + } + let mut sample = if values.len() <= TRAINING_SAMPLE_SIZE { + values.to_vec() + } else { + (0..TRAINING_SAMPLE_SIZE) + .map(|index| values[index * values.len() / TRAINING_SAMPLE_SIZE]) + .collect() + }; + sample.push(minimum); + sample.push(maximum); + sample.sort_unstable(); + (sample, minimum, maximum) +} + +fn cover_domain(mut bins: Vec, minimum: u64, maximum: u64) -> Vec { + bins[0].lower = minimum; + for index in 0..bins.len() - 1 { + let upper = bins[index + 1].lower - 1; + bins[index].offset_bits = bit_width(upper - bins[index].lower); + } + let last = bins.len() - 1; + bins[last].offset_bits = bit_width(maximum - bins[last].lower); + bins +} + +#[derive(Clone, Copy)] +struct Split { + at: usize, + gain: f64, +} + +fn best_split(sorted: &[u64], segment: &Range) -> Option { + if segment.len() < 2 || sorted[segment.start] == sorted[segment.end - 1] { + return None; + } + let whole_cost = segment.len() as f64 + * f64::from(bit_width(sorted[segment.end - 1] - sorted[segment.start])); + let mut best = None; + for candidate in 1..SPLIT_CANDIDATES { + let mut at = segment.start + segment.len() * candidate / SPLIT_CANDIDATES; + if at <= segment.start || at >= segment.end { + continue; + } + while at < segment.end && sorted[at - 1] == sorted[at] { + at += 1; + } + if at >= segment.end { + continue; + } + let left_cost = (at - segment.start) as f64 + * f64::from(bit_width(sorted[at - 1] - sorted[segment.start])); + let right_cost = + (segment.end - at) as f64 * f64::from(bit_width(sorted[segment.end - 1] - sorted[at])); + let split = Split { + at, + gain: whole_cost - left_cost - right_cost, + }; + if best.is_none_or(|current: Split| split.gain > current.gain) { + best = Some(split); + } + } + best +} + +impl Bin { + fn from_values(lower: u64, upper: u64) -> Self { + Self { + lower, + offset_bits: bit_width(upper - lower), + } + } +} + +fn bit_width(value: u64) -> u8 { + u8::try_from(u64::BITS - value.leading_zeros()).unwrap_or(64) +} + +fn low_mask(bits: u8) -> u64 { + match bits { + 0 => 0, + 64 => u64::MAX, + _ => (1_u64 << bits) - 1, + } +} + +struct BitWriter { + bytes: Vec, + pending: u64, + pending_bits: u8, + bit_len: usize, +} + +impl BitWriter { + fn with_capacity(capacity: usize) -> Self { + Self { + bytes: Vec::with_capacity(capacity), + pending: 0, + pending_bits: 0, + bit_len: 0, + } + } + + fn bit_len(&self) -> usize { + self.bit_len + } + + fn write(&mut self, value: u64, width: u8) { + self.bit_len += usize::from(width); + if width == 0 { + return; + } + let available = 64 - self.pending_bits; + self.pending |= value << self.pending_bits; + if width < available { + self.pending_bits += width; + return; + } + self.bytes.extend_from_slice(&self.pending.to_le_bytes()); + let remaining = width - available; + self.pending = if remaining == 0 { + 0 + } else { + value >> available + }; + self.pending_bits = remaining; + } + + fn finish(mut self) -> Vec { + if self.pending_bits > 0 { + let byte_count = usize::from(self.pending_bits).div_ceil(8); + self.bytes + .extend_from_slice(&self.pending.to_le_bytes()[..byte_count]); + } + self.bytes + } +} + +#[cfg(test)] +mod tests { + use vortex_error::VortexResult; + + use super::RangePackedCodec; + + #[test] + fn clustered_roundtrip_and_scalar_access() -> VortexResult<()> { + let values = (0_u64..20_000) + .map(|index| match index % 4 { + 0 => index % 17, + 1 => 1_000_000 + index % 31, + 2 => u64::MAX - index % 13, + _ => 1_000 + index % 7, + }) + .collect::>(); + let encoded = RangePackedCodec::encode(&values, 1_024)?; + assert_eq!(encoded.decode()?, values); + for index in [0, 1, 31, 32, 33, 1_023, 1_024, 19_999] { + assert_eq!(encoded.scalar_at(index)?, values[index]); + } + Ok(()) + } + + #[test] + fn full_width_roundtrip() -> VortexResult<()> { + let values = [0, u64::MAX, 1, u64::MAX - 1]; + let encoded = RangePackedCodec::encode(&values, 4)?; + assert_eq!(encoded.decode()?, values); + for (index, value) in values.into_iter().enumerate() { + assert_eq!(encoded.scalar_at(index)?, value); + } + Ok(()) + } +} From c2bb1d3eea1c019cd038295a2910242a2b2bb8ed Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 17:12:04 +0100 Subject: [PATCH 22/78] bench: measure complete fixed-bin trees Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 54 ++- .../examples/float_compressor_bench.rs | 211 ++++++++++ .../examples/support/range_packed.rs | 363 +++++++++++++++++- 3 files changed, 607 insertions(+), 21 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 2d46cf9f908..d493a5a398e 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -715,9 +715,41 @@ The CMS fixed-bin tree saves 15.2 percent against default. It remains 7.8 percen The first CMS decoder reached 8,057 MB/s. The specialized decoder increased throughput by 68.2 percent. -The complete CMS default tree decodes at 14,649 MB/s. The fixed-bin child is 7.5 percent slower. +The complete CMS default tree decodes at 14,575 MB/s. The fixed-bin child is 6.1 percent slower. -An integrated Vortex array is required for the complete-tree throughput comparison. +The benchmark now wraps the codec in an experimental Vortex array. + +This wrapper permits complete ALP decode and scalar measurements without a storage-format commitment. + +| Input | Default decode MB/s | Compact decode MB/s | Fixed-bin decode MB/s | Fused decode MB/s | +| --- | ---: | ---: | ---: | ---: | +| GloVe | 17,523 | 1,853 | 4,991 | 4,984 | +| CMS Payments | 14,575 | 5,588 | 9,342 | 9,791 | +| Food | 24,000 | 5,569 | 9,431 | 11,195 | + +| Input | Default scalar ns | Compact scalar ns | Fixed-bin scalar ns | +| --- | ---: | ---: | ---: | +| GloVe | 824 | 21,374 | 535 | +| CMS Payments | 903 | 14,473 | 383 | +| Food | 535 | 13,569 | 140 | + +The fused path writes final floats during range decode. It retains the ALP patch step. + +Fusion increased CMS decode by 4.8 percent and Food decode by 18.7 percent. It did not improve GloVe decode. + +The fixed-bin tree fails the default decode gate on all three inputs. + +CMS and Food support a Compact candidate for classic Pco bins. + +CMS uses 7.8 percent more bytes than Compact. Generic decode is 67.2 percent faster, and scalar access is 37.8 times faster. + +Food uses 5.3 percent more bytes than Compact. Generic decode is 69.4 percent faster, and scalar access is 97.3 times faster. + +The fused Food decode is 2.0 times as fast as Compact. The generic composition already gives most of the benefit. + +GloVe uses 16.8 percent more bytes than Compact because Pco also uses `IntMult(10)`. + +The evidence does not justify a fused ALP and fixed-bin array. A composable integer child has lower implementation complexity. ### Pcodec corpus gap analysis @@ -900,14 +932,16 @@ Complete these remaining steps: 1. Validate the nonlinear patch cost on the complete corpus. 2. Add a true ALP-RD child composition case if a real selected tree exposes one. 3. Validate the remaining rejected analysis cost in the complete corpus. -4. Integrate fixed-bin packing as an experimental array and measure complete CMS and GloVe trees. -5. Add real embedding datasets beyond GloVe when licenses and loaders permit them. -6. Classify more float columns where Pco beats ALP, ALP-RD, and the new schemes. -7. Prototype one lightweight scheme for the repeated real-float gap classes. -8. Run the complete `bench-vortex` compression corpus after the candidate set stabilizes. -9. Compare the geometric mean against Parquet with Zstd. -10. Calibrate all selector thresholds from complete corpus and selected-tree evidence. -11. Update this plan after each experiment. +4. Test the complete fixed-bin tree on every available classic-bin Pco win. +5. Define a Compact size threshold for the fixed-bin candidate. +6. Promote fixed-bin packing to a production array only if more unrelated classic-bin columns pass. +7. Add real embedding datasets beyond GloVe when licenses and loaders permit them. +8. Classify more float columns where Pco beats ALP, ALP-RD, and the new schemes. +9. Prototype one lightweight scheme for the repeated real-float gap classes. +10. Run the complete `bench-vortex` compression corpus after the candidate set stabilizes. +11. Compare the geometric mean against Parquet with Zstd. +12. Calibrate all selector thresholds from complete corpus and selected-tree evidence. +13. Update this plan after each experiment. ## Pull request structure diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index 6c1d8ce9797..8f5053ca05f 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -25,6 +25,10 @@ use pco::wrapped::FileCompressor; use rand::RngExt; use rand::SeedableRng; use rand::rngs::StdRng; +use vortex_alp::ALP; +use vortex_alp::ALPArrayExt; +use vortex_alp::ALPArraySlotsExt; +use vortex_alp::ALPFloat; use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::RecursiveCanonical; @@ -58,6 +62,7 @@ use vortex_utils::aliases::hash_map::HashMap; #[path = "support/range_packed.rs"] mod range_packed; +use range_packed::RangePacked; use range_packed::RangePackedCodec; const DEFAULT_ROW_COUNT: usize = 2_000_000; @@ -944,6 +949,188 @@ fn profile_range_packed( Ok(()) } +fn fixed_bin_alp_tree( + compact: &ArrayRef, + session: &VortexSession, +) -> VortexResult> { + if compact.encoding_id().as_ref() != "vortex.alp" { + return Ok(None); + } + let alp = compact.as_::(); + if alp.encoded().encoding_id().as_ref() != "vortex.pco" { + return Ok(None); + } + let primitive = alp + .encoded() + .clone() + .execute::(&mut session.create_execution_ctx())?; + let primitive = primitive + .into_array() + .cast(alp.encoded().dtype().clone())? + .execute::(&mut session.create_execution_ctx())?; + let encoded = RangePacked::from_primitive(primitive.as_view())?.into_array(); + Ok(Some( + ALP::try_new(encoded, alp.exponents(), alp.patches())?.into_array(), + )) +} + +fn measure_fixed_bin_tree( + dataset: &str, + column: &str, + expected: &ArrayRef, + compact: &ArrayRef, + session: &VortexSession, +) -> VortexResult<()> { + let Some(encoded) = fixed_bin_alp_tree(compact, session)? else { + return Ok(()); + }; + vortex_ensure!( + encoded.dtype() == expected.dtype(), + "fixed-bin tree dtype {} differs from expected {}", + encoded.dtype(), + expected.dtype() + ); + assert_arrays_eq!(encoded, expected, &mut session.create_execution_ctx()); + + let mut decode_durations = Vec::with_capacity(20); + for _ in 0..20 { + let start = Instant::now(); + black_box( + encoded + .clone() + .execute::(&mut session.create_execution_ctx())?, + ); + decode_durations.push(start.elapsed()); + } + let decode_median = percentile(&mut decode_durations, 1, 2); + let decode_throughput = expected.nbytes() as f64 / decode_median.as_secs_f64() / 1_000_000.0; + + let scalar_nanoseconds = measure_scalar_access(&encoded, session)?; + println!( + "fixed-bin-tree\t{dataset}\t{column}\t{}\t{}\t{decode_throughput:.1}\t{scalar_nanoseconds:.1}\t{}", + encoded.len(), + encoded.nbytes(), + encoding_tree(&encoded), + ); + + let mut fused_durations = Vec::with_capacity(20); + for _ in 0..20 { + let start = Instant::now(); + black_box(decode_fixed_bin_alp_tree(&encoded, session)?); + fused_durations.push(start.elapsed()); + } + let fused_median = percentile(&mut fused_durations, 1, 2); + let fused_throughput = expected.nbytes() as f64 / fused_median.as_secs_f64() / 1_000_000.0; + println!( + "fixed-bin-tree-fused\t{dataset}\t{column}\t{}\t{}\t{fused_throughput:.1}\t{}", + encoded.len(), + encoded.nbytes(), + encoding_tree(&encoded), + ); + Ok(()) +} + +fn measure_existing_tree( + dataset: &str, + column: &str, + label: &str, + expected: &ArrayRef, + encoded: &ArrayRef, + session: &VortexSession, +) -> VortexResult<()> { + assert_arrays_eq!(encoded, expected, &mut session.create_execution_ctx()); + let mut decode_durations = Vec::with_capacity(20); + for _ in 0..20 { + let start = Instant::now(); + black_box( + encoded + .clone() + .execute::(&mut session.create_execution_ctx())?, + ); + decode_durations.push(start.elapsed()); + } + let decode_median = percentile(&mut decode_durations, 1, 2); + let decode_throughput = expected.nbytes() as f64 / decode_median.as_secs_f64() / 1_000_000.0; + let scalar_nanoseconds = measure_scalar_access(encoded, session)?; + println!( + "tree-throughput\t{dataset}\t{column}\t{label}\t{}\t{}\t{decode_throughput:.1}\t{scalar_nanoseconds:.1}\t{}", + encoded.len(), + encoded.nbytes(), + encoding_tree(encoded), + ); + Ok(()) +} + +fn measure_scalar_access(encoded: &ArrayRef, session: &VortexSession) -> VortexResult { + let scalar_iterations = 200_000; + let mut scalar_index = 0usize; + let mut scalar_checksum = 0u64; + let mut scalar_ctx = session.create_execution_ctx(); + let scalar_start = Instant::now(); + for _ in 0..scalar_iterations { + scalar_index = scalar_index.wrapping_add(2_654_435_761) % encoded.len(); + let scalar = encoded.execute_scalar(scalar_index, &mut scalar_ctx)?; + let bits = match encoded.dtype().as_ptype() { + PType::F32 => u64::from( + scalar + .as_primitive() + .typed_value::() + .ok_or_else(|| vortex_err!("tree produced a null scalar"))? + .to_bits(), + ), + PType::F64 => scalar + .as_primitive() + .typed_value::() + .ok_or_else(|| vortex_err!("tree produced a null scalar"))? + .to_bits(), + ptype => { + return Err(vortex_err!( + "tree scalar benchmark does not support {ptype}" + )); + } + }; + scalar_checksum ^= black_box(bits); + } + black_box(scalar_checksum); + Ok(scalar_start.elapsed().as_secs_f64() * 1_000_000_000.0 / scalar_iterations as f64) +} + +#[expect( + clippy::cast_possible_truncation, + reason = "the range-packed child retains the ALP integer word width" +)] +fn decode_fixed_bin_alp_tree( + encoded: &ArrayRef, + session: &VortexSession, +) -> VortexResult { + let alp = encoded.as_::(); + let packed = alp.encoded().as_::(); + let validity = alp.encoded().validity()?; + let exponents = alp.exponents(); + let decoded = match encoded.dtype().as_ptype() { + PType::F32 => { + let values = RangePacked::decode_mapped(packed, |ordered| { + let encoded = ((ordered as u32) ^ (1_u32 << 31)) as i32; + ::decode_single(encoded, exponents) + })?; + PrimitiveArray::new::(values, validity) + } + PType::F64 => { + let values = RangePacked::decode_mapped(packed, |ordered| { + let encoded = (ordered ^ (1_u64 << 63)) as i64; + ::decode_single(encoded, exponents) + })?; + PrimitiveArray::new::(values, validity) + } + ptype => return Err(vortex_err!("fixed-bin ALP tree does not support {ptype}")), + }; + if let Some(patches) = alp.patches() { + decoded.patch(&patches, &mut session.create_execution_ctx()) + } else { + Ok(decoded) + } +} + fn profile_pco_array( dataset: &str, column: &str, @@ -1121,6 +1308,25 @@ fn measure_dataset( { profile_pco_array(dataset, &column.name, "root", compact_array, session)?; } + if std::env::var_os("VORTEX_BENCH_FIXED_BIN").is_some() { + measure_existing_tree( + dataset, + &column.name, + "default", + &column.array, + default_array, + session, + )?; + measure_existing_tree( + dataset, + &column.name, + "compact", + &column.array, + compact_array, + session, + )?; + measure_fixed_bin_tree(dataset, &column.name, &column.array, compact_array, session)?; + } } if std::env::var_os("VORTEX_BENCH_PROFILE_ONLY").is_some() { @@ -1245,6 +1451,11 @@ fn main() -> VortexResult<()> { println!( "fixed-bin-checkpoint\tdataset\tcolumn\tpath\trows\tpco-bytes\tfixed-bin-bytes\tbins\tencode-MB/s\tdecode-MB/s\tscalar-ns" ); + println!("fixed-bin-tree\tdataset\tcolumn\trows\tbytes\tdecode-MB/s\tscalar-ns\tencoding"); + println!("fixed-bin-tree-fused\tdataset\tcolumn\trows\tbytes\tdecode-MB/s\tencoding"); + println!( + "tree-throughput\tdataset\tcolumn\tconfig\trows\tbytes\tdecode-MB/s\tscalar-ns\tencoding" + ); println!("result\tdataset\tconfig\trows\tinput-bytes\tencoded-bytes\tencode-MB/s\tdecode-MB/s"); println!( "block-residual-profile\tdataset\tcolumn\tconfig\tpath\tptype\trows\tpatches\tblocks\taverage-residual-width\taverage-high-width\tmaximum-patch-density\tblocks-above-one-eighth\tblocks-at-one-quarter\tbytes" diff --git a/vortex-btrblocks/examples/support/range_packed.rs b/vortex-btrblocks/examples/support/range_packed.rs index 3a6945bbbc0..dd767eee294 100644 --- a/vortex-btrblocks/examples/support/range_packed.rs +++ b/vortex-btrblocks/examples/support/range_packed.rs @@ -2,11 +2,44 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::cmp::Ordering; +use std::fmt::Display; +use std::fmt::Formatter; +use std::hash::Hash; +use std::hash::Hasher; use std::mem::MaybeUninit; use std::ops::Range; +use vortex_array::Array; +use vortex_array::ArrayEq; +use vortex_array::ArrayHash; +use vortex_array::ArrayId; +use vortex_array::ArrayParts; +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::EqMode; +use vortex_array::ExecutionCtx; +use vortex_array::ExecutionResult; +use vortex_array::IntoArray; +use vortex_array::arrays::Primitive; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::buffer::BufferHandle; +use vortex_array::builtins::ArrayBuiltins; +use vortex_array::dtype::DType; +use vortex_array::dtype::Nullability; +use vortex_array::dtype::PType; +use vortex_array::scalar::Scalar; +use vortex_array::serde::ArrayChildren; +use vortex_array::validity::Validity; +use vortex_array::vtable::OperationsVTable; +use vortex_array::vtable::VTable; +use vortex_array::vtable::ValidityVTable; +use vortex_buffer::ByteBuffer; use vortex_error::VortexResult; +use vortex_error::vortex_bail; use vortex_error::vortex_ensure; +use vortex_error::vortex_panic; +use vortex_session::VortexSession; +use vortex_session::registry::CachedId; const MAX_BINS: usize = 64; const SPLIT_CANDIDATES: usize = 64; @@ -18,7 +51,7 @@ const CHECKPOINT_INTERVAL: usize = 32; const MAX_BLOCK_LEN: usize = 1_024; /// Fixed-width range identifiers with variable-width offsets and scalar checkpoints. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct RangePackedCodec { len: usize, block_len: usize, @@ -28,12 +61,305 @@ pub struct RangePackedCodec { payload: Vec, } -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] struct Bin { lower: u64, offset_bits: u8, } +/// Experimental Vortex wrapper for complete-tree benchmark measurements. +pub type RangePackedArray = Array; + +#[derive(Clone, Debug)] +pub struct RangePacked; + +#[derive(Clone, Debug)] +pub struct RangePackedData { + codec: RangePackedCodec, + physical: ByteBuffer, + ptype: PType, +} + +impl Display for RangePackedData { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "bins: {}", self.codec.bin_count()) + } +} + +impl ArrayHash for RangePackedData { + fn array_hash(&self, state: &mut H, _accuracy: EqMode) { + self.codec.hash(state); + self.ptype.hash(state); + } +} + +impl ArrayEq for RangePackedData { + fn array_eq(&self, other: &Self, _accuracy: EqMode) -> bool { + self.codec == other.codec && self.ptype == other.ptype + } +} + +impl VTable for RangePacked { + type TypedArrayData = RangePackedData; + type OperationsVTable = Self; + type ValidityVTable = Self; + + fn id(&self) -> ArrayId { + static ID: CachedId = CachedId::new("vortex.experimental.range_packed"); + *ID + } + + fn validate( + &self, + data: &Self::TypedArrayData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + vortex_ensure!(slots.is_empty(), "RangePackedArray cannot have children"); + vortex_ensure!( + dtype.is_primitive() && dtype.as_ptype() == data.ptype, + "RangePackedArray dtype differs from its physical type" + ); + vortex_ensure!(len == data.codec.len, "RangePackedArray length differs"); + vortex_ensure!( + data.physical.len() == data.codec.encoded_size(), + "RangePackedArray physical size differs" + ); + Ok(()) + } + + fn nbuffers(_array: ArrayView<'_, Self>) -> usize { + 1 + } + + fn buffer(array: ArrayView<'_, Self>, index: usize) -> BufferHandle { + if index == 0 { + BufferHandle::new_host(array.data().physical.clone()) + } else { + vortex_panic!("RangePackedArray buffer index {index} is invalid") + } + } + + fn buffer_name(_array: ArrayView<'_, Self>, index: usize) -> Option { + (index == 0).then(|| "encoded".to_string()) + } + + fn with_buffers( + &self, + array: ArrayView<'_, Self>, + buffers: &[BufferHandle], + ) -> VortexResult> { + vortex_ensure!(buffers.len() == 1, "RangePackedArray needs one buffer"); + let mut data = array.data().clone(); + data.physical = buffers[0].clone().try_to_host_sync()?; + Ok(ArrayParts::new( + self.clone(), + array.dtype().clone(), + array.len(), + data, + )) + } + + fn serialize( + _array: ArrayView<'_, Self>, + _session: &VortexSession, + ) -> VortexResult>> { + vortex_bail!("experimental RangePackedArray does not support serialization") + } + + fn deserialize( + &self, + _dtype: &DType, + _len: usize, + _metadata: &[u8], + _buffers: &[BufferHandle], + _children: &dyn ArrayChildren, + _session: &VortexSession, + ) -> VortexResult> { + vortex_bail!("experimental RangePackedArray does not support deserialization") + } + + fn slot_name(_array: ArrayView<'_, Self>, index: usize) -> String { + vortex_panic!("RangePackedArray slot index {index} is invalid") + } + + fn execute(array: Array, _ctx: &mut ExecutionCtx) -> VortexResult { + let decoded = decode_primitive(array.data())? + .into_array() + .cast(array.dtype().clone())?; + Ok(ExecutionResult::done(decoded)) + } +} + +impl OperationsVTable for RangePacked { + fn scalar_at( + array: ArrayView<'_, RangePacked>, + index: usize, + _ctx: &mut ExecutionCtx, + ) -> VortexResult { + ordered_scalar( + array.data().codec.scalar_at(index)?, + array.data().ptype, + array.dtype().nullability(), + ) + } +} + +impl ValidityVTable for RangePacked { + fn validity(array: ArrayView<'_, RangePacked>) -> VortexResult { + if array.dtype().is_nullable() { + Ok(Validity::AllValid) + } else { + Ok(Validity::NonNullable) + } + } +} + +impl RangePacked { + pub fn from_primitive(array: ArrayView<'_, Primitive>) -> VortexResult { + vortex_ensure!( + array.ptype().is_int(), + "RangePackedArray needs integer values" + ); + vortex_ensure!( + array.validity()?.definitely_no_nulls(), + "experimental RangePackedArray needs values without nulls" + ); + let ordered = ordered_primitive(array)?; + let codec = RangePackedCodec::encode(&ordered, MAX_BLOCK_LEN)?; + let physical = ByteBuffer::zeroed(codec.encoded_size()); + let data = RangePackedData { + codec, + physical, + ptype: array.ptype(), + }; + Array::try_from_parts(ArrayParts::new( + RangePacked, + array.dtype().clone(), + array.len(), + data, + )) + } + + pub fn decode_mapped(array: ArrayView<'_, RangePacked>, map: F) -> VortexResult> + where + F: FnMut(u64) -> T, + { + array.data().codec.decode_mapped(map) + } +} + +fn ordered_primitive(array: ArrayView<'_, Primitive>) -> VortexResult> { + Ok(match array.ptype() { + PType::U8 => array + .as_slice::() + .iter() + .map(|&value| u64::from(value)) + .collect(), + PType::U16 => array + .as_slice::() + .iter() + .map(|&value| u64::from(value)) + .collect(), + PType::U32 => array + .as_slice::() + .iter() + .map(|&value| u64::from(value)) + .collect(), + PType::U64 => array.as_slice::().to_vec(), + PType::I8 => array + .as_slice::() + .iter() + .map(|&value| u64::from((value as u8) ^ (1_u8 << 7))) + .collect(), + PType::I16 => array + .as_slice::() + .iter() + .map(|&value| u64::from((value as u16) ^ (1_u16 << 15))) + .collect(), + PType::I32 => array + .as_slice::() + .iter() + .map(|&value| u64::from((value as u32) ^ (1_u32 << 31))) + .collect(), + PType::I64 => array + .as_slice::() + .iter() + .map(|&value| (value as u64) ^ (1_u64 << 63)) + .collect(), + ptype => vortex_bail!("RangePackedArray does not support {ptype}"), + }) +} + +#[expect( + clippy::cast_possible_truncation, + reason = "the encoded values retain the source integer word width" +)] +fn decode_primitive(data: &RangePackedData) -> VortexResult { + let ordered = data.codec.decode()?; + Ok(match data.ptype { + PType::U8 => PrimitiveArray::from_iter( + ordered + .into_iter() + .map(u8::try_from) + .collect::, _>>()?, + ), + PType::U16 => PrimitiveArray::from_iter( + ordered + .into_iter() + .map(u16::try_from) + .collect::, _>>()?, + ), + PType::U32 => PrimitiveArray::from_iter( + ordered + .into_iter() + .map(u32::try_from) + .collect::, _>>()?, + ), + PType::U64 => PrimitiveArray::from_iter(ordered), + PType::I8 => PrimitiveArray::from_iter( + ordered + .into_iter() + .map(|value| ((value as u8) ^ (1_u8 << 7)) as i8), + ), + PType::I16 => PrimitiveArray::from_iter( + ordered + .into_iter() + .map(|value| ((value as u16) ^ (1_u16 << 15)) as i16), + ), + PType::I32 => PrimitiveArray::from_iter( + ordered + .into_iter() + .map(|value| ((value as u32) ^ (1_u32 << 31)) as i32), + ), + PType::I64 => PrimitiveArray::from_iter( + ordered + .into_iter() + .map(|value| (value ^ (1_u64 << 63)) as i64), + ), + ptype => vortex_bail!("RangePackedArray does not support {ptype}"), + }) +} + +#[expect( + clippy::cast_possible_truncation, + reason = "the encoded value retains the source integer word width" +)] +fn ordered_scalar(value: u64, ptype: PType, nullability: Nullability) -> VortexResult { + Ok(match ptype { + PType::U8 => Scalar::primitive(u8::try_from(value)?, nullability), + PType::U16 => Scalar::primitive(u16::try_from(value)?, nullability), + PType::U32 => Scalar::primitive(u32::try_from(value)?, nullability), + PType::U64 => Scalar::primitive(value, nullability), + PType::I8 => Scalar::primitive(((value as u8) ^ (1_u8 << 7)) as i8, nullability), + PType::I16 => Scalar::primitive(((value as u16) ^ (1_u16 << 15)) as i16, nullability), + PType::I32 => Scalar::primitive(((value as u32) ^ (1_u32 << 31)) as i32, nullability), + PType::I64 => Scalar::primitive((value ^ (1_u64 << 63)) as i64, nullability), + ptype => vortex_bail!("RangePackedArray does not support {ptype}"), + }) +} + impl RangePackedCodec { pub fn encode(values: &[u64], block_len: usize) -> VortexResult { vortex_ensure!( @@ -87,26 +413,37 @@ impl RangePackedCodec { } pub fn decode(&self) -> VortexResult> { + self.decode_mapped(|value| value) + } + + pub fn decode_mapped(&self, map: F) -> VortexResult> + where + F: FnMut(u64) -> T, + { if self.bins.iter().all(|bin| bin.offset_bits <= 57) { - self.decode_with_width::() + self.decode_mapped_with_width::(map) } else { - self.decode_with_width::() + self.decode_mapped_with_width::(map) } } - fn decode_with_width(&self) -> VortexResult> { + fn decode_mapped_with_width(&self, mut map: F) -> VortexResult> + where + F: FnMut(u64) -> T, + { let mut values = Vec::with_capacity(self.len); let table = DecodeTable::new(&self.bins); for block_index in 0..self.block_count() { let payload = self.block_payload(block_index)?; let value_count = self.block_value_count(block_index); let output_start = values.len(); - decode_block::( + decode_block::( payload, value_count, self.symbol_width, &table, &mut values.spare_capacity_mut()[..value_count], + &mut map, )?; // SAFETY: decode_block initialized each output slot. unsafe { values.set_len(output_start + value_count) }; @@ -252,13 +589,17 @@ fn encode_block( clippy::cast_possible_truncation, reason = "the symbol mask never exceeds 63" )] -fn decode_block( +fn decode_block( payload: &[u8], value_count: usize, symbol_width: u8, table: &DecodeTable, - values: &mut [MaybeUninit], -) -> VortexResult<()> { + values: &mut [MaybeUninit], + map: &mut F, +) -> VortexResult<()> +where + F: FnMut(u64) -> T, +{ let layout = BlockLayout::new(value_count, symbol_width, payload.len())?; let offsets = &payload[layout.offsets_start..]; let mut offset_position = 0usize; @@ -280,7 +621,7 @@ fn decode_block( unsafe { values .get_unchecked_mut(base + lane) - .write(lower.wrapping_add(offset)) + .write(map(lower.wrapping_add(offset))) }; } } @@ -297,7 +638,7 @@ fn decode_block( unsafe { values .get_unchecked_mut(index) - .write(lower.wrapping_add(offset)) + .write(map(lower.wrapping_add(offset))) }; } let offset_bytes = offset_position.div_ceil(8); From 652d9fecc44ddfd0c78ad53ecb0ff7989116075f Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 17:28:04 +0100 Subject: [PATCH 23/78] bench: optimize nullable fixed-bin trees Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 34 ++ .../examples/float_compressor_bench.rs | 244 ++++++++++--- .../examples/support/range_packed.rs | 339 +++++++++++++++--- 3 files changed, 524 insertions(+), 93 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index d493a5a398e..e0a485b2182 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -751,6 +751,40 @@ GloVe uses 16.8 percent more bytes than Compact because Pco also uses `IntMult(1 The evidence does not justify a fused ALP and fixed-bin array. A composable integer child has lower implementation complexity. +### Nullable and Delta-heavy fixed-bin cases + +The optimized nullable prototype stores only valid values in the range stream. + +It stores canonical validity bits and one 32-bit rank checkpoint per 256 logical values. + +Scalar access tests one validity bit. It then scans at most four words to find the dense value index. + +The complete Euro subjectivity tree uses `OrderedFloat(RangePacked)`. + +| Input | Default bytes | Compact bytes | Fixed-bin bytes | Default decode MB/s | Compact decode MB/s | Fixed-bin decode MB/s | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Euro subjectivity | 14,234,326 | 6,860,457 | 7,270,094 | 16,822 | 2,475 | 2,487 | +| Euro polarity | 13,505,943 | 11,638,679 | 12,796,702 | 11,517 | 1,487 | 4,767 | +| Arade F8 | 6,380,852 | 5,617,973 | 6,257,548 | 17,944 | 4,371 | 8,350 | + +Euro subjectivity uses classic Pco bins without Delta. Fixed bins use 6.0 percent more bytes and match Compact decode. + +Its scalar access takes 215 ns. Default takes 657 ns, and Compact takes 19,045 ns. + +The complete subjectivity candidate encodes at 440 MB/s. + +Euro polarity mixes lookback Delta and no-Delta chunks. Fixed bins use 10.0 percent more bytes and decode 3.2 times faster. + +Arade F8 uses consecutive Delta in Pco. Fixed bins use 11.4 percent more bytes and decode 1.9 times faster. + +These results do not support Delta in the native candidate. + +Fixed bins retain much of Pco's size benefit without Delta. They also preserve bounded scalar access. + +A ten-percent Compact size allowance includes CMS, Food, Euro subjectivity, and Euro polarity. + +Arade F8 remains outside that threshold. GloVe also remains outside because `IntMult(10)` gives Pco another size advantage. + ### Pcodec corpus gap analysis The focused benchmark reads the first two million numeric rows from each source. diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index 8f5053ca05f..0cd4b816002 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -29,12 +29,14 @@ use vortex_alp::ALP; use vortex_alp::ALPArrayExt; use vortex_alp::ALPArraySlotsExt; use vortex_alp::ALPFloat; +use vortex_alp::alp_encode; use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::RecursiveCanonical; use vortex_array::VortexSessionExecute; use vortex_array::array_session; use vortex_array::arrays::ListArray; +use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::TemporalArray; use vortex_array::arrays::VarBinViewArray; @@ -47,6 +49,8 @@ use vortex_array::validity::Validity; use vortex_arrow::ArrowSessionExt; use vortex_block_residual::BlockResidual; use vortex_block_residual::BlockResidualArraySlotsExt; +use vortex_block_residual::OrderedFloat; +use vortex_block_residual::OrderedFloatArraySlotsExt; use vortex_btrblocks::BtrBlocksCompressor; use vortex_btrblocks::BtrBlocksCompressorBuilder; use vortex_btrblocks::SchemeExt; @@ -942,38 +946,97 @@ fn profile_range_packed( scalar_duration.as_secs_f64() * 1_000_000_000.0 / scalar_iterations as f64; let encoded_bytes = u64::try_from(codec.encoded_size())? + validity_bytes; println!( - "fixed-bin-checkpoint\t{dataset}\t{column}\t{path}\t{}\t{pco_bytes}\t{encoded_bytes}\t{}\t{encode_throughput:.1}\t{decode_throughput:.1}\t{scalar_nanoseconds:.1}", + "fixed-bin-checkpoint\t{dataset}\t{column}\t{path}\t{}\t{pco_bytes}\t{encoded_bytes}\t{}\t{}\t{}\t{encode_throughput:.1}\t{decode_throughput:.1}\t{scalar_nanoseconds:.1}", ordered.len(), codec.bin_count(), + codec.max_offset_bits(), + codec.offset_widths(), ); Ok(()) } -fn fixed_bin_alp_tree( +fn fixed_bin_float_tree( compact: &ArrayRef, session: &VortexSession, ) -> VortexResult> { - if compact.encoding_id().as_ref() != "vortex.alp" { - return Ok(None); + if compact.encoding_id().as_ref() == "vortex.alp" { + let alp = compact.as_::(); + if alp.encoded().encoding_id().as_ref() != "vortex.pco" { + return Ok(None); + } + let primitive = alp + .encoded() + .clone() + .execute::(&mut session.create_execution_ctx())?; + let primitive = primitive + .into_array() + .cast(alp.encoded().dtype().clone())? + .execute::(&mut session.create_execution_ctx())?; + let encoded = + RangePacked::from_primitive(primitive.as_view(), &mut session.create_execution_ctx())? + .into_array(); + return Ok(Some( + ALP::try_new(encoded, alp.exponents(), alp.patches())?.into_array(), + )); } - let alp = compact.as_::(); - if alp.encoded().encoding_id().as_ref() != "vortex.pco" { + + if compact.encoding_id().as_ref() != "vortex.pco" || !compact.dtype().is_float() { return Ok(None); } - let primitive = alp - .encoded() + let primitive = compact .clone() .execute::(&mut session.create_execution_ctx())?; - let primitive = primitive - .into_array() - .cast(alp.encoded().dtype().clone())? - .execute::(&mut session.create_execution_ctx())?; - let encoded = RangePacked::from_primitive(primitive.as_view())?.into_array(); + let primitive = + fill_float_nulls_with_first_valid(primitive, &mut session.create_execution_ctx())?; + let ordered = OrderedFloat::from_primitive(primitive.as_view())?; + let encoded = RangePacked::from_primitive( + ordered.encoded().as_::(), + &mut session.create_execution_ctx(), + )?; Ok(Some( - ALP::try_new(encoded, alp.exponents(), alp.patches())?.into_array(), + OrderedFloat::try_new(encoded.into_array(), primitive.ptype())?.into_array(), )) } +fn fill_float_nulls_with_first_valid( + primitive: PrimitiveArray, + ctx: &mut vortex_array::ExecutionCtx, +) -> VortexResult { + let validity = primitive.validity()?; + let mask = validity.execute_mask(primitive.len(), ctx)?; + if mask.all_true() { + return Ok(primitive); + } + let first_valid = mask.first(); + Ok(match primitive.ptype() { + PType::F32 => { + let values = primitive.as_slice::(); + let fill = first_valid.map_or(0.0, |index| values[index]); + PrimitiveArray::new::( + values + .iter() + .zip(mask.iter()) + .map(|(&value, valid)| if valid { value } else { fill }) + .collect::>(), + validity, + ) + } + PType::F64 => { + let values = primitive.as_slice::(); + let fill = first_valid.map_or(0.0, |index| values[index]); + PrimitiveArray::new::( + values + .iter() + .zip(mask.iter()) + .map(|(&value, valid)| if valid { value } else { fill }) + .collect::>(), + validity, + ) + } + ptype => return Err(vortex_err!("null fill does not support {ptype}")), + }) +} + fn measure_fixed_bin_tree( dataset: &str, column: &str, @@ -981,7 +1044,7 @@ fn measure_fixed_bin_tree( compact: &ArrayRef, session: &VortexSession, ) -> VortexResult<()> { - let Some(encoded) = fixed_bin_alp_tree(compact, session)? else { + let Some(encoded) = fixed_bin_float_tree(compact, session)? else { return Ok(()); }; vortex_ensure!( @@ -1016,7 +1079,7 @@ fn measure_fixed_bin_tree( let mut fused_durations = Vec::with_capacity(20); for _ in 0..20 { let start = Instant::now(); - black_box(decode_fixed_bin_alp_tree(&encoded, session)?); + black_box(decode_fixed_bin_float_tree(&encoded, session)?); fused_durations.push(start.elapsed()); } let fused_median = percentile(&mut fused_durations, 1, 2); @@ -1027,9 +1090,54 @@ fn measure_fixed_bin_tree( encoded.nbytes(), encoding_tree(&encoded), ); + + let mut encode_durations = Vec::with_capacity(5); + for _ in 0..5 { + let start = Instant::now(); + black_box(encode_fixed_bin_float_tree( + expected.as_::(), + compact, + session, + )?); + encode_durations.push(start.elapsed()); + } + let encode_median = percentile(&mut encode_durations, 1, 2); + let encode_throughput = expected.nbytes() as f64 / encode_median.as_secs_f64() / 1_000_000.0; + println!( + "fixed-bin-tree-encode\t{dataset}\t{column}\t{}\t{}\t{encode_throughput:.1}\t{}", + encoded.len(), + encoded.nbytes(), + encoding_tree(&encoded), + ); Ok(()) } +fn encode_fixed_bin_float_tree( + primitive: vortex_array::ArrayView<'_, Primitive>, + compact: &ArrayRef, + session: &VortexSession, +) -> VortexResult { + if compact.encoding_id().as_ref() == "vortex.alp" { + let alp = alp_encode(primitive, None, &mut session.create_execution_ctx())?; + let packed = RangePacked::from_primitive( + alp.encoded().as_::(), + &mut session.create_execution_ctx(), + )?; + return Ok(ALP::try_new(packed.into_array(), alp.exponents(), alp.patches())?.into_array()); + } + + let primitive = fill_float_nulls_with_first_valid( + primitive.into_owned(), + &mut session.create_execution_ctx(), + )?; + let ordered = OrderedFloat::from_primitive(primitive.as_view())?; + let packed = RangePacked::from_primitive( + ordered.encoded().as_::(), + &mut session.create_execution_ctx(), + )?; + Ok(OrderedFloat::try_new(packed.into_array(), primitive.ptype())?.into_array()) +} + fn measure_existing_tree( dataset: &str, column: &str, @@ -1070,23 +1178,27 @@ fn measure_scalar_access(encoded: &ArrayRef, session: &VortexSession) -> VortexR for _ in 0..scalar_iterations { scalar_index = scalar_index.wrapping_add(2_654_435_761) % encoded.len(); let scalar = encoded.execute_scalar(scalar_index, &mut scalar_ctx)?; - let bits = match encoded.dtype().as_ptype() { - PType::F32 => u64::from( - scalar + let bits = if scalar.is_null() { + u64::MAX + } else { + match encoded.dtype().as_ptype() { + PType::F32 => u64::from( + scalar + .as_primitive() + .typed_value::() + .ok_or_else(|| vortex_err!("tree produced an invalid f32 scalar"))? + .to_bits(), + ), + PType::F64 => scalar .as_primitive() - .typed_value::() - .ok_or_else(|| vortex_err!("tree produced a null scalar"))? + .typed_value::() + .ok_or_else(|| vortex_err!("tree produced an invalid f64 scalar"))? .to_bits(), - ), - PType::F64 => scalar - .as_primitive() - .typed_value::() - .ok_or_else(|| vortex_err!("tree produced a null scalar"))? - .to_bits(), - ptype => { - return Err(vortex_err!( - "tree scalar benchmark does not support {ptype}" - )); + ptype => { + return Err(vortex_err!( + "tree scalar benchmark does not support {ptype}" + )); + } } }; scalar_checksum ^= black_box(bits); @@ -1099,27 +1211,60 @@ fn measure_scalar_access(encoded: &ArrayRef, session: &VortexSession) -> VortexR clippy::cast_possible_truncation, reason = "the range-packed child retains the ALP integer word width" )] -fn decode_fixed_bin_alp_tree( +fn decode_fixed_bin_float_tree( encoded: &ArrayRef, session: &VortexSession, ) -> VortexResult { + if encoded.encoding_id().as_ref() == "vortex.ordered_float" { + let ordered = encoded.as_::(); + let packed = ordered.encoded().as_::(); + let validity = ordered.encoded().validity()?; + return Ok(match encoded.dtype().as_ptype() { + PType::F32 => { + let values = RangePacked::decode_mapped( + packed, + |ordered| f32::from_bits(unordered_u32(ordered as u32)), + 0.0, + )?; + PrimitiveArray::new::(values, validity) + } + PType::F64 => { + let values = RangePacked::decode_mapped( + packed, + |ordered| f64::from_bits(unordered_u64(ordered)), + 0.0, + )?; + PrimitiveArray::new::(values, validity) + } + ptype => return Err(vortex_err!("fixed-bin float tree does not support {ptype}")), + }); + } + let alp = encoded.as_::(); let packed = alp.encoded().as_::(); let validity = alp.encoded().validity()?; let exponents = alp.exponents(); let decoded = match encoded.dtype().as_ptype() { PType::F32 => { - let values = RangePacked::decode_mapped(packed, |ordered| { - let encoded = ((ordered as u32) ^ (1_u32 << 31)) as i32; - ::decode_single(encoded, exponents) - })?; + let values = RangePacked::decode_mapped( + packed, + |ordered| { + let encoded = ((ordered as u32) ^ (1_u32 << 31)) as i32; + ::decode_single(encoded, exponents) + }, + 0.0, + )?; PrimitiveArray::new::(values, validity) } PType::F64 => { - let values = RangePacked::decode_mapped(packed, |ordered| { - let encoded = (ordered ^ (1_u64 << 63)) as i64; - ::decode_single(encoded, exponents) - })?; + let values = RangePacked::decode_mapped( + packed, + |ordered| { + let encoded = (ordered ^ (1_u64 << 63)) as i64; + ::decode_single(encoded, exponents) + }, + 0.0, + )?; PrimitiveArray::new::(values, validity) } ptype => return Err(vortex_err!("fixed-bin ALP tree does not support {ptype}")), @@ -1131,6 +1276,22 @@ fn decode_fixed_bin_alp_tree( } } +fn unordered_u32(value: u32) -> u32 { + if value & (1_u32 << 31) == 0 { + !value + } else { + value ^ (1_u32 << 31) + } +} + +fn unordered_u64(value: u64) -> u64 { + if value & (1_u64 << 63) == 0 { + !value + } else { + value ^ (1_u64 << 63) + } +} + fn profile_pco_array( dataset: &str, column: &str, @@ -1449,10 +1610,11 @@ fn main() -> VortexResult<()> { "quotient-remainder-estimate\tdataset\tcolumn\tpath\tptype\tbase\tpco-child-bytes\tremainder-entropy\tmost-common-remainder-share\tquotient-bytes\tremainder-bytes\ttotal-bytes\tquotient-bitmap-bytes\tremainder-mode-bitmap-bytes\tbitmap-total-bytes\tquotient-encoding\tremainder-encoding" ); println!( - "fixed-bin-checkpoint\tdataset\tcolumn\tpath\trows\tpco-bytes\tfixed-bin-bytes\tbins\tencode-MB/s\tdecode-MB/s\tscalar-ns" + "fixed-bin-checkpoint\tdataset\tcolumn\tpath\trows\tpco-bytes\tfixed-bin-bytes\tbins\tmax-offset-bits\toffset-widths\tencode-MB/s\tdecode-MB/s\tscalar-ns" ); println!("fixed-bin-tree\tdataset\tcolumn\trows\tbytes\tdecode-MB/s\tscalar-ns\tencoding"); println!("fixed-bin-tree-fused\tdataset\tcolumn\trows\tbytes\tdecode-MB/s\tencoding"); + println!("fixed-bin-tree-encode\tdataset\tcolumn\trows\tbytes\tencode-MB/s\tencoding"); println!( "tree-throughput\tdataset\tcolumn\tconfig\trows\tbytes\tdecode-MB/s\tscalar-ns\tencoding" ); diff --git a/vortex-btrblocks/examples/support/range_packed.rs b/vortex-btrblocks/examples/support/range_packed.rs index dd767eee294..e6ac3405dc3 100644 --- a/vortex-btrblocks/examples/support/range_packed.rs +++ b/vortex-btrblocks/examples/support/range_packed.rs @@ -48,6 +48,7 @@ const BIN_TABLE_COST_BITS: f64 = 128.0; const SYMBOL_PADDING: usize = 8; const OFFSET_PADDING: usize = 15; const CHECKPOINT_INTERVAL: usize = 32; +const VALIDITY_CHECKPOINT_INTERVAL: usize = 256; const MAX_BLOCK_LEN: usize = 1_024; /// Fixed-width range identifiers with variable-width offsets and scalar checkpoints. @@ -78,6 +79,16 @@ pub struct RangePackedData { codec: RangePackedCodec, physical: ByteBuffer, ptype: PType, + validity: Validity, + dense_validity: DenseValidity, + len: usize, +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +struct DenseValidity { + words: Vec, + checkpoints: Vec, + valid_count: usize, } impl Display for RangePackedData { @@ -90,12 +101,19 @@ impl ArrayHash for RangePackedData { fn array_hash(&self, state: &mut H, _accuracy: EqMode) { self.codec.hash(state); self.ptype.hash(state); + self.validity.array_hash(state, _accuracy); + self.dense_validity.hash(state); + self.len.hash(state); } } impl ArrayEq for RangePackedData { fn array_eq(&self, other: &Self, _accuracy: EqMode) -> bool { - self.codec == other.codec && self.ptype == other.ptype + self.codec == other.codec + && self.ptype == other.ptype + && self.validity.array_eq(&other.validity, _accuracy) + && self.dense_validity == other.dense_validity + && self.len == other.len } } @@ -121,9 +139,17 @@ impl VTable for RangePacked { dtype.is_primitive() && dtype.as_ptype() == data.ptype, "RangePackedArray dtype differs from its physical type" ); - vortex_ensure!(len == data.codec.len, "RangePackedArray length differs"); vortex_ensure!( - data.physical.len() == data.codec.encoded_size(), + dtype.nullability() == data.validity.nullability(), + "RangePackedArray dtype differs from its validity" + ); + vortex_ensure!(len == data.len, "RangePackedArray length differs"); + vortex_ensure!( + data.codec.len == data.dense_validity.valid_count, + "RangePackedArray dense length differs" + ); + vortex_ensure!( + data.physical.len() == data.codec.encoded_size() + data.dense_validity.encoded_size(), "RangePackedArray physical size differs" ); Ok(()) @@ -198,41 +224,125 @@ impl OperationsVTable for RangePacked { index: usize, _ctx: &mut ExecutionCtx, ) -> VortexResult { + if !array.data().dense_validity.is_valid(index) { + return Ok(Scalar::null(array.dtype().clone())); + } + let dense_index = array.data().dense_validity.rank(index); ordered_scalar( - array.data().codec.scalar_at(index)?, + array.data().codec.scalar_at(dense_index)?, array.data().ptype, array.dtype().nullability(), ) } } +impl DenseValidity { + fn new( + len: usize, + valid_count: usize, + validity: impl Iterator, + ) -> VortexResult { + if valid_count == len { + return Ok(Self { + words: Vec::new(), + checkpoints: Vec::new(), + valid_count, + }); + } + + let mut words = vec![0_u64; len.div_ceil(64)]; + let mut observed_len = 0usize; + for (index, valid) in validity.enumerate() { + vortex_ensure!(index < len, "dense validity exceeds its length"); + if valid { + words[index / 64] |= 1_u64 << (index % 64); + } + observed_len += 1; + } + vortex_ensure!(observed_len == len, "dense validity is too short"); + + let words_per_checkpoint = VALIDITY_CHECKPOINT_INTERVAL / 64; + let mut checkpoints = Vec::with_capacity(words.len().div_ceil(words_per_checkpoint)); + let mut rank = 0usize; + for block in words.chunks(words_per_checkpoint) { + checkpoints.push(u32::try_from(rank)?); + rank += block + .iter() + .map(|word| word.count_ones() as usize) + .sum::(); + } + vortex_ensure!(rank == valid_count, "dense validity count differs"); + Ok(Self { + words, + checkpoints, + valid_count, + }) + } + + #[inline] + fn is_valid(&self, index: usize) -> bool { + self.words.is_empty() || self.words[index / 64] & (1_u64 << (index % 64)) != 0 + } + + #[inline] + fn rank(&self, index: usize) -> usize { + if self.words.is_empty() { + return index; + } + let word_index = index / 64; + let checkpoint_index = index / VALIDITY_CHECKPOINT_INTERVAL; + let words_per_checkpoint = VALIDITY_CHECKPOINT_INTERVAL / 64; + let word_start = checkpoint_index * words_per_checkpoint; + let mut rank = self.checkpoints[checkpoint_index] as usize; + for word in &self.words[word_start..word_index] { + rank += word.count_ones() as usize; + } + let bits_before = index % 64; + if bits_before != 0 { + let mask = (1_u64 << bits_before) - 1; + rank += (self.words[word_index] & mask).count_ones() as usize; + } + rank + } + + fn encoded_size(&self) -> usize { + self.words.len() * size_of::() + self.checkpoints.len() * size_of::() + } +} + impl ValidityVTable for RangePacked { fn validity(array: ArrayView<'_, RangePacked>) -> VortexResult { - if array.dtype().is_nullable() { - Ok(Validity::AllValid) - } else { - Ok(Validity::NonNullable) - } + Ok(array.data().validity.clone()) } } impl RangePacked { - pub fn from_primitive(array: ArrayView<'_, Primitive>) -> VortexResult { + pub fn from_primitive( + array: ArrayView<'_, Primitive>, + ctx: &mut ExecutionCtx, + ) -> VortexResult { vortex_ensure!( array.ptype().is_int(), "RangePackedArray needs integer values" ); - vortex_ensure!( - array.validity()?.definitely_no_nulls(), - "experimental RangePackedArray needs values without nulls" - ); + let validity = array.validity()?; + let mask = validity.execute_mask(array.len(), ctx)?; + let dense_validity = DenseValidity::new(array.len(), mask.true_count(), mask.iter())?; let ordered = ordered_primitive(array)?; - let codec = RangePackedCodec::encode(&ordered, MAX_BLOCK_LEN)?; - let physical = ByteBuffer::zeroed(codec.encoded_size()); + let dense_ordered = ordered + .into_iter() + .zip(mask.iter()) + .filter_map(|(value, valid)| valid.then_some(value)) + .collect::>(); + let codec = RangePackedCodec::encode(&dense_ordered, MAX_BLOCK_LEN)?; + let physical = ByteBuffer::zeroed(codec.encoded_size() + dense_validity.encoded_size()); let data = RangePackedData { codec, physical, ptype: array.ptype(), + validity, + dense_validity, + len: array.len(), }; Array::try_from_parts(ArrayParts::new( RangePacked, @@ -242,11 +352,32 @@ impl RangePacked { )) } - pub fn decode_mapped(array: ArrayView<'_, RangePacked>, map: F) -> VortexResult> + pub fn decode_mapped( + array: ArrayView<'_, RangePacked>, + map: F, + null_value: T, + ) -> VortexResult> where + T: Copy, F: FnMut(u64) -> T, { - array.data().codec.decode_mapped(map) + let dense_values = array.data().codec.decode_mapped(map)?; + if array.data().dense_validity.words.is_empty() { + return Ok(dense_values); + } + let mut dense = dense_values.into_iter(); + let mut values = Vec::with_capacity(array.len()); + for index in 0..array.len() { + values.push(if array.data().dense_validity.is_valid(index) { + dense + .next() + .ok_or_else(|| vortex_error::vortex_err!("dense values are too short"))? + } else { + null_value + }); + } + vortex_ensure!(dense.next().is_none(), "dense values are too long"); + Ok(values) } } @@ -297,46 +428,75 @@ fn ordered_primitive(array: ArrayView<'_, Primitive>) -> VortexResult> reason = "the encoded values retain the source integer word width" )] fn decode_primitive(data: &RangePackedData) -> VortexResult { - let ordered = data.codec.decode()?; + let dense_ordered = data.codec.decode()?; + let ordered = if data.dense_validity.words.is_empty() { + dense_ordered + } else { + let mut dense = dense_ordered.into_iter(); + let mut values = Vec::with_capacity(data.len); + for index in 0..data.len { + values.push(if data.dense_validity.is_valid(index) { + dense + .next() + .ok_or_else(|| vortex_error::vortex_err!("dense values are too short"))? + } else { + 0 + }); + } + vortex_ensure!(dense.next().is_none(), "dense values are too long"); + values + }; + let validity = data.validity.clone(); Ok(match data.ptype { - PType::U8 => PrimitiveArray::from_iter( + PType::U8 => PrimitiveArray::new::( ordered .into_iter() .map(u8::try_from) .collect::, _>>()?, + validity, ), - PType::U16 => PrimitiveArray::from_iter( + PType::U16 => PrimitiveArray::new::( ordered .into_iter() .map(u16::try_from) .collect::, _>>()?, + validity, ), - PType::U32 => PrimitiveArray::from_iter( + PType::U32 => PrimitiveArray::new::( ordered .into_iter() .map(u32::try_from) .collect::, _>>()?, + validity, ), - PType::U64 => PrimitiveArray::from_iter(ordered), - PType::I8 => PrimitiveArray::from_iter( + PType::U64 => PrimitiveArray::new::(ordered, validity), + PType::I8 => PrimitiveArray::new::( ordered .into_iter() - .map(|value| ((value as u8) ^ (1_u8 << 7)) as i8), + .map(|value| ((value as u8) ^ (1_u8 << 7)) as i8) + .collect::>(), + validity, ), - PType::I16 => PrimitiveArray::from_iter( + PType::I16 => PrimitiveArray::new::( ordered .into_iter() - .map(|value| ((value as u16) ^ (1_u16 << 15)) as i16), + .map(|value| ((value as u16) ^ (1_u16 << 15)) as i16) + .collect::>(), + validity, ), - PType::I32 => PrimitiveArray::from_iter( + PType::I32 => PrimitiveArray::new::( ordered .into_iter() - .map(|value| ((value as u32) ^ (1_u32 << 31)) as i32), + .map(|value| ((value as u32) ^ (1_u32 << 31)) as i32) + .collect::>(), + validity, ), - PType::I64 => PrimitiveArray::from_iter( + PType::I64 => PrimitiveArray::new::( ordered .into_iter() - .map(|value| (value ^ (1_u64 << 63)) as i64), + .map(|value| (value ^ (1_u64 << 63)) as i64) + .collect::>(), + validity, ), ptype => vortex_bail!("RangePackedArray does not support {ptype}"), }) @@ -420,14 +580,20 @@ impl RangePackedCodec { where F: FnMut(u64) -> T, { - if self.bins.iter().all(|bin| bin.offset_bits <= 57) { - self.decode_mapped_with_width::(map) + let max_offset_bits = self.max_offset_bits(); + if max_offset_bits <= 40 { + self.decode_mapped_with_width::(map) + } else if max_offset_bits <= 57 { + self.decode_mapped_with_width::(map) } else { - self.decode_mapped_with_width::(map) + self.decode_mapped_with_width::(map) } } - fn decode_mapped_with_width(&self, mut map: F) -> VortexResult> + fn decode_mapped_with_width( + &self, + mut map: F, + ) -> VortexResult> where F: FnMut(u64) -> T, { @@ -437,7 +603,7 @@ impl RangePackedCodec { let payload = self.block_payload(block_index)?; let value_count = self.block_value_count(block_index); let output_start = values.len(); - decode_block::( + decode_block::( payload, value_count, self.symbol_width, @@ -498,6 +664,22 @@ impl RangePackedCodec { self.bins.len() } + pub fn max_offset_bits(&self) -> u8 { + self.bins + .iter() + .map(|bin| bin.offset_bits) + .max() + .unwrap_or_default() + } + + pub fn offset_widths(&self) -> String { + self.bins + .iter() + .map(|bin| bin.offset_bits.to_string()) + .collect::>() + .join(",") + } + fn block_count(&self) -> usize { self.len.div_ceil(self.block_len) } @@ -589,7 +771,7 @@ fn encode_block( clippy::cast_possible_truncation, reason = "the symbol mask never exceeds 63" )] -fn decode_block( +fn decode_block( payload: &[u8], value_count: usize, symbol_width: u8, @@ -608,21 +790,48 @@ where for base in (0..full_len).step_by(8) { let symbol_position = base * usize::from(symbol_width); let packed = read_bits_padded(payload, symbol_position, symbol_width * 8); - for lane in 0..8 { - let symbol = ((packed >> (lane * usize::from(symbol_width))) & symbol_mask) as usize; - debug_assert!(symbol < table.bin_count); - // SAFETY: A power-of-two bin count makes every symbol code valid. - let width = unsafe { *table.widths.get_unchecked(symbol) }; - let offset = read_offset::(offsets, offset_position, width); - offset_position += usize::from(width); - // SAFETY: A power-of-two bin count makes every symbol code valid. - let lower = unsafe { *table.lowers.get_unchecked(symbol) }; - // SAFETY: This lane is within a complete eight-value output batch. - unsafe { - values - .get_unchecked_mut(base + lane) - .write(map(lower.wrapping_add(offset))) - }; + if PARALLEL { + let mut widths = [0_u8; 8]; + let mut lowers = [0_u64; 8]; + let mut positions = [0_usize; 8]; + for lane in 0..8 { + let symbol = + ((packed >> (lane * usize::from(symbol_width))) & symbol_mask) as usize; + debug_assert!(symbol < table.bin_count); + // SAFETY: A power-of-two bin count makes every symbol code valid. + widths[lane] = unsafe { *table.widths.get_unchecked(symbol) }; + // SAFETY: A power-of-two bin count makes every symbol code valid. + lowers[lane] = unsafe { *table.lowers.get_unchecked(symbol) }; + positions[lane] = offset_position; + offset_position += usize::from(widths[lane]); + } + for lane in 0..8 { + let offset = read_offset::(offsets, positions[lane], widths[lane]); + // SAFETY: This lane is within a complete eight-value output batch. + unsafe { + values + .get_unchecked_mut(base + lane) + .write(map(lowers[lane].wrapping_add(offset))) + }; + } + } else { + for lane in 0..8 { + let symbol = + ((packed >> (lane * usize::from(symbol_width))) & symbol_mask) as usize; + debug_assert!(symbol < table.bin_count); + // SAFETY: A power-of-two bin count makes every symbol code valid. + let width = unsafe { *table.widths.get_unchecked(symbol) }; + let offset = read_offset::(offsets, offset_position, width); + offset_position += usize::from(width); + // SAFETY: A power-of-two bin count makes every symbol code valid. + let lower = unsafe { *table.lowers.get_unchecked(symbol) }; + // SAFETY: This lane is within a complete eight-value output batch. + unsafe { + values + .get_unchecked_mut(base + lane) + .write(map(lower.wrapping_add(offset))) + }; + } } } for index in full_len..value_count { @@ -911,8 +1120,14 @@ impl BitWriter { #[cfg(test)] mod tests { + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::array_session; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::assert_arrays_eq; use vortex_error::VortexResult; + use super::RangePacked; use super::RangePackedCodec; #[test] @@ -943,4 +1158,24 @@ mod tests { } Ok(()) } + + #[test] + fn nullable_dense_roundtrip_and_rank_boundaries() -> VortexResult<()> { + let expected = PrimitiveArray::from_option_iter((0_i64..600).map(|value| { + (!matches!(value, 0 | 1 | 63 | 64 | 255 | 256 | 257 | 511)).then_some(value - 300) + })); + let session = array_session(); + let mut ctx = session.create_execution_ctx(); + let encoded = RangePacked::from_primitive(expected.as_view(), &mut ctx)?; + assert_arrays_eq!(encoded, expected, &mut ctx); + for index in [0, 1, 2, 63, 64, 65, 255, 256, 257, 258, 511, 512, 599] { + let actual = encoded.execute_scalar(index, &mut ctx)?; + let expected_scalar = expected + .clone() + .into_array() + .execute_scalar(index, &mut ctx)?; + assert_eq!(actual, expected_scalar); + } + Ok(()) + } } From d417de4048fbe72e0c117dc0efc8cdc99f1ca5a9 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 17:32:22 +0100 Subject: [PATCH 24/78] refactor: move RangePacked prototype into encoding crate Signed-off-by: Will Manning --- Cargo.lock | 11 ++++++++ Cargo.toml | 2 ++ encodings/range-packed/Cargo.toml | 26 +++++++++++++++++++ .../range-packed/src/lib.rs | 2 +- vortex-btrblocks/Cargo.toml | 1 + .../examples/float_compressor_bench.rs | 7 ++--- 6 files changed, 43 insertions(+), 6 deletions(-) create mode 100644 encodings/range-packed/Cargo.toml rename vortex-btrblocks/examples/support/range_packed.rs => encodings/range-packed/src/lib.rs (99%) diff --git a/Cargo.lock b/Cargo.lock index ed8f3a51940..0ee6d61818a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9793,6 +9793,7 @@ dependencies = [ "vortex-mask", "vortex-onpair", "vortex-pco", + "vortex-range-packed", "vortex-runend", "vortex-sequence", "vortex-session", @@ -10542,6 +10543,16 @@ dependencies = [ "vortex-python-abi", ] +[[package]] +name = "vortex-range-packed" +version = "0.1.0" +dependencies = [ + "vortex-array", + "vortex-buffer", + "vortex-error", + "vortex-session", +] + [[package]] name = "vortex-row" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 4e4791a9a48..c1dda00f0a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,6 +64,7 @@ members = [ "encodings/onpair", "encodings/block-residual", "encodings/float-quant", + "encodings/range-packed", # Benchmarks "benchmarks/bench-support", "benchmarks/lance-bench", @@ -325,6 +326,7 @@ vortex-parquet-variant = { version = "0.1.0", path = "./encodings/parquet-varian vortex-pco = { version = "0.1.0", path = "./encodings/pco", default-features = false } vortex-block-residual = { version = "0.1.0", path = "./encodings/block-residual", default-features = false } vortex-float-quant = { version = "0.1.0", path = "./encodings/float-quant", default-features = false } +vortex-range-packed = { version = "0.1.0", path = "./encodings/range-packed", default-features = false } vortex-proto = { version = "0.1.0", path = "./vortex-proto", default-features = false } vortex-row = { version = "0.1.0", path = "./vortex-row", default-features = false } vortex-runend = { version = "0.1.0", path = "./encodings/runend", default-features = false } diff --git a/encodings/range-packed/Cargo.toml b/encodings/range-packed/Cargo.toml new file mode 100644 index 00000000000..cd71c593c4a --- /dev/null +++ b/encodings/range-packed/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "vortex-range-packed" +authors = { workspace = true } +categories = { workspace = true } +description = "Vortex fixed-bin range-packed integer array encoding" +edition = { workspace = true } +homepage = { workspace = true } +include = { workspace = true } +keywords = { workspace = true } +license = { workspace = true } +readme = { workspace = true } +repository = { workspace = true } +rust-version = { workspace = true } +version = { workspace = true } + +[dependencies] +vortex-array = { workspace = true } +vortex-buffer = { workspace = true } +vortex-error = { workspace = true } +vortex-session = { workspace = true } + +[dev-dependencies] +vortex-array = { workspace = true, features = ["_test-harness"] } + +[lints] +workspace = true diff --git a/vortex-btrblocks/examples/support/range_packed.rs b/encodings/range-packed/src/lib.rs similarity index 99% rename from vortex-btrblocks/examples/support/range_packed.rs rename to encodings/range-packed/src/lib.rs index e6ac3405dc3..f5ee95402b9 100644 --- a/vortex-btrblocks/examples/support/range_packed.rs +++ b/encodings/range-packed/src/lib.rs @@ -68,7 +68,7 @@ struct Bin { offset_bits: u8, } -/// Experimental Vortex wrapper for complete-tree benchmark measurements. +/// Fixed-bin range packing with bounded scalar access. pub type RangePackedArray = Array; #[derive(Clone, Debug)] diff --git a/vortex-btrblocks/Cargo.toml b/vortex-btrblocks/Cargo.toml index 33e55e4838c..d7991eaab78 100644 --- a/vortex-btrblocks/Cargo.toml +++ b/vortex-btrblocks/Cargo.toml @@ -30,6 +30,7 @@ vortex-decimal-byte-parts = { workspace = true } vortex-error = { workspace = true } vortex-fastlanes = { workspace = true } vortex-float-quant = { workspace = true } +vortex-range-packed = { workspace = true } vortex-fsst = { workspace = true } vortex-onpair = { workspace = true, optional = true } vortex-pco = { workspace = true, optional = true } diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index 0cd4b816002..4585208578b 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -63,11 +63,8 @@ use vortex_error::vortex_err; use vortex_session::VortexSession; use vortex_utils::aliases::hash_map::HashMap; -#[path = "support/range_packed.rs"] -mod range_packed; - -use range_packed::RangePacked; -use range_packed::RangePackedCodec; +use vortex_range_packed::RangePacked; +use vortex_range_packed::RangePackedCodec; const DEFAULT_ROW_COUNT: usize = 2_000_000; const CALIFORNIA_COLUMNS: [&str; 9] = [ From 0c663c2ad1670bd4c479a6dbeca79d88caeba2f8 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 17:44:23 +0100 Subject: [PATCH 25/78] feat: add serialized RangePacked array Signed-off-by: Will Manning --- Cargo.lock | 3 + encodings/range-packed/Cargo.toml | 1 + encodings/range-packed/src/array.rs | 829 ++++++++++++++++ encodings/range-packed/src/lib.rs | 881 +++++++----------- .../examples/float_compressor_bench.rs | 5 +- vortex-file/Cargo.toml | 1 + vortex-file/src/lib.rs | 1 + vortex/Cargo.toml | 1 + vortex/src/lib.rs | 5 + 9 files changed, 1166 insertions(+), 561 deletions(-) create mode 100644 encodings/range-packed/src/array.rs diff --git a/Cargo.lock b/Cargo.lock index 0ee6d61818a..a1b7e02d218 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9557,6 +9557,7 @@ dependencies = [ "vortex-metrics", "vortex-pco", "vortex-proto", + "vortex-range-packed", "vortex-runend", "vortex-scan", "vortex-sequence", @@ -10190,6 +10191,7 @@ dependencies = [ "vortex-metrics", "vortex-onpair", "vortex-pco", + "vortex-range-packed", "vortex-runend", "vortex-scan", "vortex-sequence", @@ -10547,6 +10549,7 @@ dependencies = [ name = "vortex-range-packed" version = "0.1.0" dependencies = [ + "rstest", "vortex-array", "vortex-buffer", "vortex-error", diff --git a/encodings/range-packed/Cargo.toml b/encodings/range-packed/Cargo.toml index cd71c593c4a..2264749db36 100644 --- a/encodings/range-packed/Cargo.toml +++ b/encodings/range-packed/Cargo.toml @@ -20,6 +20,7 @@ vortex-error = { workspace = true } vortex-session = { workspace = true } [dev-dependencies] +rstest = { workspace = true } vortex-array = { workspace = true, features = ["_test-harness"] } [lints] diff --git a/encodings/range-packed/src/array.rs b/encodings/range-packed/src/array.rs new file mode 100644 index 00000000000..5963c110c12 --- /dev/null +++ b/encodings/range-packed/src/array.rs @@ -0,0 +1,829 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::fmt::Display; +use std::fmt::Formatter; +use std::hash::Hash; +use std::hash::Hasher; +use std::ops::Range; + +use vortex_array::Array; +use vortex_array::ArrayEq; +use vortex_array::ArrayHash; +use vortex_array::ArrayId; +use vortex_array::ArrayParts; +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::EqMode; +use vortex_array::ExecutionCtx; +use vortex_array::ExecutionResult; +use vortex_array::IntoArray; +use vortex_array::TypedArrayRef; +use vortex_array::array_slots; +use vortex_array::arrays::Bool; +use vortex_array::arrays::BoolArray; +use vortex_array::arrays::Primitive; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::bool::BoolArrayExt; +use vortex_array::arrays::slice::SliceReduce; +use vortex_array::arrays::slice::SliceReduceAdaptor; +use vortex_array::buffer::BufferHandle; +use vortex_array::dtype::DType; +use vortex_array::dtype::Nullability; +use vortex_array::dtype::Nullability::NonNullable; +use vortex_array::dtype::PType; +use vortex_array::optimizer::rules::ParentRuleSet; +use vortex_array::scalar::Scalar; +use vortex_array::serde::ArrayChildren; +use vortex_array::validity::Validity; +use vortex_array::vtable::OperationsVTable; +use vortex_array::vtable::VTable; +use vortex_array::vtable::ValidityVTable; +use vortex_array::vtable::child_to_validity; +use vortex_error::VortexResult; +use vortex_error::vortex_bail; +use vortex_error::vortex_ensure; +use vortex_error::vortex_panic; +use vortex_session::VortexSession; +use vortex_session::registry::CachedId; + +use crate::BinTableView; +use crate::MAX_BLOCK_LEN; +use crate::RangePackedCodec; +use crate::RangePackedDecoder; +use crate::VALIDITY_CHECKPOINT_INTERVAL; +use crate::low_mask; + +const METADATA_VERSION: u8 = 1; +const METADATA_LEN: usize = 34; + +/// Fixed-bin range packing with bounded scalar access. +pub type RangePackedArray = Array; + +#[array_slots(RangePacked)] +pub struct RangePackedSlots { + #[slot(0)] + pub bin_lowers: ArrayRef, + #[slot(1)] + pub offset_widths: ArrayRef, + #[slot(2)] + pub block_offsets: ArrayRef, + #[slot(3)] + pub payload: ArrayRef, + #[slot(4)] + pub validity_bits: Option, + #[slot(5)] + pub rank_checkpoints: Option, +} + +#[derive(Clone, Debug)] +pub struct RangePackedData { + unsliced_len: usize, + dense_len: usize, + slice_start: usize, + slice_stop: usize, + payload_len: usize, + symbol_width: u8, +} + +impl Display for RangePackedData { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "bins: {}, slice: {}..{}", + self.bin_count(), + self.slice_start, + self.slice_stop + ) + } +} + +impl ArrayHash for RangePackedData { + fn array_hash(&self, state: &mut H, _accuracy: EqMode) { + self.unsliced_len.hash(state); + self.dense_len.hash(state); + self.slice_start.hash(state); + self.slice_stop.hash(state); + self.payload_len.hash(state); + self.symbol_width.hash(state); + } +} + +impl ArrayEq for RangePackedData { + fn array_eq(&self, other: &Self, _accuracy: EqMode) -> bool { + self.unsliced_len == other.unsliced_len + && self.dense_len == other.dense_len + && self.slice_start == other.slice_start + && self.slice_stop == other.slice_stop + && self.payload_len == other.payload_len + && self.symbol_width == other.symbol_width + } +} + +#[derive(Clone, Debug)] +pub struct RangePacked; + +impl VTable for RangePacked { + type TypedArrayData = RangePackedData; + type OperationsVTable = Self; + type ValidityVTable = Self; + + fn id(&self) -> ArrayId { + static ID: CachedId = CachedId::new("vortex.range_packed"); + *ID + } + + fn validate( + &self, + data: &Self::TypedArrayData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + data.validate(dtype, len, RangePackedSlotsView::from_slots(slots)) + } + + fn nbuffers(_array: ArrayView<'_, Self>) -> usize { + 0 + } + + fn buffer(_array: ArrayView<'_, Self>, index: usize) -> BufferHandle { + vortex_panic!("RangePackedArray buffer index {index} is invalid") + } + + fn buffer_name(_array: ArrayView<'_, Self>, index: usize) -> Option { + vortex_panic!("RangePackedArray buffer index {index} is invalid") + } + + fn with_buffers( + &self, + array: ArrayView<'_, Self>, + buffers: &[BufferHandle], + ) -> VortexResult> { + vortex_array::vtable::with_empty_buffers(self, array, buffers) + } + + fn serialize( + array: ArrayView<'_, Self>, + _session: &VortexSession, + ) -> VortexResult>> { + Ok(Some(RangePackedMetadata::from_data(array.data())?.encode())) + } + + fn deserialize( + &self, + dtype: &DType, + len: usize, + metadata: &[u8], + _buffers: &[BufferHandle], + children: &dyn ArrayChildren, + _session: &VortexSession, + ) -> VortexResult> { + let metadata = RangePackedMetadata::decode(metadata)?; + let unsliced_len = usize::try_from(metadata.unsliced_len)?; + let dense_len = usize::try_from(metadata.dense_len)?; + let slice_start = usize::try_from(metadata.slice_start)?; + let slice_stop = slice_start + .checked_add(len) + .ok_or_else(|| vortex_error::vortex_err!("RangePacked slice length overflows"))?; + let payload_len = usize::try_from(metadata.payload_len)?; + let bin_count = bin_count(dense_len, metadata.symbol_width)?; + let block_count = dense_len.div_ceil(MAX_BLOCK_LEN); + vortex_ensure!( + matches!(children.len(), 4 | 6), + "RangePackedArray expects four or six children" + ); + let bin_lowers = children.get(0, &primitive_dtype(PType::U64), bin_count)?; + let offset_widths = children.get(1, &primitive_dtype(PType::U8), bin_count)?; + let block_offsets = children.get(2, &primitive_dtype(PType::U32), block_count + 1)?; + let payload = children.get(3, &primitive_dtype(PType::U8), payload_len)?; + let (validity_bits, rank_checkpoints) = if children.len() == 6 { + ( + Some(children.get(4, &DType::Bool(NonNullable), unsliced_len)?), + Some(children.get( + 5, + &primitive_dtype(PType::U32), + unsliced_len.div_ceil(VALIDITY_CHECKPOINT_INTERVAL), + )?), + ) + } else { + (None, None) + }; + let data = RangePackedData { + unsliced_len, + dense_len, + slice_start, + slice_stop, + payload_len, + symbol_width: metadata.symbol_width, + }; + let slots = RangePackedSlots { + bin_lowers, + offset_widths, + block_offsets, + payload, + validity_bits, + rank_checkpoints, + }; + Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots.into_slots())) + } + + fn slot_name(_array: ArrayView<'_, Self>, index: usize) -> String { + RangePackedSlots::NAMES[index].to_string() + } + + fn execute(array: Array, _ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionResult::done( + decode_primitive(array.as_view())?.into_array(), + )) + } + + fn reduce_parent( + array: ArrayView<'_, Self>, + parent: &ArrayRef, + child_idx: usize, + ) -> VortexResult> { + RULES.evaluate(array, parent, child_idx) + } +} + +impl OperationsVTable for RangePacked { + fn scalar_at( + array: ArrayView<'_, RangePacked>, + index: usize, + _ctx: &mut ExecutionCtx, + ) -> VortexResult { + let global_index = array.data().slice_start + index; + if !is_valid(array, global_index) { + return Ok(Scalar::null(array.dtype().clone())); + } + let dense_index = dense_rank(array, global_index)?; + ordered_scalar( + with_decoder(array, |decoder| decoder.scalar_at(dense_index))?, + array.dtype().as_ptype(), + array.dtype().nullability(), + ) + } +} + +impl ValidityVTable for RangePacked { + fn validity(array: ArrayView<'_, RangePacked>) -> VortexResult { + array + .unsliced_validity() + .slice(array.data().slice_start..array.data().slice_stop) + } +} + +impl SliceReduce for RangePacked { + fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { + let data = array.data().slice(range); + Ok(Some( + Array::try_from_parts( + ArrayParts::new(RangePacked, array.dtype().clone(), data.len(), data) + .with_slots(array.slots().iter().cloned().collect()), + )? + .into_array(), + )) + } +} + +static RULES: ParentRuleSet = + ParentRuleSet::new(&[ParentRuleSet::lift(&SliceReduceAdaptor(RangePacked))]); + +pub trait RangePackedArrayExt: TypedArrayRef + RangePackedArraySlotsExt { + fn unsliced_validity(&self) -> Validity { + child_to_validity( + self.as_ref().slots()[RangePackedSlots::VALIDITY_BITS].as_ref(), + self.as_ref().dtype().nullability(), + ) + } +} + +impl> RangePackedArrayExt for T {} + +impl RangePacked { + pub fn from_primitive( + array: ArrayView<'_, Primitive>, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + vortex_ensure!( + array.ptype().is_int(), + "RangePackedArray needs integer values" + ); + let validity = array.validity()?; + let mask = validity.execute_mask(array.len(), ctx)?; + let dense_len = mask.true_count(); + let dense_ordered = ordered_primitive(array)? + .into_iter() + .zip(mask.iter()) + .filter_map(|(value, valid)| valid.then_some(value)) + .collect::>(); + let codec = RangePackedCodec::encode(&dense_ordered, MAX_BLOCK_LEN)?; + let actual_nulls = dense_len != array.len(); + let validity_bits = actual_nulls.then(|| BoolArray::from_iter(mask.iter()).into_array()); + let rank_checkpoints = actual_nulls + .then(|| rank_checkpoints(mask.iter(), dense_len)) + .transpose()? + .map(|ranks| PrimitiveArray::from_iter(ranks).into_array()); + let data = RangePackedData { + unsliced_len: array.len(), + dense_len, + slice_start: 0, + slice_stop: array.len(), + payload_len: codec.payload.len(), + symbol_width: codec.symbol_width, + }; + let slots = RangePackedSlots { + bin_lowers: PrimitiveArray::from_iter(codec.bins.iter().map(|bin| bin.lower)) + .into_array(), + offset_widths: PrimitiveArray::from_iter(codec.bins.iter().map(|bin| bin.offset_bits)) + .into_array(), + block_offsets: PrimitiveArray::from_iter(codec.block_offsets).into_array(), + payload: PrimitiveArray::from_iter(codec.payload).into_array(), + validity_bits, + rank_checkpoints, + }; + Array::try_from_parts( + ArrayParts::new( + RangePacked, + DType::Primitive(array.ptype(), validity.nullability()), + array.len(), + data, + ) + .with_slots(slots.into_slots()), + ) + } + + pub fn decode_mapped( + array: ArrayView<'_, RangePacked>, + map: F, + null_value: T, + ) -> VortexResult> + where + T: Copy, + F: FnMut(u64) -> T, + { + let dense_start = dense_rank(array, array.data().slice_start)?; + let dense_stop = dense_rank(array, array.data().slice_stop)?; + let dense_values = with_decoder(array, |decoder| { + decoder.decode_mapped_range(dense_start..dense_stop, map) + })?; + let Some(validity_bits) = array.validity_bits() else { + return Ok(dense_values); + }; + let validity_bits = validity_bits.as_::(); + let bits = validity_bits.bit_buffer_view(); + let mut dense = dense_values.into_iter(); + let mut values = Vec::with_capacity(array.len()); + for valid in bits + .slice(array.data().slice_start..array.data().slice_stop) + .iter() + { + values.push(if valid { + dense + .next() + .ok_or_else(|| vortex_error::vortex_err!("dense values are too short"))? + } else { + null_value + }); + } + vortex_ensure!(dense.next().is_none(), "dense values are too long"); + Ok(values) + } +} + +impl RangePackedData { + fn validate( + &self, + dtype: &DType, + len: usize, + slots: RangePackedSlotsView<'_>, + ) -> VortexResult<()> { + vortex_ensure!(dtype.is_int(), "RangePackedArray requires an integer dtype"); + vortex_ensure!( + self.slice_start <= self.slice_stop && self.slice_stop <= self.unsliced_len, + "RangePacked slice exceeds its source length" + ); + vortex_ensure!(len == self.len(), "RangePacked slice length is invalid"); + vortex_ensure!( + self.dense_len <= self.unsliced_len, + "RangePacked dense length exceeds its source length" + ); + let bin_count = self.bin_count(); + validate_primitive_child(slots.bin_lowers, PType::U64, bin_count, "bin lowers")?; + validate_primitive_child(slots.offset_widths, PType::U8, bin_count, "offset widths")?; + validate_primitive_child( + slots.block_offsets, + PType::U32, + self.dense_len.div_ceil(MAX_BLOCK_LEN) + 1, + "block offsets", + )?; + validate_primitive_child(slots.payload, PType::U8, self.payload_len, "payload")?; + vortex_ensure!( + slots.validity_bits.is_some() == slots.rank_checkpoints.is_some(), + "RangePacked validity and rank children must occur together" + ); + if let (Some(validity_bits), Some(ranks)) = (slots.validity_bits, slots.rank_checkpoints) { + vortex_ensure!( + dtype.is_nullable(), + "RangePacked validity requires a nullable dtype" + ); + vortex_ensure!( + validity_bits.is::() + && validity_bits.dtype() == &DType::Bool(NonNullable) + && validity_bits.len() == self.unsliced_len, + "RangePacked validity child is invalid" + ); + validate_primitive_child( + ranks, + PType::U32, + self.unsliced_len.div_ceil(VALIDITY_CHECKPOINT_INTERVAL), + "rank checkpoints", + )?; + validate_ranks( + validity_bits.as_::(), + ranks.as_::().as_slice::(), + self.dense_len, + )?; + } else { + vortex_ensure!( + self.dense_len == self.unsliced_len, + "RangePacked without validity must store every value" + ); + } + validate_bins( + slots.bin_lowers.as_::().as_slice::(), + slots.offset_widths.as_::().as_slice::(), + dtype.as_ptype(), + )?; + validate_block_offsets( + slots.block_offsets.as_::().as_slice::(), + self.payload_len, + )?; + Ok(()) + } + + fn bin_count(&self) -> usize { + if self.dense_len == 0 { + 0 + } else { + 1_usize << self.symbol_width + } + } + + fn len(&self) -> usize { + self.slice_stop - self.slice_start + } + + fn slice(&self, range: Range) -> Self { + Self { + slice_start: self.slice_start + range.start, + slice_stop: self.slice_start + range.end, + ..self.clone() + } + } +} + +fn validate_primitive_child( + child: &ArrayRef, + ptype: PType, + len: usize, + name: &str, +) -> VortexResult<()> { + vortex_ensure!( + child.is::() && child.dtype() == &primitive_dtype(ptype) && child.len() == len, + "RangePacked {name} child is invalid" + ); + Ok(()) +} + +fn validate_bins(lowers: &[u64], widths: &[u8], ptype: PType) -> VortexResult<()> { + let word_width = u8::try_from(ptype.bit_width())?; + for (index, (&lower, &width)) in lowers.iter().zip(widths).enumerate() { + vortex_ensure!(width <= word_width, "RangePacked offset width is invalid"); + if word_width < 64 { + vortex_ensure!( + lower <= low_mask(word_width), + "RangePacked bin lower exceeds its integer width" + ); + } + if let Some(&next) = lowers.get(index + 1) { + vortex_ensure!(next > lower, "RangePacked bin lowers are not ordered"); + vortex_ensure!( + next - lower - 1 <= low_mask(width), + "RangePacked bin does not cover the next boundary" + ); + } + } + Ok(()) +} + +fn validate_block_offsets(offsets: &[u32], payload_len: usize) -> VortexResult<()> { + vortex_ensure!( + offsets.first() == Some(&0), + "RangePacked offsets must start at zero" + ); + vortex_ensure!( + offsets.last().copied().map(usize::try_from).transpose()? == Some(payload_len), + "RangePacked offsets must end at the payload length" + ); + vortex_ensure!( + offsets.windows(2).all(|pair| pair[0] <= pair[1]), + "RangePacked offsets are not ordered" + ); + Ok(()) +} + +fn validate_ranks(bits: ArrayView<'_, Bool>, ranks: &[u32], dense_len: usize) -> VortexResult<()> { + let bits = bits.bit_buffer_view(); + let mut rank = 0usize; + for (checkpoint, &stored) in ranks.iter().enumerate() { + vortex_ensure!( + usize::try_from(stored)? == rank, + "RangePacked rank checkpoint is invalid" + ); + let start = checkpoint * VALIDITY_CHECKPOINT_INTERVAL; + let stop = (start + VALIDITY_CHECKPOINT_INTERVAL).min(bits.len()); + rank += bits.slice(start..stop).true_count(); + } + vortex_ensure!(rank == dense_len, "RangePacked dense length is invalid"); + Ok(()) +} + +fn rank_checkpoints( + validity: impl Iterator, + valid_count: usize, +) -> VortexResult> { + let mut ranks = Vec::new(); + let mut rank = 0usize; + for (index, valid) in validity.enumerate() { + if index.is_multiple_of(VALIDITY_CHECKPOINT_INTERVAL) { + ranks.push(u32::try_from(rank)?); + } + rank += usize::from(valid); + } + vortex_ensure!(rank == valid_count, "RangePacked validity count differs"); + Ok(ranks) +} + +#[inline] +fn is_valid(array: ArrayView<'_, RangePacked>, global_index: usize) -> bool { + array + .validity_bits() + .is_none_or(|validity| validity.as_::().bit_buffer_view().value(global_index)) +} + +fn dense_rank(array: ArrayView<'_, RangePacked>, global_index: usize) -> VortexResult { + let Some(validity) = array.validity_bits() else { + return Ok(global_index); + }; + if global_index == array.data().unsliced_len { + return Ok(array.data().dense_len); + } + let checkpoint = global_index / VALIDITY_CHECKPOINT_INTERVAL; + let checkpoint_start = checkpoint * VALIDITY_CHECKPOINT_INTERVAL; + let base = usize::try_from( + array + .rank_checkpoints() + .ok_or_else(|| vortex_error::vortex_err!("RangePacked rank child is missing"))? + .as_::() + .as_slice::()[checkpoint], + )?; + Ok(base + + validity + .as_::() + .bit_buffer_view() + .slice(checkpoint_start..global_index) + .true_count()) +} + +fn with_decoder( + array: ArrayView<'_, RangePacked>, + function: impl FnOnce(RangePackedDecoder<'_>) -> VortexResult, +) -> VortexResult { + let bin_lowers = array.bin_lowers().as_::(); + let offset_widths = array.offset_widths().as_::(); + let block_offsets = array.block_offsets().as_::(); + let payload = array.payload().as_::(); + let decoder = RangePackedDecoder::try_new( + array.data().dense_len, + MAX_BLOCK_LEN, + array.data().symbol_width, + BinTableView::Split { + lowers: bin_lowers.as_slice::(), + widths: offset_widths.as_slice::(), + }, + block_offsets.as_slice::(), + payload.as_slice::(), + )?; + function(decoder) +} + +fn ordered_primitive(array: ArrayView<'_, Primitive>) -> VortexResult> { + Ok(match array.ptype() { + PType::U8 => array + .as_slice::() + .iter() + .map(|&value| u64::from(value)) + .collect(), + PType::U16 => array + .as_slice::() + .iter() + .map(|&value| u64::from(value)) + .collect(), + PType::U32 => array + .as_slice::() + .iter() + .map(|&value| u64::from(value)) + .collect(), + PType::U64 => array.as_slice::().to_vec(), + PType::I8 => array + .as_slice::() + .iter() + .map(|&value| u64::from((value as u8) ^ (1_u8 << 7))) + .collect(), + PType::I16 => array + .as_slice::() + .iter() + .map(|&value| u64::from((value as u16) ^ (1_u16 << 15))) + .collect(), + PType::I32 => array + .as_slice::() + .iter() + .map(|&value| u64::from((value as u32) ^ (1_u32 << 31))) + .collect(), + PType::I64 => array + .as_slice::() + .iter() + .map(|&value| (value as u64) ^ (1_u64 << 63)) + .collect(), + ptype => vortex_bail!("RangePackedArray does not support {ptype}"), + }) +} + +fn decode_primitive(array: ArrayView<'_, RangePacked>) -> VortexResult { + let ordered = RangePacked::decode_mapped(array, |value| value, 0)?; + let validity = array.validity()?; + Ok(match array.dtype().as_ptype() { + PType::U8 => PrimitiveArray::new::( + ordered + .into_iter() + .map(u8::try_from) + .collect::, _>>()?, + validity, + ), + PType::U16 => PrimitiveArray::new::( + ordered + .into_iter() + .map(u16::try_from) + .collect::, _>>()?, + validity, + ), + PType::U32 => PrimitiveArray::new::( + ordered + .into_iter() + .map(u32::try_from) + .collect::, _>>()?, + validity, + ), + PType::U64 => PrimitiveArray::new::(ordered, validity), + PType::I8 => PrimitiveArray::new::( + ordered + .into_iter() + .map(|value| { + u8::try_from(value).map(|value| i8::from_le_bytes([value ^ (1_u8 << 7)])) + }) + .collect::, _>>()?, + validity, + ), + PType::I16 => PrimitiveArray::new::( + ordered + .into_iter() + .map(|value| { + u16::try_from(value) + .map(|value| i16::from_le_bytes((value ^ (1_u16 << 15)).to_le_bytes())) + }) + .collect::, _>>()?, + validity, + ), + PType::I32 => PrimitiveArray::new::( + ordered + .into_iter() + .map(|value| { + u32::try_from(value) + .map(|value| i32::from_le_bytes((value ^ (1_u32 << 31)).to_le_bytes())) + }) + .collect::, _>>()?, + validity, + ), + PType::I64 => PrimitiveArray::new::( + ordered + .into_iter() + .map(|value| i64::from_le_bytes((value ^ (1_u64 << 63)).to_le_bytes())) + .collect::>(), + validity, + ), + ptype => vortex_bail!("RangePackedArray does not support {ptype}"), + }) +} + +fn ordered_scalar(value: u64, ptype: PType, nullability: Nullability) -> VortexResult { + Ok(match ptype { + PType::U8 => Scalar::primitive(u8::try_from(value)?, nullability), + PType::U16 => Scalar::primitive(u16::try_from(value)?, nullability), + PType::U32 => Scalar::primitive(u32::try_from(value)?, nullability), + PType::U64 => Scalar::primitive(value, nullability), + PType::I8 => Scalar::primitive( + i8::from_le_bytes([u8::try_from(value)? ^ (1_u8 << 7)]), + nullability, + ), + PType::I16 => Scalar::primitive( + i16::from_le_bytes((u16::try_from(value)? ^ (1_u16 << 15)).to_le_bytes()), + nullability, + ), + PType::I32 => Scalar::primitive( + i32::from_le_bytes((u32::try_from(value)? ^ (1_u32 << 31)).to_le_bytes()), + nullability, + ), + PType::I64 => Scalar::primitive( + i64::from_le_bytes((value ^ (1_u64 << 63)).to_le_bytes()), + nullability, + ), + ptype => vortex_bail!("RangePackedArray does not support {ptype}"), + }) +} + +fn primitive_dtype(ptype: PType) -> DType { + DType::Primitive(ptype, NonNullable) +} + +fn bin_count(dense_len: usize, symbol_width: u8) -> VortexResult { + vortex_ensure!( + symbol_width <= 6, + "RangePacked symbol width exceeds six bits" + ); + Ok(if dense_len == 0 { + 0 + } else { + 1_usize << symbol_width + }) +} + +#[derive(Clone, Copy)] +struct RangePackedMetadata { + unsliced_len: u64, + dense_len: u64, + slice_start: u64, + payload_len: u64, + symbol_width: u8, +} + +impl RangePackedMetadata { + fn from_data(data: &RangePackedData) -> VortexResult { + Ok(Self { + unsliced_len: u64::try_from(data.unsliced_len)?, + dense_len: u64::try_from(data.dense_len)?, + slice_start: u64::try_from(data.slice_start)?, + payload_len: u64::try_from(data.payload_len)?, + symbol_width: data.symbol_width, + }) + } + + fn encode(self) -> Vec { + let mut bytes = Vec::with_capacity(METADATA_LEN); + bytes.push(METADATA_VERSION); + bytes.extend_from_slice(&self.unsliced_len.to_le_bytes()); + bytes.extend_from_slice(&self.dense_len.to_le_bytes()); + bytes.extend_from_slice(&self.slice_start.to_le_bytes()); + bytes.extend_from_slice(&self.payload_len.to_le_bytes()); + bytes.push(self.symbol_width); + bytes + } + + fn decode(bytes: &[u8]) -> VortexResult { + vortex_ensure!( + bytes.len() == METADATA_LEN, + "RangePacked metadata requires {METADATA_LEN} bytes" + ); + vortex_ensure!( + bytes[0] == METADATA_VERSION, + "unsupported RangePacked metadata version {}", + bytes[0] + ); + Ok(Self { + unsliced_len: read_u64(bytes, 1), + dense_len: read_u64(bytes, 9), + slice_start: read_u64(bytes, 17), + payload_len: read_u64(bytes, 25), + symbol_width: bytes[33], + }) + } +} + +fn read_u64(bytes: &[u8], offset: usize) -> u64 { + u64::from_le_bytes( + bytes[offset..offset + size_of::()] + .try_into() + .unwrap_or_else(|_| unreachable!("validated RangePacked metadata length")), + ) +} diff --git a/encodings/range-packed/src/lib.rs b/encodings/range-packed/src/lib.rs index f5ee95402b9..37738cc8dba 100644 --- a/encodings/range-packed/src/lib.rs +++ b/encodings/range-packed/src/lib.rs @@ -1,45 +1,20 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +//! Fixed-bin range packing with bounded random access. + +mod array; + use std::cmp::Ordering; -use std::fmt::Display; -use std::fmt::Formatter; use std::hash::Hash; -use std::hash::Hasher; use std::mem::MaybeUninit; use std::ops::Range; -use vortex_array::Array; -use vortex_array::ArrayEq; -use vortex_array::ArrayHash; -use vortex_array::ArrayId; -use vortex_array::ArrayParts; -use vortex_array::ArrayRef; -use vortex_array::ArrayView; -use vortex_array::EqMode; -use vortex_array::ExecutionCtx; -use vortex_array::ExecutionResult; -use vortex_array::IntoArray; -use vortex_array::arrays::Primitive; -use vortex_array::arrays::PrimitiveArray; -use vortex_array::buffer::BufferHandle; -use vortex_array::builtins::ArrayBuiltins; -use vortex_array::dtype::DType; -use vortex_array::dtype::Nullability; -use vortex_array::dtype::PType; -use vortex_array::scalar::Scalar; -use vortex_array::serde::ArrayChildren; -use vortex_array::validity::Validity; -use vortex_array::vtable::OperationsVTable; -use vortex_array::vtable::VTable; -use vortex_array::vtable::ValidityVTable; -use vortex_buffer::ByteBuffer; +pub use array::*; +use vortex_array::session::ArraySessionExt; use vortex_error::VortexResult; -use vortex_error::vortex_bail; use vortex_error::vortex_ensure; -use vortex_error::vortex_panic; use vortex_session::VortexSession; -use vortex_session::registry::CachedId; const MAX_BINS: usize = 64; const SPLIT_CANDIDATES: usize = 64; @@ -54,516 +29,223 @@ const MAX_BLOCK_LEN: usize = 1_024; /// Fixed-width range identifiers with variable-width offsets and scalar checkpoints. #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct RangePackedCodec { - len: usize, - block_len: usize, - symbol_width: u8, - bins: Vec, - block_offsets: Vec, - payload: Vec, + pub(crate) len: usize, + pub(crate) block_len: usize, + pub(crate) symbol_width: u8, + pub(crate) bins: Vec, + pub(crate) block_offsets: Vec, + pub(crate) payload: Vec, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -struct Bin { - lower: u64, - offset_bits: u8, -} - -/// Fixed-bin range packing with bounded scalar access. -pub type RangePackedArray = Array; - -#[derive(Clone, Debug)] -pub struct RangePacked; - -#[derive(Clone, Debug)] -pub struct RangePackedData { - codec: RangePackedCodec, - physical: ByteBuffer, - ptype: PType, - validity: Validity, - dense_validity: DenseValidity, - len: usize, -} - -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -struct DenseValidity { - words: Vec, - checkpoints: Vec, - valid_count: usize, -} - -impl Display for RangePackedData { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "bins: {}", self.codec.bin_count()) - } -} - -impl ArrayHash for RangePackedData { - fn array_hash(&self, state: &mut H, _accuracy: EqMode) { - self.codec.hash(state); - self.ptype.hash(state); - self.validity.array_hash(state, _accuracy); - self.dense_validity.hash(state); - self.len.hash(state); - } +pub(crate) struct Bin { + pub(crate) lower: u64, + pub(crate) offset_bits: u8, } -impl ArrayEq for RangePackedData { - fn array_eq(&self, other: &Self, _accuracy: EqMode) -> bool { - self.codec == other.codec - && self.ptype == other.ptype - && self.validity.array_eq(&other.validity, _accuracy) - && self.dense_validity == other.dense_validity - && self.len == other.len - } +/// Register the RangePacked encoding in one session. +pub fn initialize(session: &VortexSession) { + session.arrays().register(RangePacked); } -impl VTable for RangePacked { - type TypedArrayData = RangePackedData; - type OperationsVTable = Self; - type ValidityVTable = Self; - - fn id(&self) -> ArrayId { - static ID: CachedId = CachedId::new("vortex.experimental.range_packed"); - *ID - } - - fn validate( - &self, - data: &Self::TypedArrayData, - dtype: &DType, - len: usize, - slots: &[Option], - ) -> VortexResult<()> { - vortex_ensure!(slots.is_empty(), "RangePackedArray cannot have children"); - vortex_ensure!( - dtype.is_primitive() && dtype.as_ptype() == data.ptype, - "RangePackedArray dtype differs from its physical type" - ); - vortex_ensure!( - dtype.nullability() == data.validity.nullability(), - "RangePackedArray dtype differs from its validity" - ); - vortex_ensure!(len == data.len, "RangePackedArray length differs"); +impl RangePackedCodec { + pub fn encode(values: &[u64], block_len: usize) -> VortexResult { vortex_ensure!( - data.codec.len == data.dense_validity.valid_count, - "RangePackedArray dense length differs" + block_len > 0 && block_len <= MAX_BLOCK_LEN, + "range packed block length must be in 1..={MAX_BLOCK_LEN}" ); + if values.is_empty() { + return Ok(Self { + len: 0, + block_len, + symbol_width: 0, + bins: Vec::new(), + block_offsets: vec![0], + payload: Vec::new(), + }); + } + + let (sample, minimum, maximum) = training_sample(values); + let bins = cover_domain(optimize_bins(&sample), minimum, maximum); vortex_ensure!( - data.physical.len() == data.codec.encoded_size() + data.dense_validity.encoded_size(), - "RangePackedArray physical size differs" + bins.len().is_power_of_two(), + "range packed bin count must be a power of two" ); - Ok(()) - } + let symbols = assign_bins(values, &bins)?; + let symbol_width = bit_width(u64::try_from(bins.len() - 1)?); + let block_count = values.len().div_ceil(block_len); + let mut block_offsets = Vec::with_capacity(block_count + 1); + let mut payload = Vec::new(); + block_offsets.push(0); + for block_index in 0..block_count { + let start = block_index * block_len; + let stop = (start + block_len).min(values.len()); + encode_block( + &values[start..stop], + &symbols[start..stop], + &bins, + symbol_width, + &mut payload, + )?; + block_offsets.push(u32::try_from(payload.len())?); + } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { - 1 + Ok(Self { + len: values.len(), + block_len, + symbol_width, + bins, + block_offsets, + payload, + }) } - fn buffer(array: ArrayView<'_, Self>, index: usize) -> BufferHandle { - if index == 0 { - BufferHandle::new_host(array.data().physical.clone()) - } else { - vortex_panic!("RangePackedArray buffer index {index} is invalid") - } + pub fn decode(&self) -> VortexResult> { + self.decoder()?.decode_mapped(|value| value) } - fn buffer_name(_array: ArrayView<'_, Self>, index: usize) -> Option { - (index == 0).then(|| "encoded".to_string()) + pub fn decode_mapped(&self, map: F) -> VortexResult> + where + F: FnMut(u64) -> T, + { + self.decoder()?.decode_mapped(map) } - fn with_buffers( - &self, - array: ArrayView<'_, Self>, - buffers: &[BufferHandle], - ) -> VortexResult> { - vortex_ensure!(buffers.len() == 1, "RangePackedArray needs one buffer"); - let mut data = array.data().clone(); - data.physical = buffers[0].clone().try_to_host_sync()?; - Ok(ArrayParts::new( - self.clone(), - array.dtype().clone(), - array.len(), - data, - )) + pub fn scalar_at(&self, index: usize) -> VortexResult { + self.decoder()?.scalar_at(index) } - fn serialize( - _array: ArrayView<'_, Self>, - _session: &VortexSession, - ) -> VortexResult>> { - vortex_bail!("experimental RangePackedArray does not support serialization") + pub fn encoded_size(&self) -> usize { + self.bins.len() * (size_of::() + size_of::()) + + self.block_offsets.len() * size_of::() + + self.payload.len() } - fn deserialize( - &self, - _dtype: &DType, - _len: usize, - _metadata: &[u8], - _buffers: &[BufferHandle], - _children: &dyn ArrayChildren, - _session: &VortexSession, - ) -> VortexResult> { - vortex_bail!("experimental RangePackedArray does not support deserialization") + pub fn bin_count(&self) -> usize { + self.bins.len() } - fn slot_name(_array: ArrayView<'_, Self>, index: usize) -> String { - vortex_panic!("RangePackedArray slot index {index} is invalid") + pub fn max_offset_bits(&self) -> u8 { + self.bins + .iter() + .map(|bin| bin.offset_bits) + .max() + .unwrap_or_default() } - fn execute(array: Array, _ctx: &mut ExecutionCtx) -> VortexResult { - let decoded = decode_primitive(array.data())? - .into_array() - .cast(array.dtype().clone())?; - Ok(ExecutionResult::done(decoded)) + pub fn offset_widths(&self) -> String { + self.bins + .iter() + .map(|bin| bin.offset_bits.to_string()) + .collect::>() + .join(",") } -} -impl OperationsVTable for RangePacked { - fn scalar_at( - array: ArrayView<'_, RangePacked>, - index: usize, - _ctx: &mut ExecutionCtx, - ) -> VortexResult { - if !array.data().dense_validity.is_valid(index) { - return Ok(Scalar::null(array.dtype().clone())); - } - let dense_index = array.data().dense_validity.rank(index); - ordered_scalar( - array.data().codec.scalar_at(dense_index)?, - array.data().ptype, - array.dtype().nullability(), + fn decoder(&self) -> VortexResult> { + RangePackedDecoder::try_new( + self.len, + self.block_len, + self.symbol_width, + BinTableView::Interleaved(&self.bins), + &self.block_offsets, + &self.payload, ) } } -impl DenseValidity { - fn new( - len: usize, - valid_count: usize, - validity: impl Iterator, - ) -> VortexResult { - if valid_count == len { - return Ok(Self { - words: Vec::new(), - checkpoints: Vec::new(), - valid_count, - }); - } +#[derive(Clone, Copy)] +pub(crate) enum BinTableView<'a> { + Interleaved(&'a [Bin]), + Split { lowers: &'a [u64], widths: &'a [u8] }, +} - let mut words = vec![0_u64; len.div_ceil(64)]; - let mut observed_len = 0usize; - for (index, valid) in validity.enumerate() { - vortex_ensure!(index < len, "dense validity exceeds its length"); - if valid { - words[index / 64] |= 1_u64 << (index % 64); - } - observed_len += 1; - } - vortex_ensure!(observed_len == len, "dense validity is too short"); - - let words_per_checkpoint = VALIDITY_CHECKPOINT_INTERVAL / 64; - let mut checkpoints = Vec::with_capacity(words.len().div_ceil(words_per_checkpoint)); - let mut rank = 0usize; - for block in words.chunks(words_per_checkpoint) { - checkpoints.push(u32::try_from(rank)?); - rank += block - .iter() - .map(|word| word.count_ones() as usize) - .sum::(); +impl BinTableView<'_> { + fn len(self) -> usize { + match self { + Self::Interleaved(bins) => bins.len(), + Self::Split { lowers, .. } => lowers.len(), } - vortex_ensure!(rank == valid_count, "dense validity count differs"); - Ok(Self { - words, - checkpoints, - valid_count, - }) } - #[inline] - fn is_valid(&self, index: usize) -> bool { - self.words.is_empty() || self.words[index / 64] & (1_u64 << (index % 64)) != 0 + fn lower(self, index: usize) -> Option { + match self { + Self::Interleaved(bins) => bins.get(index).map(|bin| bin.lower), + Self::Split { lowers, .. } => lowers.get(index).copied(), + } } - #[inline] - fn rank(&self, index: usize) -> usize { - if self.words.is_empty() { - return index; - } - let word_index = index / 64; - let checkpoint_index = index / VALIDITY_CHECKPOINT_INTERVAL; - let words_per_checkpoint = VALIDITY_CHECKPOINT_INTERVAL / 64; - let word_start = checkpoint_index * words_per_checkpoint; - let mut rank = self.checkpoints[checkpoint_index] as usize; - for word in &self.words[word_start..word_index] { - rank += word.count_ones() as usize; - } - let bits_before = index % 64; - if bits_before != 0 { - let mask = (1_u64 << bits_before) - 1; - rank += (self.words[word_index] & mask).count_ones() as usize; + fn width(self, index: usize) -> Option { + match self { + Self::Interleaved(bins) => bins.get(index).map(|bin| bin.offset_bits), + Self::Split { widths, .. } => widths.get(index).copied(), } - rank } - fn encoded_size(&self) -> usize { - self.words.len() * size_of::() + self.checkpoints.len() * size_of::() + fn max_width(self) -> u8 { + (0..self.len()) + .map(|index| self.width(index).unwrap_or_default()) + .max() + .unwrap_or_default() } } -impl ValidityVTable for RangePacked { - fn validity(array: ArrayView<'_, RangePacked>) -> VortexResult { - Ok(array.data().validity.clone()) - } +pub(crate) struct RangePackedDecoder<'a> { + len: usize, + block_len: usize, + symbol_width: u8, + bins: BinTableView<'a>, + block_offsets: &'a [u32], + payload: &'a [u8], } -impl RangePacked { - pub fn from_primitive( - array: ArrayView<'_, Primitive>, - ctx: &mut ExecutionCtx, - ) -> VortexResult { +impl<'a> RangePackedDecoder<'a> { + pub(crate) fn try_new( + len: usize, + block_len: usize, + symbol_width: u8, + bins: BinTableView<'a>, + block_offsets: &'a [u32], + payload: &'a [u8], + ) -> VortexResult { vortex_ensure!( - array.ptype().is_int(), - "RangePackedArray needs integer values" + block_len > 0 && block_len <= MAX_BLOCK_LEN, + "range packed block length is invalid" ); - let validity = array.validity()?; - let mask = validity.execute_mask(array.len(), ctx)?; - let dense_validity = DenseValidity::new(array.len(), mask.true_count(), mask.iter())?; - let ordered = ordered_primitive(array)?; - let dense_ordered = ordered - .into_iter() - .zip(mask.iter()) - .filter_map(|(value, valid)| valid.then_some(value)) - .collect::>(); - let codec = RangePackedCodec::encode(&dense_ordered, MAX_BLOCK_LEN)?; - let physical = ByteBuffer::zeroed(codec.encoded_size() + dense_validity.encoded_size()); - let data = RangePackedData { - codec, - physical, - ptype: array.ptype(), - validity, - dense_validity, - len: array.len(), + let expected_bins = if len == 0 { + 0 + } else { + vortex_ensure!( + symbol_width <= 6, + "range packed symbol width exceeds six bits" + ); + 1_usize << symbol_width }; - Array::try_from_parts(ArrayParts::new( - RangePacked, - array.dtype().clone(), - array.len(), - data, - )) - } - - pub fn decode_mapped( - array: ArrayView<'_, RangePacked>, - map: F, - null_value: T, - ) -> VortexResult> - where - T: Copy, - F: FnMut(u64) -> T, - { - let dense_values = array.data().codec.decode_mapped(map)?; - if array.data().dense_validity.words.is_empty() { - return Ok(dense_values); - } - let mut dense = dense_values.into_iter(); - let mut values = Vec::with_capacity(array.len()); - for index in 0..array.len() { - values.push(if array.data().dense_validity.is_valid(index) { - dense - .next() - .ok_or_else(|| vortex_error::vortex_err!("dense values are too short"))? - } else { - null_value - }); - } - vortex_ensure!(dense.next().is_none(), "dense values are too long"); - Ok(values) - } -} - -fn ordered_primitive(array: ArrayView<'_, Primitive>) -> VortexResult> { - Ok(match array.ptype() { - PType::U8 => array - .as_slice::() - .iter() - .map(|&value| u64::from(value)) - .collect(), - PType::U16 => array - .as_slice::() - .iter() - .map(|&value| u64::from(value)) - .collect(), - PType::U32 => array - .as_slice::() - .iter() - .map(|&value| u64::from(value)) - .collect(), - PType::U64 => array.as_slice::().to_vec(), - PType::I8 => array - .as_slice::() - .iter() - .map(|&value| u64::from((value as u8) ^ (1_u8 << 7))) - .collect(), - PType::I16 => array - .as_slice::() - .iter() - .map(|&value| u64::from((value as u16) ^ (1_u16 << 15))) - .collect(), - PType::I32 => array - .as_slice::() - .iter() - .map(|&value| u64::from((value as u32) ^ (1_u32 << 31))) - .collect(), - PType::I64 => array - .as_slice::() - .iter() - .map(|&value| (value as u64) ^ (1_u64 << 63)) - .collect(), - ptype => vortex_bail!("RangePackedArray does not support {ptype}"), - }) -} - -#[expect( - clippy::cast_possible_truncation, - reason = "the encoded values retain the source integer word width" -)] -fn decode_primitive(data: &RangePackedData) -> VortexResult { - let dense_ordered = data.codec.decode()?; - let ordered = if data.dense_validity.words.is_empty() { - dense_ordered - } else { - let mut dense = dense_ordered.into_iter(); - let mut values = Vec::with_capacity(data.len); - for index in 0..data.len { - values.push(if data.dense_validity.is_valid(index) { - dense - .next() - .ok_or_else(|| vortex_error::vortex_err!("dense values are too short"))? - } else { - 0 - }); - } - vortex_ensure!(dense.next().is_none(), "dense values are too long"); - values - }; - let validity = data.validity.clone(); - Ok(match data.ptype { - PType::U8 => PrimitiveArray::new::( - ordered - .into_iter() - .map(u8::try_from) - .collect::, _>>()?, - validity, - ), - PType::U16 => PrimitiveArray::new::( - ordered - .into_iter() - .map(u16::try_from) - .collect::, _>>()?, - validity, - ), - PType::U32 => PrimitiveArray::new::( - ordered - .into_iter() - .map(u32::try_from) - .collect::, _>>()?, - validity, - ), - PType::U64 => PrimitiveArray::new::(ordered, validity), - PType::I8 => PrimitiveArray::new::( - ordered - .into_iter() - .map(|value| ((value as u8) ^ (1_u8 << 7)) as i8) - .collect::>(), - validity, - ), - PType::I16 => PrimitiveArray::new::( - ordered - .into_iter() - .map(|value| ((value as u16) ^ (1_u16 << 15)) as i16) - .collect::>(), - validity, - ), - PType::I32 => PrimitiveArray::new::( - ordered - .into_iter() - .map(|value| ((value as u32) ^ (1_u32 << 31)) as i32) - .collect::>(), - validity, - ), - PType::I64 => PrimitiveArray::new::( - ordered - .into_iter() - .map(|value| (value ^ (1_u64 << 63)) as i64) - .collect::>(), - validity, - ), - ptype => vortex_bail!("RangePackedArray does not support {ptype}"), - }) -} - -#[expect( - clippy::cast_possible_truncation, - reason = "the encoded value retains the source integer word width" -)] -fn ordered_scalar(value: u64, ptype: PType, nullability: Nullability) -> VortexResult { - Ok(match ptype { - PType::U8 => Scalar::primitive(u8::try_from(value)?, nullability), - PType::U16 => Scalar::primitive(u16::try_from(value)?, nullability), - PType::U32 => Scalar::primitive(u32::try_from(value)?, nullability), - PType::U64 => Scalar::primitive(value, nullability), - PType::I8 => Scalar::primitive(((value as u8) ^ (1_u8 << 7)) as i8, nullability), - PType::I16 => Scalar::primitive(((value as u16) ^ (1_u16 << 15)) as i16, nullability), - PType::I32 => Scalar::primitive(((value as u32) ^ (1_u32 << 31)) as i32, nullability), - PType::I64 => Scalar::primitive((value ^ (1_u64 << 63)) as i64, nullability), - ptype => vortex_bail!("RangePackedArray does not support {ptype}"), - }) -} - -impl RangePackedCodec { - pub fn encode(values: &[u64], block_len: usize) -> VortexResult { vortex_ensure!( - block_len > 0 && block_len <= MAX_BLOCK_LEN, - "range packed block length must be in 1..={MAX_BLOCK_LEN}" + bins.len() == expected_bins, + "range packed bin count is invalid" ); - if values.is_empty() { - return Ok(Self { - len: 0, - block_len, - symbol_width: 0, - bins: Vec::new(), - block_offsets: vec![0], - payload: Vec::new(), - }); + if let BinTableView::Split { lowers, widths } = bins { + vortex_ensure!( + lowers.len() == widths.len(), + "range packed split bin table lengths differ" + ); } - - let (sample, minimum, maximum) = training_sample(values); - let bins = cover_domain(optimize_bins(&sample), minimum, maximum); vortex_ensure!( - bins.len().is_power_of_two(), - "range packed bin count must be a power of two" + block_offsets.len() == len.div_ceil(block_len) + 1, + "range packed block offset count is invalid" + ); + vortex_ensure!( + block_offsets.first() == Some(&0) + && block_offsets + .last() + .copied() + .map(usize::try_from) + .transpose()? + == Some(payload.len()), + "range packed payload boundaries are invalid" ); - let symbols = assign_bins(values, &bins)?; - let symbol_width = bit_width(u64::try_from(bins.len() - 1)?); - let block_count = values.len().div_ceil(block_len); - let mut block_offsets = Vec::with_capacity(block_count + 1); - let mut payload = Vec::new(); - block_offsets.push(0); - for block_index in 0..block_count { - let start = block_index * block_len; - let stop = (start + block_len).min(values.len()); - encode_block( - &values[start..stop], - &symbols[start..stop], - &bins, - symbol_width, - &mut payload, - )?; - block_offsets.push(u32::try_from(payload.len())?); - } - Ok(Self { - len: values.len(), + len, block_len, symbol_width, bins, @@ -572,52 +254,88 @@ impl RangePackedCodec { }) } - pub fn decode(&self) -> VortexResult> { - self.decode_mapped(|value| value) + fn decode_mapped(&self, map: F) -> VortexResult> + where + F: FnMut(u64) -> T, + { + self.decode_mapped_range(0..self.len, map) } - pub fn decode_mapped(&self, map: F) -> VortexResult> + pub(crate) fn decode_mapped_range( + &self, + range: Range, + map: F, + ) -> VortexResult> where F: FnMut(u64) -> T, { - let max_offset_bits = self.max_offset_bits(); + vortex_ensure!( + range.start <= range.end && range.end <= self.len, + "range packed decode range exceeds array length" + ); + let max_offset_bits = self.bins.max_width(); if max_offset_bits <= 40 { - self.decode_mapped_with_width::(map) + self.decode_mapped_range_with_width::(range, map) } else if max_offset_bits <= 57 { - self.decode_mapped_with_width::(map) + self.decode_mapped_range_with_width::(range, map) } else { - self.decode_mapped_with_width::(map) + self.decode_mapped_range_with_width::(range, map) } } - fn decode_mapped_with_width( + fn decode_mapped_range_with_width( &self, + range: Range, mut map: F, ) -> VortexResult> where F: FnMut(u64) -> T, { - let mut values = Vec::with_capacity(self.len); - let table = DecodeTable::new(&self.bins); - for block_index in 0..self.block_count() { - let payload = self.block_payload(block_index)?; + let mut values = Vec::with_capacity(range.len()); + if range.is_empty() { + return Ok(values); + } + let table = DecodeTable::new(self.bins); + let first_block = range.start / self.block_len; + let last_block = range.end.div_ceil(self.block_len); + for block_index in first_block..last_block { + let block_start = block_index * self.block_len; let value_count = self.block_value_count(block_index); - let output_start = values.len(); + let local_start = range.start.saturating_sub(block_start); + let local_stop = (range.end - block_start).min(value_count); + let payload = self.block_payload(block_index)?; + if local_start == 0 && local_stop == value_count { + let output_start = values.len(); + decode_block::( + payload, + value_count, + self.symbol_width, + &table, + &mut values.spare_capacity_mut()[..value_count], + &mut map, + )?; + // SAFETY: decode_block initialized each output slot. + unsafe { values.set_len(output_start + value_count) }; + continue; + } + + let mut block_values = Vec::with_capacity(value_count); decode_block::( payload, value_count, self.symbol_width, &table, - &mut values.spare_capacity_mut()[..value_count], + &mut block_values.spare_capacity_mut()[..value_count], &mut map, )?; // SAFETY: decode_block initialized each output slot. - unsafe { values.set_len(output_start + value_count) }; + unsafe { block_values.set_len(value_count) }; + values.extend(block_values.drain(local_start..local_stop)); } Ok(values) } - pub fn scalar_at(&self, index: usize) -> VortexResult { + pub(crate) fn scalar_at(&self, index: usize) -> VortexResult { vortex_ensure!(index < self.len, "range packed index exceeds array length"); let block_index = index / self.block_len; let index_in_block = index % self.block_len; @@ -625,73 +343,49 @@ impl RangePackedCodec { let payload = self.block_payload(block_index)?; let layout = BlockLayout::new(value_count, self.symbol_width, payload.len())?; let symbol = read_symbol(payload, index_in_block, self.symbol_width)?; - let node = *self + let lower = self + .bins + .lower(symbol) + .ok_or_else(|| vortex_error::vortex_err!("range packed symbol exceeds bin table"))?; + let width = self .bins - .get(symbol) + .width(symbol) .ok_or_else(|| vortex_error::vortex_err!("range packed symbol exceeds bin table"))?; let checkpoint_index = index_in_block / CHECKPOINT_INTERVAL; let checkpoint_start = layout.checkpoints_start + checkpoint_index * size_of::(); - let checkpoint = - u16::from_le_bytes([payload[checkpoint_start], payload[checkpoint_start + 1]]); + let checkpoint_bytes = payload + .get(checkpoint_start..checkpoint_start + size_of::()) + .ok_or_else(|| vortex_error::vortex_err!("range packed checkpoint is missing"))?; + let checkpoint = u16::from_le_bytes([checkpoint_bytes[0], checkpoint_bytes[1]]); let mut offset_position = usize::from(checkpoint); let scan_start = checkpoint_index * CHECKPOINT_INTERVAL; for preceding in scan_start..index_in_block { let preceding_symbol = read_symbol(payload, preceding, self.symbol_width)?; - offset_position += usize::from( - self.bins - .get(preceding_symbol) - .ok_or_else(|| { - vortex_error::vortex_err!("range packed symbol exceeds bin table") - })? - .offset_bits, - ); + offset_position += usize::from(self.bins.width(preceding_symbol).ok_or_else(|| { + vortex_error::vortex_err!("range packed symbol exceeds bin table") + })?); } - let offset = read_bits_padded( - &payload[layout.offsets_start..], - offset_position, - node.offset_bits, - ); - Ok(node.lower.wrapping_add(offset)) - } - - pub fn encoded_size(&self) -> usize { - self.bins.len() * (size_of::() + size_of::()) - + self.block_offsets.len() * size_of::() - + self.payload.len() - } - - pub fn bin_count(&self) -> usize { - self.bins.len() - } - - pub fn max_offset_bits(&self) -> u8 { - self.bins - .iter() - .map(|bin| bin.offset_bits) - .max() - .unwrap_or_default() - } - - pub fn offset_widths(&self) -> String { - self.bins - .iter() - .map(|bin| bin.offset_bits.to_string()) - .collect::>() - .join(",") - } - - fn block_count(&self) -> usize { - self.len.div_ceil(self.block_len) + let offset = read_bits_padded(&payload[layout.offsets_start..], offset_position, width); + Ok(lower.wrapping_add(offset)) } fn block_value_count(&self, block_index: usize) -> usize { (self.len - block_index * self.block_len).min(self.block_len) } - fn block_payload(&self, block_index: usize) -> VortexResult<&[u8]> { - let start = usize::try_from(self.block_offsets[block_index])?; - let stop = usize::try_from(self.block_offsets[block_index + 1])?; - Ok(&self.payload[start..stop]) + fn block_payload(&self, block_index: usize) -> VortexResult<&'a [u8]> { + let start = + usize::try_from(*self.block_offsets.get(block_index).ok_or_else(|| { + vortex_error::vortex_err!("range packed block offset is missing") + })?)?; + let stop = + usize::try_from(*self.block_offsets.get(block_index + 1).ok_or_else(|| { + vortex_error::vortex_err!("range packed block offset is missing") + })?)?; + vortex_ensure!(start <= stop, "range packed block offsets are not ordered"); + self.payload + .get(start..stop) + .ok_or_else(|| vortex_error::vortex_err!("range packed block exceeds its payload")) } } @@ -702,12 +396,16 @@ struct DecodeTable { } impl DecodeTable { - fn new(bins: &[Bin]) -> Self { + fn new(bins: BinTableView<'_>) -> Self { let mut lowers = [0_u64; MAX_BINS]; let mut widths = [0_u8; MAX_BINS]; - for (index, bin) in bins.iter().enumerate() { - lowers[index] = bin.lower; - widths[index] = bin.offset_bits; + for index in 0..bins.len() { + lowers[index] = bins + .lower(index) + .unwrap_or_else(|| unreachable!("validated range packed bin table")); + widths[index] = bins + .width(index) + .unwrap_or_else(|| unreachable!("validated range packed bin table")); } Self { lowers, @@ -1120,15 +818,24 @@ impl BitWriter { #[cfg(test)] mod tests { + use rstest::rstest; + use vortex_array::ArrayContext; + use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::array_session; + use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; + use vortex_array::serde::SerializeOptions; + use vortex_array::serde::SerializedArray; + use vortex_buffer::ByteBufferMut; use vortex_error::VortexResult; + use vortex_session::registry::ReadContext; use super::RangePacked; use super::RangePackedCodec; + use super::initialize; #[test] fn clustered_roundtrip_and_scalar_access() -> VortexResult<()> { @@ -1178,4 +885,62 @@ mod tests { } Ok(()) } + + #[rstest] + #[case::u8(PrimitiveArray::from_iter([0_u8, 1, u8::MAX]).into_array())] + #[case::u16(PrimitiveArray::from_iter([0_u16, 1, u16::MAX]).into_array())] + #[case::u32(PrimitiveArray::from_iter([0_u32, 1, u32::MAX]).into_array())] + #[case::u64(PrimitiveArray::from_iter([0_u64, 1, u64::MAX]).into_array())] + #[case::i8(PrimitiveArray::from_iter([i8::MIN, -1, 0, i8::MAX]).into_array())] + #[case::i16(PrimitiveArray::from_iter([i16::MIN, -1, 0, i16::MAX]).into_array())] + #[case::i32(PrimitiveArray::from_iter([i32::MIN, -1, 0, i32::MAX]).into_array())] + #[case::i64(PrimitiveArray::from_iter([i64::MIN, -1, 0, i64::MAX]).into_array())] + fn integer_ptype_roundtrip(#[case] expected: ArrayRef) -> VortexResult<()> { + let session = array_session(); + let mut ctx = session.create_execution_ctx(); + let encoded = RangePacked::from_primitive(expected.as_::(), &mut ctx)?; + assert_arrays_eq!(encoded, expected, &mut ctx); + Ok(()) + } + + #[test] + fn nullable_slice_crosses_rank_and_value_blocks() -> VortexResult<()> { + let expected = PrimitiveArray::from_option_iter((0_i32..2_500).map(|value| { + (!matches!(value, 255 | 256 | 257 | 1_023 | 1_024 | 1_025)).then_some(value - 1_250) + })); + let session = array_session(); + let mut ctx = session.create_execution_ctx(); + let encoded = RangePacked::from_primitive(expected.as_view(), &mut ctx)? + .into_array() + .slice(253..1_030)?; + let expected = expected.into_array().slice(253..1_030)?; + assert_arrays_eq!(encoded, expected, &mut ctx); + Ok(()) + } + + #[test] + fn serialized_nullable_slice_roundtrip() -> VortexResult<()> { + let expected = PrimitiveArray::from_option_iter( + (0_i64..1_500).map(|value| (value % 17 != 0).then_some(value - 750)), + ); + let session = array_session(); + initialize(&session); + let mut ctx = session.create_execution_ctx(); + let encoded = RangePacked::from_primitive(expected.as_view(), &mut ctx)? + .into_array() + .slice(251..1_101)?; + let expected = expected.into_array().slice(251..1_101)?; + let dtype = encoded.dtype().clone(); + let len = encoded.len(); + let array_ctx = ArrayContext::empty(); + let serialized = encoded.serialize(&array_ctx, &session, &SerializeOptions::default())?; + let mut bytes = ByteBufferMut::empty(); + for buffer in serialized { + bytes.extend_from_slice(buffer.as_ref()); + } + let parts = SerializedArray::try_from(bytes.freeze())?; + let decoded = parts.decode(&dtype, len, &ReadContext::new(array_ctx.to_ids()), &session)?; + assert_arrays_eq!(decoded, expected, &mut ctx); + Ok(()) + } } diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index 4585208578b..df3566ef319 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -60,11 +60,10 @@ use vortex_btrblocks::schemes::integer::BlockResidualScheme; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_err; -use vortex_session::VortexSession; -use vortex_utils::aliases::hash_map::HashMap; - use vortex_range_packed::RangePacked; use vortex_range_packed::RangePackedCodec; +use vortex_session::VortexSession; +use vortex_utils::aliases::hash_map::HashMap; const DEFAULT_ROW_COUNT: usize = 2_000_000; const CALIFORNIA_COLUMNS: [&str; 9] = [ diff --git a/vortex-file/Cargo.toml b/vortex-file/Cargo.toml index 5b86c11467c..dc9d1eb6225 100644 --- a/vortex-file/Cargo.toml +++ b/vortex-file/Cargo.toml @@ -53,6 +53,7 @@ vortex-mask = { workspace = true } vortex-metrics = { workspace = true } vortex-onpair = { workspace = true } vortex-pco = { workspace = true } +vortex-range-packed = { workspace = true } vortex-runend = { workspace = true } vortex-scan = { workspace = true } vortex-sequence = { workspace = true } diff --git a/vortex-file/src/lib.rs b/vortex-file/src/lib.rs index 465c7fd6e3c..73e603fb1a8 100644 --- a/vortex-file/src/lib.rs +++ b/vortex-file/src/lib.rs @@ -192,6 +192,7 @@ pub fn register_default_encodings(session: &VortexSession) { vortex_fastlanes::initialize(session); vortex_float_quant::initialize(session); vortex_block_residual::initialize(session); + vortex_range_packed::initialize(session); vortex_runend::initialize(session); vortex_sequence::initialize(session); vortex_sparse::initialize(session); diff --git a/vortex/Cargo.toml b/vortex/Cargo.toml index 3d98384ecaa..48955ded147 100644 --- a/vortex/Cargo.toml +++ b/vortex/Cargo.toml @@ -46,6 +46,7 @@ vortex-layout = { workspace = true } vortex-mask = { workspace = true } vortex-metrics = { workspace = true } vortex-pco = { workspace = true } +vortex-range-packed = { workspace = true } vortex-proto = { workspace = true, default-features = true } vortex-runend = { workspace = true } vortex-scan = { workspace = true } diff --git a/vortex/src/lib.rs b/vortex/src/lib.rs index cb6e8aff091..f3f3db82f39 100644 --- a/vortex/src/lib.rs +++ b/vortex/src/lib.rs @@ -280,6 +280,11 @@ pub mod encodings { pub use vortex_block_residual::*; } + /// Fixed-bin range-packed integer encoding. + pub mod range_packed { + pub use vortex_range_packed::*; + } + /// Arrow-compatible run-end encoding. pub mod runend { pub use vortex_runend::*; From de7da9a0859e28bb65545511b832fd1436ee1288 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 17:58:38 +0100 Subject: [PATCH 26/78] bench: evaluate native pco replacement trees Signed-off-by: Will Manning --- .../examples/float_compressor_bench.rs | 114 +++++++++++++++++- vortex-btrblocks/src/builder.rs | 31 ++++- .../src/schemes/float/float_quant.rs | 63 +--------- vortex-btrblocks/src/schemes/float/mod.rs | 2 + .../src/schemes/float/ordered_range_packed.rs | 86 +++++++++++++ vortex-btrblocks/src/schemes/integer/mod.rs | 2 + .../src/schemes/integer/range_packed.rs | 70 +++++++++++ vortex-btrblocks/src/schemes/mod.rs | 86 +++++++++++++ vortex/src/editions/core/v2026_08.rs | 1 + 9 files changed, 389 insertions(+), 66 deletions(-) create mode 100644 vortex-btrblocks/src/schemes/float/ordered_range_packed.rs create mode 100644 vortex-btrblocks/src/schemes/integer/range_packed.rs diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index df3566ef319..3398db9e0eb 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -56,7 +56,9 @@ use vortex_btrblocks::BtrBlocksCompressorBuilder; use vortex_btrblocks::SchemeExt; use vortex_btrblocks::schemes::float::FloatQuantScheme; use vortex_btrblocks::schemes::float::OrderedBlockResidualScheme; +use vortex_btrblocks::schemes::float::OrderedRangePackedScheme; use vortex_btrblocks::schemes::integer::BlockResidualScheme; +use vortex_btrblocks::schemes::integer::RangePackedScheme; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_err; @@ -1108,6 +1110,106 @@ fn measure_fixed_bin_tree( Ok(()) } +fn measure_block_residual_float_tree( + dataset: &str, + column: &str, + expected: &ArrayRef, + compact: &ArrayRef, + session: &VortexSession, +) -> VortexResult<()> { + let Some(encoded) = block_residual_float_tree(compact, session)? else { + return Ok(()); + }; + measure_existing_tree( + dataset, + column, + "block-residual", + expected, + &encoded, + session, + )?; + + let mut encode_durations = Vec::with_capacity(5); + for _ in 0..5 { + let start = Instant::now(); + black_box(encode_block_residual_float_tree( + expected.as_::(), + compact, + session, + )?); + encode_durations.push(start.elapsed()); + } + let encode_median = percentile(&mut encode_durations, 1, 2); + let encode_throughput = expected.nbytes() as f64 / encode_median.as_secs_f64() / 1_000_000.0; + println!( + "candidate-tree-encode\t{dataset}\t{column}\tblock-residual\t{}\t{}\t{encode_throughput:.1}\t{}", + encoded.len(), + encoded.nbytes(), + encoding_tree(&encoded), + ); + Ok(()) +} + +fn block_residual_float_tree( + compact: &ArrayRef, + session: &VortexSession, +) -> VortexResult> { + if compact.encoding_id().as_ref() == "vortex.alp" { + let alp = compact.as_::(); + if alp.encoded().encoding_id().as_ref() != "vortex.pco" { + return Ok(None); + } + let primitive = alp + .encoded() + .clone() + .execute::(&mut session.create_execution_ctx())?; + let primitive = primitive + .into_array() + .cast(alp.encoded().dtype().clone())? + .execute::(&mut session.create_execution_ctx())?; + let encoded = BlockResidual::from_primitive(primitive.as_view())?.into_array(); + return Ok(Some( + ALP::try_new(encoded, alp.exponents(), alp.patches())?.into_array(), + )); + } + + if compact.encoding_id().as_ref() != "vortex.pco" || !compact.dtype().is_float() { + return Ok(None); + } + let primitive = compact + .clone() + .execute::(&mut session.create_execution_ctx())?; + let primitive = + fill_float_nulls_with_first_valid(primitive, &mut session.create_execution_ctx())?; + let ordered = OrderedFloat::from_primitive(primitive.as_view())?; + let encoded = BlockResidual::from_primitive(ordered.encoded().as_::())?; + Ok(Some( + OrderedFloat::try_new(encoded.into_array(), primitive.ptype())?.into_array(), + )) +} + +fn encode_block_residual_float_tree( + primitive: vortex_array::ArrayView<'_, Primitive>, + compact: &ArrayRef, + session: &VortexSession, +) -> VortexResult { + if compact.encoding_id().as_ref() == "vortex.alp" { + let alp = alp_encode(primitive, None, &mut session.create_execution_ctx())?; + let encoded = BlockResidual::from_primitive(alp.encoded().as_::())?; + return Ok( + ALP::try_new(encoded.into_array(), alp.exponents(), alp.patches())?.into_array(), + ); + } + + let primitive = fill_float_nulls_with_first_valid( + primitive.into_owned(), + &mut session.create_execution_ctx(), + )?; + let ordered = OrderedFloat::from_primitive(primitive.as_view())?; + let encoded = BlockResidual::from_primitive(ordered.encoded().as_::())?; + Ok(OrderedFloat::try_new(encoded.into_array(), primitive.ptype())?.into_array()) +} + fn encode_fixed_bin_float_tree( primitive: vortex_array::ArrayView<'_, Primitive>, compact: &ArrayRef, @@ -1483,6 +1585,13 @@ fn measure_dataset( session, )?; measure_fixed_bin_tree(dataset, &column.name, &column.array, compact_array, session)?; + measure_block_residual_float_tree( + dataset, + &column.name, + &column.array, + compact_array, + session, + )?; } } @@ -1535,6 +1644,8 @@ fn compressors() -> Vec<(&'static str, BtrBlocksCompressor)> { FloatQuantScheme.id(), OrderedBlockResidualScheme.id(), BlockResidualScheme.id(), + OrderedRangePackedScheme.id(), + RangePackedScheme.id(), ]; vec![ ( @@ -1568,8 +1679,8 @@ fn compressors() -> Vec<(&'static str, BtrBlocksCompressor)> { ( "prior-compact", BtrBlocksCompressorBuilder::default() - .exclude_schemes(new_scheme_ids) .with_compact() + .exclude_schemes(new_scheme_ids) .build(), ), ( @@ -1611,6 +1722,7 @@ fn main() -> VortexResult<()> { println!("fixed-bin-tree\tdataset\tcolumn\trows\tbytes\tdecode-MB/s\tscalar-ns\tencoding"); println!("fixed-bin-tree-fused\tdataset\tcolumn\trows\tbytes\tdecode-MB/s\tencoding"); println!("fixed-bin-tree-encode\tdataset\tcolumn\trows\tbytes\tencode-MB/s\tencoding"); + println!("candidate-tree-encode\tdataset\tcolumn\tconfig\trows\tbytes\tencode-MB/s\tencoding"); println!( "tree-throughput\tdataset\tcolumn\tconfig\trows\tbytes\tdecode-MB/s\tscalar-ns\tencoding" ); diff --git a/vortex-btrblocks/src/builder.rs b/vortex-btrblocks/src/builder.rs index bd6fcdb430d..f1f0fc7666f 100644 --- a/vortex-btrblocks/src/builder.rs +++ b/vortex-btrblocks/src/builder.rs @@ -133,11 +133,10 @@ impl BtrBlocksCompressorBuilder { self } - /// Adds compact encoding schemes (Zstd for strings and binary, Pco for numerics). + /// Adds compact encoding schemes for strings, binary values, and numeric values. /// /// This provides better compression ratios than the default, especially for floating-point - /// heavy datasets. Requires the `zstd` feature. When the `pco` feature is also enabled, - /// Pco schemes for integers and floats are included. + /// heavy datasets. This method requires the `zstd` feature. The `pco` feature adds Pco. /// /// # Panics /// @@ -146,7 +145,9 @@ impl BtrBlocksCompressorBuilder { pub fn with_compact(self) -> Self { let builder = self .with_new_scheme(&string::ZstdScheme) - .with_new_scheme(&binary::ZstdScheme); + .with_new_scheme(&binary::ZstdScheme) + .with_new_scheme(&integer::RangePackedScheme) + .with_new_scheme(&float::OrderedRangePackedScheme); #[cfg(feature = "pco")] let builder = builder @@ -174,11 +175,13 @@ impl BtrBlocksCompressorBuilder { )] let mut excluded: Vec = vec![ integer::BlockResidualScheme.id(), + integer::RangePackedScheme.id(), integer::SparseScheme.id(), integer::IntRLEScheme.id(), float::ALPRDScheme.id(), float::FloatQuantScheme.id(), float::OrderedBlockResidualScheme.id(), + float::OrderedRangePackedScheme.id(), float::FloatRLEScheme.id(), float::NullDominatedSparseScheme.id(), string::StringDictScheme.id(), @@ -228,6 +231,7 @@ impl BtrBlocksCompressorBuilder { #[cfg(test)] mod tests { use vortex_array::VTable; + use vortex_error::VortexResult; use vortex_fastlanes::FoR; use super::*; @@ -316,6 +320,25 @@ mod tests { ); } + #[test] + #[cfg(feature = "zstd")] + fn compact_adds_range_packed_after_default_schemes() -> VortexResult<()> { + let builder = BtrBlocksCompressorBuilder::default().with_compact(); + let integer_index = builder + .schemes + .iter() + .position(|scheme| scheme.id() == integer::RangePackedScheme.id()) + .ok_or_else(|| vortex_error::vortex_err!("compact lacks integer RangePacked"))?; + let float_index = builder + .schemes + .iter() + .position(|scheme| scheme.id() == float::OrderedRangePackedScheme.id()) + .ok_or_else(|| vortex_error::vortex_err!("compact lacks ordered RangePacked"))?; + assert!(integer_index >= ALL_SCHEMES.len()); + assert!(float_index > integer_index); + Ok(()) + } + #[test] fn cuda_compatible_uses_fsst_for_strings() { let builder = BtrBlocksCompressorBuilder::default().only_cuda_compatible(); diff --git a/vortex-btrblocks/src/schemes/float/float_quant.rs b/vortex-btrblocks/src/schemes/float/float_quant.rs index 0bfe5c6ee11..4079703bfd6 100644 --- a/vortex-btrblocks/src/schemes/float/float_quant.rs +++ b/vortex-btrblocks/src/schemes/float/float_quant.rs @@ -3,8 +3,6 @@ //! Lossless float quantization with a fixed frame-of-reference child. -use std::ops::Range; - use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; @@ -12,10 +10,8 @@ use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::VTable; use vortex_array::arrays::Primitive; -use vortex_array::arrays::PrimitiveArray; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::PType; -use vortex_array::match_each_float_ptype; use vortex_array::scalar::Scalar; use vortex_array::validity::Validity; use vortex_compressor::scheme::CompressionEstimate; @@ -35,10 +31,7 @@ use crate::CascadingCompressor; use crate::CompressorContext; use crate::Scheme; use crate::normalize_null_values; - -const SAMPLE_BLOCK_LEN: usize = 64; -const MIN_SAMPLE_BLOCKS: usize = 16; -const SAMPLE_BLOCK_MULTIPLE: usize = 16; +use crate::schemes::sample_primitive_one_percent; /// FloatQuant split with a fixed frame-of-reference primary child. #[derive(Debug, Copy, Clone, PartialEq, Eq)] @@ -68,7 +61,7 @@ impl Scheme for FloatQuantScheme { } CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( |_compressor, data, _best_so_far, _compress_ctx, exec_ctx| { - let sample = float_quant_sample(data.array_as_primitive(), exec_ctx)?; + let sample = sample_primitive_one_percent(data.array_as_primitive(), exec_ctx)?; let Some(analysis) = analyze_float_quant(sample.as_view()) else { return Ok(EstimateVerdict::Skip); }; @@ -225,55 +218,3 @@ fn ordered_u64(bits: u64) -> u64 { !bits } } - -fn float_quant_sample( - primitive: vortex_array::ArrayView<'_, Primitive>, - exec_ctx: &mut ExecutionCtx, -) -> VortexResult { - let sample_blocks = (primitive.len() / 100 / SAMPLE_BLOCK_LEN) - .next_multiple_of(SAMPLE_BLOCK_MULTIPLE) - .max(MIN_SAMPLE_BLOCKS); - let sample_len = sample_blocks * SAMPLE_BLOCK_LEN; - if primitive.len() <= sample_len { - return normalize_null_values(primitive, exec_ctx); - } - - let ranges = float_quant_sample_ranges(primitive.len(), sample_blocks); - let validity = primitive.validity()?; - if validity.definitely_no_nulls() { - return Ok(match_each_float_ptype!(primitive.ptype(), |T| { - let values = primitive.as_slice::(); - let mut sample = Vec::with_capacity(sample_len); - for range in ranges { - sample.extend_from_slice(&values[range]); - } - PrimitiveArray::from_iter(sample) - })); - } - - let validity = validity.execute_mask(primitive.len(), exec_ctx)?; - Ok(match_each_float_ptype!(primitive.ptype(), |T| { - let values = primitive.as_slice::(); - PrimitiveArray::from_option_iter( - ranges - .into_iter() - .flatten() - .map(|index| validity.value(index).then_some(values[index])), - ) - })) -} - -fn float_quant_sample_ranges(len: usize, sample_blocks: usize) -> Vec> { - let partition_len = len / sample_blocks; - let long_partitions = len % sample_blocks; - let mut partition_start = 0; - (0..sample_blocks) - .map(|partition_index| { - let current_partition_len = - partition_len + usize::from(partition_index < long_partitions); - let start = partition_start + (current_partition_len - SAMPLE_BLOCK_LEN) / 2; - partition_start += current_partition_len; - start..start + SAMPLE_BLOCK_LEN - }) - .collect() -} diff --git a/vortex-btrblocks/src/schemes/float/mod.rs b/vortex-btrblocks/src/schemes/float/mod.rs index e2f8aecb398..85bf1958d2b 100644 --- a/vortex-btrblocks/src/schemes/float/mod.rs +++ b/vortex-btrblocks/src/schemes/float/mod.rs @@ -7,6 +7,7 @@ mod alp; mod alprd; mod float_quant; mod ordered_block_residual; +mod ordered_range_packed; mod rle; mod sparse; @@ -17,6 +18,7 @@ pub use alp::ALPScheme; pub use alprd::ALPRDScheme; pub use float_quant::FloatQuantScheme; pub use ordered_block_residual::OrderedBlockResidualScheme; +pub use ordered_range_packed::OrderedRangePackedScheme; #[cfg(feature = "pco")] pub use pco::PcoScheme; pub use rle::FloatRLEScheme; diff --git a/vortex-btrblocks/src/schemes/float/ordered_range_packed.rs b/vortex-btrblocks/src/schemes/float/ordered_range_packed.rs new file mode 100644 index 00000000000..66f34414ad3 --- /dev/null +++ b/vortex-btrblocks/src/schemes/float/ordered_range_packed.rs @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Compact float compression with ordered IEEE bits and fixed range packing. + +use vortex_array::ArrayId; +use vortex_array::ArrayRef; +use vortex_array::Canonical; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::VTable; +use vortex_array::arrays::Primitive; +use vortex_block_residual::OrderedFloat; +use vortex_block_residual::OrderedFloatArraySlotsExt; +use vortex_compressor::scheme::CompressionEstimate; +use vortex_compressor::scheme::DeferredEstimate; +use vortex_compressor::scheme::EstimateVerdict; +use vortex_error::VortexResult; +use vortex_range_packed::RangePacked; + +use crate::ArrayAndStats; +use crate::CascadingCompressor; +use crate::CompressorContext; +use crate::Scheme; +use crate::schemes::range_packed_compact_verdict; +use crate::schemes::sample_primitive_one_percent; + +/// Compress floats as fixed-bin packed ordered IEEE bits. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct OrderedRangePackedScheme; + +impl Scheme for OrderedRangePackedScheme { + fn scheme_name(&self) -> &'static str { + "vortex.float.ordered_range_packed" + } + + fn matches(&self, canonical: &Canonical) -> bool { + canonical.dtype().is_float() + } + + fn produced_encodings(&self) -> Vec { + vec![OrderedFloat.id(), RangePacked.id()] + } + + fn num_children(&self) -> usize { + 1 + } + + fn expected_compression_ratio( + &self, + _data: &ArrayAndStats, + compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> CompressionEstimate { + if compress_ctx.finished_cascading() { + return CompressionEstimate::Verdict(EstimateVerdict::Skip); + } + CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( + |_compressor, data, best_native, _compress_ctx, exec_ctx| { + let sample = sample_primitive_one_percent(data.array_as_primitive(), exec_ctx)?; + let before_nbytes = sample.nbytes(); + let ordered = OrderedFloat::from_primitive(sample.as_view())?; + let encoded = + RangePacked::from_primitive(ordered.encoded().as_::(), exec_ctx)?; + Ok(range_packed_compact_verdict( + before_nbytes, + encoded.nbytes(), + best_native, + )) + }, + ))) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + data: &ArrayAndStats, + _compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, + ) -> VortexResult { + let primitive = data.array_as_primitive(); + let ordered = OrderedFloat::from_primitive(primitive)?; + let encoded = RangePacked::from_primitive(ordered.encoded().as_::(), exec_ctx)?; + Ok(OrderedFloat::try_new(encoded.into_array(), primitive.ptype())?.into_array()) + } +} diff --git a/vortex-btrblocks/src/schemes/integer/mod.rs b/vortex-btrblocks/src/schemes/integer/mod.rs index 43ff8bbb6cd..6a3930d77b7 100644 --- a/vortex-btrblocks/src/schemes/integer/mod.rs +++ b/vortex-btrblocks/src/schemes/integer/mod.rs @@ -8,6 +8,7 @@ mod block_residual; #[cfg(feature = "unstable_encodings")] mod delta; mod for_; +mod range_packed; mod rle; mod runend; mod sequence; @@ -25,6 +26,7 @@ pub use delta::DeltaScheme; pub use for_::FoRScheme; #[cfg(feature = "pco")] pub use pco::PcoScheme; +pub use range_packed::RangePackedScheme; pub use rle::IntRLEScheme; pub(crate) use rle::rle_compress; #[cfg(feature = "unstable_encodings")] diff --git a/vortex-btrblocks/src/schemes/integer/range_packed.rs b/vortex-btrblocks/src/schemes/integer/range_packed.rs new file mode 100644 index 00000000000..7f675e3842a --- /dev/null +++ b/vortex-btrblocks/src/schemes/integer/range_packed.rs @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Compact integer compression with fixed bin identifiers and packed offsets. + +use vortex_array::ArrayId; +use vortex_array::ArrayRef; +use vortex_array::Canonical; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::VTable; +use vortex_compressor::scheme::CompressionEstimate; +use vortex_compressor::scheme::DeferredEstimate; +use vortex_error::VortexResult; +use vortex_range_packed::RangePacked; + +use crate::ArrayAndStats; +use crate::CascadingCompressor; +use crate::CompressorContext; +use crate::Scheme; +use crate::schemes::range_packed_compact_verdict; +use crate::schemes::sample_primitive_one_percent; + +/// Compress integers with one fixed range table and packed offsets. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct RangePackedScheme; + +impl Scheme for RangePackedScheme { + fn scheme_name(&self) -> &'static str { + "vortex.int.range_packed" + } + + fn matches(&self, canonical: &Canonical) -> bool { + canonical.dtype().is_int() + } + + fn produced_encodings(&self) -> Vec { + vec![RangePacked.id()] + } + + fn expected_compression_ratio( + &self, + _data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> CompressionEstimate { + CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( + |_compressor, data, best_native, _compress_ctx, exec_ctx| { + let sample = sample_primitive_one_percent(data.array_as_primitive(), exec_ctx)?; + let before_nbytes = sample.nbytes(); + let encoded = RangePacked::from_primitive(sample.as_view(), exec_ctx)?; + Ok(range_packed_compact_verdict( + before_nbytes, + encoded.nbytes(), + best_native, + )) + }, + ))) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + data: &ArrayAndStats, + _compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, + ) -> VortexResult { + Ok(RangePacked::from_primitive(data.array_as_primitive(), exec_ctx)?.into_array()) + } +} diff --git a/vortex-btrblocks/src/schemes/mod.rs b/vortex-btrblocks/src/schemes/mod.rs index 765cc6804e0..0e5f3478b15 100644 --- a/vortex-btrblocks/src/schemes/mod.rs +++ b/vortex-btrblocks/src/schemes/mod.rs @@ -13,8 +13,14 @@ pub mod temporal; pub(crate) mod patches; +use std::ops::Range; + +use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; +use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::NativePType; +use vortex_array::match_each_native_ptype; use vortex_compressor::builtins::BinaryDictScheme; use vortex_compressor::builtins::FloatDictScheme; use vortex_compressor::builtins::IntDictScheme; @@ -22,10 +28,90 @@ use vortex_compressor::builtins::StringDictScheme; use vortex_compressor::scheme::AncestorExclusion; use vortex_compressor::scheme::ChildSelection; use vortex_compressor::scheme::DescendantExclusion; +use vortex_compressor::scheme::EstimateScore; +use vortex_compressor::scheme::EstimateVerdict; use vortex_compressor::scheme::SchemeExt; +use vortex_error::VortexResult; +use crate::normalize_null_values; use crate::schemes::integer::SparseScheme; +const SAMPLE_BLOCK_LEN: usize = 64; +const MIN_SAMPLE_BLOCKS: usize = 16; +const SAMPLE_BLOCK_MULTIPLE: usize = 16; +const RANGE_PACKED_NATIVE_SIZE_ADVANTAGE: f64 = 1.10; +const RANGE_PACKED_PCO_SIZE_ALLOWANCE: f64 = 1.10; + +fn range_packed_compact_verdict( + before_nbytes: u64, + after_nbytes: u64, + best_native: Option, +) -> EstimateVerdict { + if after_nbytes == 0 || after_nbytes >= before_nbytes { + return EstimateVerdict::Skip; + } + let ratio = before_nbytes as f64 / after_nbytes as f64; + if best_native + .and_then(EstimateScore::finite_ratio) + .is_some_and(|best| ratio < best * RANGE_PACKED_NATIVE_SIZE_ADVANTAGE) + { + return EstimateVerdict::Skip; + } + EstimateVerdict::Ratio(ratio * RANGE_PACKED_PCO_SIZE_ALLOWANCE) +} + +fn sample_primitive_one_percent( + primitive: ArrayView<'_, Primitive>, + exec_ctx: &mut ExecutionCtx, +) -> VortexResult { + let sample_blocks = (primitive.len() / 100 / SAMPLE_BLOCK_LEN) + .next_multiple_of(SAMPLE_BLOCK_MULTIPLE) + .max(MIN_SAMPLE_BLOCKS); + let sample_len = sample_blocks * SAMPLE_BLOCK_LEN; + if primitive.len() <= sample_len { + return normalize_null_values(primitive, exec_ctx); + } + + let ranges = one_percent_sample_ranges(primitive.len(), sample_blocks); + let validity = primitive.validity()?; + if validity.definitely_no_nulls() { + return Ok(match_each_native_ptype!(primitive.ptype(), |T| { + let values = primitive.as_slice::(); + let mut sample = Vec::with_capacity(sample_len); + for range in ranges { + sample.extend_from_slice(&values[range]); + } + PrimitiveArray::from_iter(sample) + })); + } + + let validity = validity.execute_mask(primitive.len(), exec_ctx)?; + Ok(match_each_native_ptype!(primitive.ptype(), |T| { + let values = primitive.as_slice::(); + PrimitiveArray::from_option_iter( + ranges + .into_iter() + .flatten() + .map(|index| validity.value(index).then_some(values[index])), + ) + })) +} + +fn one_percent_sample_ranges(len: usize, sample_blocks: usize) -> Vec> { + let partition_len = len / sample_blocks; + let long_partitions = len % sample_blocks; + let mut partition_start = 0; + (0..sample_blocks) + .map(|partition_index| { + let current_partition_len = + partition_len + usize::from(partition_index < long_partitions); + let start = partition_start + (current_partition_len - SAMPLE_BLOCK_LEN) / 2; + partition_start += current_partition_len; + start..start + SAMPLE_BLOCK_LEN + }) + .collect() +} + fn sample_primitive_blocks( values: &[T], all_valid: bool, diff --git a/vortex/src/editions/core/v2026_08.rs b/vortex/src/editions/core/v2026_08.rs index b64faf56025..5903db06a5e 100644 --- a/vortex/src/editions/core/v2026_08.rs +++ b/vortex/src/editions/core/v2026_08.rs @@ -30,6 +30,7 @@ pub static DECLARATION: EditionDeclaration = EditionDeclaration { EditionMember::array(&"vortex.float_quant"), EditionMember::array(&"vortex.map"), EditionMember::array(&"vortex.ordered_float"), + EditionMember::array(&"vortex.range_packed"), EditionMember::aggregate(&"vortex.bounded_max"), EditionMember::aggregate(&"vortex.bounded_min"), EditionMember::aggregate(&"vortex.max"), From e2c3cf67cc2d1f48332d68970f54ee5dcbd31bb9 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 17:59:25 +0100 Subject: [PATCH 27/78] refactor: remove RangePacked selector experiment Signed-off-by: Will Manning --- .../examples/float_compressor_bench.rs | 4 - vortex-btrblocks/src/builder.rs | 31 +------ vortex-btrblocks/src/schemes/float/mod.rs | 2 - .../src/schemes/float/ordered_range_packed.rs | 86 ------------------- vortex-btrblocks/src/schemes/integer/mod.rs | 2 - .../src/schemes/integer/range_packed.rs | 70 --------------- vortex-btrblocks/src/schemes/mod.rs | 23 ----- 7 files changed, 4 insertions(+), 214 deletions(-) delete mode 100644 vortex-btrblocks/src/schemes/float/ordered_range_packed.rs delete mode 100644 vortex-btrblocks/src/schemes/integer/range_packed.rs diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index 3398db9e0eb..0035a9db1ad 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -56,9 +56,7 @@ use vortex_btrblocks::BtrBlocksCompressorBuilder; use vortex_btrblocks::SchemeExt; use vortex_btrblocks::schemes::float::FloatQuantScheme; use vortex_btrblocks::schemes::float::OrderedBlockResidualScheme; -use vortex_btrblocks::schemes::float::OrderedRangePackedScheme; use vortex_btrblocks::schemes::integer::BlockResidualScheme; -use vortex_btrblocks::schemes::integer::RangePackedScheme; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_err; @@ -1644,8 +1642,6 @@ fn compressors() -> Vec<(&'static str, BtrBlocksCompressor)> { FloatQuantScheme.id(), OrderedBlockResidualScheme.id(), BlockResidualScheme.id(), - OrderedRangePackedScheme.id(), - RangePackedScheme.id(), ]; vec![ ( diff --git a/vortex-btrblocks/src/builder.rs b/vortex-btrblocks/src/builder.rs index f1f0fc7666f..bd6fcdb430d 100644 --- a/vortex-btrblocks/src/builder.rs +++ b/vortex-btrblocks/src/builder.rs @@ -133,10 +133,11 @@ impl BtrBlocksCompressorBuilder { self } - /// Adds compact encoding schemes for strings, binary values, and numeric values. + /// Adds compact encoding schemes (Zstd for strings and binary, Pco for numerics). /// /// This provides better compression ratios than the default, especially for floating-point - /// heavy datasets. This method requires the `zstd` feature. The `pco` feature adds Pco. + /// heavy datasets. Requires the `zstd` feature. When the `pco` feature is also enabled, + /// Pco schemes for integers and floats are included. /// /// # Panics /// @@ -145,9 +146,7 @@ impl BtrBlocksCompressorBuilder { pub fn with_compact(self) -> Self { let builder = self .with_new_scheme(&string::ZstdScheme) - .with_new_scheme(&binary::ZstdScheme) - .with_new_scheme(&integer::RangePackedScheme) - .with_new_scheme(&float::OrderedRangePackedScheme); + .with_new_scheme(&binary::ZstdScheme); #[cfg(feature = "pco")] let builder = builder @@ -175,13 +174,11 @@ impl BtrBlocksCompressorBuilder { )] let mut excluded: Vec = vec![ integer::BlockResidualScheme.id(), - integer::RangePackedScheme.id(), integer::SparseScheme.id(), integer::IntRLEScheme.id(), float::ALPRDScheme.id(), float::FloatQuantScheme.id(), float::OrderedBlockResidualScheme.id(), - float::OrderedRangePackedScheme.id(), float::FloatRLEScheme.id(), float::NullDominatedSparseScheme.id(), string::StringDictScheme.id(), @@ -231,7 +228,6 @@ impl BtrBlocksCompressorBuilder { #[cfg(test)] mod tests { use vortex_array::VTable; - use vortex_error::VortexResult; use vortex_fastlanes::FoR; use super::*; @@ -320,25 +316,6 @@ mod tests { ); } - #[test] - #[cfg(feature = "zstd")] - fn compact_adds_range_packed_after_default_schemes() -> VortexResult<()> { - let builder = BtrBlocksCompressorBuilder::default().with_compact(); - let integer_index = builder - .schemes - .iter() - .position(|scheme| scheme.id() == integer::RangePackedScheme.id()) - .ok_or_else(|| vortex_error::vortex_err!("compact lacks integer RangePacked"))?; - let float_index = builder - .schemes - .iter() - .position(|scheme| scheme.id() == float::OrderedRangePackedScheme.id()) - .ok_or_else(|| vortex_error::vortex_err!("compact lacks ordered RangePacked"))?; - assert!(integer_index >= ALL_SCHEMES.len()); - assert!(float_index > integer_index); - Ok(()) - } - #[test] fn cuda_compatible_uses_fsst_for_strings() { let builder = BtrBlocksCompressorBuilder::default().only_cuda_compatible(); diff --git a/vortex-btrblocks/src/schemes/float/mod.rs b/vortex-btrblocks/src/schemes/float/mod.rs index 85bf1958d2b..e2f8aecb398 100644 --- a/vortex-btrblocks/src/schemes/float/mod.rs +++ b/vortex-btrblocks/src/schemes/float/mod.rs @@ -7,7 +7,6 @@ mod alp; mod alprd; mod float_quant; mod ordered_block_residual; -mod ordered_range_packed; mod rle; mod sparse; @@ -18,7 +17,6 @@ pub use alp::ALPScheme; pub use alprd::ALPRDScheme; pub use float_quant::FloatQuantScheme; pub use ordered_block_residual::OrderedBlockResidualScheme; -pub use ordered_range_packed::OrderedRangePackedScheme; #[cfg(feature = "pco")] pub use pco::PcoScheme; pub use rle::FloatRLEScheme; diff --git a/vortex-btrblocks/src/schemes/float/ordered_range_packed.rs b/vortex-btrblocks/src/schemes/float/ordered_range_packed.rs deleted file mode 100644 index 66f34414ad3..00000000000 --- a/vortex-btrblocks/src/schemes/float/ordered_range_packed.rs +++ /dev/null @@ -1,86 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! Compact float compression with ordered IEEE bits and fixed range packing. - -use vortex_array::ArrayId; -use vortex_array::ArrayRef; -use vortex_array::Canonical; -use vortex_array::ExecutionCtx; -use vortex_array::IntoArray; -use vortex_array::VTable; -use vortex_array::arrays::Primitive; -use vortex_block_residual::OrderedFloat; -use vortex_block_residual::OrderedFloatArraySlotsExt; -use vortex_compressor::scheme::CompressionEstimate; -use vortex_compressor::scheme::DeferredEstimate; -use vortex_compressor::scheme::EstimateVerdict; -use vortex_error::VortexResult; -use vortex_range_packed::RangePacked; - -use crate::ArrayAndStats; -use crate::CascadingCompressor; -use crate::CompressorContext; -use crate::Scheme; -use crate::schemes::range_packed_compact_verdict; -use crate::schemes::sample_primitive_one_percent; - -/// Compress floats as fixed-bin packed ordered IEEE bits. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub struct OrderedRangePackedScheme; - -impl Scheme for OrderedRangePackedScheme { - fn scheme_name(&self) -> &'static str { - "vortex.float.ordered_range_packed" - } - - fn matches(&self, canonical: &Canonical) -> bool { - canonical.dtype().is_float() - } - - fn produced_encodings(&self) -> Vec { - vec![OrderedFloat.id(), RangePacked.id()] - } - - fn num_children(&self) -> usize { - 1 - } - - fn expected_compression_ratio( - &self, - _data: &ArrayAndStats, - compress_ctx: CompressorContext, - _exec_ctx: &mut ExecutionCtx, - ) -> CompressionEstimate { - if compress_ctx.finished_cascading() { - return CompressionEstimate::Verdict(EstimateVerdict::Skip); - } - CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |_compressor, data, best_native, _compress_ctx, exec_ctx| { - let sample = sample_primitive_one_percent(data.array_as_primitive(), exec_ctx)?; - let before_nbytes = sample.nbytes(); - let ordered = OrderedFloat::from_primitive(sample.as_view())?; - let encoded = - RangePacked::from_primitive(ordered.encoded().as_::(), exec_ctx)?; - Ok(range_packed_compact_verdict( - before_nbytes, - encoded.nbytes(), - best_native, - )) - }, - ))) - } - - fn compress( - &self, - _compressor: &CascadingCompressor, - data: &ArrayAndStats, - _compress_ctx: CompressorContext, - exec_ctx: &mut ExecutionCtx, - ) -> VortexResult { - let primitive = data.array_as_primitive(); - let ordered = OrderedFloat::from_primitive(primitive)?; - let encoded = RangePacked::from_primitive(ordered.encoded().as_::(), exec_ctx)?; - Ok(OrderedFloat::try_new(encoded.into_array(), primitive.ptype())?.into_array()) - } -} diff --git a/vortex-btrblocks/src/schemes/integer/mod.rs b/vortex-btrblocks/src/schemes/integer/mod.rs index 6a3930d77b7..43ff8bbb6cd 100644 --- a/vortex-btrblocks/src/schemes/integer/mod.rs +++ b/vortex-btrblocks/src/schemes/integer/mod.rs @@ -8,7 +8,6 @@ mod block_residual; #[cfg(feature = "unstable_encodings")] mod delta; mod for_; -mod range_packed; mod rle; mod runend; mod sequence; @@ -26,7 +25,6 @@ pub use delta::DeltaScheme; pub use for_::FoRScheme; #[cfg(feature = "pco")] pub use pco::PcoScheme; -pub use range_packed::RangePackedScheme; pub use rle::IntRLEScheme; pub(crate) use rle::rle_compress; #[cfg(feature = "unstable_encodings")] diff --git a/vortex-btrblocks/src/schemes/integer/range_packed.rs b/vortex-btrblocks/src/schemes/integer/range_packed.rs deleted file mode 100644 index 7f675e3842a..00000000000 --- a/vortex-btrblocks/src/schemes/integer/range_packed.rs +++ /dev/null @@ -1,70 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! Compact integer compression with fixed bin identifiers and packed offsets. - -use vortex_array::ArrayId; -use vortex_array::ArrayRef; -use vortex_array::Canonical; -use vortex_array::ExecutionCtx; -use vortex_array::IntoArray; -use vortex_array::VTable; -use vortex_compressor::scheme::CompressionEstimate; -use vortex_compressor::scheme::DeferredEstimate; -use vortex_error::VortexResult; -use vortex_range_packed::RangePacked; - -use crate::ArrayAndStats; -use crate::CascadingCompressor; -use crate::CompressorContext; -use crate::Scheme; -use crate::schemes::range_packed_compact_verdict; -use crate::schemes::sample_primitive_one_percent; - -/// Compress integers with one fixed range table and packed offsets. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub struct RangePackedScheme; - -impl Scheme for RangePackedScheme { - fn scheme_name(&self) -> &'static str { - "vortex.int.range_packed" - } - - fn matches(&self, canonical: &Canonical) -> bool { - canonical.dtype().is_int() - } - - fn produced_encodings(&self) -> Vec { - vec![RangePacked.id()] - } - - fn expected_compression_ratio( - &self, - _data: &ArrayAndStats, - _compress_ctx: CompressorContext, - _exec_ctx: &mut ExecutionCtx, - ) -> CompressionEstimate { - CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |_compressor, data, best_native, _compress_ctx, exec_ctx| { - let sample = sample_primitive_one_percent(data.array_as_primitive(), exec_ctx)?; - let before_nbytes = sample.nbytes(); - let encoded = RangePacked::from_primitive(sample.as_view(), exec_ctx)?; - Ok(range_packed_compact_verdict( - before_nbytes, - encoded.nbytes(), - best_native, - )) - }, - ))) - } - - fn compress( - &self, - _compressor: &CascadingCompressor, - data: &ArrayAndStats, - _compress_ctx: CompressorContext, - exec_ctx: &mut ExecutionCtx, - ) -> VortexResult { - Ok(RangePacked::from_primitive(data.array_as_primitive(), exec_ctx)?.into_array()) - } -} diff --git a/vortex-btrblocks/src/schemes/mod.rs b/vortex-btrblocks/src/schemes/mod.rs index 0e5f3478b15..0b9baab0b16 100644 --- a/vortex-btrblocks/src/schemes/mod.rs +++ b/vortex-btrblocks/src/schemes/mod.rs @@ -28,8 +28,6 @@ use vortex_compressor::builtins::StringDictScheme; use vortex_compressor::scheme::AncestorExclusion; use vortex_compressor::scheme::ChildSelection; use vortex_compressor::scheme::DescendantExclusion; -use vortex_compressor::scheme::EstimateScore; -use vortex_compressor::scheme::EstimateVerdict; use vortex_compressor::scheme::SchemeExt; use vortex_error::VortexResult; @@ -39,27 +37,6 @@ use crate::schemes::integer::SparseScheme; const SAMPLE_BLOCK_LEN: usize = 64; const MIN_SAMPLE_BLOCKS: usize = 16; const SAMPLE_BLOCK_MULTIPLE: usize = 16; -const RANGE_PACKED_NATIVE_SIZE_ADVANTAGE: f64 = 1.10; -const RANGE_PACKED_PCO_SIZE_ALLOWANCE: f64 = 1.10; - -fn range_packed_compact_verdict( - before_nbytes: u64, - after_nbytes: u64, - best_native: Option, -) -> EstimateVerdict { - if after_nbytes == 0 || after_nbytes >= before_nbytes { - return EstimateVerdict::Skip; - } - let ratio = before_nbytes as f64 / after_nbytes as f64; - if best_native - .and_then(EstimateScore::finite_ratio) - .is_some_and(|best| ratio < best * RANGE_PACKED_NATIVE_SIZE_ADVANTAGE) - { - return EstimateVerdict::Skip; - } - EstimateVerdict::Ratio(ratio * RANGE_PACKED_PCO_SIZE_ALLOWANCE) -} - fn sample_primitive_one_percent( primitive: ArrayView<'_, Primitive>, exec_ctx: &mut ExecutionCtx, From 3d47f2c700be782e635c6331bb9fae092db8deba Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 18:23:18 +0100 Subject: [PATCH 28/78] bench: isolate numeric file topology costs Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 81 +++++++++++++------ .../examples/float_compressor_bench.rs | 25 +++++- 2 files changed, 82 insertions(+), 24 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index e0a485b2182..ed4322faad8 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -24,7 +24,9 @@ Remove `RangeEntropyArray`, `RangeEntropyScheme`, and `BitSplitCodec` from the f The `wm/pcodec-entropy-experiments` branch preserves the complete entropy and bit-split prototypes. -Keep fixed-bin range packing as an experimental Compact candidate. It is not a default candidate. +Keep `RangePackedArray` and its manual benchmark as experimental work. + +Do not add RangePacked to the default or Compact selector. Do not add adjacent Delta, Delta-of-delta, Delta with lookback, or convolution Delta. @@ -46,6 +48,10 @@ Direct integer BlockResidual supports every integer type. The default selector a The retained schemes win on specific structures. They do not replace ALP or ALP-RD across general float data. +FloatQuant now passes the speed gates for zero-secondary and one-bit-secondary inputs. + +BlockResidual now passes the direct speed gates for 32-bit and 64-bit integers. + The GloVe result identifies a separate entropy gap inside the ALP integer child. Generic quotient and remainder trees do not close that gap. @@ -238,9 +244,9 @@ The fixed tree was `FloatQuant(FoR(BitPacked), BitPacked)`. The secondary used o | Configuration | Bytes | Encode MB/s | Decode MB/s | Scalar access ns | | --- | ---: | ---: | ---: | ---: | -| Prior ALP-RD default | 14,057,966 | 564.5 | 12,234.8 | 187 | -| Default with FloatQuant | 9,004,032 | 633.5 | 13,462.8 | 174 | -| Compact Pco | 6,171,139 | 268.4 | 3,054.5 | Not measured | +| Prior ALP-RD default | 14,057,966 | 558.0 | 11,420 | 250 | +| Default with FloatQuant | 9,004,032 | 603.2 | 13,060 | 209 | +| Compact Pco | 6,171,139 | 255.5 | 2,914 | Not measured | FloatQuant reduced size by 36.0 percent. It was 2.9 percent larger than the zero-secondary tree. @@ -250,15 +256,15 @@ The first generic decode path reached 8,375 MB/s. The fused pair kernel increased decode throughput by 59.4 percent. -The complete default encoded 12.2 percent faster and decoded 10.0 percent faster than the prior default. +The complete default encoded 8.1 percent faster and decoded 14.4 percent faster than the prior default. -Scalar access latency decreased by 7.0 percent. +Scalar access latency decreased by 16.4 percent. -The direct two-child scheme compressed at 6,469 MB/s. +The direct two-child scheme compressed at 6,298 MB/s. The fused kernel supports aligned, patch-free BitPacked secondary children of any width. -Decode throughput was 12,900 MB/s with a one-bit secondary. +Decode throughput was 13,010 MB/s with a one-bit secondary. It was 12,410 MB/s with a 16-bit secondary. @@ -308,8 +314,8 @@ The direct benchmark uses two million block-local values. | Logical type and tree | Encode GB/s | Decode GB/s | Scalar access ns | | --- | ---: | ---: | ---: | -| `u64` BlockResidual | 5.00 | 36.43 | 125 | -| `u64` FoR plus BitPacked | 4.25 | 35.37 | 115 | +| `u64` BlockResidual | 5.11 | 37.39 | 84 | +| `u64` FoR plus BitPacked | 4.51 | 36.22 | 125 | | `i16` BlockResidual | 1.39 | 31.86 | 125 | | `i16` FoR plus BitPacked | 1.15 | 42.14 | 125 | @@ -664,7 +670,30 @@ Bitmap patches improve size, but they do not match the Pco child. The results reject a general quotient and remainder array with ordinary children. -They support a small-alphabet entropy experiment and a fixed-bin Compact experiment. +They support a small-alphabet entropy experiment and a fixed integer split experiment. + +### Exact ALP child comparison + +The benchmark replaces a Compact `ALP(Pco)` child with the same latent integers in BlockResidual. + +This comparison separates float transform quality from integer child compression. + +| Input | Current default bytes | `ALP(BlockResidual)` bytes | Current decode MB/s | Candidate decode MB/s | Candidate encode MB/s | +| --- | ---: | ---: | ---: | ---: | ---: | +| CMS Payments | 4,146,124 | 3,918,239 | 14,202 | 20,280 | 2,605 | +| GloVe | 6,274,834 | 6,298,751 | 16,738 | 19,310 | 1,201 | + +The CMS candidate is 5.5 percent smaller and 42.8 percent faster to decode. + +The current selector rejects it because the 64-bit BlockResidual score includes a 1.20 factor. + +This result makes the CMS miss a final selector calibration issue. + +The GloVe candidate is 0.4 percent larger than the current default. + +It does not recover the 15.7 percent Pco size advantage. + +GloVe still requires a better integer split or small-alphabet backend for the ALP child. ### Fixed-bin range backend experiments @@ -717,7 +746,7 @@ The first CMS decoder reached 8,057 MB/s. The specialized decoder increased thro The complete CMS default tree decodes at 14,575 MB/s. The fixed-bin child is 6.1 percent slower. -The benchmark now wraps the codec in an experimental Vortex array. +The benchmark now wraps the codec in a serialized Vortex array. This wrapper permits complete ALP decode and scalar measurements without a storage-format commitment. @@ -739,7 +768,7 @@ Fusion increased CMS decode by 4.8 percent and Food decode by 18.7 percent. It d The fixed-bin tree fails the default decode gate on all three inputs. -CMS and Food support a Compact candidate for classic Pco bins. +CMS and Food show that fixed bins can trade size for faster access than Pco. CMS uses 7.8 percent more bytes than Compact. Generic decode is 67.2 percent faster, and scalar access is 37.8 times faster. @@ -749,7 +778,13 @@ The fused Food decode is 2.0 times as fast as Compact. The generic composition a GloVe uses 16.8 percent more bytes than Compact because Pco also uses `IntMult(10)`. -The evidence does not justify a fused ALP and fixed-bin array. A composable integer child has lower implementation complexity. +The evidence does not justify a fused ALP and fixed-bin array. + +The temporary Compact selector increased GloVe encode time by 7.8 percent when it rejected RangePacked. + +The selector also increased aggregate CMS bytes by 1.1 percent after it replaced Pco. + +RangePacked therefore remains outside all writer selectors. ### Nullable and Delta-heavy fixed-bin cases @@ -966,15 +1001,15 @@ Complete these remaining steps: 1. Validate the nonlinear patch cost on the complete corpus. 2. Add a true ALP-RD child composition case if a real selected tree exposes one. 3. Validate the remaining rejected analysis cost in the complete corpus. -4. Test the complete fixed-bin tree on every available classic-bin Pco win. -5. Define a Compact size threshold for the fixed-bin candidate. -6. Promote fixed-bin packing to a production array only if more unrelated classic-bin columns pass. -7. Add real embedding datasets beyond GloVe when licenses and loaders permit them. -8. Classify more float columns where Pco beats ALP, ALP-RD, and the new schemes. -9. Prototype one lightweight scheme for the repeated real-float gap classes. -10. Run the complete `bench-vortex` compression corpus after the candidate set stabilizes. -11. Compare the geometric mean against Parquet with Zstd. -12. Calibrate all selector thresholds from complete corpus and selected-tree evidence. +4. Add real embedding datasets beyond GloVe when licenses and loaders permit them. +5. Classify more float columns where Pco beats ALP, ALP-RD, and the new schemes. +6. Prototype a lightweight integer split for Pco `IntMult` wins under ALP. +7. Prototype one lightweight scheme for repeated real-float Pco wins. +8. Run the complete `bench-vortex` compression corpus after the candidate set stabilizes. +9. Compare the geometric mean against Parquet with Zstd. +10. Calibrate all selector thresholds from complete corpus and selected-tree evidence. +11. Revisit the 32-bit and 64-bit BlockResidual score factors with selected-tree evidence. +12. Keep RangePacked outside writer selectors unless a new decode design changes the result. 13. Update this plan after each experiment. ## Pull request structure diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index 0035a9db1ad..d5983a5283f 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -35,6 +35,7 @@ use vortex_array::IntoArray; use vortex_array::RecursiveCanonical; use vortex_array::VortexSessionExecute; use vortex_array::array_session; +use vortex_array::arrays::ChunkedArray; use vortex_array::arrays::ListArray; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; @@ -515,9 +516,31 @@ fn encode_all( columns: &[Column], session: &VortexSession, ) -> VortexResult> { + let chunk_rows = std::env::var("VORTEX_BENCH_CHUNK_ROWS") + .ok() + .map(|value| { + value + .parse::() + .map_err(|error| vortex_err!("invalid VORTEX_BENCH_CHUNK_ROWS: {error}")) + }) + .transpose()?; columns .iter() - .map(|column| compressor.compress(&column.array, &mut session.create_execution_ctx())) + .map(|column| { + let Some(chunk_rows) = chunk_rows else { + return compressor.compress(&column.array, &mut session.create_execution_ctx()); + }; + vortex_ensure!(chunk_rows > 0, "VORTEX_BENCH_CHUNK_ROWS must be positive"); + let chunks = (0..column.array.len()) + .step_by(chunk_rows) + .map(|start| { + let stop = (start + chunk_rows).min(column.array.len()); + let chunk = column.array.slice(start..stop)?; + compressor.compress(&chunk, &mut session.create_execution_ctx()) + }) + .collect::>>()?; + Ok(ChunkedArray::try_new(chunks, column.array.dtype().clone())?.into_array()) + }) .collect() } From 8e2b55f82a9be55acb813bda9f7afe708d506948 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 19:09:16 +0100 Subject: [PATCH 29/78] perf: flatten BlockResidual file layout Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 89 +++- .../src/block_residual_array.rs | 424 +++++++++++------- .../block-residual/src/ordered_float_array.rs | 2 +- .../examples/float_compressor_bench.rs | 32 +- .../schemes/float/scheme_selection_tests.rs | 15 +- .../schemes/integer/scheme_selection_tests.rs | 16 +- ...den__unstable__string_fsst_structured.snap | 6 +- 7 files changed, 374 insertions(+), 210 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index ed4322faad8..d9d50448f9d 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -8,6 +8,22 @@ Keep native Vortex arrays, bounded metadata, fast canonical decode, and bounded Pco remains a compression oracle. The new arrays do not preserve Pco byte compatibility. +The production target is the default compressor, not Compact. + +Use Compact to identify numeric columns where the default compressor leaves a material size gap. + +For each repeated gap, identify the Pco model that creates the gain. + +Transfer that model into a native array only when it preserves these Default properties: + +- Full decode remains close to the displaced encoding. +- Compression throughput remains close to the displaced encoding. +- Scalar access remains O(1) or bounded O(log N). +- Rejected candidates add little selector cost. +- The size gain remains material after native array overhead. + +Compact can use experimental schemes during evaluation. Default selection remains the release decision. + ## Current decision Focus the production work on these encodings: @@ -82,9 +98,11 @@ Rare wide residuals use sorted positions and packed high bits. Scalar access uses one packed read and one binary search over the patch positions. -Child arrays store references, widths, offsets, packed words, patch positions, and patch high bits. +One aligned parent buffer stores references, widths, offsets, packed words, patch positions, and patch high bits. -Fixed metadata stores only lengths and slice bounds. Variable tables remain in child arrays. +Fixed metadata stores only lengths and slice bounds. + +Typed zero-copy views point into the parent buffer after a file read. The array supports canonical decode, scalar access, slice reduction, serialization, and validation. @@ -92,7 +110,7 @@ The `OrderedFloat(BlockResidual)` execute kernel combines residual decode with t The residual payload uses the logical integer width. A 16-bit array uses 16-bit FastLanes pack and unpack operations. -The serialized residual payload remains a `u64` child. The logical type defines the packed word interpretation. +The serialized residual payload remains a `u64` section. The logical type defines the packed word interpretation. The production codec uses one reference per block. The multi-reference prototype did not justify its encode and decode costs. @@ -347,7 +365,7 @@ Compact gives priority to the Pco size point. Narrow BlockResidual therefore has The 48.8 percent saving against FoR plus BitPacked supports one more narrow decode optimization pass. -The BlockResidual estimator measures its sampled tree exactly, with all child arrays. +The BlockResidual estimator measures its sampled tree exactly, with all payload sections. The incumbent and outer tree estimates remain approximate. Trial compression previously mis-ranked Dict and ALP on Taxi tips. @@ -469,6 +487,66 @@ One width factor does not predict both signed and unsigned results. Retain the current factor until the final corpus calibration uses selected-tree evidence. +### Serialized BlockResidual topology + +The first serialized layout stored nine primitive child arrays per BlockResidual array. + +The codec kernels remained fast, but the file reader executed nine extra array nodes for each chunk. + +An intermediate layout replaced those children with nine direct parent buffers. + +That layout improved file decode, but each buffer still produced a separate file segment. + +The final layout stores every table and payload in one aligned parent buffer. + +The section order preserves native alignment without padding: + +1. Block references and residual words use 64-bit sections. +2. Residual and patch offsets use 32-bit sections. +3. Patch positions use one 16-bit section. +4. Widths and packed patch high bits use byte sections. + +Typed zero-copy views share the same allocation. Scalar access and bulk decode read those views directly. + +The single-encoding benchmark uses two million block-local values. + +| Logical type | Encode GB/s | Decode GB/s | Scalar access ns | +| --- | ---: | ---: | ---: | +| `u64` | 5.23 | 36.87 | 83 | +| `u32` | 2.79 | 46.69 | 38 | +| `i32` | 2.78 | 32.23 | 41 | +| `i16` | 1.42 | 32.18 | 83 | + +The direct 8-bit and 16-bit selector exclusion remains valid. + +The broad file benchmark uses three iterations across eight datasets. + +Positive throughput changes indicate faster execution. + +| Dataset | Size change | Encode throughput change | Decode throughput change | +| --- | ---: | ---: | ---: | +| Taxi | -3.24 percent | -1.06 percent | +2.51 percent | +| GloVe | 0.00 percent | +2.45 percent | +0.53 percent | +| Arade | -0.19 percent | +0.47 percent | +1.72 percent | +| Bimbo | -1.38 percent | +0.57 percent | +0.05 percent | +| CMSprovider | -1.41 percent | +0.20 percent | -0.06 percent | +| Euro2016 | -2.68 percent | +1.68 percent | -5.74 percent | +| Food | -1.76 percent | +0.50 percent | -3.50 percent | +| HashTags | -0.63 percent | +1.77 percent | -9.71 percent | +| Geometric mean | -1.42 percent | +0.82 percent | -1.86 percent | + +Five datasets decoded at parity or faster. + +The focused five-iteration runs showed smaller HashTags decode losses of 3.7 to 5.6 percent. + +The focused Euro2016 decode loss ranged from 3.8 to 5.1 percent. + +Focused encode throughput remained within one percent of the prior default. + +This layout recovers most of the Compact-like size gain without a material aggregate throughput loss. + +The remaining threshold calibration must use selected-column evidence and the complete corpus. + ### Patch-density sweep The u32 input uses one large outlier at each configured stride. @@ -976,6 +1054,9 @@ This round completed these steps: - Added complete temporal, FSST, Sparse, RunEnd, list, and ALP composition benchmarks. - Added selection tests for BlockResidual under ALP, Sparse, and RunEnd. - Added exact allocation-free BlockResidual size estimates. +- Replaced nine BlockResidual child arrays with one aligned parent payload buffer. +- Removed eight file segments from each serialized BlockResidual array. +- Revalidated size and throughput across eight file datasets. - Added an all-valid fast path for locality sample copies. - Rejected a four-block short-column estimate after a real mis-ranking. - Added patch-density statistics to the BlockResidual profile. diff --git a/encodings/block-residual/src/block_residual_array.rs b/encodings/block-residual/src/block_residual_array.rs index 1bc17d0a157..9982e728f0a 100644 --- a/encodings/block-residual/src/block_residual_array.rs +++ b/encodings/block-residual/src/block_residual_array.rs @@ -27,7 +27,6 @@ use vortex_array::arrays::slice::SliceReduceAdaptor; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; -use vortex_array::dtype::Nullability::NonNullable; use vortex_array::dtype::PType; use vortex_array::optimizer::rules::ParentRuleSet; use vortex_array::scalar::Scalar; @@ -38,7 +37,11 @@ use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityVTable; use vortex_array::vtable::child_to_validity; use vortex_array::vtable::validity_to_child; +use vortex_buffer::Alignment; +use vortex_buffer::Buffer; use vortex_buffer::BufferMut; +use vortex_buffer::ByteBuffer; +use vortex_buffer::ByteBufferMut; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; @@ -55,7 +58,7 @@ use crate::codec::packed_words_as_native; use crate::codec::read_wide_bits; const BLOCK_LEN: usize = 1024; -const METADATA_VERSION: u8 = 1; +const METADATA_VERSION: u8 = 2; const METADATA_LEN: usize = 41; /// Ordered unsigned integers with one reference and packed residuals per block. @@ -64,24 +67,6 @@ pub type BlockResidualArray = Array; #[array_slots(BlockResidual)] pub struct BlockResidualSlots { #[slot(0)] - pub bases: ArrayRef, - #[slot(1)] - pub residual_widths: ArrayRef, - #[slot(2)] - pub high_widths: ArrayRef, - #[slot(3)] - pub residual_starts: ArrayRef, - #[slot(4)] - pub patch_starts: ArrayRef, - #[slot(5)] - pub high_starts: ArrayRef, - #[slot(6)] - pub residual_words: ArrayRef, - #[slot(7)] - pub patch_positions: ArrayRef, - #[slot(8)] - pub patch_highs: ArrayRef, - #[slot(9)] pub validity: Option, } @@ -90,9 +75,16 @@ pub struct BlockResidualData { unsliced_len: usize, slice_start: usize, slice_stop: usize, - residual_word_count: usize, - patch_count: usize, - patch_high_count: usize, + payload: ByteBuffer, + bases: Buffer, + residual_widths: Buffer, + high_widths: Buffer, + residual_starts: Buffer, + patch_starts: Buffer, + high_starts: Buffer, + residual_words: Buffer, + patch_positions: Buffer, + patch_highs: Buffer, } impl Display for BlockResidualData { @@ -108,24 +100,20 @@ impl Display for BlockResidualData { } impl ArrayHash for BlockResidualData { - fn array_hash(&self, state: &mut H, _accuracy: EqMode) { + fn array_hash(&self, state: &mut H, accuracy: EqMode) { self.unsliced_len.hash(state); self.slice_start.hash(state); self.slice_stop.hash(state); - self.residual_word_count.hash(state); - self.patch_count.hash(state); - self.patch_high_count.hash(state); + self.payload.array_hash(state, accuracy); } } impl ArrayEq for BlockResidualData { - fn array_eq(&self, other: &Self, _accuracy: EqMode) -> bool { + fn array_eq(&self, other: &Self, accuracy: EqMode) -> bool { self.unsliced_len == other.unsliced_len && self.slice_start == other.slice_start && self.slice_stop == other.slice_stop - && self.residual_word_count == other.residual_word_count - && self.patch_count == other.patch_count - && self.patch_high_count == other.patch_high_count + && self.payload.array_eq(&other.payload, accuracy) } } @@ -188,15 +176,21 @@ impl VTable for BlockResidual { } fn nbuffers(_array: ArrayView<'_, Self>) -> usize { - 0 + 1 } - fn buffer(_array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { - vortex_panic!("BlockResidualArray buffer index {idx} out of bounds") + fn buffer(array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { + match idx { + 0 => BufferHandle::new_host(array.payload.clone()), + _ => vortex_panic!("BlockResidualArray buffer index {idx} out of bounds"), + } } fn buffer_name(_array: ArrayView<'_, Self>, idx: usize) -> Option { - vortex_panic!("BlockResidualArray buffer_name {idx} out of bounds") + match idx { + 0 => Some("payload".to_string()), + _ => vortex_panic!("BlockResidualArray buffer_name {idx} out of bounds"), + } } fn with_buffers( @@ -204,7 +198,13 @@ impl VTable for BlockResidual { array: ArrayView<'_, Self>, buffers: &[BufferHandle], ) -> VortexResult> { - vortex_array::vtable::with_empty_buffers(self, array, buffers) + vortex_ensure!(buffers.len() == 1, "BlockResidualArray expects one buffer"); + let mut data = array.data().clone(); + data.replace_payload(&buffers[0])?; + Ok( + ArrayParts::new(self.clone(), array.dtype().clone(), array.len(), data) + .with_slots(array.slots().iter().cloned().collect()), + ) } fn serialize( @@ -221,7 +221,7 @@ impl VTable for BlockResidual { dtype: &DType, len: usize, metadata: &[u8], - _buffers: &[BufferHandle], + buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult> { @@ -234,42 +234,25 @@ impl VTable for BlockResidual { let residual_word_count = usize::try_from(metadata.residual_word_count)?; let patch_count = usize::try_from(metadata.patch_count)?; let patch_high_count = usize::try_from(metadata.patch_high_count)?; - let block_count = unsliced_len.div_ceil(BLOCK_LEN); - let bases = children.get(0, &primitive_dtype(PType::U64), block_count)?; - let residual_widths = children.get(1, &primitive_dtype(PType::U8), block_count)?; - let high_widths = children.get(2, &primitive_dtype(PType::U8), block_count)?; - let residual_starts = children.get(3, &primitive_dtype(PType::U32), block_count + 1)?; - let patch_starts = children.get(4, &primitive_dtype(PType::U32), block_count + 1)?; - let high_starts = children.get(5, &primitive_dtype(PType::U32), block_count + 1)?; - let residual_words = children.get(6, &primitive_dtype(PType::U64), residual_word_count)?; - let patch_positions = children.get(7, &primitive_dtype(PType::U16), patch_count)?; - let patch_highs = children.get(8, &primitive_dtype(PType::U8), patch_high_count)?; + vortex_ensure!(buffers.len() == 1, "BlockResidualArray expects one buffer"); let validity = match children.len() { - 9 => Validity::from(dtype.nullability()), - 10 => Validity::Array(children.get(9, &Validity::DTYPE, unsliced_len)?), - count => vortex_bail!("BlockResidualArray expects nine or ten children, got {count}"), + 0 => Validity::from(dtype.nullability()), + 1 => Validity::Array(children.get(0, &Validity::DTYPE, unsliced_len)?), + count => vortex_bail!("BlockResidualArray expects zero or one child, got {count}"), }; let slots = BlockResidualSlots { - bases, - residual_widths, - high_widths, - residual_starts, - patch_starts, - high_starts, - residual_words, - patch_positions, - patch_highs, validity: validity_to_child(&validity, unsliced_len), } .into_slots(); - let data = BlockResidualData { + let data = BlockResidualData::try_new( unsliced_len, slice_start, slice_stop, residual_word_count, patch_count, patch_high_count, - }; + host_payload(&buffers[0])?, + )?; Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } @@ -353,7 +336,7 @@ impl SliceReduce for BlockResidual { static RULES: ParentRuleSet = ParentRuleSet::new(&[ParentRuleSet::lift(&SliceReduceAdaptor(BlockResidual))]); -pub trait BlockResidualArrayExt: TypedArrayRef + BlockResidualArraySlotsExt { +pub trait BlockResidualArrayExt: TypedArrayRef { fn unsliced_validity(&self) -> Validity { child_to_validity( self.as_ref().slots()[BlockResidualSlots::VALIDITY].as_ref(), @@ -361,6 +344,51 @@ pub trait BlockResidualArrayExt: TypedArrayRef + BlockResidualArr ) } + /// Return the reference value for each block. + fn bases(&self) -> &[u64] { + &self.bases + } + + /// Return the packed residual width for each block. + fn residual_widths(&self) -> &[u8] { + &self.residual_widths + } + + /// Return the packed patch width for each block. + fn high_widths(&self) -> &[u8] { + &self.high_widths + } + + /// Return the residual payload offsets. + fn residual_starts(&self) -> &[u32] { + &self.residual_starts + } + + /// Return the patch position offsets. + fn patch_starts(&self) -> &[u32] { + &self.patch_starts + } + + /// Return the patch high-bit offsets. + fn high_starts(&self) -> &[u32] { + &self.high_starts + } + + /// Return the packed residual payload. + fn residual_words(&self) -> &[u64] { + &self.residual_words + } + + /// Return the patch positions. + fn patch_positions(&self) -> &[u16] { + &self.patch_positions + } + + /// Return the packed patch high bits. + fn patch_highs(&self) -> &[u8] { + &self.patch_highs + } + /// Decode the logical slice. fn decompress(&self, ctx: &mut ExecutionCtx) -> VortexResult { decompress_array(self.to_owned().as_view(), ctx) @@ -443,24 +471,17 @@ impl BlockResidual { validity: Validity, ptype: PType, ) -> VortexResult { - let data = BlockResidualData { - unsliced_len: parts.len, - slice_start: 0, - slice_stop: parts.len, - residual_word_count: parts.residual_words.len(), - patch_count: parts.patch_positions.len(), - patch_high_count: parts.patch_highs.len(), - }; + let payload = payload_from_parts(&parts)?; + let data = BlockResidualData::try_new( + parts.len, + 0, + parts.len, + parts.residual_words.len(), + parts.patch_positions.len(), + parts.patch_highs.len(), + payload, + )?; let slots = BlockResidualSlots { - bases: PrimitiveArray::from_iter(parts.bases).into_array(), - residual_widths: PrimitiveArray::from_iter(parts.residual_widths).into_array(), - high_widths: PrimitiveArray::from_iter(parts.high_widths).into_array(), - residual_starts: PrimitiveArray::from_iter(parts.residual_starts).into_array(), - patch_starts: PrimitiveArray::from_iter(parts.patch_starts).into_array(), - high_starts: PrimitiveArray::from_iter(parts.high_starts).into_array(), - residual_words: PrimitiveArray::from_iter(parts.residual_words).into_array(), - patch_positions: PrimitiveArray::from_iter(parts.patch_positions).into_array(), - patch_highs: PrimitiveArray::from_iter(parts.patch_highs).into_array(), validity: validity_to_child(&validity, data.unsliced_len), } .into_slots(); @@ -519,11 +540,54 @@ fn ordered_values(array: ArrayView<'_, Primitive>) -> VortexResult> { } impl BlockResidualData { + fn try_new( + unsliced_len: usize, + slice_start: usize, + slice_stop: usize, + residual_word_count: usize, + patch_count: usize, + patch_high_count: usize, + payload: ByteBuffer, + ) -> VortexResult { + let block_count = unsliced_len.div_ceil(BLOCK_LEN); + let mut offset = 0; + let bases = take_payload(&payload, &mut offset, block_count, "bases")?; + let residual_words = + take_payload(&payload, &mut offset, residual_word_count, "residual words")?; + let residual_starts = + take_payload(&payload, &mut offset, block_count + 1, "residual starts")?; + let patch_starts = take_payload(&payload, &mut offset, block_count + 1, "patch starts")?; + let high_starts = take_payload(&payload, &mut offset, block_count + 1, "high starts")?; + let patch_positions = take_payload(&payload, &mut offset, patch_count, "patch positions")?; + let residual_widths = take_payload(&payload, &mut offset, block_count, "residual widths")?; + let high_widths = take_payload(&payload, &mut offset, block_count, "high widths")?; + let patch_highs = take_payload(&payload, &mut offset, patch_high_count, "patch highs")?; + vortex_ensure!( + offset == payload.len(), + "block residual payload contains trailing bytes" + ); + Ok(Self { + unsliced_len, + slice_start, + slice_stop, + payload, + bases, + residual_widths, + high_widths, + residual_starts, + patch_starts, + high_starts, + residual_words, + patch_positions, + patch_highs, + }) + } + fn validate( &self, dtype: &DType, len: usize, - slots: BlockResidualSlotsView<'_>, + _slots: BlockResidualSlotsView<'_>, validity: &Validity, ) -> VortexResult<()> { vortex_ensure!( @@ -536,22 +600,18 @@ impl BlockResidualData { ); vortex_ensure!(len == self.len(), "block residual slice length is invalid"); let block_count = self.unsliced_len.div_ceil(BLOCK_LEN); - for (child, ptype, child_len) in [ - (slots.bases, PType::U64, block_count), - (slots.residual_widths, PType::U8, block_count), - (slots.high_widths, PType::U8, block_count), - (slots.residual_starts, PType::U32, block_count + 1), - (slots.patch_starts, PType::U32, block_count + 1), - (slots.high_starts, PType::U32, block_count + 1), - (slots.residual_words, PType::U64, self.residual_word_count), - (slots.patch_positions, PType::U16, self.patch_count), - (slots.patch_highs, PType::U8, self.patch_high_count), - ] { - vortex_ensure!( - child.dtype() == &primitive_dtype(ptype) && child.len() == child_len, - "block residual child has an invalid dtype or length" - ); - } + vortex_ensure!( + self.bases.len() == block_count + && self.residual_widths.len() == block_count + && self.high_widths.len() == block_count, + "block residual block tables have invalid lengths" + ); + vortex_ensure!( + self.residual_starts.len() == block_count + 1 + && self.patch_starts.len() == block_count + 1 + && self.high_starts.len() == block_count + 1, + "block residual offset tables have invalid lengths" + ); if let Some(validity_len) = validity.maybe_len() { vortex_ensure!( validity_len == self.unsliced_len, @@ -572,56 +632,98 @@ impl BlockResidualData { ..self.clone() } } + + fn replace_payload(&mut self, buffer: &BufferHandle) -> VortexResult<()> { + *self = Self::try_new( + self.unsliced_len, + self.slice_start, + self.slice_stop, + self.residual_words.len(), + self.patch_positions.len(), + self.patch_highs.len(), + host_payload(buffer)?, + )?; + Ok(()) + } } -struct ExecutedBlockResidual { - bases: PrimitiveArray, - residual_widths: PrimitiveArray, - high_widths: PrimitiveArray, - residual_starts: PrimitiveArray, - patch_starts: PrimitiveArray, - high_starts: PrimitiveArray, - residual_words: PrimitiveArray, - patch_positions: PrimitiveArray, - patch_highs: PrimitiveArray, +fn host_payload(buffer: &BufferHandle) -> VortexResult { + buffer + .clone() + .ensure_aligned(Alignment::of::())? + .try_into_host_sync() } -impl ExecutedBlockResidual { - fn new(array: ArrayView<'_, BlockResidual>, ctx: &mut ExecutionCtx) -> VortexResult { - macro_rules! execute_child { - ($accessor:ident) => { - array.$accessor().clone().execute::(ctx)? - }; - } - Ok(Self { - bases: execute_child!(bases), - residual_widths: execute_child!(residual_widths), - high_widths: execute_child!(high_widths), - residual_starts: execute_child!(residual_starts), - patch_starts: execute_child!(patch_starts), - high_starts: execute_child!(high_starts), - residual_words: execute_child!(residual_words), - patch_positions: execute_child!(patch_positions), - patch_highs: execute_child!(patch_highs), - }) - } +fn take_payload( + payload: &ByteBuffer, + offset: &mut usize, + len: usize, + name: &str, +) -> VortexResult> { + let nbytes = len + .checked_mul(size_of::()) + .ok_or_else(|| vortex_error::vortex_err!("block residual {name} size overflows"))?; + let stop = offset + .checked_add(nbytes) + .ok_or_else(|| vortex_error::vortex_err!("block residual {name} offset overflows"))?; + vortex_ensure!( + stop <= payload.len(), + "block residual {name} exceeds the payload" + ); + let bytes = payload.slice_with_alignment(*offset..stop, Alignment::of::()); + *offset = stop; + Ok(Buffer::from_byte_buffer(bytes)) +} + +fn payload_from_parts(parts: &BlockResidualParts) -> VortexResult { + let total_nbytes = [ + size_of_val(parts.bases.as_slice()), + size_of_val(parts.residual_words.as_slice()), + size_of_val(parts.residual_starts.as_slice()), + size_of_val(parts.patch_starts.as_slice()), + size_of_val(parts.high_starts.as_slice()), + size_of_val(parts.patch_positions.as_slice()), + size_of_val(parts.residual_widths.as_slice()), + size_of_val(parts.high_widths.as_slice()), + size_of_val(parts.patch_highs.as_slice()), + ] + .into_iter() + .try_fold(0_usize, |total, nbytes| total.checked_add(nbytes)) + .ok_or_else(|| vortex_error::vortex_err!("block residual payload size overflows"))?; + let mut payload = ByteBufferMut::with_capacity_aligned(total_nbytes, Alignment::of::()); + append_native(&mut payload, &parts.bases); + append_native(&mut payload, &parts.residual_words); + append_native(&mut payload, &parts.residual_starts); + append_native(&mut payload, &parts.patch_starts); + append_native(&mut payload, &parts.high_starts); + append_native(&mut payload, &parts.patch_positions); + append_native(&mut payload, &parts.residual_widths); + append_native(&mut payload, &parts.high_widths); + append_native(&mut payload, &parts.patch_highs); + Ok(payload.freeze()) +} + +fn append_native(payload: &mut ByteBufferMut, values: &[T]) { + // SAFETY: NativePType values contain no padding and permit every initialized bit pattern. + let bytes = + unsafe { std::slice::from_raw_parts(values.as_ptr().cast::(), size_of_val(values)) }; + payload.extend_from_slice(bytes); } fn decode_array_values( array: ArrayView<'_, BlockResidual>, - ctx: &mut ExecutionCtx, + _ctx: &mut ExecutionCtx, mut transform: impl FnMut(U) -> T, ) -> VortexResult { - let children = ExecutedBlockResidual::new(array, ctx)?; - let bases = children.bases.as_slice::(); - let residual_widths = children.residual_widths.as_slice::(); - let high_widths = children.high_widths.as_slice::(); - let residual_starts = children.residual_starts.as_slice::(); - let patch_starts = children.patch_starts.as_slice::(); - let high_starts = children.high_starts.as_slice::(); - let residual_words = children.residual_words.as_slice::(); - let patch_positions = children.patch_positions.as_slice::(); - let patch_highs = children.patch_highs.as_slice::(); + let bases = array.bases(); + let residual_widths = array.residual_widths(); + let high_widths = array.high_widths(); + let residual_starts = array.residual_starts(); + let patch_starts = array.patch_starts(); + let high_starts = array.high_starts(); + let residual_words = array.residual_words(); + let patch_positions = array.patch_positions(); + let patch_highs = array.patch_highs(); let logical_range = array.data().slice_start..array.data().slice_stop; let mut direct_values = DIRECT_OUTPUT.then(|| BufferMut::::with_capacity(logical_range.len())); @@ -981,14 +1083,13 @@ pub(crate) fn decompress_ordered_f64( fn scalar_from_array( array: ArrayView<'_, BlockResidual>, index: usize, - ctx: &mut ExecutionCtx, + _ctx: &mut ExecutionCtx, ) -> VortexResult { let source_index = array.data().slice_start + index; let block_index = source_index / BLOCK_LEN; let index_in_block = source_index % BLOCK_LEN; - let children = ExecutedBlockResidual::new(array, ctx)?; - let residual_width = children.residual_widths.as_slice::()[block_index]; - let high_width = children.high_widths.as_slice::()[block_index]; + let residual_width = array.residual_widths()[block_index]; + let high_width = array.high_widths()[block_index]; let logical_width = array.dtype().as_ptype().bit_width(); vortex_ensure!( usize::from(residual_width) <= logical_width @@ -996,9 +1097,9 @@ fn scalar_from_array( && usize::from(residual_width) + usize::from(high_width) <= logical_width, "block residual bit widths are invalid" ); - let residual_words = children.residual_words.as_slice::(); + let residual_words = array.residual_words(); let residual_payload = payload_range( - children.residual_starts.as_slice::(), + array.residual_starts(), block_index, residual_words.len(), "residual", @@ -1031,21 +1132,11 @@ fn scalar_from_array( _ => vortex_bail!("block residual logical bit width is invalid"), }; - let positions = children.patch_positions.as_slice::(); - let patch_payload = payload_range( - children.patch_starts.as_slice::(), - block_index, - positions.len(), - "patch", - )?; + let positions = array.patch_positions(); + let patch_payload = payload_range(array.patch_starts(), block_index, positions.len(), "patch")?; let block_positions = &positions[patch_payload]; - let highs = children.patch_highs.as_slice::(); - let high_payload = payload_range( - children.high_starts.as_slice::(), - block_index, - highs.len(), - "patch high", - )?; + let highs = array.patch_highs(); + let high_payload = payload_range(array.high_starts(), block_index, highs.len(), "patch high")?; validate_patch_header( residual_width, high_width, @@ -1064,7 +1155,7 @@ fn scalar_from_array( }; residual |= high << residual_width; } - Ok(children.bases.as_slice::()[block_index].wrapping_add(residual)) + Ok(array.bases()[block_index].wrapping_add(residual)) } fn unpack_single_residual(width: u8, packed_words: &[u64], index: usize) -> u64 { @@ -1148,9 +1239,9 @@ impl BlockResidualMetadata { Ok(Self { unsliced_len: u64::try_from(data.unsliced_len)?, slice_start: u64::try_from(data.slice_start)?, - residual_word_count: u64::try_from(data.residual_word_count)?, - patch_count: u64::try_from(data.patch_count)?, - patch_high_count: u64::try_from(data.patch_high_count)?, + residual_word_count: u64::try_from(data.residual_words.len())?, + patch_count: u64::try_from(data.patch_positions.len())?, + patch_high_count: u64::try_from(data.patch_highs.len())?, }) } @@ -1201,10 +1292,6 @@ impl BlockResidualMetadata { } } -fn primitive_dtype(ptype: PType) -> DType { - DType::Primitive(ptype, NonNullable) -} - #[cfg(test)] mod tests { use vortex_array::ArrayContext; @@ -1222,7 +1309,7 @@ mod tests { use vortex_session::registry::ReadContext; use super::BlockResidual; - use super::BlockResidualArraySlotsExt; + use super::BlockResidualArrayExt; #[test] fn roundtrip_and_scalar_access() -> VortexResult<()> { @@ -1343,12 +1430,7 @@ mod tests { let session = array_session(); crate::initialize(&session); let mut ctx = session.create_execution_ctx(); - let widths = encoded - .residual_widths() - .clone() - .execute::(&mut ctx)?; - - assert_eq!(widths.as_slice::()[0], 0); + assert_eq!(encoded.residual_widths()[0], 0); assert_eq!( encoded .execute_scalar(1_023, &mut ctx)? diff --git a/encodings/block-residual/src/ordered_float_array.rs b/encodings/block-residual/src/ordered_float_array.rs index 39e363410f8..ab018615191 100644 --- a/encodings/block-residual/src/ordered_float_array.rs +++ b/encodings/block-residual/src/ordered_float_array.rs @@ -412,7 +412,7 @@ mod tests { use super::OrderedFloat; use super::OrderedFloatArraySlotsExt; use crate::BlockResidual; - use crate::BlockResidualArraySlotsExt; + use crate::BlockResidualArrayExt; #[test] fn roundtrip_special_values() -> VortexResult<()> { diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index d5983a5283f..27e3de528a0 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -49,7 +49,7 @@ use vortex_array::extension::datetime::TimeUnit; use vortex_array::validity::Validity; use vortex_arrow::ArrowSessionExt; use vortex_block_residual::BlockResidual; -use vortex_block_residual::BlockResidualArraySlotsExt; +use vortex_block_residual::BlockResidualArrayExt; use vortex_block_residual::OrderedFloat; use vortex_block_residual::OrderedFloatArraySlotsExt; use vortex_btrblocks::BtrBlocksCompressor; @@ -448,36 +448,22 @@ fn profile_block_residual_array( config: &str, path: &str, array: &ArrayRef, - session: &VortexSession, ) -> VortexResult<()> { if let Some(residuals) = array.as_typed::() { - let mut ctx = session.create_execution_ctx(); - let residual_widths = residuals - .residual_widths() - .clone() - .execute::(&mut ctx)?; - let high_widths = residuals - .high_widths() - .clone() - .execute::(&mut ctx)?; + let residual_widths = residuals.residual_widths(); + let high_widths = residuals.high_widths(); let blocks = residual_widths.len(); let average_residual_width = residual_widths - .as_slice::() .iter() .map(|&width| f64::from(width)) .sum::() / blocks as f64; let average_high_width = high_widths - .as_slice::() .iter() .map(|&width| f64::from(width)) .sum::() / blocks as f64; - let patch_starts = residuals - .patch_starts() - .clone() - .execute::(&mut ctx)?; - let patch_starts = patch_starts.as_slice::(); + let patch_starts = residuals.patch_starts(); let mut maximum_patch_density = 0.0_f64; let mut blocks_above_one_eighth = 0usize; let mut blocks_at_one_quarter = 0usize; @@ -505,7 +491,6 @@ fn profile_block_residual_array( config, &format!("{path}/{child_index}"), child, - session, )?; } Ok(()) @@ -1538,14 +1523,7 @@ fn measure_dataset( *config, "integer-block-residual-only" | "ordered-block-residual-only" ) { - profile_block_residual_array( - dataset, - &column.name, - config, - "root", - array, - session, - )?; + profile_block_residual_array(dataset, &column.name, config, "root", array)?; } } } diff --git a/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs index 4615a6cf54f..90196d1acf7 100644 --- a/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs @@ -28,6 +28,12 @@ use vortex_float_quant::FloatQuantArraySlotsExt; use vortex_session::VortexSession; use crate::BtrBlocksCompressor; +#[cfg(feature = "unstable_encodings")] +use crate::BtrBlocksCompressorBuilder; +#[cfg(feature = "unstable_encodings")] +use crate::SchemeExt; +#[cfg(feature = "unstable_encodings")] +use crate::schemes::integer::DeltaScheme; static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); @@ -250,8 +256,13 @@ fn test_block_residual_composes_with_alp() -> VortexResult<()> { (block * 1_000_000 + residual) as f64 }); let array = PrimitiveArray::from_iter(values).into_array(); - let compressed = - BtrBlocksCompressor::default().compress(&array, &mut SESSION.create_execution_ctx())?; + #[cfg(not(feature = "unstable_encodings"))] + let compressor = BtrBlocksCompressor::default(); + #[cfg(feature = "unstable_encodings")] + let compressor = BtrBlocksCompressorBuilder::default() + .exclude_schemes([DeltaScheme::default().id()]) + .build(); + let compressed = compressor.compress(&array, &mut SESSION.create_execution_ctx())?; assert!( compressed.is::(), diff --git a/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs index c8999b6acab..2ec13b6d7ea 100644 --- a/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs @@ -30,6 +30,12 @@ use vortex_session::VortexSession; use vortex_sparse::Sparse; use crate::BtrBlocksCompressor; +#[cfg(feature = "unstable_encodings")] +use crate::BtrBlocksCompressorBuilder; +#[cfg(feature = "unstable_encodings")] +use crate::SchemeExt; +#[cfg(feature = "unstable_encodings")] +use crate::schemes::integer::DeltaScheme; static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[test] @@ -62,8 +68,14 @@ fn test_block_residual_compressed() -> VortexResult<()> { }) .collect::>(); let array = PrimitiveArray::from_iter(values); - let compressed = BtrBlocksCompressor::default() - .compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; + #[cfg(not(feature = "unstable_encodings"))] + let compressor = BtrBlocksCompressor::default(); + #[cfg(feature = "unstable_encodings")] + let compressor = BtrBlocksCompressorBuilder::default() + .exclude_schemes([DeltaScheme::default().id()]) + .build(); + let compressed = + compressor.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; assert!( compressed.is::(), diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__string_fsst_structured.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__string_fsst_structured.snap index 327f050b0d7..ddbd6548e12 100644 --- a/vortex-btrblocks/tests/snapshots/golden__unstable__string_fsst_structured.snap +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__string_fsst_structured.snap @@ -3,7 +3,7 @@ source: vortex-btrblocks/tests/golden.rs expression: rendered --- input: utf8, len=16384, nbytes=653785 -root: vortex.fsst(utf8, len=16384) nbytes=151382 +root: vortex.fsst(utf8, len=16384) nbytes=141400 metadata: len: 16384, nsymbols: 223 uncompressed_lengths: vortex.sparse(u8, len=16384) nbytes=3154 metadata: fill_value: 24u8 @@ -11,5 +11,5 @@ root: vortex.fsst(utf8, len=16384) nbytes=151382 metadata: ptype: u16 patch_values: vortex.constant(u8, len=1575) nbytes=2 metadata: scalar: 23u8 - codes_offsets: fastlanes.bitpacked(u32, len=16385) nbytes=36992 - metadata: bit_width: 17, offset: 0 + codes_offsets: vortex.block_residual(u32, len=16385) nbytes=27010 + metadata: blocks: 17, slice: 0..16385 From 01b5304e00a4771b127d8e21e8d9f090522f21eb Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 19:10:53 +0100 Subject: [PATCH 30/78] bench: compare compact compression Signed-off-by: Will Manning --- benchmarks/compress-bench/README.md | 7 +++++++ benchmarks/compress-bench/src/main.rs | 6 +++++- benchmarks/compress-bench/src/vortex.rs | 14 ++++++++++++-- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/benchmarks/compress-bench/README.md b/benchmarks/compress-bench/README.md index bf2d3efc1db..0b3c14be4b9 100644 --- a/benchmarks/compress-bench/README.md +++ b/benchmarks/compress-bench/README.md @@ -15,6 +15,13 @@ See [`src/main.rs`](./src/main.rs) for the dataset list and CLI flags (`--format cargo run -p compress-bench --profile release_debug ``` +Compare the default compressor, Compact, and Parquet with Zstd: + +```bash +cargo run -p compress-bench --profile release_debug -- \ + --formats vortex,vortex-compact,parquet +``` + GPU decompression is opt-in and runs only the existing benchmark names allow-listed in `src/main.rs`: diff --git a/benchmarks/compress-bench/src/main.rs b/benchmarks/compress-bench/src/main.rs index 6e81a8a4376..99af8db81cc 100644 --- a/benchmarks/compress-bench/src/main.rs +++ b/benchmarks/compress-bench/src/main.rs @@ -137,7 +137,11 @@ fn get_compressor( } match format { - Format::OnDiskVortex => Box::new(VortexCompressor::new(!vortex_without_new_numeric)), + Format::OnDiskVortex => Box::new(VortexCompressor::new( + Format::OnDiskVortex, + !vortex_without_new_numeric, + )), + Format::VortexCompact => Box::new(VortexCompressor::new(Format::VortexCompact, true)), Format::Parquet => Box::new(ParquetCompressor::new()), #[cfg(feature = "lance")] Format::Lance => Box::new(LanceCompressor), diff --git a/benchmarks/compress-bench/src/vortex.rs b/benchmarks/compress-bench/src/vortex.rs index caf135165e2..40de621d1c2 100644 --- a/benchmarks/compress-bench/src/vortex.rs +++ b/benchmarks/compress-bench/src/vortex.rs @@ -21,6 +21,7 @@ use vortex::file::VortexWriteOptions; use vortex::file::WriteOptionsSessionExt; use vortex::file::WriteStrategyBuilder; use vortex_arrow::ArrowSessionExt; +use vortex_bench::CompactionStrategy; use vortex_bench::Format; use vortex_bench::SESSION; use vortex_bench::compress::Compressor; @@ -34,18 +35,27 @@ use vortex_btrblocks::schemes::integer::BlockResidualScheme; /// Compressor implementation for Vortex format. pub struct VortexCompressor { + format: Format, new_numeric_schemes: bool, } impl VortexCompressor { - pub fn new(new_numeric_schemes: bool) -> Self { + pub fn new(format: Format, new_numeric_schemes: bool) -> Self { + assert!(matches!( + format, + Format::OnDiskVortex | Format::VortexCompact + )); Self { + format, new_numeric_schemes, } } fn write_options(&self) -> VortexWriteOptions { let options = SESSION.write_options(); + if self.format == Format::VortexCompact { + return CompactionStrategy::Compact.apply_options(options); + } if self.new_numeric_schemes { return options; } @@ -65,7 +75,7 @@ impl VortexCompressor { #[async_trait] impl Compressor for VortexCompressor { fn format(&self) -> Format { - Format::OnDiskVortex + self.format } async fn compress(&self, parquet_path: &Path) -> Result<(u64, Duration)> { From c26ffb63ab7a3f420b47bb789e71b77062e37a79 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 19:16:08 +0100 Subject: [PATCH 31/78] bench: serialize compact compression timings Signed-off-by: Will Manning --- vortex-bench/src/measurements.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vortex-bench/src/measurements.rs b/vortex-bench/src/measurements.rs index d6d4ad85b32..905f120a06a 100644 --- a/vortex-bench/src/measurements.rs +++ b/vortex-bench/src/measurements.rs @@ -351,11 +351,11 @@ pub struct CompressionTimingMeasurement { impl ToJson for CompressionTimingMeasurement { fn to_json(&self) -> serde_json::Value { let (name, engine) = match self.format { - Format::OnDiskVortex => (self.name.to_string(), Engine::Vortex), + Format::OnDiskVortex | Format::VortexCompact => (self.name.to_string(), Engine::Vortex), Format::Parquet => (format!("parquet_rs-zstd {}", self.name), Engine::Vortex), Format::Lance => (format!("lance {}", self.name), Engine::Vortex), _ => vortex_panic!( - "CompressionTimingMeasurement only supports vortex, lance, and parquet formats" + "CompressionTimingMeasurement only supports Vortex, Lance, and Parquet formats" ), }; From 2c1ac5ed1a0153d0827921b361ec0860f3799e33 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 19:57:38 +0100 Subject: [PATCH 32/78] bench: classify remaining compact numeric gaps Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 173 ++++++++++++++++-- .../examples/float_compressor_bench.rs | 17 +- 2 files changed, 175 insertions(+), 15 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index d9d50448f9d..abf1e912c6e 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -24,6 +24,10 @@ Transfer that model into a native array only when it preserves these Default pro Compact can use experimental schemes during evaluation. Default selection remains the release decision. +Measure each candidate against both the prior Default tree and the Compact tree. + +Use Compact gap recovery as the size metric. Use the displaced Default tree as the speed baseline. + ## Current decision Focus the production work on these encodings: @@ -76,6 +80,10 @@ The current selector factors and nonlinear patch cost remain provisional. Final calibration requires the complete corpus and selected-tree evidence. +The current branch recovers a small share of Compact's aggregate numeric advantage. + +The remaining work targets repeated Compact mechanisms with native, bounded-access trees. + ## OrderedFloatArray `OrderedFloatArray` maps IEEE float bits to unsigned integers with the same order. @@ -547,6 +555,82 @@ This layout recovers most of the Compact-like size gain without a material aggre The remaining threshold calibration must use selected-column evidence and the complete corpus. +### Compact transfer baseline + +The local corpus comparison uses three iterations on all 16 available datasets. + +The proposed Default includes FloatQuant and BlockResidual. The prior Default excludes all new numeric schemes. + +The numeric scope contains eight numeric datasets. The real scope adds two TPC-H comment variants. + +Positive throughput changes indicate faster execution. + +| Scope | Size change | Encode throughput change | Decode throughput change | Proposed gap above Compact | Proposed gap above Parquet with Zstd | +| --- | ---: | ---: | ---: | ---: | ---: | +| Numeric, 8 datasets | -0.84 percent | -0.35 percent | -2.10 percent | +40.84 percent | +1.46 percent | +| Real, 10 datasets | -1.12 percent | -0.47 percent | -1.85 percent | +37.92 percent | +2.28 percent | +| All, 16 datasets | -0.70 percent | -1.45 percent | -1.85 percent | +22.26 percent | -0.84 percent | + +The proposed Default preserves the aggregate speed and Parquet size constraints. + +It recovers little of Compact's numeric advantage. Compact remains 40.84 percent smaller across the numeric scope. + +Taxi gains 3.23 percent, CMS gains 1.34 percent, and Euro2016 gains 2.42 percent against the prior Default. + +GloVe size does not change. The remaining datasets change by less than one percent. + +This result defines the next objective. Recover repeated Compact gains without a large loss in Default throughput or scalar access. + +### Column attribution for Compact wins + +The profile reads up to two million numeric rows from seven Public BI files and GloVe. + +Twenty-nine float columns give Compact a size advantage of at least ten percent. + +The prior Default uses 190,686,999 bytes across those columns. Compact uses 143,939,147 bytes. + +Compact saves 24.52 percent, or 46,747,852 bytes. + +The Pco trees use these main modes: + +- Classic bins without Delta cover 20,200,768 profiled values. +- Classic bins with consecutive Delta cover 9,311,364 values. +- `IntMult` with consecutive Delta covers 5,048,574 values. +- `IntMult` without Delta covers 4,469,863 values. +- `FloatMult` with consecutive Delta covers 3,999,998 values. +- FloatQuant variants cover about four million values. +- Classic bins with lookback Delta cover 1,345,593 values. + +Classic bins without Delta form the largest transferable target. + +`IntMult` forms a separate target for ALP integer children. GloVe demonstrates this target with `IntMult(10)`. + +Consecutive Delta explains several Arade wins. It conflicts with the Default random-access and decode goals. + +The largest single gap is Euro2016 `subjectivity_confidence`. + +The prior Default uses 14,234,326 bytes. Compact uses 6,860,457 bytes with classic bins and no Delta. + +CMS standard-deviation columns also use classic bins without Delta. + +CMS payment fields use `IntMult`. CMS submitted-charge fields use FloatQuant. + +Food `volume_total_bytes` uses classic bins inside an ALP child. + +The column attribution separates numeric Compact gains from file-level string compression gains. + +### Selector calibration miss on a Compact gap + +The CMS ALP integer child exposes a current BlockResidual threshold miss. + +The current Default tree uses 11,064,308 bytes and decodes at 24.98 GB/s. + +`ALP(BlockResidual)` uses 10,716,519 bytes and decodes at 22.27 GB/s. + +The candidate is 3.1 percent smaller and 10.8 percent slower to decode. + +The current 1.20 factor rejects it. Final threshold calibration must decide whether this trade belongs in Default. + ### Patch-density sweep The u32 input uses one large outlier at each configured stride. @@ -898,6 +982,64 @@ A ten-percent Compact size allowance includes CMS, Food, Euro subjectivity, and Arade F8 remains outside that threshold. GloVe also remains outside because `IntMult(10)` gives Pco another size advantage. +### Alternate fixed-bin layouts + +The byte-aligned prototype rounds every bin offset width to a byte boundary. + +Euro2016 `subjectivity_confidence` provides the comparison. + +| Backend | Bytes | Encode MB/s | Decode MB/s | Scalar ns | +| --- | ---: | ---: | ---: | ---: | +| Serial bit-packed bins | 7,238,842 | 541.8 | 3,824.4 | 31.1 | +| Serial byte-aligned bins | 7,835,311 | 474.4 | 3,528.3 | 25.0 | + +Byte alignment improves scalar access. It worsens size, encode throughput, and bulk decode throughput. + +Cross-byte offset extraction is not the main bulk decode cost. + +The grouped-bin prototype stores bit-sliced identifiers and one offset stream per bin. + +It uses rank checkpoints every 256 values. Scalar access scans at most four 64-bit words. + +| Input and backend | Bytes | Encode MB/s | Decode MB/s | Scalar ns | +| --- | ---: | ---: | ---: | ---: | +| Euro serial bins | 7,238,842 | 555.2 | 3,949.5 | 32.1 | +| Euro grouped bins | 7,257,069 | 461.0 | 5,665.8 | 25.0 | +| CMS serial bins | 10,650,059 | 1,256.0 | 13,699.0 | 36.8 | +| CMS grouped bins | 10,670,330 | 929.9 | 6,283.7 | 30.3 | + +Grouped bins improve Euro bulk decode by 43 percent. The complete Default tree still decodes about four times faster. + +CMS uses similar offset widths across its bins. The grouped scatter path then loses more than half of serial-bin throughput. + +Grouped bins do not generalize. The prototype remains outside production code. + +### Multi-reference block prototype + +The prototype stores up to four quantile references per 1,024-value block. + +Two-bit FastLanes identifiers select one reference for each value. FastLanes stores the residuals. + +The final experiment uses packed `u8` identifiers and a direct four-entry reference lookup. + +| Euro candidate | Bytes | Encode MB/s | Decode MB/s | Scalar ns | +| --- | ---: | ---: | ---: | ---: | +| Default ALP | 14,234,326 | Not isolated | 21,759.7 | 523.6 | +| One-reference BlockResidual | 13,293,263 | 2,171.1 | 24,382.6 | 198.9 | +| Four references with patches | 11,353,335 | 1,026.5 | 11,215.0 | 19.2 | +| Four references without patches | 13,117,584 | 1,357.9 | 12,960.4 | 7.4 | +| Compact Pco | 6,860,457 | Not isolated | 2,467.8 | 18,027.2 | + +Packed `u8` identifiers materially improve the multi-reference decoder. + +The patched form is 20.2 percent smaller than Default. Its decode throughput is 48.5 percent lower. + +The patch-free form is 7.8 percent smaller. Its decode throughput is 40.4 percent lower. + +Patch application is not the main cost. Reference-ID expansion and per-value reference lookup dominate. + +The one-reference BlockResidual tree is both faster and simpler. The multi-reference prototype is rejected for Default. + ### Pcodec corpus gap analysis The focused benchmark reads the first two million numeric rows from each source. @@ -1072,6 +1214,12 @@ This round completed these steps: - Added direct paired FastLanes packing for both FloatQuant children. - Added fused two-child FloatQuant decode and width-sweep benchmarks. - Validated selected and rejected two-child FloatQuant paths. +- Added Compact as a first-class format in the compression benchmark. +- Compared the prior Default, proposed Default, Compact, and Parquet across all 16 local datasets. +- Attributed the largest Compact numeric gaps to Pco modes at the column level. +- Rejected byte-aligned fixed bins after a direct size and throughput comparison. +- Rejected grouped fixed bins after Euro2016 and CMS comparisons. +- Rejected packed multi-reference blocks after patched and patch-free comparisons. The Pco mode profile and quotient and remainder experiments are complete. @@ -1079,19 +1227,18 @@ The nonzero-secondary FloatQuant implementation and focused validation are compl Complete these remaining steps: -1. Validate the nonlinear patch cost on the complete corpus. -2. Add a true ALP-RD child composition case if a real selected tree exposes one. -3. Validate the remaining rejected analysis cost in the complete corpus. -4. Add real embedding datasets beyond GloVe when licenses and loaders permit them. -5. Classify more float columns where Pco beats ALP, ALP-RD, and the new schemes. -6. Prototype a lightweight integer split for Pco `IntMult` wins under ALP. -7. Prototype one lightweight scheme for repeated real-float Pco wins. -8. Run the complete `bench-vortex` compression corpus after the candidate set stabilizes. -9. Compare the geometric mean against Parquet with Zstd. -10. Calibrate all selector thresholds from complete corpus and selected-tree evidence. -11. Revisit the 32-bit and 64-bit BlockResidual score factors with selected-tree evidence. -12. Keep RangePacked outside writer selectors unless a new decode design changes the result. -13. Update this plan after each experiment. +1. Prototype a lightweight integer split for Pco `IntMult` wins under ALP. +2. Prototype one bounded-access scheme for repeated classic-bin wins without Delta. +3. Add a true ALP-RD child composition case if a real selected tree exposes one. +4. Validate the remaining rejected analysis cost after the candidate set stabilizes. +5. Add real embedding datasets beyond GloVe when licenses and loaders permit them. +6. Classify more float columns where Pco beats ALP, ALP-RD, and the new schemes. +7. Run the complete compression corpus after the candidate set stabilizes. +8. Compare the final geometric mean against Compact and Parquet with Zstd. +9. Calibrate all selector thresholds from complete corpus and selected-tree evidence. +10. Revisit the 32-bit and 64-bit BlockResidual factors with selected-tree evidence. +11. Keep RangePacked outside writer selectors unless a new decode design changes the result. +12. Update this plan after each experiment. ## Pull request structure diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index 27e3de528a0..4640edcf054 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -925,8 +925,9 @@ fn profile_range_packed( let encode_median = percentile(&mut encode_durations, 1, 2); vortex_ensure!(codec.decode()? == ordered, "range packed decode differs"); - let mut decode_durations = Vec::with_capacity(20); - for _ in 0..20 { + let decode_iterations = codec_decode_iterations()?; + let mut decode_durations = Vec::with_capacity(decode_iterations); + for _ in 0..decode_iterations { let start = Instant::now(); black_box(codec.decode()?); decode_durations.push(start.elapsed()); @@ -959,6 +960,18 @@ fn profile_range_packed( Ok(()) } +fn codec_decode_iterations() -> VortexResult { + std::env::var("VORTEX_BENCH_CODEC_DECODE_ITERATIONS") + .ok() + .map(|value| { + value.parse::().map_err(|error| { + vortex_err!("invalid VORTEX_BENCH_CODEC_DECODE_ITERATIONS: {error}") + }) + }) + .transpose() + .map(|iterations| iterations.unwrap_or(20)) +} + fn fixed_bin_float_tree( compact: &ArrayRef, session: &VortexSession, From aca7413bfc72e6d5cb41fa4aa067305bb3d83d0f Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 20:00:12 +0100 Subject: [PATCH 33/78] bench: preserve latent widths in split estimates Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 4 ++ .../examples/float_compressor_bench.rs | 39 ++++++++++++++----- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index abf1e912c6e..a2f6cb65de0 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -817,6 +817,10 @@ The Pco advantage comes from a different encoding of the same ALP integer child. The first prototype splits the exact ALP integer child with a constant base. +The estimator stores each child at the source latent width. GloVe `i32` latents produce `u32` quotient and remainder children. + +The native-width correction did not change the GloVe byte totals. Each selected child still uses the same packed bit width. + | Candidate | Bytes | | --- | ---: | | Pco `IntMult(10)` child | 4,518,870 | diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index 4640edcf054..10fd318c687 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -877,16 +877,16 @@ fn profile_quotient_remainder( let most_common_remainder_share = remainder_counts.values().copied().max().unwrap_or_default() as f64 / ordered.len() as f64; - let quotients = PrimitiveArray::from_iter(ordered.iter().map(|value| value / base)); - let remainders = PrimitiveArray::from_iter(ordered.iter().map(|value| value % base)); - let quotient_bitmap_bytes = - estimate_bitmap_patches(quotients.as_slice::(), ptype.bit_width()); - let remainder_mode_bitmap_bytes = - estimate_mode_bitmap(remainders.as_slice::(), ptype.bit_width()); - let quotient = - compressor.compress("ients.into_array(), &mut session.create_execution_ctx())?; + let quotients = ordered.iter().map(|value| value / base).collect::>(); + let remainders = ordered.iter().map(|value| value % base).collect::>(); + let quotient_bitmap_bytes = estimate_bitmap_patches("ients, ptype.bit_width()); + let remainder_mode_bitmap_bytes = estimate_mode_bitmap(&remainders, ptype.bit_width()); + let quotient = compressor.compress( + &latent_array("ients, ptype)?.into_array(), + &mut session.create_execution_ctx(), + )?; let remainder = compressor.compress( - &remainders.into_array(), + &latent_array(&remainders, ptype)?.into_array(), &mut session.create_execution_ctx(), )?; println!( @@ -902,6 +902,27 @@ fn profile_quotient_remainder( Ok(()) } +fn latent_array(values: &[u64], ptype: PType) -> VortexResult { + match ptype.bit_width() { + 16 => Ok(PrimitiveArray::from_iter( + values + .iter() + .map(|&value| u16::try_from(value)) + .collect::, _>>()?, + )), + 32 => Ok(PrimitiveArray::from_iter( + values + .iter() + .map(|&value| u32::try_from(value)) + .collect::, _>>()?, + )), + 64 => Ok(PrimitiveArray::from_iter(values.iter().copied())), + width => Err(vortex_err!( + "quotient and remainder profiling does not support {width}-bit values" + )), + } +} + fn profile_range_packed( dataset: &str, column: &str, From 40903b33d5bc5db5a606b9cfe43c4d5840e07702 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 20:25:41 +0100 Subject: [PATCH 34/78] bench: prototype bounded IntMult layouts Signed-off-by: Will Manning --- Cargo.lock | 1 + docs/plans/native-pcodec.md | 47 +- vortex-btrblocks/Cargo.toml | 1 + .../examples/float_compressor_bench.rs | 256 +++- .../float_compressor_bench/int_mult_codec.rs | 1057 +++++++++++++++++ 5 files changed, 1359 insertions(+), 3 deletions(-) create mode 100644 vortex-btrblocks/examples/float_compressor_bench/int_mult_codec.rs diff --git a/Cargo.lock b/Cargo.lock index a1b7e02d218..eaa9a68f150 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9770,6 +9770,7 @@ dependencies = [ "arrow-schema 58.4.0", "arrow-select 58.4.0", "codspeed-divan-compat", + "fastlanes", "insta", "itertools 0.14.0", "parquet 58.4.0", diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index a2f6cb65de0..eca6491db07 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -838,6 +838,47 @@ The results reject a general quotient and remainder array with ordinary children They support a small-alphabet entropy experiment and a fixed integer split experiment. +### Bounded IntMult prototypes + +The direct prototypes use 1,024-value blocks and exact quotient and remainder splits. + +Each quotient block stores one reference, FastLanes-packed low bits, and bounded bitmap patches. + +The GloVe variants tested three remainder layouts: + +- A mode value with bitmap exceptions. +- Gap positions with a maximum scan of 1,024 values. +- One exception bit per value pair with one payload byte per exceptional pair. + +| GloVe base-ten child | Bytes | Decode MB/s | Scalar ns | +| --- | ---: | ---: | ---: | +| Bitmap remainder | 5,197,126 | 10,400 to 11,200 | 14 | +| Gap positions | 5,348,130 | 10,800 to 12,400 | 78 to 88 | +| Pair bytes | 5,184,309 | 10,600 to 10,700 | Not measured | + +The current Default GloVe tree decodes at 16,700 to 17,500 MB/s. + +The bitmap variant encodes at 650 to 666 MB/s. It applies 142,861 quotient patches and 298,831 remainder exceptions. + +The gap layout loses size and scalar speed. The pair layout does not improve bulk decode. + +The CMS prototype stores every remainder with one dense FastLanes stream. This layout avoids sparse remainder expansion. + +| CMS payment child, base ten | Bytes | Encode MB/s | Decode MB/s | Scalar ns | +| --- | ---: | ---: | ---: | ---: | +| Pco child | 9,141,872 | Not isolated | Not isolated | Not isolated | +| Dense remainder prototype | 9,811,885 | 879 | 8,094 | 19 | + +The complete Default CMS tree uses 10,494,404 bytes and decodes at 25,226 MB/s. + +The dense prototype applies 853,533 quotient patches across two million values. + +A diagnostic decode without quotient patches reaches 12,628 MB/s. The split and reconstruction costs still miss the Default decode target. + +These results reject the tested IntMult layouts for Default. The layouts retain bounded scalar access, but their bulk decode cost is too high. + +The result does not reject an IntMult backend for Compact. Compact accepts a different throughput and random-access tradeoff. + ### Exact ALP child comparison The benchmark replaces a Compact `ALP(Pco)` child with the same latent integers in BlockResidual. @@ -1224,6 +1265,8 @@ This round completed these steps: - Rejected byte-aligned fixed bins after a direct size and throughput comparison. - Rejected grouped fixed bins after Euro2016 and CMS comparisons. - Rejected packed multi-reference blocks after patched and patch-free comparisons. +- Rejected three bounded IntMult remainder layouts on GloVe. +- Rejected the dense-remainder IntMult layout on CMS Payments. The Pco mode profile and quotient and remainder experiments are complete. @@ -1231,8 +1274,8 @@ The nonzero-secondary FloatQuant implementation and focused validation are compl Complete these remaining steps: -1. Prototype a lightweight integer split for Pco `IntMult` wins under ALP. -2. Prototype one bounded-access scheme for repeated classic-bin wins without Delta. +1. Prototype one bounded-access scheme for repeated classic-bin wins without Delta. +2. Keep IntMult outside Default unless a new design removes the split reconstruction cost. 3. Add a true ALP-RD child composition case if a real selected tree exposes one. 4. Validate the remaining rejected analysis cost after the candidate set stabilizes. 5. Add real embedding datasets beyond GloVe when licenses and loaders permit them. diff --git a/vortex-btrblocks/Cargo.toml b/vortex-btrblocks/Cargo.toml index d7991eaab78..308cac2c98d 100644 --- a/vortex-btrblocks/Cargo.toml +++ b/vortex-btrblocks/Cargo.toml @@ -46,6 +46,7 @@ arrow-array = { workspace = true } arrow-schema = { workspace = true } arrow-select = { workspace = true } divan = { workspace = true } +fastlanes = { workspace = true } insta = { workspace = true } parquet = { workspace = true } rstest = { workspace = true } diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index 10fd318c687..c9212fd2e0d 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -1,6 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +#[path = "float_compressor_bench/int_mult_codec.rs"] +mod int_mult_codec; + use std::fs::File; use std::hint::black_box; use std::io::BufRead; @@ -20,6 +23,7 @@ use pco::data_types::Number; use pco::metadata::ChunkLatentVarMeta; use pco::metadata::DeltaEncoding; use pco::metadata::DynBins; +use pco::metadata::DynLatent; use pco::metadata::Mode; use pco::wrapped::FileCompressor; use rand::RngExt; @@ -66,6 +70,9 @@ use vortex_range_packed::RangePackedCodec; use vortex_session::VortexSession; use vortex_utils::aliases::hash_map::HashMap; +use crate::int_mult_codec::IntMultCodec32; +use crate::int_mult_codec::IntMultDenseCodec64; + const DEFAULT_ROW_COUNT: usize = 2_000_000; const CALIFORNIA_COLUMNS: [&str; 9] = [ "longitude", @@ -587,7 +594,7 @@ fn optional_bin_profile(meta: Option<&ChunkLatentVarMeta>) -> BinProfile { fn mode_name(mode: &Mode) -> String { match mode { Mode::Classic => "classic".to_string(), - Mode::IntMult(_) => "int-mult".to_string(), + Mode::IntMult(base) => format!("int-mult-{}", dyn_latent_value(*base)), Mode::FloatMult(_) => "float-mult".to_string(), Mode::FloatQuant(k) => format!("float-quant-{k}"), Mode::Dict(_) => "dict".to_string(), @@ -595,6 +602,16 @@ fn mode_name(mode: &Mode) -> String { } } +fn dyn_latent_value(value: DynLatent) -> u64 { + match value { + DynLatent::U8(value) => u64::from(value), + DynLatent::U16(value) => u64::from(value), + DynLatent::U32(value) => u64::from(value), + DynLatent::U64(value) => value, + _ => unreachable!("unsupported Pco latent type"), + } +} + fn delta_name(delta: &DeltaEncoding) -> String { match delta { DeltaEncoding::NoOp => "none".to_string(), @@ -899,6 +916,13 @@ fn profile_quotient_remainder( encoding_tree(&remainder), ); } + if std::env::var_os("VORTEX_BENCH_INT_MULT").is_some() { + match ptype.bit_width() { + 32 => profile_int_mult_codec(dataset, column, path, ptype, pco_bytes, ordered)?, + 64 => profile_int_mult_dense_codec64(dataset, column, path, ptype, pco_bytes, ordered)?, + _ => {} + } + } Ok(()) } @@ -923,6 +947,223 @@ fn latent_array(values: &[u64], ptype: PType) -> VortexResult { } } +fn profile_int_mult_codec( + dataset: &str, + column: &str, + path: &str, + ptype: PType, + pco_bytes: u64, + ordered: &[u64], +) -> VortexResult<()> { + let values = ordered + .iter() + .map(|&value| u32::try_from(value)) + .collect::, _>>()?; + for base in [2, 5, 10, 16, 100, 1_000] { + let mut encode_durations = Vec::with_capacity(5); + let mut codec = IntMultCodec32::encode(&values, base)?; + for _ in 0..5 { + let start = Instant::now(); + codec = IntMultCodec32::encode(black_box(&values), base)?; + encode_durations.push(start.elapsed()); + } + let encode_median = percentile(&mut encode_durations, 1, 2); + + vortex_ensure!(codec.decode() == values, "IntMult codec decode differs"); + let decode_iterations = codec_decode_iterations()?; + let mut decode_durations = Vec::with_capacity(decode_iterations); + for _ in 0..decode_iterations { + let start = Instant::now(); + black_box(codec.decode()); + decode_durations.push(start.elapsed()); + } + let decode_median = percentile(&mut decode_durations, 1, 2); + + let scalar_iterations = 200_000; + let mut scalar_index = 0usize; + let mut scalar_checksum = 0u32; + let scalar_start = Instant::now(); + for _ in 0..scalar_iterations { + scalar_index = scalar_index.wrapping_add(2_654_435_761) % values.len(); + scalar_checksum ^= black_box(codec.scalar_at(scalar_index)?); + } + black_box(scalar_checksum); + let scalar_duration = scalar_start.elapsed(); + + let input_bytes = values.len() * ptype.byte_width(); + let encode_throughput = input_bytes as f64 / encode_median.as_secs_f64() / 1_000_000.0; + let decode_throughput = input_bytes as f64 / decode_median.as_secs_f64() / 1_000_000.0; + let scalar_nanoseconds = + scalar_duration.as_secs_f64() * 1_000_000_000.0 / scalar_iterations as f64; + println!( + "int-mult-checkpoint\t{dataset}\t{column}\t{path}\t{ptype}\t{base}\t{}\t{pco_bytes}\t{}\t{}\t{}\t{encode_throughput:.1}\t{decode_throughput:.1}\t{scalar_nanoseconds:.1}", + values.len(), + codec.encoded_size(), + codec.quotient_patch_count(), + codec.remainder_exception_count(), + ); + let mut gap_decode_durations = Vec::with_capacity(decode_iterations); + for _ in 0..decode_iterations { + let start = Instant::now(); + black_box(codec.decode_gaps()); + gap_decode_durations.push(start.elapsed()); + } + let gap_decode_median = percentile(&mut gap_decode_durations, 1, 2); + let gap_decode_throughput = + input_bytes as f64 / gap_decode_median.as_secs_f64() / 1_000_000.0; + let mut gap_scalar_index = 0usize; + let mut gap_scalar_checksum = 0u32; + let gap_scalar_start = Instant::now(); + for _ in 0..scalar_iterations { + gap_scalar_index = gap_scalar_index.wrapping_add(2_654_435_761) % values.len(); + gap_scalar_checksum ^= black_box(codec.scalar_at_gaps(gap_scalar_index)?); + } + black_box(gap_scalar_checksum); + let gap_scalar_nanoseconds = + gap_scalar_start.elapsed().as_secs_f64() * 1_000_000_000.0 / scalar_iterations as f64; + let (quotient_gap_bytes, remainder_gap_bytes) = codec.gap_bytes(); + let (quotient_bitmap_bytes, remainder_bitmap_bytes) = codec.bitmap_bytes(); + println!( + "int-mult-gaps-checkpoint\t{dataset}\t{column}\t{path}\t{ptype}\t{base}\t{}\t{pco_bytes}\t{}\t{quotient_gap_bytes}\t{remainder_gap_bytes}\t{quotient_bitmap_bytes}\t{remainder_bitmap_bytes}\t{encode_throughput:.1}\t{gap_decode_throughput:.1}\t{gap_scalar_nanoseconds:.1}", + values.len(), + codec.encoded_size_gaps(), + ); + if base <= 16 { + let mut pair_decode_durations = Vec::with_capacity(decode_iterations); + for _ in 0..decode_iterations { + let start = Instant::now(); + black_box(codec.decode_pairs()); + pair_decode_durations.push(start.elapsed()); + } + let pair_decode_median = percentile(&mut pair_decode_durations, 1, 2); + let pair_decode_throughput = + input_bytes as f64 / pair_decode_median.as_secs_f64() / 1_000_000.0; + println!( + "int-mult-pairs-checkpoint\t{dataset}\t{column}\t{path}\t{ptype}\t{base}\t{}\t{pco_bytes}\t{}\t{encode_throughput:.1}\t{pair_decode_throughput:.1}", + values.len(), + codec.encoded_size_pairs(), + ); + } + if base == 10 { + profile_int_mult_decode_breakdown(&codec, input_bytes, decode_iterations); + } + } + Ok(()) +} + +fn profile_int_mult_decode_breakdown( + codec: &IntMultCodec32, + input_bytes: usize, + iterations: usize, +) { + type DecodeVariant = fn(&IntMultCodec32) -> Vec; + let variants: [(&str, DecodeVariant); 9] = [ + ("full", IntMultCodec32::decode), + ( + "no-quotient-patches", + IntMultCodec32::decode_without_quotient_patches, + ), + ( + "no-remainder-exceptions", + IntMultCodec32::decode_without_remainder_exceptions, + ), + ("no-exceptions", IntMultCodec32::decode_without_exceptions), + ("gaps-full", IntMultCodec32::decode_gaps), + ( + "gaps-no-quotient-patches", + IntMultCodec32::decode_gaps_without_quotient_patches, + ), + ( + "gaps-no-remainder-exceptions", + IntMultCodec32::decode_gaps_without_remainder_exceptions, + ), + ( + "gaps-no-exceptions", + IntMultCodec32::decode_gaps_without_exceptions, + ), + ("pairs-full", IntMultCodec32::decode_pairs), + ]; + for (variant, decode) in variants { + let mut durations = Vec::with_capacity(iterations); + for _ in 0..iterations { + let start = Instant::now(); + black_box(decode(codec)); + durations.push(start.elapsed()); + } + let median = percentile(&mut durations, 1, 2); + let throughput = input_bytes as f64 / median.as_secs_f64() / 1_000_000.0; + println!("int-mult-decode-breakdown\t{variant}\t{throughput:.1}"); + } +} + +fn profile_int_mult_dense_codec64( + dataset: &str, + column: &str, + path: &str, + ptype: PType, + pco_bytes: u64, + values: &[u64], +) -> VortexResult<()> { + for base in [2, 5, 10, 16, 100, 1_000] { + let mut encode_durations = Vec::with_capacity(5); + let mut codec = IntMultDenseCodec64::encode(values, base)?; + for _ in 0..5 { + let start = Instant::now(); + codec = IntMultDenseCodec64::encode(black_box(values), base)?; + encode_durations.push(start.elapsed()); + } + let encode_median = percentile(&mut encode_durations, 1, 2); + + vortex_ensure!(codec.decode() == values, "dense IntMult decode differs"); + let decode_iterations = codec_decode_iterations()?; + let mut decode_durations = Vec::with_capacity(decode_iterations); + for _ in 0..decode_iterations { + let start = Instant::now(); + black_box(codec.decode()); + decode_durations.push(start.elapsed()); + } + let decode_median = percentile(&mut decode_durations, 1, 2); + + let scalar_iterations = 200_000; + let mut scalar_index = 0usize; + let mut scalar_checksum = 0u64; + let scalar_start = Instant::now(); + for _ in 0..scalar_iterations { + scalar_index = scalar_index.wrapping_add(2_654_435_761) % values.len(); + scalar_checksum ^= black_box(codec.scalar_at(scalar_index)?); + } + black_box(scalar_checksum); + let scalar_duration = scalar_start.elapsed(); + + let input_bytes = values.len() * ptype.byte_width(); + let encode_throughput = input_bytes as f64 / encode_median.as_secs_f64() / 1_000_000.0; + let decode_throughput = input_bytes as f64 / decode_median.as_secs_f64() / 1_000_000.0; + let scalar_nanoseconds = + scalar_duration.as_secs_f64() * 1_000_000_000.0 / scalar_iterations as f64; + println!( + "int-mult-dense64-checkpoint\t{dataset}\t{column}\t{path}\t{ptype}\t{base}\t{}\t{pco_bytes}\t{}\t{}\t{encode_throughput:.1}\t{decode_throughput:.1}\t{scalar_nanoseconds:.1}", + values.len(), + codec.encoded_size(), + codec.quotient_patch_count(), + ); + if base == 10 { + let mut no_patch_durations = Vec::with_capacity(decode_iterations); + for _ in 0..decode_iterations { + let start = Instant::now(); + black_box(codec.decode_without_quotient_patches()); + no_patch_durations.push(start.elapsed()); + } + let no_patch_median = percentile(&mut no_patch_durations, 1, 2); + let no_patch_throughput = + input_bytes as f64 / no_patch_median.as_secs_f64() / 1_000_000.0; + println!( + "int-mult-decode-breakdown\tdense64-no-quotient-patches\t{no_patch_throughput:.1}" + ); + } + } + Ok(()) +} + fn profile_range_packed( dataset: &str, column: &str, @@ -1747,6 +1988,19 @@ fn main() -> VortexResult<()> { println!( "quotient-remainder-estimate\tdataset\tcolumn\tpath\tptype\tbase\tpco-child-bytes\tremainder-entropy\tmost-common-remainder-share\tquotient-bytes\tremainder-bytes\ttotal-bytes\tquotient-bitmap-bytes\tremainder-mode-bitmap-bytes\tbitmap-total-bytes\tquotient-encoding\tremainder-encoding" ); + println!( + "int-mult-checkpoint\tdataset\tcolumn\tpath\tptype\tbase\trows\tpco-child-bytes\tint-mult-bytes\tquotient-patches\tremainder-exceptions\tencode-MB/s\tdecode-MB/s\tscalar-ns" + ); + println!( + "int-mult-gaps-checkpoint\tdataset\tcolumn\tpath\tptype\tbase\trows\tpco-child-bytes\tint-mult-bytes\tquotient-gap-bytes\tremainder-gap-bytes\tquotient-bitmap-bytes\tremainder-bitmap-bytes\tencode-MB/s\tdecode-MB/s\tscalar-ns" + ); + println!( + "int-mult-pairs-checkpoint\tdataset\tcolumn\tpath\tptype\tbase\trows\tpco-child-bytes\tint-mult-bytes\tencode-MB/s\tdecode-MB/s" + ); + println!("int-mult-decode-breakdown\tvariant\tdecode-MB/s"); + println!( + "int-mult-dense64-checkpoint\tdataset\tcolumn\tpath\tptype\tbase\trows\tpco-child-bytes\tint-mult-bytes\tquotient-patches\tencode-MB/s\tdecode-MB/s\tscalar-ns" + ); println!( "fixed-bin-checkpoint\tdataset\tcolumn\tpath\trows\tpco-bytes\tfixed-bin-bytes\tbins\tmax-offset-bits\toffset-widths\tencode-MB/s\tdecode-MB/s\tscalar-ns" ); diff --git a/vortex-btrblocks/examples/float_compressor_bench/int_mult_codec.rs b/vortex-btrblocks/examples/float_compressor_bench/int_mult_codec.rs new file mode 100644 index 00000000000..a724044e956 --- /dev/null +++ b/vortex-btrblocks/examples/float_compressor_bench/int_mult_codec.rs @@ -0,0 +1,1057 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::mem::size_of; + +use fastlanes::BitPacking; +use vortex_error::VortexResult; + +const CHUNK_LEN: usize = 1_024; +const BITMAP_WORDS: usize = CHUNK_LEN / u64::BITS as usize; +const STREAM_PADDING: usize = 7; +const SERIALIZED_BLOCK_METADATA_BYTES: usize = 24; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct IntMultCodec32 { + len: usize, + base: u32, + blocks: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct IntMultDenseCodec64 { + len: usize, + base: u64, + blocks: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct IntMultDenseBlock64 { + len: u16, + quotient_base: u64, + quotient_width: u8, + quotient_high_width: u8, + quotient_lows: Vec, + quotient_patch_bitmap: Vec, + quotient_highs: Vec, + remainders: Vec, + quotient_patch_count: u16, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct IntMultBlock32 { + len: u16, + quotient_base: u32, + quotient_width: u8, + quotient_high_width: u8, + quotient_lows: Vec, + quotient_patch_bitmap: Vec, + quotient_patch_gaps: Vec, + quotient_highs: Vec, + remainder_mode: u32, + remainder_width: u8, + remainder_exception_bitmap: Vec, + remainder_exception_gaps: Vec, + remainder_pair_bitmap: Vec, + remainder_pair_values: Vec, + remainder_exceptions: Vec, + quotient_patch_count: u16, + remainder_exception_count: u16, +} + +impl IntMultCodec32 { + pub fn encode(values: &[u32], base: u32) -> VortexResult { + vortex_error::vortex_ensure!(base > 1, "IntMult base must exceed one"); + let blocks = values + .chunks(CHUNK_LEN) + .map(|block| encode_block(block, base)) + .collect::>>()?; + Ok(Self { + len: values.len(), + base, + blocks, + }) + } + + pub fn decode(&self) -> Vec { + self.decode_variant::() + } + + pub fn decode_gaps(&self) -> Vec { + self.decode_variant::() + } + + pub fn decode_pairs(&self) -> Vec { + self.decode_variant::() + } + + pub fn decode_gaps_without_quotient_patches(&self) -> Vec { + self.decode_variant::() + } + + pub fn decode_gaps_without_remainder_exceptions(&self) -> Vec { + self.decode_variant::() + } + + pub fn decode_gaps_without_exceptions(&self) -> Vec { + self.decode_variant::() + } + + pub fn decode_without_quotient_patches(&self) -> Vec { + self.decode_variant::() + } + + pub fn decode_without_remainder_exceptions(&self) -> Vec { + self.decode_variant::() + } + + pub fn decode_without_exceptions(&self) -> Vec { + self.decode_variant::() + } + + fn decode_variant< + const QUOTIENT_PATCHES: bool, + const REMAINDER_EXCEPTIONS: bool, + const GAPS: bool, + const PAIRS: bool, + >( + &self, + ) -> Vec { + let mut values = Vec::with_capacity(self.len); + let mut quotients = [0_u32; CHUNK_LEN]; + let mut remainder_exceptions = [0_u8; CHUNK_LEN]; + for block in &self.blocks { + if block.quotient_width == 0 { + quotients.fill(0); + } else { + // SAFETY: The encoder creates one complete FastLanes chunk. + unsafe { + u32::unchecked_unpack( + usize::from(block.quotient_width), + &block.quotient_lows, + &mut quotients, + ); + } + } + + if QUOTIENT_PATCHES { + let mut high_index = 0usize; + let mut apply = |position| { + let high = read_bits( + &block.quotient_highs, + high_index * usize::from(block.quotient_high_width), + block.quotient_high_width, + ); + quotients[position] |= high << block.quotient_width; + high_index += 1; + }; + if GAPS { + for_each_gap(&block.quotient_patch_gaps, &mut apply); + } else { + for_each_set_bit(&block.quotient_patch_bitmap, &mut apply); + } + } + let block_len = usize::from(block.len); + for quotient in &mut quotients[..block_len] { + *quotient = quotient + .wrapping_add(block.quotient_base) + .wrapping_mul(self.base) + .wrapping_add(block.remainder_mode); + } + if REMAINDER_EXCEPTIONS { + let mut exception_index = 0usize; + if PAIRS { + let mut pair_value_index = 0usize; + for_each_set_bit(&block.remainder_pair_bitmap, |pair_index| { + let position = pair_index * 2; + // SAFETY: The encoder stores one byte for every set pair bit. + let pair = + unsafe { *block.remainder_pair_values.get_unchecked(pair_value_index) }; + quotients[position] = quotients[position] + .wrapping_sub(block.remainder_mode) + .wrapping_add(u32::from(pair & 0x0f)); + if position + 1 < block_len { + quotients[position + 1] = quotients[position + 1] + .wrapping_sub(block.remainder_mode) + .wrapping_add(u32::from(pair >> 4)); + } + pair_value_index += 1; + }); + } else if GAPS { + for_each_gap(&block.remainder_exception_gaps, |position| { + let remainder = read_bits( + &block.remainder_exceptions, + exception_index * usize::from(block.remainder_width), + block.remainder_width, + ); + quotients[position] = quotients[position] + .wrapping_sub(block.remainder_mode) + .wrapping_add(remainder); + exception_index += 1; + }); + } else { + if block.remainder_width == 4 { + let exception_count = usize::from(block.remainder_exception_count); + for index in 0..exception_count.div_ceil(2) { + // SAFETY: The encoder stores two exceptions in each payload byte. + let byte = unsafe { *block.remainder_exceptions.get_unchecked(index) }; + remainder_exceptions[index * 2] = byte & 0x0f; + remainder_exceptions[index * 2 + 1] = byte >> 4; + } + for_each_set_bit(&block.remainder_exception_bitmap, |position| { + let remainder = u32::from(remainder_exceptions[exception_index]); + quotients[position] = quotients[position] + .wrapping_sub(block.remainder_mode) + .wrapping_add(remainder); + exception_index += 1; + }); + } else { + for_each_set_bit(&block.remainder_exception_bitmap, |position| { + let remainder = read_bits( + &block.remainder_exceptions, + exception_index * usize::from(block.remainder_width), + block.remainder_width, + ); + quotients[position] = quotients[position] + .wrapping_sub(block.remainder_mode) + .wrapping_add(remainder); + exception_index += 1; + }); + } + } + } + values.extend_from_slice("ients[..block_len]); + } + values + } + + pub fn scalar_at(&self, index: usize) -> VortexResult { + self.scalar_at_variant::(index) + } + + pub fn scalar_at_gaps(&self, index: usize) -> VortexResult { + self.scalar_at_variant::(index) + } + + fn scalar_at_variant(&self, index: usize) -> VortexResult { + vortex_error::vortex_ensure!( + index < self.len, + "index {index} is out of bounds for length {}", + self.len + ); + let block = &self.blocks[index / CHUNK_LEN]; + let index_in_block = index % CHUNK_LEN; + let mut quotient_residual = if block.quotient_width == 0 { + 0 + } else { + // SAFETY: The encoder creates one complete FastLanes chunk. + unsafe { + u32::unchecked_unpack_single( + usize::from(block.quotient_width), + &block.quotient_lows, + index_in_block, + ) + } + }; + let quotient_rank = if GAPS { + gap_rank(&block.quotient_patch_gaps, index_in_block) + } else if bitmap_value(&block.quotient_patch_bitmap, index_in_block) { + Some(bitmap_rank(&block.quotient_patch_bitmap, index_in_block)) + } else { + None + }; + if let Some(rank) = quotient_rank { + let high = read_bits( + &block.quotient_highs, + rank * usize::from(block.quotient_high_width), + block.quotient_high_width, + ); + quotient_residual |= high << block.quotient_width; + } + let quotient = block.quotient_base.wrapping_add(quotient_residual); + + let remainder_rank = if GAPS { + gap_rank(&block.remainder_exception_gaps, index_in_block) + } else if bitmap_value(&block.remainder_exception_bitmap, index_in_block) { + Some(bitmap_rank( + &block.remainder_exception_bitmap, + index_in_block, + )) + } else { + None + }; + let remainder = if let Some(rank) = remainder_rank { + read_bits( + &block.remainder_exceptions, + rank * usize::from(block.remainder_width), + block.remainder_width, + ) + } else { + block.remainder_mode + }; + Ok(quotient.wrapping_mul(self.base).wrapping_add(remainder)) + } + + pub fn encoded_size(&self) -> usize { + size_of::() + + self + .blocks + .iter() + .map(|block| { + SERIALIZED_BLOCK_METADATA_BYTES + + block.quotient_lows.len() * size_of::() + + block.quotient_patch_bitmap.len() * size_of::() + + block.quotient_highs.len() + + block.remainder_exception_bitmap.len() * size_of::() + + block.remainder_exceptions.len() + }) + .sum::() + } + + pub fn encoded_size_gaps(&self) -> usize { + size_of::() + + self + .blocks + .iter() + .map(|block| { + SERIALIZED_BLOCK_METADATA_BYTES + + block.quotient_lows.len() * size_of::() + + block.quotient_patch_gaps.len() + + block.quotient_highs.len() + + block.remainder_exception_gaps.len() + + block.remainder_exceptions.len() + }) + .sum::() + } + + pub fn encoded_size_pairs(&self) -> usize { + size_of::() + + self + .blocks + .iter() + .map(|block| { + SERIALIZED_BLOCK_METADATA_BYTES + + block.quotient_lows.len() * size_of::() + + block.quotient_patch_bitmap.len() * size_of::() + + block.quotient_highs.len() + + block.remainder_pair_bitmap.len() * size_of::() + + block.remainder_pair_values.len() + }) + .sum::() + } + + pub fn quotient_patch_count(&self) -> usize { + self.blocks + .iter() + .map(|block| usize::from(block.quotient_patch_count)) + .sum() + } + + pub fn remainder_exception_count(&self) -> usize { + self.blocks + .iter() + .map(|block| usize::from(block.remainder_exception_count)) + .sum() + } + + pub fn gap_bytes(&self) -> (usize, usize) { + self.blocks + .iter() + .fold((0, 0), |(quotient, remainder), block| { + ( + quotient + block.quotient_patch_gaps.len(), + remainder + block.remainder_exception_gaps.len(), + ) + }) + } + + pub fn bitmap_bytes(&self) -> (usize, usize) { + self.blocks + .iter() + .fold((0, 0), |(quotient, remainder), block| { + ( + quotient + block.quotient_patch_bitmap.len() * size_of::(), + remainder + block.remainder_exception_bitmap.len() * size_of::(), + ) + }) + } +} + +impl IntMultDenseCodec64 { + pub fn encode(values: &[u64], base: u64) -> VortexResult { + vortex_error::vortex_ensure!(base > 1, "IntMult base must exceed one"); + let blocks = values + .chunks(CHUNK_LEN) + .map(|block| encode_dense_block64(block, base)) + .collect::>>()?; + Ok(Self { + len: values.len(), + base, + blocks, + }) + } + + pub fn decode(&self) -> Vec { + self.decode_with_quotient_patches(true) + } + + pub fn decode_without_quotient_patches(&self) -> Vec { + self.decode_with_quotient_patches(false) + } + + fn decode_with_quotient_patches(&self, apply_patches: bool) -> Vec { + let mut values = Vec::with_capacity(self.len); + let mut quotients = [0_u64; CHUNK_LEN]; + let mut remainders = [0_u64; CHUNK_LEN]; + for block in &self.blocks { + if block.quotient_width == 0 { + quotients.fill(0); + } else { + // SAFETY: The encoder creates one complete FastLanes chunk. + unsafe { + u64::unchecked_unpack( + usize::from(block.quotient_width), + &block.quotient_lows, + &mut quotients, + ); + } + } + if apply_patches { + let mut high_index = 0usize; + for_each_set_bit(&block.quotient_patch_bitmap, |position| { + let high = read_bits64( + &block.quotient_highs, + high_index * usize::from(block.quotient_high_width), + block.quotient_high_width, + ); + quotients[position] |= high << block.quotient_width; + high_index += 1; + }); + } + + let remainder_width = bit_width64(self.base - 1); + if remainder_width == 0 { + remainders.fill(0); + } else { + // SAFETY: The encoder creates one complete FastLanes chunk. + unsafe { + u64::unchecked_unpack( + usize::from(remainder_width), + &block.remainders, + &mut remainders, + ); + } + } + + let block_len = usize::from(block.len); + for index in 0..block_len { + quotients[index] = quotients[index] + .wrapping_add(block.quotient_base) + .wrapping_mul(self.base) + .wrapping_add(remainders[index]); + } + values.extend_from_slice("ients[..block_len]); + } + values + } + + pub fn scalar_at(&self, index: usize) -> VortexResult { + vortex_error::vortex_ensure!( + index < self.len, + "index {index} is out of bounds for length {}", + self.len + ); + let block = &self.blocks[index / CHUNK_LEN]; + let index_in_block = index % CHUNK_LEN; + let mut quotient_residual = if block.quotient_width == 0 { + 0 + } else { + // SAFETY: The encoder creates one complete FastLanes chunk. + unsafe { + u64::unchecked_unpack_single( + usize::from(block.quotient_width), + &block.quotient_lows, + index_in_block, + ) + } + }; + if bitmap_value(&block.quotient_patch_bitmap, index_in_block) { + let rank = bitmap_rank(&block.quotient_patch_bitmap, index_in_block); + let high = read_bits64( + &block.quotient_highs, + rank * usize::from(block.quotient_high_width), + block.quotient_high_width, + ); + quotient_residual |= high << block.quotient_width; + } + let remainder_width = bit_width64(self.base - 1); + let remainder = if remainder_width == 0 { + 0 + } else { + // SAFETY: The encoder creates one complete FastLanes chunk. + unsafe { + u64::unchecked_unpack_single( + usize::from(remainder_width), + &block.remainders, + index_in_block, + ) + } + }; + Ok(quotient_residual + .wrapping_add(block.quotient_base) + .wrapping_mul(self.base) + .wrapping_add(remainder)) + } + + pub fn encoded_size(&self) -> usize { + size_of::() + + self + .blocks + .iter() + .map(|block| { + SERIALIZED_BLOCK_METADATA_BYTES + + block.quotient_lows.len() * size_of::() + + block.quotient_patch_bitmap.len() * size_of::() + + block.quotient_highs.len() + + block.remainders.len() * size_of::() + }) + .sum::() + } + + pub fn quotient_patch_count(&self) -> usize { + self.blocks + .iter() + .map(|block| usize::from(block.quotient_patch_count)) + .sum() + } +} + +fn encode_block(values: &[u32], base: u32) -> VortexResult { + let mut quotients = Vec::with_capacity(CHUNK_LEN); + let mut remainders = Vec::with_capacity(CHUNK_LEN); + for &value in values { + quotients.push(value / base); + remainders.push(value % base); + } + + let quotient_base = quotients.iter().copied().min().unwrap_or_default(); + let mut quotient_residuals = quotients + .iter() + .map(|"ient| quotient - quotient_base) + .collect::>(); + let (quotient_width, quotient_high_width, quotient_patch_count) = + choose_quotient_width("ient_residuals, values.len()); + quotient_residuals.resize(CHUNK_LEN, 0); + let quotient_mask = low_mask(quotient_width); + let quotient_lows = quotient_residuals + .iter() + .map(|&residual| residual & quotient_mask) + .collect::>(); + + let mut quotient_patch_bitmap = vec![0_u64; BITMAP_WORDS]; + let mut quotient_patch_positions = Vec::with_capacity(quotient_patch_count); + let mut quotient_highs = + BitWriter::with_capacity(quotient_patch_count * usize::from(quotient_high_width)); + if quotient_high_width > 0 { + for (position, &residual) in quotient_residuals[..values.len()].iter().enumerate() { + let high = residual >> quotient_width; + if high != 0 { + set_bitmap(&mut quotient_patch_bitmap, position); + quotient_patch_positions.push(u16::try_from(position)?); + quotient_highs.write(high, quotient_high_width); + } + } + } + if quotient_patch_count == 0 { + quotient_patch_bitmap.clear(); + } + + let remainder_mode = mode(&remainders, base)?; + let remainder_width = bit_width(base - 1); + let mut remainder_exception_bitmap = vec![0_u64; BITMAP_WORDS]; + let remainder_exception_count = remainders + .iter() + .filter(|&&remainder| remainder != remainder_mode) + .count(); + let mut remainder_exceptions = + BitWriter::with_capacity(remainder_exception_count * usize::from(remainder_width)); + let mut remainder_exception_positions = Vec::with_capacity(remainder_exception_count); + for (position, &remainder) in remainders.iter().enumerate() { + if remainder != remainder_mode { + set_bitmap(&mut remainder_exception_bitmap, position); + remainder_exception_positions.push(u16::try_from(position)?); + remainder_exceptions.write(remainder, remainder_width); + } + } + if remainder_exception_count == 0 { + remainder_exception_bitmap.clear(); + } + + let mut remainder_pair_bitmap = vec![0_u64; BITMAP_WORDS / 2]; + let mut remainder_pair_values = Vec::with_capacity(values.len().div_ceil(2)); + let mut remainder_pair_count = 0usize; + if base <= 16 { + for (pair_index, pair) in remainders.chunks(2).enumerate() { + let left_exception = pair[0] != remainder_mode; + let right_exception = pair + .get(1) + .is_some_and(|&remainder| remainder != remainder_mode); + if !left_exception && !right_exception { + continue; + } + set_bitmap(&mut remainder_pair_bitmap, pair_index); + let right = pair.get(1).copied().unwrap_or(remainder_mode); + remainder_pair_values.push(u8::try_from(pair[0] | (right << 4))?); + remainder_pair_count += 1; + } + } + if remainder_pair_count == 0 { + remainder_pair_bitmap.clear(); + } + + Ok(IntMultBlock32 { + len: u16::try_from(values.len())?, + quotient_base, + quotient_width, + quotient_high_width, + quotient_lows: fast_pack("ient_lows, quotient_width), + quotient_patch_bitmap, + quotient_patch_gaps: encode_gaps("ient_patch_positions), + quotient_highs: quotient_highs.finish(), + remainder_mode, + remainder_width, + remainder_exception_bitmap, + remainder_exception_gaps: encode_gaps(&remainder_exception_positions), + remainder_pair_bitmap, + remainder_pair_values, + remainder_exceptions: remainder_exceptions.finish(), + quotient_patch_count: u16::try_from(quotient_patch_count)?, + remainder_exception_count: u16::try_from(remainder_exception_count)?, + }) +} + +fn encode_dense_block64(values: &[u64], base: u64) -> VortexResult { + let mut quotient_residuals = Vec::with_capacity(CHUNK_LEN); + let mut remainders = Vec::with_capacity(CHUNK_LEN); + let quotient_base = values + .iter() + .map(|value| value / base) + .min() + .unwrap_or_default(); + for &value in values { + quotient_residuals.push(value / base - quotient_base); + remainders.push(value % base); + } + let (quotient_width, quotient_high_width, quotient_patch_count) = + choose_quotient_width64("ient_residuals, values.len()); + quotient_residuals.resize(CHUNK_LEN, 0); + remainders.resize(CHUNK_LEN, 0); + let quotient_mask = low_mask64(quotient_width); + let quotient_lows = quotient_residuals + .iter() + .map(|&residual| residual & quotient_mask) + .collect::>(); + + let mut quotient_patch_bitmap = vec![0_u64; BITMAP_WORDS]; + let mut quotient_highs = + BitWriter64::with_capacity(quotient_patch_count * usize::from(quotient_high_width)); + if quotient_high_width > 0 { + for (position, &residual) in quotient_residuals[..values.len()].iter().enumerate() { + let high = residual >> quotient_width; + if high != 0 { + set_bitmap(&mut quotient_patch_bitmap, position); + quotient_highs.write(high, quotient_high_width); + } + } + } + if quotient_patch_count == 0 { + quotient_patch_bitmap.clear(); + } + + Ok(IntMultDenseBlock64 { + len: u16::try_from(values.len())?, + quotient_base, + quotient_width, + quotient_high_width, + quotient_lows: fast_pack64("ient_lows, quotient_width), + quotient_patch_bitmap, + quotient_highs: quotient_highs.finish(), + remainders: fast_pack64(&remainders, bit_width64(base - 1)), + quotient_patch_count: u16::try_from(quotient_patch_count)?, + }) +} + +fn choose_quotient_width(residuals: &[u32], value_count: usize) -> (u8, u8, usize) { + let mut width_counts = [0_usize; 33]; + let mut maximum_width = 0u8; + for &residual in residuals { + let width = bit_width(residual); + width_counts[usize::from(width)] += 1; + maximum_width = maximum_width.max(width); + } + + let mut patch_count = value_count; + let mut best = (maximum_width, 0_u8, 0_usize); + let mut best_bits = CHUNK_LEN * usize::from(maximum_width); + for residual_width in 0..=maximum_width { + patch_count -= width_counts[usize::from(residual_width)]; + let high_width = if patch_count == 0 { + 0 + } else { + maximum_width - residual_width + }; + let bits = CHUNK_LEN * usize::from(residual_width) + + usize::from(patch_count > 0) * CHUNK_LEN + + patch_count * usize::from(high_width); + if bits < best_bits { + best = (residual_width, high_width, patch_count); + best_bits = bits; + } + } + best +} + +fn choose_quotient_width64(residuals: &[u64], value_count: usize) -> (u8, u8, usize) { + let mut width_counts = [0_usize; 65]; + let mut maximum_width = 0u8; + for &residual in residuals { + let width = bit_width64(residual); + width_counts[usize::from(width)] += 1; + maximum_width = maximum_width.max(width); + } + + let mut patch_count = value_count; + let mut best = (maximum_width, 0_u8, 0_usize); + let mut best_bits = CHUNK_LEN * usize::from(maximum_width); + for residual_width in 0..=maximum_width { + patch_count -= width_counts[usize::from(residual_width)]; + let high_width = if patch_count == 0 { + 0 + } else { + maximum_width - residual_width + }; + let bits = CHUNK_LEN * usize::from(residual_width) + + usize::from(patch_count > 0) * CHUNK_LEN + + patch_count * usize::from(high_width); + if bits < best_bits { + best = (residual_width, high_width, patch_count); + best_bits = bits; + } + } + best +} + +fn mode(values: &[u32], base: u32) -> VortexResult { + let mut counts = vec![0_u16; usize::try_from(base)?]; + for &value in values { + counts[usize::try_from(value)?] += 1; + } + Ok(counts + .iter() + .enumerate() + .max_by_key(|(_, count)| **count) + .map(|(value, _)| u32::try_from(value)) + .transpose()? + .unwrap_or_default()) +} + +fn fast_pack(values: &[u32], width: u8) -> Vec { + if width == 0 { + return Vec::new(); + } + let mut packed = vec![0_u32; CHUNK_LEN * usize::from(width) / u32::BITS as usize]; + // SAFETY: Both slices have the exact lengths required for one FastLanes chunk. + unsafe { u32::unchecked_pack(usize::from(width), values, &mut packed) }; + packed +} + +fn fast_pack64(values: &[u64], width: u8) -> Vec { + if width == 0 { + return Vec::new(); + } + let mut packed = vec![0_u64; CHUNK_LEN * usize::from(width) / u64::BITS as usize]; + // SAFETY: Both slices have the exact lengths required for one FastLanes chunk. + unsafe { u64::unchecked_pack(usize::from(width), values, &mut packed) }; + packed +} + +fn bit_width(value: u32) -> u8 { + u8::try_from(u32::BITS - value.leading_zeros()).unwrap_or(u8::MAX) +} + +fn low_mask(width: u8) -> u32 { + if width == 32 { + u32::MAX + } else if width == 0 { + 0 + } else { + (1_u32 << width) - 1 + } +} + +fn bit_width64(value: u64) -> u8 { + u8::try_from(u64::BITS - value.leading_zeros()).unwrap_or(u8::MAX) +} + +fn low_mask64(width: u8) -> u64 { + if width == 64 { + u64::MAX + } else if width == 0 { + 0 + } else { + (1_u64 << width) - 1 + } +} + +fn set_bitmap(bitmap: &mut [u64], index: usize) { + bitmap[index / 64] |= 1_u64 << (index % 64); +} + +fn bitmap_value(bitmap: &[u64], index: usize) -> bool { + bitmap + .get(index / 64) + .is_some_and(|word| word & (1_u64 << (index % 64)) != 0) +} + +fn bitmap_rank(bitmap: &[u64], index: usize) -> usize { + let full_words = index / 64; + let prior = bitmap[..full_words] + .iter() + .map(|word| word.count_ones() as usize) + .sum::(); + let bit_index = index % 64; + let current = bitmap + .get(full_words) + .map(|word| (word & ((1_u64 << bit_index).wrapping_sub(1))).count_ones() as usize) + .unwrap_or_default(); + prior + current +} + +fn encode_gaps(positions: &[u16]) -> Vec { + let mut gaps = Vec::with_capacity(positions.len()); + let mut next_position = 0usize; + for &position in positions { + let mut gap = usize::from(position) - next_position; + while gap >= 255 { + gaps.push(255); + gap -= 255; + } + gaps.push(u8::try_from(gap).unwrap_or_else(|_| unreachable!("gap is below 255"))); + next_position = usize::from(position) + 1; + } + gaps +} + +#[inline(always)] +fn for_each_gap(gaps: &[u8], mut function: impl FnMut(usize)) { + let mut next_position = 0usize; + let mut gap = 0usize; + for &part in gaps { + gap += usize::from(part); + if part < 255 { + let position = next_position + gap; + function(position); + next_position = position + 1; + gap = 0; + } + } +} + +fn gap_rank(gaps: &[u8], target: usize) -> Option { + let mut next_position = 0usize; + let mut gap = 0usize; + let mut rank = 0usize; + for &part in gaps { + gap += usize::from(part); + if part < 255 { + let position = next_position + gap; + if position >= target { + return (position == target).then_some(rank); + } + next_position = position + 1; + gap = 0; + rank += 1; + } + } + None +} + +#[inline(always)] +fn for_each_set_bit(bitmap: &[u64], mut function: impl FnMut(usize)) { + for (word_index, &word) in bitmap.iter().enumerate() { + let mut remaining = word; + while remaining != 0 { + let bit = remaining.trailing_zeros() as usize; + function(word_index * 64 + bit); + remaining &= remaining - 1; + } + } +} + +#[inline(always)] +fn read_bits(bytes: &[u8], bit_offset: usize, width: u8) -> u32 { + if width == 0 { + return 0; + } + let byte_offset = bit_offset / 8; + let shift = bit_offset % 8; + // SAFETY: Each nonempty stream contains seven readable padding bytes. + let word = unsafe { + bytes + .as_ptr() + .add(byte_offset) + .cast::() + .read_unaligned() + }; + u32::try_from((u64::from_le(word) >> shift) & u64::from(low_mask(width))) + .unwrap_or_else(|_| unreachable!("masked bit stream value fits u32")) +} + +#[inline(always)] +fn read_bits64(bytes: &[u8], bit_offset: usize, width: u8) -> u64 { + if width == 0 { + return 0; + } + let byte_offset = bit_offset / 8; + let shift = bit_offset % 8; + // SAFETY: Each nonempty stream contains fifteen readable padding bytes. + let low = unsafe { + bytes + .as_ptr() + .add(byte_offset) + .cast::() + .read_unaligned() + }; + let mut value = u64::from_le(low) >> shift; + if shift + usize::from(width) > 64 { + // SAFETY: Each nonempty stream contains fifteen readable padding bytes. + let high = unsafe { + bytes + .as_ptr() + .add(byte_offset + size_of::()) + .cast::() + .read_unaligned() + }; + value |= u64::from_le(high) << (64 - shift); + } + value & low_mask64(width) +} + +struct BitWriter { + bytes: Vec, + bit_len: usize, +} + +struct BitWriter64 { + bytes: Vec, + bit_len: usize, +} + +impl BitWriter64 { + fn with_capacity(bits: usize) -> Self { + Self { + bytes: Vec::with_capacity(bits.div_ceil(8) + 15), + bit_len: 0, + } + } + + fn write(&mut self, value: u64, width: u8) { + for bit in 0..width { + let position = self.bit_len + usize::from(bit); + let byte_index = position / 8; + if byte_index == self.bytes.len() { + self.bytes.push(0); + } + self.bytes[byte_index] |= (((value >> bit) & 1) as u8) << (position % 8); + } + self.bit_len += usize::from(width); + } + + fn finish(mut self) -> Vec { + if self.bit_len == 0 { + return Vec::new(); + } + self.bytes.extend_from_slice(&[0; 15]); + self.bytes + } +} + +impl BitWriter { + fn with_capacity(bits: usize) -> Self { + Self { + bytes: Vec::with_capacity(bits.div_ceil(8) + STREAM_PADDING), + bit_len: 0, + } + } + + fn write(&mut self, value: u32, width: u8) { + for bit in 0..width { + let position = self.bit_len + usize::from(bit); + let byte_index = position / 8; + if byte_index == self.bytes.len() { + self.bytes.push(0); + } + self.bytes[byte_index] |= (((value >> bit) & 1) as u8) << (position % 8); + } + self.bit_len += usize::from(width); + } + + fn finish(mut self) -> Vec { + if self.bit_len == 0 { + return Vec::new(); + } + self.bytes.extend_from_slice(&[0; STREAM_PADDING]); + self.bytes + } +} + +#[cfg(test)] +mod tests { + use vortex_error::VortexResult; + + use super::IntMultCodec32; + use super::IntMultDenseCodec64; + + #[test] + fn roundtrip_and_scalar_access() -> VortexResult<()> { + let values = (0_u32..20_000) + .map(|index| match index % 5 { + 0 => u32::MAX - index, + 1 => 2_000_000_000 + index % 10, + _ => 1_000_000 + (index % 31) * 10, + }) + .collect::>(); + for base in [2, 10, 100, 1_000] { + let codec = IntMultCodec32::encode(&values, base)?; + assert_eq!(codec.decode(), values); + assert_eq!(codec.decode_gaps(), values); + if base <= 16 { + assert_eq!(codec.decode_pairs(), values); + } + for index in [0, 1, 1_023, 1_024, values.len() - 1] { + assert_eq!(codec.scalar_at(index)?, values[index]); + assert_eq!(codec.scalar_at_gaps(index)?, values[index]); + } + } + Ok(()) + } + + #[test] + fn dense_u64_roundtrip_and_scalar_access() -> VortexResult<()> { + let values = (0_u64..20_000) + .map(|index| match index % 4 { + 0 => u64::MAX - index, + 1 => (1_u64 << 63) + index % 10, + _ => 1_000_000 + (index % 31) * 10, + }) + .collect::>(); + for base in [2, 10, 100, 1_000] { + let codec = IntMultDenseCodec64::encode(&values, base)?; + assert_eq!(codec.decode(), values); + for index in [0, 1, 1_023, 1_024, values.len() - 1] { + assert_eq!(codec.scalar_at(index)?, values[index]); + } + } + Ok(()) + } +} From 56e013af0ccab1ae2835cf95ceb762d1b4fb179e Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 21:08:54 +0100 Subject: [PATCH 35/78] bench: evaluate centered block residuals Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 53 ++- encodings/block-residual/src/codec.rs | 91 ++++- .../examples/float_compressor_bench.rs | 362 ++++++++++++++++++ 3 files changed, 489 insertions(+), 17 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index eca6491db07..ca8fe058704 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -1085,6 +1085,35 @@ Patch application is not the main cost. Reference-ID expansion and per-value ref The one-reference BlockResidual tree is both faster and simpler. The multi-reference prototype is rejected for Default. +### Centered block residual prototype + +Euro2016 `subjectivity_confidence` contains a large cluster near `1.0`. + +About 36.5 percent of the first two million values equal `1.0`. Many other values sit close to that value. + +The prototype selects one median reference per 1,024-value block. It ZigZag-encodes each ordered-float distance from that reference. + +BlockResidual stores the transformed distances. The decoder fuses residual reconstruction, inverse ZigZag, and the ordered-float inverse. + +Null values use the block reference as their payload. They produce zero residuals and keep logical positions. + +The prototype tested extra patch costs from 16 through 96 bits. A 32-bit cost gives the best measured size and speed tradeoff. + +| Euro subjectivity tree | Bytes | Encode MB/s | Decode MB/s | Scalar ns | +| --- | ---: | ---: | ---: | ---: | +| Current Default | 14,234,326 | 358 | 22,664 | 511 | +| `OrderedFloat(BlockResidual)` | 13,293,263 | 2,171 | 24,128 | 196 | +| Centered block residual | 13,071,137 | 1,068 | 14,028 | Direct codec: 12 | +| Compact Pco | 6,860,457 | 424 | 2,711 | 17,582 | + +The centered form uses 1.7 percent fewer bytes than ordinary BlockResidual. + +Its decode throughput is 41.9 percent lower than ordinary BlockResidual. Its encode throughput is also about half as fast. + +The result rejects centered BlockResidual for Default. The extra transform does not recover enough of the Compact gap. + +Ordinary BlockResidual gives a strict size and throughput win on this column. The final selector calibration must include this case. + ### Pcodec corpus gap analysis The focused benchmark reads the first two million numeric rows from each source. @@ -1267,6 +1296,7 @@ This round completed these steps: - Rejected packed multi-reference blocks after patched and patch-free comparisons. - Rejected three bounded IntMult remainder layouts on GloVe. - Rejected the dense-remainder IntMult layout on CMS Payments. +- Rejected centered block residuals after a complete Euro subjectivity comparison. The Pco mode profile and quotient and remainder experiments are complete. @@ -1274,18 +1304,17 @@ The nonzero-secondary FloatQuant implementation and focused validation are compl Complete these remaining steps: -1. Prototype one bounded-access scheme for repeated classic-bin wins without Delta. -2. Keep IntMult outside Default unless a new design removes the split reconstruction cost. -3. Add a true ALP-RD child composition case if a real selected tree exposes one. -4. Validate the remaining rejected analysis cost after the candidate set stabilizes. -5. Add real embedding datasets beyond GloVe when licenses and loaders permit them. -6. Classify more float columns where Pco beats ALP, ALP-RD, and the new schemes. -7. Run the complete compression corpus after the candidate set stabilizes. -8. Compare the final geometric mean against Compact and Parquet with Zstd. -9. Calibrate all selector thresholds from complete corpus and selected-tree evidence. -10. Revisit the 32-bit and 64-bit BlockResidual factors with selected-tree evidence. -11. Keep RangePacked outside writer selectors unless a new decode design changes the result. -12. Update this plan after each experiment. +1. Keep IntMult outside Default unless a new design removes the split reconstruction cost. +2. Add a true ALP-RD child composition case if a real selected tree exposes one. +3. Validate the remaining rejected analysis cost after the candidate set stabilizes. +4. Add real embedding datasets beyond GloVe when licenses and loaders permit them. +5. Classify more float columns where Pco beats ALP, ALP-RD, and the new schemes. +6. Run the complete compression corpus after the candidate set stabilizes. +7. Compare the final geometric mean against Compact and Parquet with Zstd. +8. Calibrate all selector thresholds from complete corpus and selected-tree evidence. +9. Revisit the 32-bit and 64-bit BlockResidual factors with selected-tree evidence. +10. Keep RangePacked outside writer selectors unless a new decode design changes the result. +11. Update this plan after each experiment. ## Pull request structure diff --git a/encodings/block-residual/src/codec.rs b/encodings/block-residual/src/codec.rs index 8c91b14ccff..cf8a4ea9d95 100644 --- a/encodings/block-residual/src/codec.rs +++ b/encodings/block-residual/src/codec.rs @@ -55,13 +55,33 @@ impl BlockResidualCodec { } pub(crate) fn encode_with_word_width(values: &[u64], word_width: u8) -> VortexResult { + Self::encode_with_word_width_and_patch_penalty( + values, + word_width, + PATCH_DECODE_PENALTY_BITS, + ) + } + + /// Encode values with an additional encoded-bit cost for each patch. + pub fn encode_with_patch_penalty( + values: &[u64], + patch_penalty_bits: usize, + ) -> VortexResult { + Self::encode_with_word_width_and_patch_penalty(values, 64, patch_penalty_bits) + } + + fn encode_with_word_width_and_patch_penalty( + values: &[u64], + word_width: u8, + patch_penalty_bits: usize, + ) -> VortexResult { vortex_error::vortex_ensure!( matches!(word_width, 8 | 16 | 32 | 64), "block residual word width is invalid" ); let blocks = values .chunks(CHUNK_LEN) - .map(|block| encode_block(block, word_width)) + .map(|block| encode_block(block, word_width, patch_penalty_bits)) .collect::>>()?; Ok(Self { len: values.len(), @@ -301,6 +321,52 @@ impl BlockResidualCodec { Ok(values) } + /// Decode centered ordered `f64` latents with fused inverse transforms. + pub fn decode_centered_ordered_f64(&self, references: &[u64]) -> VortexResult> { + vortex_error::vortex_ensure!( + references.len() == self.blocks.len(), + "centered residual reference count differs from block count" + ); + let mut values = Vec::with_capacity(self.len); + let mut residuals = [0_u64; CHUNK_LEN]; + for (block_index, block) in self.blocks.iter().enumerate() { + residuals.fill(0); + if block.residual_width > 0 { + // SAFETY: The encoder creates one complete FastLanes chunk. + unsafe { + u64::unchecked_unpack( + usize::from(block.residual_width), + &block.residuals, + &mut residuals, + ); + } + } + let mut high_bit_position = 0_usize; + for &position in &block.patch_positions { + // SAFETY: The encoder appends fifteen readable padding bytes. + let high = unsafe { + read_wide_bits(&block.patch_highs, high_bit_position, block.high_width) + }; + residuals[usize::from(position)] |= high << block.residual_width; + high_bit_position += usize::from(block.high_width); + } + let reference = references[block_index]; + let block_len = usize::from(block.len); + values.extend(residuals[..block_len].iter().map(|&residual| { + let zigzag = residual.wrapping_add(block.base); + let delta = ((zigzag >> 1) as i64) ^ -((zigzag & 1) as i64); + let ordered = reference.wrapping_add(delta as u64); + let bits = if ordered & (1_u64 << 63) == 0 { + !ordered + } else { + ordered ^ (1_u64 << 63) + }; + f64::from_bits(bits) + })); + } + Ok(values) + } + /// Decode one value with direct packed access and a binary patch search. pub fn scalar_at(&self, index: usize) -> VortexResult { vortex_error::vortex_ensure!( @@ -385,7 +451,11 @@ impl BlockResidualCodec { } } -fn encode_block(values: &[u64], word_width: u8) -> VortexResult { +fn encode_block( + values: &[u64], + word_width: u8, + patch_penalty_bits: usize, +) -> VortexResult { let base = values.iter().copied().min().unwrap_or(0); let mut residuals = Vec::with_capacity(CHUNK_LEN); let mut width_counts = [0usize; 65]; @@ -399,7 +469,12 @@ fn encode_block(values: &[u64], word_width: u8) -> VortexResult(values: &[T], transform: impl Fn(T) -> u64) -> BlockW width_counts[usize::from(width)] += 1; maximum_width = maximum_width.max(width); } - choose_width(&width_counts, maximum_width, values.len()) + choose_width( + &width_counts, + maximum_width, + values.len(), + PATCH_DECODE_PENALTY_BITS, + ) } fn choose_width( width_counts: &[usize; 65], maximum_width: u8, value_count: usize, + patch_penalty_bits: usize, ) -> BlockWidthPlan { let mut patch_count = value_count; let mut best = (usize::MAX, maximum_width, 0u8, 0usize); @@ -448,7 +529,7 @@ fn choose_width( }; let cost_bits = usize::from(residual_width) * CHUNK_LEN + patch_count * (u16::BITS as usize + usize::from(high_width)) - + patch_count * PATCH_DECODE_PENALTY_BITS + + patch_count * patch_penalty_bits + u64::BITS as usize + SERIALIZED_BLOCK_METADATA_BYTES * 8 + usize::from(patch_count > 0) * HIGH_PADDING * 8; diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index c9212fd2e0d..fa6e802b010 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -54,6 +54,7 @@ use vortex_array::validity::Validity; use vortex_arrow::ArrowSessionExt; use vortex_block_residual::BlockResidual; use vortex_block_residual::BlockResidualArrayExt; +use vortex_block_residual::BlockResidualCodec; use vortex_block_residual::OrderedFloat; use vortex_block_residual::OrderedFloatArraySlotsExt; use vortex_btrblocks::BtrBlocksCompressor; @@ -923,6 +924,355 @@ fn profile_quotient_remainder( _ => {} } } + if std::env::var_os("VORTEX_BENCH_CENTERED_RESIDUAL").is_some() && ptype.bit_width() == 64 { + profile_centered_residual(dataset, column, path, ptype, pco_bytes, ordered)?; + } + Ok(()) +} + +#[derive(Clone, Copy)] +enum CenterReference { + Median, + UpperQuartile, + NinetiethPercentile, + Mode, +} + +impl CenterReference { + fn name(self) -> &'static str { + match self { + Self::Median => "median", + Self::UpperQuartile => "upper-quartile", + Self::NinetiethPercentile => "ninetieth-percentile", + Self::Mode => "mode", + } + } + + fn select(self, values: &[u64]) -> u64 { + let mut sorted = values.to_vec(); + sorted.sort_unstable(); + match self { + Self::Median => sorted[sorted.len() / 2], + Self::UpperQuartile => sorted[sorted.len() * 3 / 4], + Self::NinetiethPercentile => sorted[sorted.len() * 9 / 10], + Self::Mode => { + let mut best_value = sorted[0]; + let mut best_count = 1_usize; + let mut current_value = sorted[0]; + let mut current_count = 1_usize; + for &value in &sorted[1..] { + if value == current_value { + current_count += 1; + } else { + if current_count > best_count { + best_value = current_value; + best_count = current_count; + } + current_value = value; + current_count = 1; + } + } + if current_count > best_count { + current_value + } else { + best_value + } + } + } + } +} + +fn zigzag_distance(value: u64, reference: u64) -> u64 { + let delta = value.wrapping_sub(reference) as i64; + ((delta as u64) << 1) ^ ((delta >> 63) as u64) +} + +fn inverse_zigzag_distance(value: u64, reference: u64) -> u64 { + let delta = ((value >> 1) as i64) ^ -((value & 1) as i64); + reference.wrapping_add(delta as u64) +} + +struct CenteredResidualCodec64 { + len: usize, + references: Vec, + residuals: BlockResidualCodec, +} + +impl CenteredResidualCodec64 { + fn encode( + values: &[u64], + strategy: CenterReference, + patch_penalty_bits: usize, + ) -> VortexResult { + let mut references = Vec::with_capacity(values.len().div_ceil(1_024)); + let mut transformed = Vec::with_capacity(values.len()); + for block in values.chunks(1_024) { + let reference = strategy.select(block); + references.push(reference); + transformed.extend(block.iter().map(|&value| zigzag_distance(value, reference))); + } + Ok(Self { + len: values.len(), + references, + residuals: BlockResidualCodec::encode_with_patch_penalty( + &transformed, + patch_penalty_bits, + )?, + }) + } + + fn decode(&self) -> VortexResult> { + let mut values = self.residuals.decode()?; + for (block_index, block) in values.chunks_mut(1_024).enumerate() { + let reference = self.references[block_index]; + for value in block { + *value = inverse_zigzag_distance(*value, reference); + } + } + Ok(values) + } + + fn decode_fused_f64(&self) -> VortexResult> { + self.residuals.decode_centered_ordered_f64(&self.references) + } + + fn scalar_at(&self, index: usize) -> VortexResult { + vortex_ensure!(index < self.len, "centered residual index exceeds length"); + let residual = self.residuals.scalar_at(index)?; + Ok(inverse_zigzag_distance( + residual, + self.references[index / 1_024], + )) + } + + fn encoded_size(&self) -> usize { + self.residuals + .encoded_size() + .saturating_add(self.references.len() * size_of::()) + } + + fn patch_count(&self) -> usize { + self.residuals.patch_count() + } +} + +fn encode_centered_nullable_f64( + primitive: &PrimitiveArray, + patch_penalty_bits: usize, + session: &VortexSession, +) -> VortexResult { + let mask = primitive + .validity()? + .execute_mask(primitive.len(), &mut session.create_execution_ctx())?; + let validity = mask.iter().collect::>(); + let ordered = primitive + .as_slice::() + .iter() + .copied() + .map(ordered_f64) + .collect::>(); + let mut references = Vec::with_capacity(primitive.len().div_ceil(1_024)); + let mut transformed = Vec::with_capacity(primitive.len()); + for (block_index, block) in ordered.chunks(1_024).enumerate() { + let start = block_index * 1_024; + let valid_block = &validity[start..start + block.len()]; + let valid_values = block + .iter() + .zip(valid_block) + .filter_map(|(&value, &valid)| valid.then_some(value)) + .collect::>(); + let reference = if valid_values.is_empty() { + 0 + } else { + CenterReference::Median.select(&valid_values) + }; + references.push(reference); + transformed.extend(block.iter().zip(valid_block).map(|(&value, &valid)| { + zigzag_distance(if valid { value } else { reference }, reference) + })); + } + Ok(CenteredResidualCodec64 { + len: primitive.len(), + references, + residuals: BlockResidualCodec::encode_with_patch_penalty(&transformed, patch_penalty_bits)?, + }) +} + +fn profile_centered_residual( + dataset: &str, + column: &str, + path: &str, + ptype: PType, + pco_bytes: u64, + values: &[u64], +) -> VortexResult<()> { + for strategy in [ + CenterReference::Median, + CenterReference::UpperQuartile, + CenterReference::NinetiethPercentile, + CenterReference::Mode, + ] { + let mut references = Vec::with_capacity(values.len().div_ceil(1_024)); + let mut transformed = Vec::with_capacity(values.len()); + for block in values.chunks(1_024) { + let reference = strategy.select(block); + references.push(reference); + transformed.extend(block.iter().map(|&value| zigzag_distance(value, reference))); + } + let penalties: &[usize] = if matches!(strategy, CenterReference::Median) { + &[16, 32, 64, 96] + } else { + &[16] + }; + for &patch_penalty_bits in penalties { + let codec = + BlockResidualCodec::encode_with_patch_penalty(&transformed, patch_penalty_bits)?; + let encoded_bytes = codec + .encoded_size() + .saturating_add(references.len() * size_of::()); + println!( + "centered-residual-estimate\t{dataset}\t{column}\t{path}\t{ptype}\t{}\t{patch_penalty_bits}\t{}\t{pco_bytes}\t{encoded_bytes}\t{}", + strategy.name(), + values.len(), + codec.patch_count(), + ); + if matches!(strategy, CenterReference::Median | CenterReference::Mode) { + profile_centered_residual_codec( + dataset, + column, + path, + ptype, + pco_bytes, + values, + strategy, + patch_penalty_bits, + )?; + } + } + } + Ok(()) +} + +#[expect( + clippy::too_many_arguments, + reason = "the benchmark output retains all dataset attribution fields" +)] +fn profile_centered_residual_codec( + dataset: &str, + column: &str, + path: &str, + ptype: PType, + pco_bytes: u64, + values: &[u64], + strategy: CenterReference, + patch_penalty_bits: usize, +) -> VortexResult<()> { + let mut encode_durations = Vec::with_capacity(5); + let mut codec = CenteredResidualCodec64::encode(values, strategy, patch_penalty_bits)?; + for _ in 0..5 { + let start = Instant::now(); + codec = CenteredResidualCodec64::encode(black_box(values), strategy, patch_penalty_bits)?; + encode_durations.push(start.elapsed()); + } + let encode_median = percentile(&mut encode_durations, 1, 2); + vortex_ensure!( + codec.decode()? == values, + "centered residual decode differs" + ); + + let decode_iterations = codec_decode_iterations()?; + let mut decode_durations = Vec::with_capacity(decode_iterations); + for _ in 0..decode_iterations { + let start = Instant::now(); + black_box(codec.decode()?); + decode_durations.push(start.elapsed()); + } + let decode_median = percentile(&mut decode_durations, 1, 2); + + let mut fused_decode_durations = Vec::with_capacity(decode_iterations); + for _ in 0..decode_iterations { + let start = Instant::now(); + black_box(codec.decode_fused_f64()?); + fused_decode_durations.push(start.elapsed()); + } + let fused_decode_median = percentile(&mut fused_decode_durations, 1, 2); + + let scalar_iterations = 200_000; + let mut scalar_index = 0_usize; + let mut scalar_checksum = 0_u64; + let scalar_start = Instant::now(); + for _ in 0..scalar_iterations { + scalar_index = scalar_index.wrapping_add(2_654_435_761) % values.len(); + scalar_checksum ^= black_box(codec.scalar_at(scalar_index)?); + } + black_box(scalar_checksum); + let scalar_duration = scalar_start.elapsed(); + + let input_bytes = values.len() * ptype.byte_width(); + let encode_throughput = input_bytes as f64 / encode_median.as_secs_f64() / 1_000_000.0; + let decode_throughput = input_bytes as f64 / decode_median.as_secs_f64() / 1_000_000.0; + let fused_decode_throughput = + input_bytes as f64 / fused_decode_median.as_secs_f64() / 1_000_000.0; + let scalar_nanoseconds = + scalar_duration.as_secs_f64() * 1_000_000_000.0 / scalar_iterations as f64; + println!( + "centered-residual-checkpoint\t{dataset}\t{column}\t{path}\t{ptype}\t{}\t{patch_penalty_bits}\t{}\t{pco_bytes}\t{}\t{}\t{encode_throughput:.1}\t{decode_throughput:.1}\t{fused_decode_throughput:.1}\t{scalar_nanoseconds:.1}", + strategy.name(), + values.len(), + codec.encoded_size(), + codec.patch_count(), + ); + Ok(()) +} + +fn profile_centered_float_tree( + dataset: &str, + column: &str, + primitive: &PrimitiveArray, + session: &VortexSession, +) -> VortexResult<()> { + if primitive.ptype() != PType::F64 { + return Ok(()); + } + let filled = + fill_float_nulls_with_first_valid(primitive.clone(), &mut session.create_execution_ctx())?; + let ordered_array = OrderedFloat::from_primitive(filled.as_view())?; + let ordinary = BlockResidual::from_primitive(ordered_array.encoded().as_::())?; + let validity_bytes = ordinary + .into_array() + .children() + .iter() + .map(ArrayRef::nbytes) + .sum::(); + for patch_penalty_bits in [32, 64, 96] { + let mut codec = encode_centered_nullable_f64(primitive, patch_penalty_bits, session)?; + let mut encode_durations = Vec::with_capacity(5); + for _ in 0..5 { + let start = Instant::now(); + codec = + encode_centered_nullable_f64(black_box(primitive), patch_penalty_bits, session)?; + encode_durations.push(start.elapsed()); + } + let encode_median = percentile(&mut encode_durations, 1, 2); + + let decode_iterations = codec_decode_iterations()?; + let mut decode_durations = Vec::with_capacity(decode_iterations); + for _ in 0..decode_iterations { + let start = Instant::now(); + black_box(codec.decode_fused_f64()?); + decode_durations.push(start.elapsed()); + } + let decode_median = percentile(&mut decode_durations, 1, 2); + let input_bytes = primitive.len() * size_of::(); + let encode_throughput = input_bytes as f64 / encode_median.as_secs_f64() / 1_000_000.0; + let decode_throughput = input_bytes as f64 / decode_median.as_secs_f64() / 1_000_000.0; + let encoded_bytes = u64::try_from(codec.encoded_size())?.saturating_add(validity_bytes); + println!( + "centered-float-tree\t{dataset}\t{column}\t{patch_penalty_bits}\t{}\t{encoded_bytes}\t{}\t{encode_throughput:.1}\t{decode_throughput:.1}", + primitive.len(), + codec.patch_count(), + ); + } Ok(()) } @@ -1841,6 +2191,9 @@ fn measure_dataset( { profile_pco_array(dataset, &column.name, "root", compact_array, session)?; } + if std::env::var_os("VORTEX_BENCH_CENTERED_RESIDUAL").is_some() { + profile_centered_float_tree(dataset, &column.name, primitive, session)?; + } if std::env::var_os("VORTEX_BENCH_FIXED_BIN").is_some() { measure_existing_tree( dataset, @@ -2001,6 +2354,15 @@ fn main() -> VortexResult<()> { println!( "int-mult-dense64-checkpoint\tdataset\tcolumn\tpath\tptype\tbase\trows\tpco-child-bytes\tint-mult-bytes\tquotient-patches\tencode-MB/s\tdecode-MB/s\tscalar-ns" ); + println!( + "centered-residual-estimate\tdataset\tcolumn\tpath\tptype\treference\tpatch-penalty-bits\trows\tpco-child-bytes\tcentered-bytes\tpatches" + ); + println!( + "centered-residual-checkpoint\tdataset\tcolumn\tpath\tptype\treference\tpatch-penalty-bits\trows\tpco-child-bytes\tcentered-bytes\tpatches\tencode-MB/s\tdecode-MB/s\tfused-f64-decode-MB/s\tscalar-ns" + ); + println!( + "centered-float-tree\tdataset\tcolumn\tpatch-penalty-bits\trows\tbytes\tpatches\tencode-MB/s\tdecode-MB/s" + ); println!( "fixed-bin-checkpoint\tdataset\tcolumn\tpath\trows\tpco-bytes\tfixed-bin-bytes\tbins\tmax-offset-bits\toffset-widths\tencode-MB/s\tdecode-MB/s\tscalar-ns" ); From 940e9772c222babf50e472f48203b24f0e89310a Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 21:10:25 +0100 Subject: [PATCH 36/78] chore: remove rejected centered residual prototype Signed-off-by: Will Manning --- encodings/block-residual/src/codec.rs | 91 +---- .../examples/float_compressor_bench.rs | 362 ------------------ 2 files changed, 5 insertions(+), 448 deletions(-) diff --git a/encodings/block-residual/src/codec.rs b/encodings/block-residual/src/codec.rs index cf8a4ea9d95..8c91b14ccff 100644 --- a/encodings/block-residual/src/codec.rs +++ b/encodings/block-residual/src/codec.rs @@ -55,33 +55,13 @@ impl BlockResidualCodec { } pub(crate) fn encode_with_word_width(values: &[u64], word_width: u8) -> VortexResult { - Self::encode_with_word_width_and_patch_penalty( - values, - word_width, - PATCH_DECODE_PENALTY_BITS, - ) - } - - /// Encode values with an additional encoded-bit cost for each patch. - pub fn encode_with_patch_penalty( - values: &[u64], - patch_penalty_bits: usize, - ) -> VortexResult { - Self::encode_with_word_width_and_patch_penalty(values, 64, patch_penalty_bits) - } - - fn encode_with_word_width_and_patch_penalty( - values: &[u64], - word_width: u8, - patch_penalty_bits: usize, - ) -> VortexResult { vortex_error::vortex_ensure!( matches!(word_width, 8 | 16 | 32 | 64), "block residual word width is invalid" ); let blocks = values .chunks(CHUNK_LEN) - .map(|block| encode_block(block, word_width, patch_penalty_bits)) + .map(|block| encode_block(block, word_width)) .collect::>>()?; Ok(Self { len: values.len(), @@ -321,52 +301,6 @@ impl BlockResidualCodec { Ok(values) } - /// Decode centered ordered `f64` latents with fused inverse transforms. - pub fn decode_centered_ordered_f64(&self, references: &[u64]) -> VortexResult> { - vortex_error::vortex_ensure!( - references.len() == self.blocks.len(), - "centered residual reference count differs from block count" - ); - let mut values = Vec::with_capacity(self.len); - let mut residuals = [0_u64; CHUNK_LEN]; - for (block_index, block) in self.blocks.iter().enumerate() { - residuals.fill(0); - if block.residual_width > 0 { - // SAFETY: The encoder creates one complete FastLanes chunk. - unsafe { - u64::unchecked_unpack( - usize::from(block.residual_width), - &block.residuals, - &mut residuals, - ); - } - } - let mut high_bit_position = 0_usize; - for &position in &block.patch_positions { - // SAFETY: The encoder appends fifteen readable padding bytes. - let high = unsafe { - read_wide_bits(&block.patch_highs, high_bit_position, block.high_width) - }; - residuals[usize::from(position)] |= high << block.residual_width; - high_bit_position += usize::from(block.high_width); - } - let reference = references[block_index]; - let block_len = usize::from(block.len); - values.extend(residuals[..block_len].iter().map(|&residual| { - let zigzag = residual.wrapping_add(block.base); - let delta = ((zigzag >> 1) as i64) ^ -((zigzag & 1) as i64); - let ordered = reference.wrapping_add(delta as u64); - let bits = if ordered & (1_u64 << 63) == 0 { - !ordered - } else { - ordered ^ (1_u64 << 63) - }; - f64::from_bits(bits) - })); - } - Ok(values) - } - /// Decode one value with direct packed access and a binary patch search. pub fn scalar_at(&self, index: usize) -> VortexResult { vortex_error::vortex_ensure!( @@ -451,11 +385,7 @@ impl BlockResidualCodec { } } -fn encode_block( - values: &[u64], - word_width: u8, - patch_penalty_bits: usize, -) -> VortexResult { +fn encode_block(values: &[u64], word_width: u8) -> VortexResult { let base = values.iter().copied().min().unwrap_or(0); let mut residuals = Vec::with_capacity(CHUNK_LEN); let mut width_counts = [0usize; 65]; @@ -469,12 +399,7 @@ fn encode_block( } residuals.resize(CHUNK_LEN, 0); - let width_plan = choose_width( - &width_counts, - maximum_width, - values.len(), - patch_penalty_bits, - ); + let width_plan = choose_width(&width_counts, maximum_width, values.len()); materialize_block( values, @@ -504,19 +429,13 @@ fn estimate_block(values: &[T], transform: impl Fn(T) -> u64) -> BlockW width_counts[usize::from(width)] += 1; maximum_width = maximum_width.max(width); } - choose_width( - &width_counts, - maximum_width, - values.len(), - PATCH_DECODE_PENALTY_BITS, - ) + choose_width(&width_counts, maximum_width, values.len()) } fn choose_width( width_counts: &[usize; 65], maximum_width: u8, value_count: usize, - patch_penalty_bits: usize, ) -> BlockWidthPlan { let mut patch_count = value_count; let mut best = (usize::MAX, maximum_width, 0u8, 0usize); @@ -529,7 +448,7 @@ fn choose_width( }; let cost_bits = usize::from(residual_width) * CHUNK_LEN + patch_count * (u16::BITS as usize + usize::from(high_width)) - + patch_count * patch_penalty_bits + + patch_count * PATCH_DECODE_PENALTY_BITS + u64::BITS as usize + SERIALIZED_BLOCK_METADATA_BYTES * 8 + usize::from(patch_count > 0) * HIGH_PADDING * 8; diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index fa6e802b010..c9212fd2e0d 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -54,7 +54,6 @@ use vortex_array::validity::Validity; use vortex_arrow::ArrowSessionExt; use vortex_block_residual::BlockResidual; use vortex_block_residual::BlockResidualArrayExt; -use vortex_block_residual::BlockResidualCodec; use vortex_block_residual::OrderedFloat; use vortex_block_residual::OrderedFloatArraySlotsExt; use vortex_btrblocks::BtrBlocksCompressor; @@ -924,355 +923,6 @@ fn profile_quotient_remainder( _ => {} } } - if std::env::var_os("VORTEX_BENCH_CENTERED_RESIDUAL").is_some() && ptype.bit_width() == 64 { - profile_centered_residual(dataset, column, path, ptype, pco_bytes, ordered)?; - } - Ok(()) -} - -#[derive(Clone, Copy)] -enum CenterReference { - Median, - UpperQuartile, - NinetiethPercentile, - Mode, -} - -impl CenterReference { - fn name(self) -> &'static str { - match self { - Self::Median => "median", - Self::UpperQuartile => "upper-quartile", - Self::NinetiethPercentile => "ninetieth-percentile", - Self::Mode => "mode", - } - } - - fn select(self, values: &[u64]) -> u64 { - let mut sorted = values.to_vec(); - sorted.sort_unstable(); - match self { - Self::Median => sorted[sorted.len() / 2], - Self::UpperQuartile => sorted[sorted.len() * 3 / 4], - Self::NinetiethPercentile => sorted[sorted.len() * 9 / 10], - Self::Mode => { - let mut best_value = sorted[0]; - let mut best_count = 1_usize; - let mut current_value = sorted[0]; - let mut current_count = 1_usize; - for &value in &sorted[1..] { - if value == current_value { - current_count += 1; - } else { - if current_count > best_count { - best_value = current_value; - best_count = current_count; - } - current_value = value; - current_count = 1; - } - } - if current_count > best_count { - current_value - } else { - best_value - } - } - } - } -} - -fn zigzag_distance(value: u64, reference: u64) -> u64 { - let delta = value.wrapping_sub(reference) as i64; - ((delta as u64) << 1) ^ ((delta >> 63) as u64) -} - -fn inverse_zigzag_distance(value: u64, reference: u64) -> u64 { - let delta = ((value >> 1) as i64) ^ -((value & 1) as i64); - reference.wrapping_add(delta as u64) -} - -struct CenteredResidualCodec64 { - len: usize, - references: Vec, - residuals: BlockResidualCodec, -} - -impl CenteredResidualCodec64 { - fn encode( - values: &[u64], - strategy: CenterReference, - patch_penalty_bits: usize, - ) -> VortexResult { - let mut references = Vec::with_capacity(values.len().div_ceil(1_024)); - let mut transformed = Vec::with_capacity(values.len()); - for block in values.chunks(1_024) { - let reference = strategy.select(block); - references.push(reference); - transformed.extend(block.iter().map(|&value| zigzag_distance(value, reference))); - } - Ok(Self { - len: values.len(), - references, - residuals: BlockResidualCodec::encode_with_patch_penalty( - &transformed, - patch_penalty_bits, - )?, - }) - } - - fn decode(&self) -> VortexResult> { - let mut values = self.residuals.decode()?; - for (block_index, block) in values.chunks_mut(1_024).enumerate() { - let reference = self.references[block_index]; - for value in block { - *value = inverse_zigzag_distance(*value, reference); - } - } - Ok(values) - } - - fn decode_fused_f64(&self) -> VortexResult> { - self.residuals.decode_centered_ordered_f64(&self.references) - } - - fn scalar_at(&self, index: usize) -> VortexResult { - vortex_ensure!(index < self.len, "centered residual index exceeds length"); - let residual = self.residuals.scalar_at(index)?; - Ok(inverse_zigzag_distance( - residual, - self.references[index / 1_024], - )) - } - - fn encoded_size(&self) -> usize { - self.residuals - .encoded_size() - .saturating_add(self.references.len() * size_of::()) - } - - fn patch_count(&self) -> usize { - self.residuals.patch_count() - } -} - -fn encode_centered_nullable_f64( - primitive: &PrimitiveArray, - patch_penalty_bits: usize, - session: &VortexSession, -) -> VortexResult { - let mask = primitive - .validity()? - .execute_mask(primitive.len(), &mut session.create_execution_ctx())?; - let validity = mask.iter().collect::>(); - let ordered = primitive - .as_slice::() - .iter() - .copied() - .map(ordered_f64) - .collect::>(); - let mut references = Vec::with_capacity(primitive.len().div_ceil(1_024)); - let mut transformed = Vec::with_capacity(primitive.len()); - for (block_index, block) in ordered.chunks(1_024).enumerate() { - let start = block_index * 1_024; - let valid_block = &validity[start..start + block.len()]; - let valid_values = block - .iter() - .zip(valid_block) - .filter_map(|(&value, &valid)| valid.then_some(value)) - .collect::>(); - let reference = if valid_values.is_empty() { - 0 - } else { - CenterReference::Median.select(&valid_values) - }; - references.push(reference); - transformed.extend(block.iter().zip(valid_block).map(|(&value, &valid)| { - zigzag_distance(if valid { value } else { reference }, reference) - })); - } - Ok(CenteredResidualCodec64 { - len: primitive.len(), - references, - residuals: BlockResidualCodec::encode_with_patch_penalty(&transformed, patch_penalty_bits)?, - }) -} - -fn profile_centered_residual( - dataset: &str, - column: &str, - path: &str, - ptype: PType, - pco_bytes: u64, - values: &[u64], -) -> VortexResult<()> { - for strategy in [ - CenterReference::Median, - CenterReference::UpperQuartile, - CenterReference::NinetiethPercentile, - CenterReference::Mode, - ] { - let mut references = Vec::with_capacity(values.len().div_ceil(1_024)); - let mut transformed = Vec::with_capacity(values.len()); - for block in values.chunks(1_024) { - let reference = strategy.select(block); - references.push(reference); - transformed.extend(block.iter().map(|&value| zigzag_distance(value, reference))); - } - let penalties: &[usize] = if matches!(strategy, CenterReference::Median) { - &[16, 32, 64, 96] - } else { - &[16] - }; - for &patch_penalty_bits in penalties { - let codec = - BlockResidualCodec::encode_with_patch_penalty(&transformed, patch_penalty_bits)?; - let encoded_bytes = codec - .encoded_size() - .saturating_add(references.len() * size_of::()); - println!( - "centered-residual-estimate\t{dataset}\t{column}\t{path}\t{ptype}\t{}\t{patch_penalty_bits}\t{}\t{pco_bytes}\t{encoded_bytes}\t{}", - strategy.name(), - values.len(), - codec.patch_count(), - ); - if matches!(strategy, CenterReference::Median | CenterReference::Mode) { - profile_centered_residual_codec( - dataset, - column, - path, - ptype, - pco_bytes, - values, - strategy, - patch_penalty_bits, - )?; - } - } - } - Ok(()) -} - -#[expect( - clippy::too_many_arguments, - reason = "the benchmark output retains all dataset attribution fields" -)] -fn profile_centered_residual_codec( - dataset: &str, - column: &str, - path: &str, - ptype: PType, - pco_bytes: u64, - values: &[u64], - strategy: CenterReference, - patch_penalty_bits: usize, -) -> VortexResult<()> { - let mut encode_durations = Vec::with_capacity(5); - let mut codec = CenteredResidualCodec64::encode(values, strategy, patch_penalty_bits)?; - for _ in 0..5 { - let start = Instant::now(); - codec = CenteredResidualCodec64::encode(black_box(values), strategy, patch_penalty_bits)?; - encode_durations.push(start.elapsed()); - } - let encode_median = percentile(&mut encode_durations, 1, 2); - vortex_ensure!( - codec.decode()? == values, - "centered residual decode differs" - ); - - let decode_iterations = codec_decode_iterations()?; - let mut decode_durations = Vec::with_capacity(decode_iterations); - for _ in 0..decode_iterations { - let start = Instant::now(); - black_box(codec.decode()?); - decode_durations.push(start.elapsed()); - } - let decode_median = percentile(&mut decode_durations, 1, 2); - - let mut fused_decode_durations = Vec::with_capacity(decode_iterations); - for _ in 0..decode_iterations { - let start = Instant::now(); - black_box(codec.decode_fused_f64()?); - fused_decode_durations.push(start.elapsed()); - } - let fused_decode_median = percentile(&mut fused_decode_durations, 1, 2); - - let scalar_iterations = 200_000; - let mut scalar_index = 0_usize; - let mut scalar_checksum = 0_u64; - let scalar_start = Instant::now(); - for _ in 0..scalar_iterations { - scalar_index = scalar_index.wrapping_add(2_654_435_761) % values.len(); - scalar_checksum ^= black_box(codec.scalar_at(scalar_index)?); - } - black_box(scalar_checksum); - let scalar_duration = scalar_start.elapsed(); - - let input_bytes = values.len() * ptype.byte_width(); - let encode_throughput = input_bytes as f64 / encode_median.as_secs_f64() / 1_000_000.0; - let decode_throughput = input_bytes as f64 / decode_median.as_secs_f64() / 1_000_000.0; - let fused_decode_throughput = - input_bytes as f64 / fused_decode_median.as_secs_f64() / 1_000_000.0; - let scalar_nanoseconds = - scalar_duration.as_secs_f64() * 1_000_000_000.0 / scalar_iterations as f64; - println!( - "centered-residual-checkpoint\t{dataset}\t{column}\t{path}\t{ptype}\t{}\t{patch_penalty_bits}\t{}\t{pco_bytes}\t{}\t{}\t{encode_throughput:.1}\t{decode_throughput:.1}\t{fused_decode_throughput:.1}\t{scalar_nanoseconds:.1}", - strategy.name(), - values.len(), - codec.encoded_size(), - codec.patch_count(), - ); - Ok(()) -} - -fn profile_centered_float_tree( - dataset: &str, - column: &str, - primitive: &PrimitiveArray, - session: &VortexSession, -) -> VortexResult<()> { - if primitive.ptype() != PType::F64 { - return Ok(()); - } - let filled = - fill_float_nulls_with_first_valid(primitive.clone(), &mut session.create_execution_ctx())?; - let ordered_array = OrderedFloat::from_primitive(filled.as_view())?; - let ordinary = BlockResidual::from_primitive(ordered_array.encoded().as_::())?; - let validity_bytes = ordinary - .into_array() - .children() - .iter() - .map(ArrayRef::nbytes) - .sum::(); - for patch_penalty_bits in [32, 64, 96] { - let mut codec = encode_centered_nullable_f64(primitive, patch_penalty_bits, session)?; - let mut encode_durations = Vec::with_capacity(5); - for _ in 0..5 { - let start = Instant::now(); - codec = - encode_centered_nullable_f64(black_box(primitive), patch_penalty_bits, session)?; - encode_durations.push(start.elapsed()); - } - let encode_median = percentile(&mut encode_durations, 1, 2); - - let decode_iterations = codec_decode_iterations()?; - let mut decode_durations = Vec::with_capacity(decode_iterations); - for _ in 0..decode_iterations { - let start = Instant::now(); - black_box(codec.decode_fused_f64()?); - decode_durations.push(start.elapsed()); - } - let decode_median = percentile(&mut decode_durations, 1, 2); - let input_bytes = primitive.len() * size_of::(); - let encode_throughput = input_bytes as f64 / encode_median.as_secs_f64() / 1_000_000.0; - let decode_throughput = input_bytes as f64 / decode_median.as_secs_f64() / 1_000_000.0; - let encoded_bytes = u64::try_from(codec.encoded_size())?.saturating_add(validity_bytes); - println!( - "centered-float-tree\t{dataset}\t{column}\t{patch_penalty_bits}\t{}\t{encoded_bytes}\t{}\t{encode_throughput:.1}\t{decode_throughput:.1}", - primitive.len(), - codec.patch_count(), - ); - } Ok(()) } @@ -2191,9 +1841,6 @@ fn measure_dataset( { profile_pco_array(dataset, &column.name, "root", compact_array, session)?; } - if std::env::var_os("VORTEX_BENCH_CENTERED_RESIDUAL").is_some() { - profile_centered_float_tree(dataset, &column.name, primitive, session)?; - } if std::env::var_os("VORTEX_BENCH_FIXED_BIN").is_some() { measure_existing_tree( dataset, @@ -2354,15 +2001,6 @@ fn main() -> VortexResult<()> { println!( "int-mult-dense64-checkpoint\tdataset\tcolumn\tpath\tptype\tbase\trows\tpco-child-bytes\tint-mult-bytes\tquotient-patches\tencode-MB/s\tdecode-MB/s\tscalar-ns" ); - println!( - "centered-residual-estimate\tdataset\tcolumn\tpath\tptype\treference\tpatch-penalty-bits\trows\tpco-child-bytes\tcentered-bytes\tpatches" - ); - println!( - "centered-residual-checkpoint\tdataset\tcolumn\tpath\tptype\treference\tpatch-penalty-bits\trows\tpco-child-bytes\tcentered-bytes\tpatches\tencode-MB/s\tdecode-MB/s\tfused-f64-decode-MB/s\tscalar-ns" - ); - println!( - "centered-float-tree\tdataset\tcolumn\tpatch-penalty-bits\trows\tbytes\tpatches\tencode-MB/s\tdecode-MB/s" - ); println!( "fixed-bin-checkpoint\tdataset\tcolumn\tpath\trows\tpco-bytes\tfixed-bin-bytes\tbins\tmax-offset-bits\toffset-widths\tencode-MB/s\tdecode-MB/s\tscalar-ns" ); From 7310914476589f2bd4c843c3cd39807a855b4d57 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Wed, 19 Aug 2026 21:19:02 +0100 Subject: [PATCH 37/78] perf: calibrate BlockResidual selection Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 46 +++++++++++++++++++ .../src/schemes/integer/block_residual.rs | 16 +++---- 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index ca8fe058704..be07f046783 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -179,6 +179,16 @@ It does not create packed payloads, patch payloads, temporary ordered integers, All-valid samples use direct block copies. Nullable samples retain the validity mask. +`BlockResidualScheme` requires at least a 1.05 compression ratio before outer comparison. + +The nonlinear patch cost represents the measured decode cliff for dense patches. + +The selector does not apply another width-specific size factor. + +Direct 32-bit and 64-bit BlockResidual decode now matches or exceeds the main FoR paths. + +BlockResidual does not occur below ZigZag. That composition lost decode throughput on GloVe for a small size reduction. + The residual scheme requires a 1.05 compression ratio. Its adjusted score includes a 1.02 decode-cost factor. `BlockResidualScheme` uses the same locality probe for integer arrays. @@ -555,6 +565,42 @@ This layout recovers most of the Compact-like size gain without a material aggre The remaining threshold calibration must use selected-column evidence and the complete corpus. +### BlockResidual selector calibration + +The final numeric trial removes the prior 1.10 and 1.20 width factors. + +The 1.05 minimum ratio and nonlinear patch cost remain active. + +An initial trial selected `ALP(ZigZag(BlockResidual))` on GloVe. + +That tree reduced size by 1.2 percent and reduced decode throughput by 19.9 percent. + +No other selected corpus tree placed BlockResidual below ZigZag. An ancestor exclusion removes that composition. + +The final run reads two million rows from seven Public BI files and GloVe. + +| Dataset | Size change | Encode throughput change | Decode throughput change | +| --- | ---: | ---: | ---: | +| Arade | -3.13 percent | -4.44 percent | +8.80 percent | +| Bimbo | -2.97 percent | -0.36 percent | +1.78 percent | +| CMSprovider 1 | -1.63 percent | -3.42 percent | -1.29 percent | +| CMSprovider 2 | -1.56 percent | -3.32 percent | -0.65 percent | +| Euro2016 | -11.47 percent | -3.61 percent | +6.23 percent | +| Food | -6.89 percent | -0.85 percent | -4.00 percent | +| HashTags | -5.83 percent | -0.47 percent | +4.36 percent | +| GloVe | 0.00 percent | +0.43 percent | +3.63 percent | +| Geometric mean | -4.25 percent | -2.02 percent | +2.28 percent | + +The prior width factors reduced size by 2.28 percent in the same eight-dataset scope. + +They changed encode throughput by -0.49 percent and decode throughput by +3.30 percent. + +The new policy recovers another 1.97 percent of geometric-mean size. + +It keeps every dataset throughput change far inside the 20-percent gate. + +The complete file corpus remains necessary before the policy becomes final. + ### Compact transfer baseline The local corpus comparison uses three iterations on all 16 available datasets. diff --git a/vortex-btrblocks/src/schemes/integer/block_residual.rs b/vortex-btrblocks/src/schemes/integer/block_residual.rs index a45d5a7bffd..9b944f40a5d 100644 --- a/vortex-btrblocks/src/schemes/integer/block_residual.rs +++ b/vortex-btrblocks/src/schemes/integer/block_residual.rs @@ -25,6 +25,7 @@ use vortex_compressor::scheme::DeferredEstimate; use vortex_compressor::scheme::EstimateVerdict; use vortex_error::VortexResult; +use super::ZigZagScheme; use crate::ArrayAndStats; use crate::CascadingCompressor; use crate::CompressorContext; @@ -75,6 +76,10 @@ impl Scheme for BlockResidualScheme { ancestor: BinaryDictScheme.id(), children: ChildSelection::One(1), }, + AncestorExclusion { + ancestor: ZigZagScheme.id(), + children: ChildSelection::One(0), + }, ] } @@ -102,16 +107,7 @@ impl Scheme for BlockResidualScheme { if ratio < MIN_COMPRESSION_RATIO { return Ok(EstimateVerdict::Skip); } - let speed_penalty = match sample.ptype().bit_width() { - 32 => 1.10, - 64 => 1.20, - _ => unreachable!("BlockResidual only matches 32-bit and 64-bit integers"), - }; - let adjusted_ratio = ratio / speed_penalty; - if adjusted_ratio < MIN_COMPRESSION_RATIO { - return Ok(EstimateVerdict::Skip); - } - Ok(EstimateVerdict::Ratio(adjusted_ratio)) + Ok(EstimateVerdict::Ratio(ratio)) }, ))) } From 1fd7beafcfdd2ee71fdfd7c52c23e3f9c8a824e9 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 04:35:25 +0100 Subject: [PATCH 38/78] feat: add composable IntMult transform Signed-off-by: Will Manning --- Cargo.lock | 14 + Cargo.toml | 2 + encodings/int-mult/Cargo.toml | 27 ++ encodings/int-mult/src/array.rs | 554 ++++++++++++++++++++++++++++++ encodings/int-mult/src/lib.rs | 17 + encodings/int-mult/src/rules.rs | 10 + encodings/int-mult/src/slice.rs | 27 ++ encodings/range-packed/Cargo.toml | 1 + encodings/range-packed/src/lib.rs | 103 +++++- vortex-file/Cargo.toml | 1 + vortex-file/src/lib.rs | 1 + vortex/Cargo.toml | 1 + vortex/src/lib.rs | 5 + 13 files changed, 762 insertions(+), 1 deletion(-) create mode 100644 encodings/int-mult/Cargo.toml create mode 100644 encodings/int-mult/src/array.rs create mode 100644 encodings/int-mult/src/lib.rs create mode 100644 encodings/int-mult/src/rules.rs create mode 100644 encodings/int-mult/src/slice.rs diff --git a/Cargo.lock b/Cargo.lock index eaa9a68f150..ed8a5f8dcc0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9550,6 +9550,7 @@ dependencies = [ "vortex-flatbuffers", "vortex-float-quant", "vortex-fsst", + "vortex-int-mult", "vortex-io", "vortex-ipc", "vortex-layout", @@ -10186,6 +10187,7 @@ dependencies = [ "vortex-flatbuffers", "vortex-float-quant", "vortex-fsst", + "vortex-int-mult", "vortex-io", "vortex-layout", "vortex-mask", @@ -10270,6 +10272,17 @@ dependencies = [ "vortex-utils", ] +[[package]] +name = "vortex-int-mult" +version = "0.1.0" +dependencies = [ + "rstest", + "vortex-array", + "vortex-buffer", + "vortex-error", + "vortex-session", +] + [[package]] name = "vortex-io" version = "0.1.0" @@ -10554,6 +10567,7 @@ dependencies = [ "vortex-array", "vortex-buffer", "vortex-error", + "vortex-int-mult", "vortex-session", ] diff --git a/Cargo.toml b/Cargo.toml index c1dda00f0a0..7dd52ad12d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,6 +64,7 @@ members = [ "encodings/onpair", "encodings/block-residual", "encodings/float-quant", + "encodings/int-mult", "encodings/range-packed", # Benchmarks "benchmarks/bench-support", @@ -326,6 +327,7 @@ vortex-parquet-variant = { version = "0.1.0", path = "./encodings/parquet-varian vortex-pco = { version = "0.1.0", path = "./encodings/pco", default-features = false } vortex-block-residual = { version = "0.1.0", path = "./encodings/block-residual", default-features = false } vortex-float-quant = { version = "0.1.0", path = "./encodings/float-quant", default-features = false } +vortex-int-mult = { version = "0.1.0", path = "./encodings/int-mult", default-features = false } vortex-range-packed = { version = "0.1.0", path = "./encodings/range-packed", default-features = false } vortex-proto = { version = "0.1.0", path = "./vortex-proto", default-features = false } vortex-row = { version = "0.1.0", path = "./vortex-row", default-features = false } diff --git a/encodings/int-mult/Cargo.toml b/encodings/int-mult/Cargo.toml new file mode 100644 index 00000000000..2bc5bfcd295 --- /dev/null +++ b/encodings/int-mult/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "vortex-int-mult" +authors = { workspace = true } +categories = { workspace = true } +description = "Vortex integer multiplier transform array encoding" +edition = { workspace = true } +homepage = { workspace = true } +include = { workspace = true } +keywords = { workspace = true } +license = { workspace = true } +readme = { workspace = true } +repository = { workspace = true } +rust-version = { workspace = true } +version = { workspace = true } + +[dependencies] +vortex-array = { workspace = true } +vortex-buffer = { workspace = true } +vortex-error = { workspace = true } +vortex-session = { workspace = true } + +[dev-dependencies] +rstest = { workspace = true } +vortex-array = { workspace = true, features = ["_test-harness"] } + +[lints] +workspace = true diff --git a/encodings/int-mult/src/array.rs b/encodings/int-mult/src/array.rs new file mode 100644 index 00000000000..2743a8e6d41 --- /dev/null +++ b/encodings/int-mult/src/array.rs @@ -0,0 +1,554 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::fmt::Display; +use std::fmt::Formatter; +use std::hash::Hash; +use std::hash::Hasher; + +use vortex_array::Array; +use vortex_array::ArrayEq; +use vortex_array::ArrayHash; +use vortex_array::ArrayId; +use vortex_array::ArrayParts; +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::EqMode; +use vortex_array::ExecutionCtx; +use vortex_array::ExecutionResult; +use vortex_array::IntoArray; +use vortex_array::TypedArrayRef; +use vortex_array::array_slots; +use vortex_array::arrays::Primitive; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::buffer::BufferHandle; +use vortex_array::dtype::DType; +use vortex_array::dtype::NativePType; +use vortex_array::dtype::Nullability::NonNullable; +use vortex_array::dtype::PType; +use vortex_array::match_each_integer_ptype; +use vortex_array::scalar::Scalar; +use vortex_array::serde::ArrayChildren; +use vortex_array::vtable::OperationsVTable; +use vortex_array::vtable::VTable; +use vortex_array::vtable::ValidityChild; +use vortex_array::vtable::ValidityVTableFromChild; +use vortex_buffer::Buffer; +use vortex_buffer::BufferMut; +use vortex_error::VortexExpect; +use vortex_error::VortexResult; +use vortex_error::vortex_bail; +use vortex_error::vortex_ensure; +use vortex_error::vortex_panic; +use vortex_session::VortexSession; +use vortex_session::registry::CachedId; + +use crate::rules::RULES; + +const METADATA_VERSION: u8 = 1; +const METADATA_LEN: usize = 9; + +/// An integer transform represented as multiplied and additive child arrays. +pub type IntMultArray = Array; + +#[array_slots(IntMult)] +pub struct IntMultSlots { + /// Values multiplied by the array base. + #[slot(0)] + pub primary: ArrayRef, + /// Values added after multiplication. + #[slot(1)] + pub secondary: ArrayRef, +} + +#[derive(Clone, Debug)] +pub struct IntMultData { + base: u64, +} + +impl Display for IntMultData { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "base: {}", self.base) + } +} + +impl ArrayHash for IntMultData { + fn array_hash(&self, state: &mut H, _accuracy: EqMode) { + self.base.hash(state); + } +} + +impl ArrayEq for IntMultData { + fn array_eq(&self, other: &Self, _accuracy: EqMode) -> bool { + self.base == other.base + } +} + +#[derive(Clone, Debug)] +pub struct IntMult; + +impl VTable for IntMult { + type TypedArrayData = IntMultData; + type OperationsVTable = Self; + type ValidityVTable = ValidityVTableFromChild; + + fn id(&self) -> ArrayId { + static ID: CachedId = CachedId::new("vortex.int_mult"); + *ID + } + + fn validate( + &self, + data: &Self::TypedArrayData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + validate_children(data.base, dtype, len, IntMultSlotsView::from_slots(slots)) + } + + fn nbuffers(_array: ArrayView<'_, Self>) -> usize { + 0 + } + + fn buffer(_array: ArrayView<'_, Self>, index: usize) -> BufferHandle { + vortex_panic!("IntMultArray buffer index {index} is invalid") + } + + fn buffer_name(_array: ArrayView<'_, Self>, index: usize) -> Option { + vortex_panic!("IntMultArray buffer index {index} is invalid") + } + + fn with_buffers( + &self, + array: ArrayView<'_, Self>, + buffers: &[BufferHandle], + ) -> VortexResult> { + vortex_array::vtable::with_empty_buffers(self, array, buffers) + } + + fn serialize( + array: ArrayView<'_, Self>, + _session: &VortexSession, + ) -> VortexResult>> { + let mut metadata = Vec::with_capacity(METADATA_LEN); + metadata.push(METADATA_VERSION); + metadata.extend_from_slice(&array.data().base.to_le_bytes()); + Ok(Some(metadata)) + } + + fn deserialize( + &self, + dtype: &DType, + len: usize, + metadata: &[u8], + _buffers: &[BufferHandle], + children: &dyn ArrayChildren, + _session: &VortexSession, + ) -> VortexResult> { + vortex_ensure!( + metadata.len() == METADATA_LEN, + "IntMult metadata requires {METADATA_LEN} bytes" + ); + vortex_ensure!( + metadata[0] == METADATA_VERSION, + "unsupported IntMult metadata version {}", + metadata[0] + ); + vortex_ensure!(children.len() == 2, "IntMult requires two children"); + + let mut base_bytes = [0_u8; size_of::()]; + base_bytes.copy_from_slice(&metadata[1..]); + let base = u64::from_le_bytes(base_bytes); + let ptype = PType::try_from(dtype)?; + ensure_base_fits(base, ptype)?; + let primary = children.get(0, dtype, len)?; + let secondary_dtype = DType::Primitive(ptype, NonNullable); + let secondary = children.get(1, &secondary_dtype, len)?; + let slots = IntMultSlots { primary, secondary }.into_slots(); + Ok( + ArrayParts::new(self.clone(), dtype.clone(), len, IntMultData { base }) + .with_slots(slots), + ) + } + + fn slot_name(_array: ArrayView<'_, Self>, index: usize) -> String { + IntMultSlots::NAMES[index].to_string() + } + + fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { + let primary = array.primary().clone().execute::(ctx)?; + let secondary = array.secondary().clone().execute::(ctx)?; + Ok(ExecutionResult::done( + decode_primitive(primary, secondary, array.base())?.into_array(), + )) + } + + fn reduce_parent( + array: ArrayView<'_, Self>, + parent: &ArrayRef, + child_idx: usize, + ) -> VortexResult> { + RULES.evaluate(array, parent, child_idx) + } +} + +impl OperationsVTable for IntMult { + fn scalar_at( + array: ArrayView<'_, IntMult>, + index: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + let primary = array.primary().execute_scalar(index, ctx)?; + if primary.is_null() { + return Ok(Scalar::null(array.dtype().clone())); + } + let secondary = array.secondary().execute_scalar(index, ctx)?; + let primary = primary.as_primitive(); + let secondary = secondary.as_primitive(); + let base = array.base(); + let nullability = array.dtype().nullability(); + Ok(match_each_integer_ptype!(primary.ptype(), |T| { + let primary = primary + .typed_value::() + .vortex_expect("validated IntMult primary scalar"); + let secondary = secondary + .typed_value::() + .vortex_expect("validated IntMult secondary scalar"); + let base = T::try_from(base).vortex_expect("validated IntMult base"); + let value = if array.base() == 1 { + ::wrapping_add(primary, secondary) + } else { + T::wrapping_mul_add(base, primary, secondary) + }; + Scalar::primitive(value, nullability) + })) + } +} + +impl ValidityChild for IntMult { + fn validity_child(array: ArrayView<'_, IntMult>) -> ArrayRef { + array.primary().clone() + } +} + +pub trait IntMultArrayExt: TypedArrayRef + IntMultArraySlotsExt { + /// Return the multiplier that reconstructs each value. + fn base(&self) -> u64 { + self.deref().base + } +} + +impl> IntMultArrayExt for T {} + +impl IntMult { + /// Construct an integer multiplication array from generic child arrays. + pub fn try_new( + primary: ArrayRef, + secondary: ArrayRef, + base: u64, + ) -> VortexResult { + let dtype = primary.dtype().clone(); + let len = primary.len(); + let slots = IntMultSlots { primary, secondary }.into_slots(); + Array::try_from_parts( + ArrayParts::new(IntMult, dtype, len, IntMultData { base }).with_slots(slots), + ) + } + + /// Split integers into quotient and remainder child arrays. + pub fn from_primitive( + array: ArrayView<'_, Primitive>, + base: u64, + ) -> VortexResult { + let ptype = array.ptype(); + ensure_base_fits(base, ptype)?; + let validity = array.validity()?; + let (primary, secondary) = match_each_integer_ptype!(ptype, |T| { + let base = T::try_from(base).vortex_expect("validated IntMult base"); + let (primary, secondary) = split_buffer(array.as_slice::(), base); + ( + PrimitiveArray::new(primary, validity).into_array(), + PrimitiveArray::new(secondary, NonNullable.into()).into_array(), + ) + }); + Self::try_new(primary, secondary, base) + } +} + +fn validate_children( + base: u64, + dtype: &DType, + len: usize, + slots: IntMultSlotsView<'_>, +) -> VortexResult<()> { + let ptype = PType::try_from(dtype)?; + ensure_base_fits(base, ptype)?; + vortex_ensure!( + slots.primary.dtype() == dtype, + "IntMult primary dtype {} differs from {dtype}", + slots.primary.dtype() + ); + vortex_ensure!(slots.primary.len() == len, "IntMult primary length differs"); + let secondary_dtype = DType::Primitive(ptype, NonNullable); + vortex_ensure!( + slots.secondary.dtype() == &secondary_dtype, + "IntMult secondary dtype {} differs from {secondary_dtype}", + slots.secondary.dtype() + ); + vortex_ensure!( + slots.secondary.len() == len, + "IntMult secondary length differs" + ); + Ok(()) +} + +fn ensure_base_fits(base: u64, ptype: PType) -> VortexResult<()> { + vortex_ensure!(ptype.is_int(), "IntMult requires integers"); + vortex_ensure!(base >= 1, "IntMult base must be positive"); + let maximum = match ptype { + PType::U8 => u64::from(u8::MAX), + PType::U16 => u64::from(u16::MAX), + PType::U32 => u64::from(u32::MAX), + PType::U64 => u64::MAX, + PType::I8 => i8::MAX as u64, + PType::I16 => i16::MAX as u64, + PType::I32 => i32::MAX as u64, + PType::I64 => i64::MAX as u64, + _ => vortex_bail!("IntMult requires integers"), + }; + vortex_ensure!(base <= maximum, "IntMult base does not fit {ptype}"); + Ok(()) +} + +fn split_buffer(values: &[T], base: T) -> (Buffer, Buffer) +where + T: NativePType + Copy + std::ops::Div + std::ops::Rem, +{ + let mut primary = BufferMut::with_capacity(values.len()); + let mut secondary = BufferMut::with_capacity(values.len()); + for value in values { + primary.push(*value / base); + secondary.push(*value % base); + } + (primary.freeze(), secondary.freeze()) +} + +fn decode_primitive( + primary: PrimitiveArray, + secondary: PrimitiveArray, + base: u64, +) -> VortexResult { + let ptype = primary.ptype(); + ensure_base_fits(base, ptype)?; + let validity = primary.validity()?; + Ok(match_each_integer_ptype!(ptype, |T| { + let base = T::try_from(base).vortex_expect("validated IntMult base"); + let values = compose_buffer( + primary.into_buffer_mut::(), + secondary.into_buffer::(), + base, + ); + PrimitiveArray::new(values, validity) + })) +} + +fn compose_buffer(mut primary: BufferMut, secondary: Buffer, base: T) -> Buffer +where + T: NativePType + WrappingMulAdd, +{ + if ::is_one(base) { + for (primary, secondary) in primary.as_mut_slice().iter_mut().zip(secondary.as_slice()) { + *primary = ::wrapping_add(*primary, *secondary); + } + return primary.freeze(); + } + + for (primary, secondary) in primary.as_mut_slice().iter_mut().zip(secondary.as_slice()) { + *primary = T::wrapping_mul_add(base, *primary, *secondary); + } + primary.freeze() +} + +trait WrappingMulAdd: Copy { + fn is_one(value: Self) -> bool; + + fn wrapping_add(lhs: Self, rhs: Self) -> Self; + + fn wrapping_mul_add(base: Self, primary: Self, secondary: Self) -> Self; +} + +macro_rules! impl_wrapping_mul_add { + ($type:ty) => { + impl WrappingMulAdd for $type { + #[inline] + fn is_one(value: Self) -> bool { + value == 1 + } + + #[inline] + fn wrapping_add(lhs: Self, rhs: Self) -> Self { + lhs.wrapping_add(rhs) + } + + #[inline] + fn wrapping_mul_add(base: Self, primary: Self, secondary: Self) -> Self { + base.wrapping_mul(primary).wrapping_add(secondary) + } + } + }; +} + +impl_wrapping_mul_add!(u8); +impl_wrapping_mul_add!(u16); +impl_wrapping_mul_add!(u32); +impl_wrapping_mul_add!(u64); +impl_wrapping_mul_add!(i8); +impl_wrapping_mul_add!(i16); +impl_wrapping_mul_add!(i32); +impl_wrapping_mul_add!(i64); + +#[cfg(test)] +mod tests { + use rstest::rstest; + use vortex_array::ArrayContext; + use vortex_array::ArrayRef; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::array_session; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::assert_arrays_eq; + use vortex_array::dtype::NativePType; + use vortex_array::serde::SerializeOptions; + use vortex_array::serde::SerializedArray; + use vortex_array::validity::Validity; + use vortex_buffer::ByteBufferMut; + use vortex_buffer::buffer; + use vortex_session::registry::ReadContext; + + use super::*; + + fn round_trip(values: Vec, base: u64) -> VortexResult<()> + where + T: NativePType + Copy, + { + let input = PrimitiveArray::from_iter(values); + let encoded = IntMult::from_primitive(input.as_view(), base)?; + let decoded = encoded + .into_array() + .execute::(&mut array_session().create_execution_ctx())?; + assert_arrays_eq!(decoded, input, &mut array_session().create_execution_ctx()); + Ok(()) + } + + #[rstest] + #[case(vec![0_u8, 1, 6, 7, 8, 254, 255], 7)] + #[case(vec![0_u16, 1, 6, 7, 8, 65_534, 65_535], 7)] + #[case(vec![0_u32, 1, 6, 7, 8, u32::MAX - 1, u32::MAX], 7)] + #[case(vec![0_u64, 1, 6, 7, 8, u64::MAX - 1, u64::MAX], 7)] + #[case(vec![i8::MIN, -8, -7, -1, 0, 1, 7, 8, i8::MAX], 7)] + #[case(vec![i16::MIN, -8, -7, -1, 0, 1, 7, 8, i16::MAX], 7)] + #[case(vec![i32::MIN, -8, -7, -1, 0, 1, 7, 8, i32::MAX], 7)] + #[case(vec![i64::MIN, -8, -7, -1, 0, 1, 7, 8, i64::MAX], 7)] + fn round_trip_unsigned(#[case] values: Vec, #[case] base: u64) -> VortexResult<()> + where + T: NativePType + Copy, + { + round_trip(values, base) + } + + #[test] + fn exposes_exact_children() -> VortexResult<()> { + let input = PrimitiveArray::from_iter([0_u32, 11, 22, 33, 44]); + let encoded = IntMult::from_primitive(input.as_view(), 10)?; + assert_arrays_eq!( + encoded.primary(), + PrimitiveArray::from_iter([0_u32, 1, 2, 3, 4]), + &mut array_session().create_execution_ctx() + ); + assert_arrays_eq!( + encoded.secondary(), + PrimitiveArray::from_iter([0_u32, 1, 2, 3, 4]), + &mut array_session().create_execution_ctx() + ); + Ok(()) + } + + #[test] + fn base_one_adds_generic_children() -> VortexResult<()> { + let primary = PrimitiveArray::from_iter([100_u32, 200, 300]).into_array(); + let secondary = PrimitiveArray::from_iter([1_u32, 2, 3]).into_array(); + let encoded = IntMult::try_new(primary, secondary, 1)?; + assert_arrays_eq!( + encoded, + PrimitiveArray::from_iter([101_u32, 202, 303]), + &mut array_session().create_execution_ctx() + ); + Ok(()) + } + + #[test] + fn preserves_validity() -> VortexResult<()> { + let input = PrimitiveArray::new( + buffer![10_u32, 20, 30, 40], + Validity::from_iter([true, false, true, false]), + ); + let encoded = IntMult::from_primitive(input.as_view(), 10)?; + let mut ctx = array_session().create_execution_ctx(); + assert!( + encoded + .clone() + .into_array() + .execute_scalar(1, &mut ctx)? + .is_null() + ); + let decoded = encoded.into_array().execute::(&mut ctx)?; + assert_arrays_eq!(decoded, input, &mut ctx); + Ok(()) + } + + #[test] + fn slices_children() -> VortexResult<()> { + let input = PrimitiveArray::from_iter([10_u32, 21, 32, 43, 54]); + let encoded = IntMult::from_primitive(input.as_view(), 10)?; + let sliced = encoded.into_array().slice(1..4)?; + assert_arrays_eq!( + sliced, + PrimitiveArray::from_iter([21_u32, 32, 43]), + &mut array_session().create_execution_ctx() + ); + Ok(()) + } + + #[test] + fn serialized_generic_children_round_trip() -> VortexResult<()> { + let session = array_session(); + crate::initialize(&session); + let primary = PrimitiveArray::from_option_iter([Some(100_u32), None, Some(300)]); + let secondary = PrimitiveArray::from_iter([1_u32, 2, 3]); + let encoded = + IntMult::try_new(primary.into_array(), secondary.into_array(), 1)?.into_array(); + let dtype = encoded.dtype().clone(); + let len = encoded.len(); + let array_ctx = ArrayContext::empty(); + let serialized = encoded.serialize(&array_ctx, &session, &SerializeOptions::default())?; + let mut bytes = ByteBufferMut::empty(); + for buffer in serialized { + bytes.extend_from_slice(buffer.as_ref()); + } + let parts = SerializedArray::try_from(bytes.freeze())?; + let decoded = parts.decode(&dtype, len, &ReadContext::new(array_ctx.to_ids()), &session)?; + let expected: ArrayRef = + PrimitiveArray::from_option_iter([Some(101_u32), None, Some(303)]).into_array(); + assert_arrays_eq!(decoded, expected, &mut session.create_execution_ctx()); + Ok(()) + } + + #[rstest] + #[case(0)] + #[case(256)] + fn rejects_invalid_u8_base(#[case] base: u64) { + let input = PrimitiveArray::from_iter([1_u8, 2, 3]); + assert!(IntMult::from_primitive(input.as_view(), base).is_err()); + } +} diff --git a/encodings/int-mult/src/lib.rs b/encodings/int-mult/src/lib.rs new file mode 100644 index 00000000000..1050c146f8e --- /dev/null +++ b/encodings/int-mult/src/lib.rs @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Integer multiplication transform with independent multiplied and additive children. + +mod array; +mod rules; +mod slice; + +pub use array::*; +use vortex_array::session::ArraySessionExt; +use vortex_session::VortexSession; + +/// Register the integer multiplication encoding in one session. +pub fn initialize(session: &VortexSession) { + session.arrays().register(IntMult); +} diff --git a/encodings/int-mult/src/rules.rs b/encodings/int-mult/src/rules.rs new file mode 100644 index 00000000000..9d2be58f5ba --- /dev/null +++ b/encodings/int-mult/src/rules.rs @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use vortex_array::arrays::slice::SliceReduceAdaptor; +use vortex_array::optimizer::rules::ParentRuleSet; + +use crate::IntMult; + +pub(crate) static RULES: ParentRuleSet = + ParentRuleSet::new(&[ParentRuleSet::lift(&SliceReduceAdaptor(IntMult))]); diff --git a/encodings/int-mult/src/slice.rs b/encodings/int-mult/src/slice.rs new file mode 100644 index 00000000000..0ff8bffa94b --- /dev/null +++ b/encodings/int-mult/src/slice.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::ops::Range; + +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::IntoArray; +use vortex_array::arrays::slice::SliceReduce; +use vortex_error::VortexResult; + +use crate::IntMult; +use crate::IntMultArrayExt; +use crate::IntMultArraySlotsExt; + +impl SliceReduce for IntMult { + fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { + Ok(Some( + IntMult::try_new( + array.primary().slice(range.clone())?, + array.secondary().slice(range)?, + array.base(), + )? + .into_array(), + )) + } +} diff --git a/encodings/range-packed/Cargo.toml b/encodings/range-packed/Cargo.toml index 2264749db36..a1323327a58 100644 --- a/encodings/range-packed/Cargo.toml +++ b/encodings/range-packed/Cargo.toml @@ -17,6 +17,7 @@ version = { workspace = true } vortex-array = { workspace = true } vortex-buffer = { workspace = true } vortex-error = { workspace = true } +vortex-int-mult = { workspace = true } vortex-session = { workspace = true } [dev-dependencies] diff --git a/encodings/range-packed/src/lib.rs b/encodings/range-packed/src/lib.rs index 37738cc8dba..bedbe39bc57 100644 --- a/encodings/range-packed/src/lib.rs +++ b/encodings/range-packed/src/lib.rs @@ -11,9 +11,15 @@ use std::mem::MaybeUninit; use std::ops::Range; pub use array::*; +use vortex_array::IntoArray; +use vortex_array::arrays::DictArray; +use vortex_array::arrays::PrimitiveArray; use vortex_array::session::ArraySessionExt; +use vortex_array::validity::Validity; use vortex_error::VortexResult; use vortex_error::vortex_ensure; +use vortex_int_mult::IntMult; +use vortex_int_mult::IntMultArray; use vortex_session::VortexSession; const MAX_BINS: usize = 64; @@ -43,6 +49,73 @@ pub(crate) struct Bin { pub(crate) offset_bits: u8, } +/// A fixed-bin split before compression of its component arrays. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RangeDecomposition { + bin_starts: Vec, + codes: Vec, + offsets: Vec, +} + +impl RangeDecomposition { + /// Fit at most 64 bins and split values into bin codes and offsets. + pub fn encode(values: &[u64]) -> VortexResult { + if values.is_empty() { + return Ok(Self { + bin_starts: Vec::new(), + codes: Vec::new(), + offsets: Vec::new(), + }); + } + + let (sample, minimum, maximum) = training_sample(values); + let bins = cover_domain(optimize_bins_arbitrary(&sample), minimum, maximum); + let codes = assign_bins(values, &bins)?; + let offsets = values + .iter() + .zip(&codes) + .map(|(&value, &code)| value - bins[usize::from(code)].lower) + .collect(); + Ok(Self { + bin_starts: bins.iter().map(|bin| bin.lower).collect(), + codes, + offsets, + }) + } + + /// Return the selected bin starts. + pub fn bin_starts(&self) -> &[u64] { + &self.bin_starts + } + + /// Return the per-value bin codes. + pub fn codes(&self) -> &[u8] { + &self.codes + } + + /// Return each value's offset from its selected bin start. + pub fn offsets(&self) -> &[u64] { + &self.offsets + } + + /// Return the number of bits required for each fixed-width code. + pub fn code_width(&self) -> u8 { + self.bin_starts + .len() + .checked_sub(1) + .map_or(0, |maximum| bit_width(maximum as u64)) + } + + /// Compose the raw components from a dictionary and an IntMult array. + pub fn into_array(self, validity: Validity) -> VortexResult { + let codes = PrimitiveArray::new(self.codes, validity).into_array(); + let starts = PrimitiveArray::from_iter(self.bin_starts).into_array(); + let references = DictArray::try_new(codes, starts)?.into_array(); + let offsets = PrimitiveArray::from_iter(self.offsets).into_array(); + IntMult::try_new(references, offsets, 1) + } +} + /// Register the RangePacked encoding in one session. pub fn initialize(session: &VortexSession) { session.arrays().register(RangePacked); @@ -621,6 +694,17 @@ fn assign_bins(values: &[u64], bins: &[Bin]) -> VortexResult> { } fn optimize_bins(sorted: &[u64]) -> Vec { + optimize_bins_with_count_policy(sorted, usize::is_power_of_two) +} + +fn optimize_bins_arbitrary(sorted: &[u64]) -> Vec { + optimize_bins_with_count_policy(sorted, |_| true) +} + +fn optimize_bins_with_count_policy( + sorted: &[u64], + allowed_count: impl Fn(usize) -> bool, +) -> Vec { let mut segments = Vec::with_capacity(MAX_BINS); segments.push(0..sorted.len()); let mut best_segments = segments.clone(); @@ -649,7 +733,7 @@ fn optimize_bins(sorted: &[u64]) -> Vec { segments.push(split.at..segment.end); segments.sort_unstable_by_key(|segment| segment.start); let cost = partition_cost(sorted, &segments); - if segments.len().is_power_of_two() && cost < best_cost { + if allowed_count(segments.len()) && cost < best_cost { best_cost = cost; best_segments.clone_from(&segments); } @@ -833,10 +917,27 @@ mod tests { use vortex_error::VortexResult; use vortex_session::registry::ReadContext; + use super::RangeDecomposition; use super::RangePacked; use super::RangePackedCodec; use super::initialize; + #[test] + fn decomposition_uses_arbitrary_bin_count() -> VortexResult<()> { + let values = (0_u64..10) + .flat_map(|cluster| (0_u64..1_000).map(move |_| cluster * 1_000_000_000)) + .collect::>(); + let decomposition = RangeDecomposition::encode(&values)?; + assert_eq!(decomposition.bin_starts().len(), 10); + assert_eq!(decomposition.code_width(), 4); + assert_arrays_eq!( + decomposition.into_array(vortex_array::validity::Validity::NonNullable)?, + PrimitiveArray::from_iter(values), + &mut array_session().create_execution_ctx() + ); + Ok(()) + } + #[test] fn clustered_roundtrip_and_scalar_access() -> VortexResult<()> { let values = (0_u64..20_000) diff --git a/vortex-file/Cargo.toml b/vortex-file/Cargo.toml index dc9d1eb6225..7eeeca59ec0 100644 --- a/vortex-file/Cargo.toml +++ b/vortex-file/Cargo.toml @@ -47,6 +47,7 @@ vortex-fastlanes = { workspace = true } vortex-flatbuffers = { workspace = true, features = ["file"] } vortex-float-quant = { workspace = true } vortex-fsst = { workspace = true } +vortex-int-mult = { workspace = true } vortex-io = { workspace = true } vortex-layout = { workspace = true } vortex-mask = { workspace = true } diff --git a/vortex-file/src/lib.rs b/vortex-file/src/lib.rs index 73e603fb1a8..ed27018089c 100644 --- a/vortex-file/src/lib.rs +++ b/vortex-file/src/lib.rs @@ -191,6 +191,7 @@ pub fn register_default_encodings(session: &VortexSession) { vortex_decimal_byte_parts::initialize(session); vortex_fastlanes::initialize(session); vortex_float_quant::initialize(session); + vortex_int_mult::initialize(session); vortex_block_residual::initialize(session); vortex_range_packed::initialize(session); vortex_runend::initialize(session); diff --git a/vortex/Cargo.toml b/vortex/Cargo.toml index 48955ded147..30106c914d0 100644 --- a/vortex/Cargo.toml +++ b/vortex/Cargo.toml @@ -40,6 +40,7 @@ vortex-file = { workspace = true, optional = true, default-features = true } vortex-flatbuffers = { workspace = true } vortex-float-quant = { workspace = true } vortex-fsst = { workspace = true } +vortex-int-mult = { workspace = true } vortex-io = { workspace = true } vortex-ipc = { workspace = true } vortex-layout = { workspace = true } diff --git a/vortex/src/lib.rs b/vortex/src/lib.rs index f3f3db82f39..92d52efddfd 100644 --- a/vortex/src/lib.rs +++ b/vortex/src/lib.rs @@ -265,6 +265,11 @@ pub mod encodings { pub use vortex_float_quant::*; } + /// Integer multiplication transform with generic multiplied and additive children. + pub mod int_mult { + pub use vortex_int_mult::*; + } + /// Fast Static Symbol Table string encoding. pub mod fsst { pub use vortex_fsst::*; From ca570c3b135e1ade9f4cf99a6b6daf35610adcdc Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 04:46:16 +0100 Subject: [PATCH 39/78] perf: fuse base-one dictionary addition Signed-off-by: Will Manning --- Cargo.lock | 2 + .../src/bitpacking/array/unpack_iter.rs | 4 +- encodings/int-mult/Cargo.toml | 2 + encodings/int-mult/src/array.rs | 160 ++++++++++++++++++ vortex/benches/single_encoding_throughput.rs | 109 ++++++++++++ 5 files changed, 275 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ed8a5f8dcc0..bc4980686a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10276,10 +10276,12 @@ dependencies = [ name = "vortex-int-mult" version = "0.1.0" dependencies = [ + "num-traits", "rstest", "vortex-array", "vortex-buffer", "vortex-error", + "vortex-fastlanes", "vortex-session", ] diff --git a/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs b/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs index 3c77e146ad0..2c79c6173f2 100644 --- a/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs +++ b/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs @@ -221,8 +221,8 @@ impl> UnpackedChunks { }); } - /// Walk every unpacked chunk in array order, reusing the internal scratch buffer. - pub(crate) fn for_each_unpacked_chunk(&mut self, mut f: F) + /// Walk every unpacked chunk in logical order and reuse one scratch buffer. + pub fn for_each_unpacked_chunk(&mut self, mut f: F) where F: FnMut(&mut [T], Range), { diff --git a/encodings/int-mult/Cargo.toml b/encodings/int-mult/Cargo.toml index 2bc5bfcd295..273e7f22da5 100644 --- a/encodings/int-mult/Cargo.toml +++ b/encodings/int-mult/Cargo.toml @@ -14,9 +14,11 @@ rust-version = { workspace = true } version = { workspace = true } [dependencies] +num-traits = { workspace = true } vortex-array = { workspace = true } vortex-buffer = { workspace = true } vortex-error = { workspace = true } +vortex-fastlanes = { workspace = true } vortex-session = { workspace = true } [dev-dependencies] diff --git a/encodings/int-mult/src/array.rs b/encodings/int-mult/src/array.rs index 2743a8e6d41..5c91d7fb114 100644 --- a/encodings/int-mult/src/array.rs +++ b/encodings/int-mult/src/array.rs @@ -6,6 +6,7 @@ use std::fmt::Formatter; use std::hash::Hash; use std::hash::Hasher; +use num_traits::AsPrimitive; use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; @@ -19,8 +20,10 @@ use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::TypedArrayRef; use vortex_array::array_slots; +use vortex_array::arrays::Dict; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::dict::DictArraySlotsExt; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; @@ -40,6 +43,8 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; +use vortex_fastlanes::BitPacked; +use vortex_fastlanes::BitPackedArrayExt; use vortex_session::VortexSession; use vortex_session::registry::CachedId; @@ -177,6 +182,11 @@ impl VTable for IntMult { } fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { + if array.base() == 1 + && let Some(decoded) = decode_dict_add(array.as_view(), ctx)? + { + return Ok(ExecutionResult::done(decoded.into_array())); + } let primary = array.primary().clone().execute::(ctx)?; let secondary = array.secondary().clone().execute::(ctx)?; Ok(ExecutionResult::done( @@ -193,6 +203,118 @@ impl VTable for IntMult { } } +fn decode_dict_add( + array: ArrayView<'_, IntMult>, + ctx: &mut ExecutionCtx, +) -> VortexResult> { + let Some(dict) = array.primary().as_typed::() else { + return Ok(None); + }; + let starts = dict.values().clone().execute::(ctx)?; + if !starts.validity()?.definitely_no_nulls() { + return Ok(None); + } + let secondary = array.secondary().clone().execute::(ctx)?; + + if let Some(codes) = dict.codes().as_typed::() { + let validity = BitPackedArrayExt::validity(&codes); + if codes.patches().is_none() && validity.definitely_no_nulls() { + return decode_bitpacked_dict(starts, secondary, codes, validity).map(Some); + } + } + + let codes = dict.codes().clone().execute::(ctx)?; + if !codes.validity()?.definitely_no_nulls() { + return Ok(None); + } + decode_primitive_dict(starts, secondary, codes).map(Some) +} + +fn decode_bitpacked_dict( + starts: PrimitiveArray, + secondary: PrimitiveArray, + codes: ArrayView<'_, BitPacked>, + validity: vortex_array::validity::Validity, +) -> VortexResult { + match_each_integer_ptype!(secondary.ptype(), |T| { + let starts = starts.as_slice::(); + let output = add_bitpacked_codes(starts, secondary.into_buffer_mut::(), codes)?; + VortexResult::Ok(PrimitiveArray::new(output, validity)) + }) +} + +fn decode_primitive_dict( + starts: PrimitiveArray, + secondary: PrimitiveArray, + codes: PrimitiveArray, +) -> VortexResult { + let validity = codes.validity()?; + match_each_integer_ptype!(secondary.ptype(), |T| { + let starts = starts.as_slice::(); + match_each_integer_ptype!(codes.ptype(), |C| { + let output = add_primitive_codes( + starts, + secondary.into_buffer_mut::(), + codes.as_slice::(), + )?; + VortexResult::Ok(PrimitiveArray::new(output, validity)) + }) + }) +} + +fn add_primitive_codes( + starts: &[T], + mut output: BufferMut, + codes: &[C], +) -> VortexResult> +where + T: NativePType + WrappingMulAdd, + C: NativePType + AsPrimitive, +{ + for (value, code) in output.as_mut_slice().iter_mut().zip(codes) { + let code: usize = code.as_(); + let Some(start) = starts.get(code) else { + vortex_bail!("IntMult dictionary code is invalid"); + }; + *value = ::wrapping_add(*start, *value); + } + Ok(output) +} + +fn add_bitpacked_codes( + starts: &[T], + mut output: BufferMut, + codes: ArrayView<'_, BitPacked>, +) -> VortexResult> +where + T: NativePType + WrappingMulAdd, +{ + vortex_ensure!(!starts.is_empty(), "IntMult dictionary values are empty"); + if codes.bit_width() == 0 { + for value in output.as_mut_slice() { + *value = ::wrapping_add(starts[0], *value); + } + return Ok(output); + } + + match_each_integer_ptype!(codes.dtype().as_ptype(), |C| { + let mut invalid_code = false; + let mut chunks = codes.unpacked_chunks::()?; + chunks.for_each_unpacked_chunk(|codes, range| { + for (value, code) in output.as_mut_slice()[range].iter_mut().zip(codes) { + let code: usize = code.as_(); + let Some(start) = starts.get(code) else { + invalid_code = true; + continue; + }; + *value = ::wrapping_add(*start, *value); + } + }); + vortex_ensure!(!invalid_code, "IntMult dictionary code is invalid"); + VortexResult::Ok(output) + }) +} + impl OperationsVTable for IntMult { fn scalar_at( array: ArrayView<'_, IntMult>, @@ -416,6 +538,7 @@ mod tests { use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::array_session; + use vortex_array::arrays::DictArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::dtype::NativePType; @@ -424,6 +547,7 @@ mod tests { use vortex_array::validity::Validity; use vortex_buffer::ByteBufferMut; use vortex_buffer::buffer; + use vortex_fastlanes::bitpack_compress::bitpack_encode_unchecked; use vortex_session::registry::ReadContext; use super::*; @@ -487,6 +611,42 @@ mod tests { Ok(()) } + #[test] + fn base_one_adds_bitpacked_dictionary_references() -> VortexResult<()> { + let codes = PrimitiveArray::from_iter([0_u8, 1, 2, 1, 0]); + // SAFETY: Two bits represent every code. + let codes = unsafe { bitpack_encode_unchecked(codes, 2) }?.into_array(); + let starts = PrimitiveArray::from_iter([100_u32, 1_000, 10_000]).into_array(); + let references = DictArray::try_new(codes, starts)?.into_array(); + let offsets = PrimitiveArray::from_iter([1_u32, 2, 3, 4, 5]).into_array(); + let encoded = IntMult::try_new(references, offsets, 1)?; + + assert_arrays_eq!( + encoded, + PrimitiveArray::from_iter([101_u32, 1_002, 10_003, 1_004, 105]), + &mut array_session().create_execution_ctx() + ); + Ok(()) + } + + #[test] + fn base_one_adds_zero_width_dictionary_references() -> VortexResult<()> { + let codes = PrimitiveArray::from_iter([0_u8; 3]); + // SAFETY: Zero bits represent the only code. + let codes = unsafe { bitpack_encode_unchecked(codes, 0) }?.into_array(); + let starts = PrimitiveArray::from_iter([100_u32]).into_array(); + let references = DictArray::try_new(codes, starts)?.into_array(); + let offsets = PrimitiveArray::from_iter([1_u32, 2, 3]).into_array(); + let encoded = IntMult::try_new(references, offsets, 1)?; + + assert_arrays_eq!( + encoded, + PrimitiveArray::from_iter([101_u32, 102, 103]), + &mut array_session().create_execution_ctx() + ); + Ok(()) + } + #[test] fn preserves_validity() -> VortexResult<()> { let input = PrimitiveArray::new( diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index 60084b1a770..fa1c135d117 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -20,6 +20,7 @@ use vortex::VortexSessionDefault; use vortex::array::Canonical; use vortex::array::ExecutionCtx; use vortex::array::IntoArray; +use vortex::array::arrays::DictArray; use vortex::array::arrays::Primitive; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::VarBinViewArray; @@ -47,7 +48,9 @@ use vortex::encodings::float_quant::FloatQuantArraySlotsExt; use vortex::encodings::float_quant::analyze_float_quant; use vortex::encodings::fsst::fsst_compress; use vortex::encodings::fsst::fsst_train_compressor; +use vortex::encodings::int_mult::IntMult; use vortex::encodings::pco::Pco; +use vortex::encodings::range_packed::RangeDecomposition; use vortex::encodings::runend::RunEnd; use vortex::encodings::sequence::sequence_encode; use vortex::encodings::zigzag::zigzag_encode; @@ -236,6 +239,40 @@ fn setup_block_local_i16_array() -> PrimitiveArray { })) } +fn setup_int_mult_i32_array() -> PrimitiveArray { + PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + let primary = (index as i32).wrapping_mul(7_919) % 1_000_000; + let secondary = index as i32 % 10 - 5; + primary.wrapping_mul(10).wrapping_add(secondary) + })) +} + +fn setup_ranged_u64_array() -> PrimitiveArray { + PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + let cluster = index % 10; + cluster * 1_000_000_000 + index.wrapping_mul(7_919) % 1_024 + })) +} + +fn encode_decomposed_range_tree(array: &PrimitiveArray) -> vortex::array::ArrayRef { + let decomposition = RangeDecomposition::encode(array.as_slice::()).unwrap(); + let code_width = decomposition.code_width(); + let codes = PrimitiveArray::from_iter(decomposition.codes().iter().copied()); + // SAFETY: The decomposition computes the exact code width. + let codes = unsafe { bitpack_encode_unchecked(codes, code_width) } + .unwrap() + .into_array(); + let starts = PrimitiveArray::from_iter(decomposition.bin_starts().iter().copied()).into_array(); + let references = DictArray::try_new(codes, starts).unwrap().into_array(); + let offsets = PrimitiveArray::from_iter(decomposition.offsets().iter().copied()); + let offsets = BlockResidual::from_primitive(offsets.as_view()) + .unwrap() + .into_array(); + IntMult::try_new(references, offsets, 1) + .unwrap() + .into_array() +} + fn encode_for_bitpacked_tree(array: &PrimitiveArray, bit_width: u8) -> vortex::array::ArrayRef { let mut ctx = SESSION.create_execution_ctx(); let encoded = FoR::encode(array.clone(), &mut ctx).unwrap(); @@ -590,6 +627,78 @@ fn bench_ordered_float_decompress_f64(bencher: Bencher) { .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); } +#[divan::bench(name = "int_mult_split_compress_i32")] +fn bench_int_mult_split_compress_i32(bencher: Bencher) { + let array = setup_int_mult_i32_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) + .with_inputs(|| &array) + .bench_refs(|array| IntMult::from_primitive(array.as_view(), 10).unwrap()); +} + +#[divan::bench(name = "int_mult_split_decompress_i32")] +fn bench_int_mult_split_decompress_i32(bencher: Bencher) { + let encoded = IntMult::from_primitive(setup_int_mult_i32_array().as_view(), 10) + .unwrap() + .into_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "int_mult_split_scalar_at_i32")] +fn bench_int_mult_split_scalar_at_i32(bencher: Bencher) { + let encoded = IntMult::from_primitive(setup_int_mult_i32_array().as_view(), 10) + .unwrap() + .into_array(); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + +#[divan::bench(name = "decomposed_range_compress_u64")] +fn bench_decomposed_range_compress_u64(bencher: Bencher) { + let array = setup_ranged_u64_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| &array) + .bench_refs(|array| encode_decomposed_range_tree(array)); +} + +#[divan::bench(name = "decomposed_range_decompress_u64")] +fn bench_decomposed_range_decompress_u64(bencher: Bencher) { + let encoded = encode_decomposed_range_tree(&setup_ranged_u64_array()); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "decomposed_range_scalar_at_u64")] +fn bench_decomposed_range_scalar_at_u64(bencher: Bencher) { + let encoded = encode_decomposed_range_tree(&setup_ranged_u64_array()); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + #[divan::bench(name = "block_residual_compress_u64")] fn bench_block_residual_compress_u64(bencher: Bencher) { let float_array = setup_random_walk_array(); From 6713869eb41509ee0cc34a3bc668708f9eaaa3b1 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 04:49:50 +0100 Subject: [PATCH 40/78] docs: update native numeric compression plan Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 166 ++++++++++++++++++++++++++++++++---- 1 file changed, 148 insertions(+), 18 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index be07f046783..cb1873153e2 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -35,6 +35,7 @@ Focus the production work on these encodings: - `OrderedFloatArray`. - `BlockResidualArray` for all integer types, with one reference per 1,024-value block. - `FloatQuantArray`. +- `IntMultArray` as a composable integer transform. Keep `FloatQuantScheme`, `OrderedBlockResidualScheme`, and `BlockResidualScheme` as BtrBlocks candidates. @@ -44,17 +45,29 @@ Remove `RangeEntropyArray`, `RangeEntropyScheme`, and `BitSplitCodec` from the f The `wm/pcodec-entropy-experiments` branch preserves the complete entropy and bit-split prototypes. -Keep `RangePackedArray` and its manual benchmark as experimental work. +Keep the fused `RangePackedArray` and its manual benchmark as experimental work. -Do not add RangePacked to the default or Compact selector. +Prototype fixed bins as a composed tree instead: + +- `Dict(BitPacked(bin_codes), bin_starts)` reconstructs one reference per value. +- `BlockResidual(offsets)` stores each distance from the selected reference. +- `IntMult(base=1, references, offsets)` adds both components. + +The prototype permits any bin count from one through 64. + +The BtrBlocks selector and final child choices remain incomplete. + +Do not add the fused RangePacked array to the Default or Compact selector. Do not add adjacent Delta, Delta-of-delta, Delta with lookback, or convolution Delta. ## Current state -The branch implements `OrderedFloatArray`, `BlockResidualArray`, and `FloatQuantArray`. +The branch implements `OrderedFloatArray`, `BlockResidualArray`, `FloatQuantArray`, and `IntMultArray`. -The default candidate set includes their three BtrBlocks schemes. +The default candidate set includes BtrBlocks schemes for the first three arrays. + +IntMult does not have a BtrBlocks scheme yet. `OrderedFloat(BlockResidual)` now supports `f32` and `f64` inputs. @@ -78,6 +91,16 @@ Generic quotient and remainder trees do not close that gap. The current selector factors and nonlinear patch cost remain provisional. +`IntMultArray` owns only the quotient and remainder transform. + +Its two children remain generic arrays. Child compression owns patches and other integer models. + +The range decomposition prototype uses `IntMult(base=1)` as generic addition. + +Its fused decode path skips multiplication and avoids materialization of dictionary references. + +Neither IntMult nor decomposed fixed bins participate in the Default selector yet. + Final calibration requires the complete corpus and selected-tree evidence. The current branch recovers a small share of Compact's aggregate numeric advantage. @@ -147,6 +170,34 @@ Native `f32` and two-child selection meet the selected-column and rejected-colum Final selector factors still require broad corpus validation. +## IntMultArray + +`IntMultArray` reconstructs each integer as `base * primary + secondary`. + +The array supports every signed and unsigned integer type. + +The primary and secondary children can use any compatible Vortex array encoding. + +The primary child owns validity. The secondary child is nonnullable. + +The array supports canonical decode, scalar access, slices, serialization, and validation. + +`IntMult::from_primitive` creates quotient and remainder children with the source integer width. + +A future selector can compress both children through specialized or recursive integer schemes. + +IntMult does not own exception positions or exception payloads. + +Child encodings can use `PatchArray`, `BlockResidual`, or other integer arrays when those trees win. + +When the base equals one, bulk decode and scalar access skip multiplication. + +For a dictionary primary, the fused path decodes the secondary into the output buffer. + +It then unpacks dictionary codes and adds the selected reference directly. + +This path avoids a full materialized array of references. + ## Selection policy `FloatQuantScheme` uses the normal sample comparison against ALP, ALP-RD, dictionary, sparse, and RLE schemes. @@ -921,7 +972,11 @@ The dense prototype applies 853,533 quotient patches across two million values. A diagnostic decode without quotient patches reaches 12,628 MB/s. The split and reconstruction costs still miss the Default decode target. -These results reject the tested IntMult layouts for Default. The layouts retain bounded scalar access, but their bulk decode cost is too high. +These results reject the tested bespoke IntMult layouts for Default. + +The layouts retain bounded scalar access, but their bulk decode cost is too high. + +They do not reject a pure IntMult transform with independently compressed children. The result does not reject an IntMult backend for Compact. Compact accepts a different throughput and random-access tradeoff. @@ -1039,6 +1094,56 @@ The selector also increased aggregate CMS bytes by 1.1 percent after it replaced RangePacked therefore remains outside all writer selectors. +### Composable fixed-bin prototype + +The prior RangePacked prototypes fused bins, offset streams, checkpoints, and reconstruction into one codec. + +The current prototype separates the model from storage. + +`RangeDecomposition` fits any number of bins through 64. + +It produces three logical components: + +- A small array of bin starts. +- One fixed-width bin code per value. +- One integer offset from the selected start per value. + +The manual benchmark builds this exact tree: + +`IntMult(base=1, Dict(BitPacked(codes), starts), BlockResidual(offsets))`. + +The first IntMult decode materialized every dictionary reference before addition. + +A generic dictionary-add path increased decode throughput from 12.06 GB/s to 12.62 GB/s. + +A packed-code specialization increased median decode throughput to 14.27 GB/s. + +The specialization unpacks codes and adds starts directly into the decoded offset buffer. + +It does not multiply because the IntMult base equals one. + +The benchmark uses two million `u64` values in ten separated clusters. + +| Operation | Result | +| --- | ---: | +| Encode | 1.20 GB/s | +| Decode | 14.27 GB/s | +| Scalar access | 332 ns | + +A separate base-ten IntMult benchmark uses two million `i32` values and primitive children. + +| Operation | Result | +| --- | ---: | +| Split encode | 2.88 GB/s | +| Decode | 30.67 GB/s | +| Scalar access | 125 ns | + +These values establish low transform overhead and bounded scalar access. + +They do not establish a Default win because the range benchmark lacks incumbent size and throughput comparisons. + +The next experiment must compare complete trees on the real Pco gap columns. + ### Nullable and Delta-heavy fixed-bin cases The optimized nullable prototype stores only valid values in the range stream. @@ -1344,23 +1449,44 @@ This round completed these steps: - Rejected the dense-remainder IntMult layout on CMS Payments. - Rejected centered block residuals after a complete Euro subjectivity comparison. -The Pco mode profile and quotient and remainder experiments are complete. +The Pco mode profile and the first quotient and remainder experiments are complete. + +The composable IntMult follow-up remains incomplete. The nonzero-secondary FloatQuant implementation and focused validation are complete. -Complete these remaining steps: +Complete these next experiments in order: + +1. Measure the decomposed fixed-bin tree on CMS, Food, Euro2016, GloVe, and other no-Delta Pco wins. +2. Compare complete tree size and throughput against the displaced Default tree and Compact Pco. +3. Compare `BlockResidual` and `FoR(BitPacked)` offset children. Use recursive compression as an oracle for other trees. +4. Prototype pure IntMult on the exact GloVe and CMS ALP children. +5. Compress both IntMult children independently through normal integer schemes. +6. Test BlockResidual as a general compression candidate for patch positions. +7. Add a specialized range scheme only after a complete tree passes the column gates. +8. Profile another direct narrow BlockResidual decode optimization for `i16`. +9. Classify more float columns where Pco beats ALP, ALP-RD, and the new schemes. +10. Validate rejected-candidate analysis cost after the candidate set stabilizes. + +Use packed bin codes and primitive bin starts unless evidence supports more complexity. -1. Keep IntMult outside Default unless a new design removes the split reconstruction cost. -2. Add a true ALP-RD child composition case if a real selected tree exposes one. -3. Validate the remaining rejected analysis cost after the candidate set stabilizes. -4. Add real embedding datasets beyond GloVe when licenses and loaders permit them. -5. Classify more float columns where Pco beats ALP, ALP-RD, and the new schemes. -6. Run the complete compression corpus after the candidate set stabilizes. -7. Compare the final geometric mean against Compact and Parquet with Zstd. -8. Calibrate all selector thresholds from complete corpus and selected-tree evidence. -9. Revisit the 32-bit and 64-bit BlockResidual factors with selected-tree evidence. -10. Keep RangePacked outside writer selectors unless a new decode design changes the result. -11. Update this plan after each experiment. +Let ordinary child arrays own patches and exception payloads. + +Add an outer OrderedFloat fused decode only if the generic composition misses the decode gate. + +Retain the direct 8-bit and 16-bit selector exclusions until benchmark evidence changes them. + +Keep fused RangePacked and RangeEntropy outside Default. + +After the candidate set stabilizes, complete these final steps: + +1. Run the complete compression corpus. +2. Compare the final geometric mean against Compact and Parquet with Zstd. +3. Calibrate all selector thresholds from complete corpus and selected-tree evidence. +4. Revisit the BlockResidual factors with selected-tree evidence. +5. Add a true ALP-RD child composition case if a real selected tree exposes one. +6. Add more real embedding datasets when licenses and loaders permit them. +7. Update this plan after each experiment. ## Pull request structure @@ -1368,6 +1494,10 @@ Prepare one focused stack for `OrderedFloatArray`, `BlockResidualArray`, and `Or Prepare a separate stack for `FloatQuantArray` and `FloatQuantScheme`. +Prepare a separate primitive-array change for `IntMultArray`. + +Prepare a separate BtrBlocks change for decomposed fixed bins only after real-data validation. + Keep entropy, bit-split, and alternate residual models on `wm/pcodec-entropy-experiments`. ## References From 7d89719bab7e12648ee778539603829248e6ab72 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 05:18:52 +0100 Subject: [PATCH 41/78] perf: optimize FloatQuant split encoding Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 70 ++++++- .../block-residual/src/ordered_float_array.rs | 42 ++++ encodings/float-quant/src/array.rs | 79 ++++---- vortex/benches/single_encoding_throughput.rs | 186 ++++++++++++++++++ 4 files changed, 329 insertions(+), 48 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index cb1873153e2..86c06e9f05a 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -87,7 +87,7 @@ BlockResidual now passes the direct speed gates for 32-bit and 64-bit integers. The GloVe result identifies a separate entropy gap inside the ALP integer child. -Generic quotient and remainder trees do not close that gap. +The previously tested quotient and remainder trees do not close that gap. The current selector factors and nonlinear patch cost remain provisional. @@ -240,7 +240,9 @@ Direct 32-bit and 64-bit BlockResidual decode now matches or exceeds the main Fo BlockResidual does not occur below ZigZag. That composition lost decode throughput on GloVe for a small size reduction. -The residual scheme requires a 1.05 compression ratio. Its adjusted score includes a 1.02 decode-cost factor. +The ordered-float residual scheme requires a 1.05 compression ratio. + +Its adjusted score includes a 1.02 decode-cost factor. `BlockResidualScheme` uses the same locality probe for integer arrays. @@ -250,12 +252,7 @@ The scheme does not run inside trial compression for an outer scheme. Generic 64 The selected outer scheme can still choose BlockResidual for its full child. -The integer selector divides the measured compression ratio by these decode-cost factors: - -- 1.10 for 32-bit integers. -- 1.20 for 64-bit integers. - -A 1.20 factor requires about 16.7 percent fewer estimated bytes. It does not require 20 percent fewer bytes. +The integer BlockResidual scheme does not use a width-specific decode-cost factor. The block planner adds 16 synthetic cost bits per patch. @@ -275,6 +272,63 @@ The selector excludes BlockResidual from dictionary-code children. A complete Bl Both schemes remain eligible to displace ALP or ALP-RD when their sample size scores win. +## Single-encoding performance ledger + +This snapshot dates from 2026-08-20. + +Each benchmark uses two million logical values and 100 Divan samples. + +Encode and decode results report median logical input throughput. + +Scalar results report median random-access latency. + +### Primitive transforms + +These cases use canonical primitive children. They isolate each outer array transform. + +| Array and input | Encode | Decode | Scalar access | +| --- | ---: | ---: | ---: | +| OrderedFloat `f32` | 52.38 GB/s | 60.47 GB/s | 82 ns | +| OrderedFloat `f64` | 58.78 GB/s | 42.78 GB/s | 90 ns | +| IntMult base-ten `i32` | 2.83 GB/s | 30.43 GB/s | 125 ns | +| IntMult base-ten `u64` | 5.67 GB/s | 25.24 GB/s | 140 ns | +| FloatQuant zero-secondary `f32` | 11.66 GB/s | 31.17 GB/s | 83 ns | +| FloatQuant zero-secondary `f64` | 19.02 GB/s | 25.15 GB/s | 83 ns | +| FloatQuant one-bit-secondary `f64` | 9.26 GB/s | 8.14 GB/s | 167 ns | + +IntMult encode includes quotient and remainder creation. It excludes compression of both children. + +FloatQuant zero-secondary encode uses separate validation and transform passes. + +The prior fallible per-value loop reached 1.99 GB/s for `f32` and 3.72 GB/s for `f64`. + +The two-pass implementation improved those results by 5.9 times and 5.1 times. + +### Production child trees + +These cases include the current compressed children and fused decode paths. + +| Array tree and input | Encode | Decode | Scalar access | +| --- | ---: | ---: | ---: | +| BlockResidual `i16` | 1.41 GB/s | 32.03 GB/s | 83 ns | +| BlockResidual `i32` | 2.80 GB/s | 33.07 GB/s | 57 ns | +| BlockResidual `u32` | 2.79 GB/s | 45.99 GB/s | 60 ns | +| BlockResidual `u64` | 5.18 GB/s | 37.72 GB/s | 83 ns | +| OrderedFloat with BlockResidual `f32` | 2.43 GB/s | 29.64 GB/s | 78 ns | +| OrderedFloat with BlockResidual `f64` | 2.70 GB/s | 20.44 GB/s | 125 ns | +| FloatQuant with packed primary `f32` | 7.59 GB/s | 18.21 GB/s | 125 ns | +| FloatQuant with packed primary `f64` | 8.30 GB/s | 14.77 GB/s | 125 ns | +| FloatQuant with two packed children `f64` | 4.21 GB/s | 13.05 GB/s | 209 ns | +| Decomposed fixed bins `u64` | 1.20 GB/s | 14.27 GB/s | 332 ns | + +The FloatQuant scheme includes analysis and direct tree construction. + +It encodes at 5.19 GB/s for `f32` and 7.07 GB/s for `f64` on selected inputs. + +The decomposed fixed-bin row uses ten separated clusters and a BlockResidual offset child. + +The fixed-bin result still lacks complete Default and Compact comparisons on real columns. + ## Performance requirements A default candidate must meet these limits: diff --git a/encodings/block-residual/src/ordered_float_array.rs b/encodings/block-residual/src/ordered_float_array.rs index ab018615191..f177e46c680 100644 --- a/encodings/block-residual/src/ordered_float_array.rs +++ b/encodings/block-residual/src/ordered_float_array.rs @@ -400,6 +400,7 @@ fn unordered_u64(value: u64) -> u64 { #[cfg(test)] mod tests { + use vortex_array::ArrayContext; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::array_session; @@ -407,7 +408,13 @@ mod tests { use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::dtype::PType; + use vortex_array::serde::SerializeOptions; + use vortex_array::serde::SerializedArray; + use vortex_array::validity::Validity; + use vortex_buffer::Buffer; + use vortex_buffer::ByteBufferMut; use vortex_error::VortexResult; + use vortex_session::registry::ReadContext; use super::OrderedFloat; use super::OrderedFloatArraySlotsExt; @@ -471,4 +478,39 @@ mod tests { ); Ok(()) } + + #[test] + fn nullable_serialized_slice_roundtrip() -> VortexResult<()> { + let primitive = PrimitiveArray::new( + Buffer::from(vec![f64::NEG_INFINITY, -0.0, 0.0, 42.25, f64::INFINITY]), + Validity::from_iter([true, false, true, true, true]), + ); + let encoded = OrderedFloat::from_primitive(primitive.as_view())?; + let session = array_session(); + crate::initialize(&session); + let mut ctx = session.create_execution_ctx(); + assert!(encoded.execute_scalar(1, &mut ctx)?.is_null()); + + let sliced = encoded.into_array().slice(1..5)?; + let expected = primitive.into_array().slice(1..5)?; + let dtype = sliced.dtype().clone(); + let len = sliced.len(); + let array_context = ArrayContext::empty(); + let serialized = + sliced.serialize(&array_context, &session, &SerializeOptions::default())?; + let mut bytes = ByteBufferMut::empty(); + for buffer in serialized { + bytes.extend_from_slice(buffer.as_ref()); + } + let decoded = SerializedArray::try_from(bytes.freeze())?.decode( + &dtype, + len, + &ReadContext::new(array_context.to_ids()), + &session, + )?; + + assert!(decoded.is::()); + assert_arrays_eq!(decoded, expected, &mut ctx); + Ok(()) + } } diff --git a/encodings/float-quant/src/array.rs b/encodings/float-quant/src/array.rs index 7724adf2d76..9580dccc0de 100644 --- a/encodings/float-quant/src/array.rs +++ b/encodings/float-quant/src/array.rs @@ -589,65 +589,53 @@ fn split_f64(values: &[f64], k: u8) -> VortexResult<(Vec, Vec)> { fn split_primary_f32(values: &[f32], k: u8) -> VortexResult> { vortex_ensure!(k > 0 && k <= 23, "FloatQuant f32 k must be in 1..=23"); let low_mask = (1_u32 << k) - 1; - values + vortex_ensure!( + values.iter().all(|value| value.to_bits() & low_mask == 0), + "FloatQuant constant secondary requires zero low bits" + ); + Ok(values .iter() - .map(|value| { - let bits = value.to_bits(); - vortex_ensure!( - bits & low_mask == 0, - "FloatQuant constant secondary requires zero low bits" - ); - Ok(ordered_u32(bits) >> k) - }) - .collect() + .map(|value| ordered_u32(value.to_bits()) >> k) + .collect()) } fn split_primary_f64(values: &[f64], k: u8) -> VortexResult> { vortex_ensure!(k > 0 && k <= 52, "FloatQuant f64 k must be in 1..=52"); let low_mask = (1_u64 << k) - 1; - values + vortex_ensure!( + values.iter().all(|value| value.to_bits() & low_mask == 0), + "FloatQuant constant secondary requires zero low bits" + ); + Ok(values .iter() - .map(|value| { - let bits = value.to_bits(); - vortex_ensure!( - bits & low_mask == 0, - "FloatQuant constant secondary requires zero low bits" - ); - Ok(ordered_u64(bits) >> k) - }) - .collect() + .map(|value| ordered_u64(value.to_bits()) >> k) + .collect()) } fn split_primary_for_f32(values: &[f32], k: u8, primary_min: u32) -> VortexResult> { vortex_ensure!(k > 0 && k <= 23, "FloatQuant f32 k must be in 1..=23"); let low_mask = (1_u32 << k) - 1; - values + vortex_ensure!( + values.iter().all(|value| value.to_bits() & low_mask == 0), + "FloatQuant constant secondary requires zero low bits" + ); + Ok(values .iter() - .map(|value| { - let bits = value.to_bits(); - vortex_ensure!( - bits & low_mask == 0, - "FloatQuant constant secondary requires zero low bits" - ); - Ok((ordered_u32(bits) >> k) - primary_min) - }) - .collect() + .map(|value| (ordered_u32(value.to_bits()) >> k) - primary_min) + .collect()) } fn split_primary_for_f64(values: &[f64], k: u8, primary_min: u64) -> VortexResult> { vortex_ensure!(k > 0 && k <= 52, "FloatQuant f64 k must be in 1..=52"); let low_mask = (1_u64 << k) - 1; - values + vortex_ensure!( + values.iter().all(|value| value.to_bits() & low_mask == 0), + "FloatQuant constant secondary requires zero low bits" + ); + Ok(values .iter() - .map(|value| { - let bits = value.to_bits(); - vortex_ensure!( - bits & low_mask == 0, - "FloatQuant constant secondary requires zero low bits" - ); - Ok((ordered_u64(bits) >> k) - primary_min) - }) - .collect() + .map(|value| (ordered_u64(value.to_bits()) >> k) - primary_min) + .collect()) } fn join_f32(primary: u32, secondary: u32, k: u8) -> f32 { @@ -961,6 +949,17 @@ mod tests { Ok(()) } + #[test] + fn implicit_zero_secondary_rejects_nonzero_low_bits() { + let f32_values = PrimitiveArray::from_iter([f32::from_bits(1.0_f32.to_bits() | 1)]); + assert!(FloatQuant::from_primitive_constant_secondary(f32_values.as_view(), 8).is_err()); + assert!(FloatQuant::primary_for_primitive(f32_values.as_view(), 8, 0).is_err()); + + let f64_values = PrimitiveArray::from_iter([f64::from_bits(1.0_f64.to_bits() | 1)]); + assert!(FloatQuant::from_primitive_constant_secondary(f64_values.as_view(), 29).is_err()); + assert!(FloatQuant::primary_for_primitive(f64_values.as_view(), 29, 0).is_err()); + } + #[test] fn fastlanes_pair_decode_roundtrip_and_slice() -> VortexResult<()> { let original = PrimitiveArray::from_iter((0_u32..4097).map(|index| { diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index fa1c135d117..323fcf3044b 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -247,6 +247,14 @@ fn setup_int_mult_i32_array() -> PrimitiveArray { })) } +fn setup_int_mult_u64_array() -> PrimitiveArray { + PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + let primary = index.wrapping_mul(7_919) % 1_000_000_000; + let secondary = index % 10; + primary.wrapping_mul(10).wrapping_add(secondary) + })) +} + fn setup_ranged_u64_array() -> PrimitiveArray { PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { let cluster = index % 10; @@ -627,6 +635,62 @@ fn bench_ordered_float_decompress_f64(bencher: Bencher) { .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); } +#[divan::bench(name = "ordered_float_scalar_at_f64")] +fn bench_ordered_float_scalar_at_f64(bencher: Bencher) { + let encoded = OrderedFloat::from_primitive(setup_random_walk_array().as_view()) + .unwrap() + .into_array(); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + +#[divan::bench(name = "ordered_float_compress_f32")] +fn bench_ordered_float_compress_f32(bencher: Bencher) { + let float_array = setup_ordered_f32_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) + .with_inputs(|| &float_array) + .bench_refs(|array| OrderedFloat::from_primitive(array.as_view()).unwrap()); +} + +#[divan::bench(name = "ordered_float_decompress_f32")] +fn bench_ordered_float_decompress_f32(bencher: Bencher) { + let encoded = OrderedFloat::from_primitive(setup_ordered_f32_array().as_view()) + .unwrap() + .into_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "ordered_float_scalar_at_f32")] +fn bench_ordered_float_scalar_at_f32(bencher: Bencher) { + let encoded = OrderedFloat::from_primitive(setup_ordered_f32_array().as_view()) + .unwrap() + .into_array(); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + #[divan::bench(name = "int_mult_split_compress_i32")] fn bench_int_mult_split_compress_i32(bencher: Bencher) { let array = setup_int_mult_i32_array(); @@ -665,6 +729,44 @@ fn bench_int_mult_split_scalar_at_i32(bencher: Bencher) { .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); } +#[divan::bench(name = "int_mult_split_compress_u64")] +fn bench_int_mult_split_compress_u64(bencher: Bencher) { + let array = setup_int_mult_u64_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| &array) + .bench_refs(|array| IntMult::from_primitive(array.as_view(), 10).unwrap()); +} + +#[divan::bench(name = "int_mult_split_decompress_u64")] +fn bench_int_mult_split_decompress_u64(bencher: Bencher) { + let encoded = IntMult::from_primitive(setup_int_mult_u64_array().as_view(), 10) + .unwrap() + .into_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "int_mult_split_scalar_at_u64")] +fn bench_int_mult_split_scalar_at_u64(bencher: Bencher) { + let encoded = IntMult::from_primitive(setup_int_mult_u64_array().as_view(), 10) + .unwrap() + .into_array(); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + #[divan::bench(name = "decomposed_range_compress_u64")] fn bench_decomposed_range_compress_u64(bencher: Bencher) { let array = setup_ranged_u64_array(); @@ -1233,6 +1335,25 @@ fn bench_float_quant_split_decompress_f64(bencher: Bencher) { .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); } +#[divan::bench(name = "float_quant_split_scalar_at_f64")] +fn bench_float_quant_split_scalar_at_f64(bencher: Bencher) { + let encoded = + FloatQuant::from_primitive_constant_secondary(setup_widened_f32_array().as_view(), 29) + .unwrap() + .into_array(); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + #[divan::bench(name = "float_quant_materialized_tree_compress_f64")] fn bench_float_quant_tree_compress_f64(bencher: Bencher) { let float_array = setup_widened_f32_array(); @@ -1276,6 +1397,51 @@ fn bench_float_quant_tree_scalar_at_f64(bencher: Bencher) { .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); } +#[divan::bench(name = "float_quant_split_compress_f32")] +fn bench_float_quant_split_compress_f32(bencher: Bencher) { + let float_array = setup_quantized_f32_array(); + let k = analyze_float_quant(float_array.as_view()).unwrap().k; + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) + .with_inputs(|| &float_array) + .bench_refs(|array| { + FloatQuant::from_primitive_constant_secondary(array.as_view(), k).unwrap() + }); +} + +#[divan::bench(name = "float_quant_split_decompress_f32")] +fn bench_float_quant_split_decompress_f32(bencher: Bencher) { + let float_array = setup_quantized_f32_array(); + let k = analyze_float_quant(float_array.as_view()).unwrap().k; + let encoded = FloatQuant::from_primitive_constant_secondary(float_array.as_view(), k) + .unwrap() + .into_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "float_quant_split_scalar_at_f32")] +fn bench_float_quant_split_scalar_at_f32(bencher: Bencher) { + let float_array = setup_quantized_f32_array(); + let k = analyze_float_quant(float_array.as_view()).unwrap().k; + let encoded = FloatQuant::from_primitive_constant_secondary(float_array.as_view(), k) + .unwrap() + .into_array(); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + #[divan::bench(name = "float_quant_materialized_tree_compress_f32")] fn bench_float_quant_tree_compress_f32(bencher: Bencher) { let float_array = setup_quantized_f32_array(); @@ -1414,6 +1580,26 @@ fn bench_float_quant_nonzero_secondary_split_decompress_f64(bencher: Bencher) { .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); } +#[divan::bench(name = "float_quant_nonzero_secondary_split_scalar_at_f64")] +fn bench_float_quant_nonzero_secondary_split_scalar_at_f64(bencher: Bencher) { + let float_array = setup_nonzero_secondary_array(); + let k = analyze_float_quant(float_array.as_view()).unwrap().k; + let encoded = FloatQuant::from_primitive(float_array.as_view(), k) + .unwrap() + .into_array(); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + #[divan::bench(name = "float_quant_nonzero_secondary_tree_compress_f64")] fn bench_float_quant_nonzero_secondary_tree_compress_f64(bencher: Bencher) { let float_array = setup_nonzero_secondary_array(); From eb3cf3ec2837fe343b365931db85d6a427379a72 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 05:38:57 +0100 Subject: [PATCH 42/78] feat: complete numeric array type coverage Signed-off-by: Will Manning --- Cargo.lock | 1 + docs/plans/native-pcodec.md | 112 ++++- encodings/block-residual/Cargo.toml | 1 + .../src/block_residual_array.rs | 152 +++++- .../block-residual/src/ordered_float_array.rs | 147 +++++- encodings/float-quant/src/array.rs | 268 +++++++++- encodings/int-mult/src/array.rs | 15 +- .../src/schemes/float/float_quant.rs | 43 ++ .../schemes/float/scheme_selection_tests.rs | 36 ++ .../golden__default__int_monotone_jitter.snap | 6 +- ...lden__default__string_fsst_structured.snap | 18 - ...n__default__temporal_timestamp_micros.snap | 34 +- vortex/benches/single_encoding_throughput.rs | 474 ++++++++++++------ 13 files changed, 1058 insertions(+), 249 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bc4980686a9..58ea90d2576 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9757,6 +9757,7 @@ name = "vortex-block-residual" version = "0.1.0" dependencies = [ "fastlanes", + "rstest", "vortex-array", "vortex-buffer", "vortex-error", diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 86c06e9f05a..80b06ff7f45 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -69,13 +69,13 @@ The default candidate set includes BtrBlocks schemes for the first three arrays. IntMult does not have a BtrBlocks scheme yet. -`OrderedFloat(BlockResidual)` now supports `f32` and `f64` inputs. +`OrderedFloat(BlockResidual)` now supports `f16`, `f32`, and `f64` inputs. The float scheme applies `OrderedFloat` first, then `BlockResidual` to the unsigned child. The serialized tree uses `OrderedFloat(BlockResidual(...))` because the outer array restores the float dtype. -The FloatQuant candidate accepts native `f32` and `f64` inputs. +The FloatQuant candidate accepts native `f16`, `f32`, and `f64` inputs. Direct integer BlockResidual supports every integer type. The default selector accepts only 32-bit and 64-bit inputs. @@ -107,6 +107,35 @@ The current branch recovers a small share of Compact's aggregate numeric advanta The remaining work targets repeated Compact mechanisms with native, bounded-access trees. +## Array support and validation + +The array API and the Default selector use separate type policies. + +An array supports each natural logical type unless the transform has a structural restriction. + +The Default selector can exclude a supported type when measured costs do not justify selection. + +| Array | Supported logical types | Default policy | +| --- | --- | --- | +| OrderedFloat | `f16`, `f32`, `f64` | All float types remain eligible. | +| BlockResidual | `u8`, `u16`, `u32`, `u64`, `i8`, `i16`, `i32`, `i64` | Direct selection uses 32-bit and 64-bit integers. | +| FloatQuant | `f16`, `f32`, `f64` | All float types remain eligible. | +| IntMult | `u8`, `u16`, `u32`, `u64`, `i8`, `i16`, `i32`, `i64` | No Default scheme exists yet. | + +OrderedFloat validates the logical float type, unsigned child width, child nullability, child length, and empty metadata. + +FloatQuant validates the metadata version, split width, latent child types, child nullability, and child lengths. + +IntMult validates the metadata version, positive base, base range, matching child types, child nullability, and child lengths. + +BlockResidual validates every payload offset table before decode or scalar access. + +It also validates bases, bit widths, packed word counts, patch counts, patch order, patch bounds, and validity length. + +Bit-exact tests cover signed zero values, infinities, and NaN payloads for every float width. + +Round-trip tests cover every signed and unsigned integer width. + ## OrderedFloatArray `OrderedFloatArray` maps IEEE float bits to unsigned integers with the same order. @@ -115,7 +144,9 @@ The transform preserves every bit pattern. It also preserves nulls and signed ze The array stores one unsigned child. Empty metadata identifies the transform. -The array supports canonical decode, scalar access, slice reduction, serialization, and validation. +The array supports `f16`, `f32`, and `f64` values. + +It supports canonical decode, scalar access, slice reduction, serialization, and validation. ## BlockResidualArray @@ -164,7 +195,7 @@ The kernel unpacks both aligned children and reconstructs each float in one pass The array supports exact IEEE bit-pattern round trips, nulls, slices, scalar access, and serialization. -The automatic scheme accepts `f32` and `f64` inputs. +The automatic scheme accepts `f16`, `f32`, and `f64` inputs. Native `f32` and two-child selection meet the selected-column and rejected-column throughput limits. @@ -200,6 +231,20 @@ This path avoids a full materialized array of references. ## Selection policy +Fast rejection is a strong preference for schemes in normal Default recursion. + +A cheap rejection path reads a bounded sample and avoids child compression when the model does not fit. + +This path lets Default test more encodings with little throughput loss on rejected columns. + +Fast rejection is not a hard gate. + +A scheme with costly analysis can enter Default when corpus evidence shows larger size or decode gains. + +Specialized schemes remain useful when they bypass depth limits, recursive search, or an unfused common tree. + +Measure rejected-column cost separately from selected-column encode cost. + `FloatQuantScheme` uses the normal sample comparison against ALP, ALP-RD, dictionary, sparse, and RLE schemes. The sample builds the exact one-child or two-child fixed tree. @@ -288,10 +333,18 @@ These cases use canonical primitive children. They isolate each outer array tran | Array and input | Encode | Decode | Scalar access | | --- | ---: | ---: | ---: | +| OrderedFloat `f16` | 59.11 GB/s | 59.09 GB/s | 77 ns | | OrderedFloat `f32` | 52.38 GB/s | 60.47 GB/s | 82 ns | | OrderedFloat `f64` | 58.78 GB/s | 42.78 GB/s | 90 ns | -| IntMult base-ten `i32` | 2.83 GB/s | 30.43 GB/s | 125 ns | -| IntMult base-ten `u64` | 5.67 GB/s | 25.24 GB/s | 140 ns | +| IntMult base-ten `i8` | 0.72 GB/s | 29.98 GB/s | 125 ns | +| IntMult base-ten `i16` | 1.41 GB/s | 29.68 GB/s | 105 ns | +| IntMult base-ten `i32` | 2.82 GB/s | 29.84 GB/s | 100 ns | +| IntMult base-ten `i64` | 5.57 GB/s | 24.80 GB/s | 169 ns | +| IntMult base-ten `u8` | 0.70 GB/s | 29.63 GB/s | 125 ns | +| IntMult base-ten `u16` | 1.42 GB/s | 30.59 GB/s | 160 ns | +| IntMult base-ten `u32` | 2.84 GB/s | 30.51 GB/s | 136 ns | +| IntMult base-ten `u64` | 5.66 GB/s | 24.57 GB/s | 150 ns | +| FloatQuant zero-secondary `f16` | 6.84 GB/s | 31.24 GB/s | 80 ns | | FloatQuant zero-secondary `f32` | 11.66 GB/s | 31.17 GB/s | 83 ns | | FloatQuant zero-secondary `f64` | 19.02 GB/s | 25.15 GB/s | 83 ns | | FloatQuant one-bit-secondary `f64` | 9.26 GB/s | 8.14 GB/s | 167 ns | @@ -310,12 +363,18 @@ These cases include the current compressed children and fused decode paths. | Array tree and input | Encode | Decode | Scalar access | | --- | ---: | ---: | ---: | -| BlockResidual `i16` | 1.41 GB/s | 32.03 GB/s | 83 ns | -| BlockResidual `i32` | 2.80 GB/s | 33.07 GB/s | 57 ns | -| BlockResidual `u32` | 2.79 GB/s | 45.99 GB/s | 60 ns | -| BlockResidual `u64` | 5.18 GB/s | 37.72 GB/s | 83 ns | +| BlockResidual `i8` | 0.56 GB/s | 30.53 GB/s | 44 ns | +| BlockResidual `i16` | 1.40 GB/s | 31.31 GB/s | 39 ns | +| BlockResidual `i32` | 2.72 GB/s | 33.89 GB/s | 40 ns | +| BlockResidual `i64` | 4.93 GB/s | 35.88 GB/s | 42 ns | +| BlockResidual `u8` | 0.55 GB/s | 30.14 GB/s | 35 ns | +| BlockResidual `u16` | 1.31 GB/s | 32.55 GB/s | 61 ns | +| BlockResidual `u32` | 2.67 GB/s | 46.61 GB/s | 48 ns | +| BlockResidual `u64` | 4.97 GB/s | 35.56 GB/s | 40 ns | +| OrderedFloat with BlockResidual `f16` | 1.26 GB/s | 20.50 GB/s | 79 ns | | OrderedFloat with BlockResidual `f32` | 2.43 GB/s | 29.64 GB/s | 78 ns | | OrderedFloat with BlockResidual `f64` | 2.70 GB/s | 20.44 GB/s | 125 ns | +| FloatQuant with packed primary `f16` | 3.42 GB/s | 19.39 GB/s | 129 ns | | FloatQuant with packed primary `f32` | 7.59 GB/s | 18.21 GB/s | 125 ns | | FloatQuant with packed primary `f64` | 8.30 GB/s | 14.77 GB/s | 125 ns | | FloatQuant with two packed children `f64` | 4.21 GB/s | 13.05 GB/s | 209 ns | @@ -583,6 +642,31 @@ The prior default compressed at 394 MB/s. The difference is 1.2 percent. The Compact size is specific to this synthetic pattern. It does not establish a general real-float result. +### Native f16 coverage + +The audit added `f16` support to OrderedFloat, OrderedFloat with BlockResidual, FloatQuant, and FloatQuantScheme. + +The test corpus covers zero low bits, nonzero secondary bits, signed zero values, infinities, and NaN payloads. + +The quantized input contains two million `f16` values with four zero low mantissa bits. + +| Configuration | Bytes | Encode MB/s | Decode MB/s | +| --- | ---: | ---: | ---: | +| Prior default | 1,500,800 | 368.1 | 7,268 | +| Default with FloatQuant | 1,500,672 | 1,018 | 19,450 | + +The selected Default tree is `FloatQuant(FoR(BitPacked))`. + +It uses nearly the same space as the prior dictionary-like tree. + +Compression throughput increased by 2.77 times. Decode throughput increased by 2.68 times. + +On general rejected `f16` values, median compression throughput decreased by 1.8 percent. + +The `f16` BlockResidual tree decoded at 20.50 GB/s. Its generic inverse transform lacks a fused narrow path. + +Retain `f16` eligibility. Revisit its selector factors during final corpus calibration. + ### Direct integer comparison after decode specialization The direct benchmark compares two million block-local values against a whole-column FoR plus BitPacked tree. @@ -1502,6 +1586,12 @@ This round completed these steps: - Rejected three bounded IntMult remainder layouts on GloVe. - Rejected the dense-remainder IntMult layout on CMS Payments. - Rejected centered block residuals after a complete Euro subjectivity comparison. +- Audited constructor and deserialization validation for all four production arrays. +- Added round-trip tests for every supported integer and float type. +- Added single-encoding benchmarks for every supported integer and float type. +- Added `f16` support to OrderedFloat, FloatQuant, and both Default schemes. +- Verified zero-secondary and nonzero-secondary `f16` FloatQuant trees. +- Updated stale golden trees after the BlockResidual payload and selector changes. The Pco mode profile and the first quotient and remainder experiments are complete. @@ -1521,6 +1611,8 @@ Complete these next experiments in order: 8. Profile another direct narrow BlockResidual decode optimization for `i16`. 9. Classify more float columns where Pco beats ALP, ALP-RD, and the new schemes. 10. Validate rejected-candidate analysis cost after the candidate set stabilizes. +11. Prefer cheap rejection tests when they preserve the best candidate model. +12. Require stronger corpus evidence when a useful scheme needs costly analysis. Use packed bin codes and primitive bin starts unless evidence supports more complexity. diff --git a/encodings/block-residual/Cargo.toml b/encodings/block-residual/Cargo.toml index 42e7c06fbd5..e0bb52093d0 100644 --- a/encodings/block-residual/Cargo.toml +++ b/encodings/block-residual/Cargo.toml @@ -24,4 +24,5 @@ vortex-error = { workspace = true } vortex-session = { workspace = true } [dev-dependencies] +rstest = { workspace = true } vortex-array = { workspace = true, features = ["_test-harness"] } diff --git a/encodings/block-residual/src/block_residual_array.rs b/encodings/block-residual/src/block_residual_array.rs index 9982e728f0a..266b27bed74 100644 --- a/encodings/block-residual/src/block_residual_array.rs +++ b/encodings/block-residual/src/block_residual_array.rs @@ -612,6 +612,80 @@ impl BlockResidualData { && self.high_starts.len() == block_count + 1, "block residual offset tables have invalid lengths" ); + validate_offset_table( + &self.residual_starts, + block_count, + self.residual_words.len(), + "residual", + )?; + validate_offset_table( + &self.patch_starts, + block_count, + self.patch_positions.len(), + "patch", + )?; + validate_offset_table( + &self.high_starts, + block_count, + self.patch_highs.len(), + "patch high", + )?; + let logical_width = dtype.as_ptype().bit_width(); + let maximum = if logical_width == 64 { + u64::MAX + } else { + (1_u64 << logical_width) - 1 + }; + for block_index in 0..block_count { + vortex_ensure!( + self.bases[block_index] <= maximum, + "block residual base exceeds its logical type" + ); + let residual_width = self.residual_widths[block_index]; + let high_width = self.high_widths[block_index]; + vortex_ensure!( + usize::from(residual_width) <= logical_width + && usize::from(high_width) <= logical_width + && usize::from(residual_width) + usize::from(high_width) <= logical_width, + "block residual bit widths are invalid" + ); + let residual_range = payload_range( + &self.residual_starts, + block_index, + self.residual_words.len(), + "residual", + )?; + vortex_ensure!( + residual_range.len() == BLOCK_LEN * usize::from(residual_width) / 64, + "block residual word count is invalid" + ); + let patch_range = payload_range( + &self.patch_starts, + block_index, + self.patch_positions.len(), + "patch", + )?; + let high_range = payload_range( + &self.high_starts, + block_index, + self.patch_highs.len(), + "patch high", + )?; + let positions = &self.patch_positions[patch_range]; + validate_patch_header( + residual_width, + high_width, + positions.len(), + high_range.len(), + )?; + let block_start = block_index * BLOCK_LEN; + let block_len = (self.unsliced_len - block_start).min(BLOCK_LEN); + let mut previous = None; + for &position in positions { + validate_patch_position(block_len, previous, position)?; + previous = Some(position); + } + } if let Some(validity_len) = validity.maybe_len() { vortex_ensure!( validity_len == self.unsliced_len, @@ -1202,6 +1276,28 @@ fn validate_patch_position( Ok(()) } +fn validate_offset_table( + starts: &[u32], + block_count: usize, + payload_len: usize, + name: &str, +) -> VortexResult<()> { + vortex_ensure!( + starts.len() == block_count + 1, + "block residual {name} offsets have an invalid length" + ); + vortex_ensure!( + starts.first() == Some(&0) + && starts.last().copied().map(usize::try_from).transpose()? == Some(payload_len), + "block residual {name} offsets do not cover the payload" + ); + vortex_ensure!( + starts.windows(2).all(|window| window[0] <= window[1]), + "block residual {name} offsets are not ordered" + ); + Ok(()) +} + fn payload_range( starts: &[u32], block_index: usize, @@ -1294,12 +1390,15 @@ impl BlockResidualMetadata { #[cfg(test)] mod tests { + use rstest::rstest; use vortex_array::ArrayContext; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::array_session; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; + use vortex_array::dtype::NativePType; + use vortex_array::dtype::PType; use vortex_array::serde::SerializeOptions; use vortex_array::serde::SerializedArray; use vortex_array::validity::Validity; @@ -1310,6 +1409,7 @@ mod tests { use super::BlockResidual; use super::BlockResidualArrayExt; + use crate::BlockResidualCodec; #[test] fn roundtrip_and_scalar_access() -> VortexResult<()> { @@ -1360,27 +1460,51 @@ mod tests { Ok(()) } - #[test] - fn narrow_integer_roundtrip() -> VortexResult<()> { - let signed = PrimitiveArray::from_iter((0..2_050).map(|index| { - let value = (index * 7919) % 65_521; - value - 32_760 - })); - let unsigned = PrimitiveArray::from_iter((0..2_050).map(|index| { - let value = (index * 7907) % 65_521; - value as u16 - })); - let signed_encoded = BlockResidual::from_primitive(signed.as_view())?; - let unsigned_encoded = BlockResidual::from_primitive(unsigned.as_view())?; + #[rstest] + #[case(vec![0_u8, 1, u8::MAX])] + #[case(vec![0_u16, 1, u16::MAX])] + #[case(vec![0_u32, 1, u32::MAX])] + #[case(vec![0_u64, 1, u64::MAX])] + #[case(vec![i8::MIN, -1, 0, 1, i8::MAX])] + #[case(vec![i16::MIN, -1, 0, 1, i16::MAX])] + #[case(vec![i32::MIN, -1, 0, 1, i32::MAX])] + #[case(vec![i64::MIN, -1, 0, 1, i64::MAX])] + fn integer_ptype_roundtrip(#[case] values: Vec) -> VortexResult<()> + where + T: NativePType + Copy, + { + let primitive = PrimitiveArray::from_iter(values); + let encoded = BlockResidual::from_primitive(primitive.as_view())?; let session = array_session(); crate::initialize(&session); let mut ctx = session.create_execution_ctx(); - assert_arrays_eq!(signed_encoded, signed.into_array(), &mut ctx); - assert_arrays_eq!(unsigned_encoded, unsigned.into_array(), &mut ctx); + assert_arrays_eq!(encoded, primitive, &mut ctx); Ok(()) } + #[test] + fn rejects_components_outside_logical_width() -> VortexResult<()> { + let mut parts = BlockResidualCodec::encode(&[0_u64, 1, 2])?.into_parts()?; + parts.bases[0] = u64::from(u8::MAX) + 1; + assert!(BlockResidual::try_new(parts, Validity::NonNullable, PType::U8).is_err()); + Ok(()) + } + + #[test] + fn rejects_invalid_component_offsets() -> VortexResult<()> { + let mut parts = BlockResidualCodec::encode(&[0_u64, 1, 2])?.into_parts()?; + parts.residual_starts[0] = 1; + assert!(BlockResidual::try_new(parts, Validity::NonNullable, PType::U64).is_err()); + Ok(()) + } + + #[test] + fn rejects_non_integer_input() { + let primitive = PrimitiveArray::from_iter([0.0_f32, 1.0, 2.0]); + assert!(BlockResidual::from_primitive(primitive.as_view()).is_err()); + } + #[test] fn u32_direct_decode_roundtrip() -> VortexResult<()> { let primitive = PrimitiveArray::from_iter((0..2_050_u32).map(|index| { diff --git a/encodings/block-residual/src/ordered_float_array.rs b/encodings/block-residual/src/ordered_float_array.rs index f177e46c680..d1682c32b2d 100644 --- a/encodings/block-residual/src/ordered_float_array.rs +++ b/encodings/block-residual/src/ordered_float_array.rs @@ -26,6 +26,7 @@ use vortex_array::arrays::slice::SliceReduceAdaptor; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::PType; +use vortex_array::dtype::half::f16; use vortex_array::optimizer::rules::ParentRuleSet; use vortex_array::scalar::Scalar; use vortex_array::serde::ArrayChildren; @@ -101,8 +102,8 @@ impl VTable for OrderedFloat { ) -> VortexResult<()> { let ptype = PType::try_from(dtype)?; vortex_ensure!( - matches!(ptype, PType::F32 | PType::F64), - "OrderedFloatArray requires f32 or f64" + matches!(ptype, PType::F16 | PType::F32 | PType::F64), + "OrderedFloatArray requires f16, f32, or f64" ); let encoded = OrderedFloatSlotsView::from_slots(slots).encoded; let expected = DType::Primitive(ordered_ptype(ptype)?, dtype.nullability()); @@ -175,6 +176,7 @@ impl VTable for OrderedFloat { fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { let decoded = if let Some(block_residual) = array.encoded().as_typed::() { match array.dtype().as_ptype() { + PType::F16 => decode_primitive(array.as_view(), ctx)?, PType::F32 => decompress_ordered_f32(block_residual, ctx)?, PType::F64 => decompress_ordered_f64(block_residual, ctx)?, ptype => vortex_bail!("unsupported OrderedFloat ptype {ptype}"), @@ -205,6 +207,15 @@ impl OperationsVTable for OrderedFloat { return Ok(Scalar::null(array.dtype().clone())); } Ok(match array.dtype().as_ptype() { + PType::F16 => Scalar::primitive( + f16::from_bits(unordered_u16( + scalar + .as_primitive() + .typed_value::() + .vortex_expect("validated ordered float scalar"), + )), + array.dtype().nullability(), + ), PType::F32 => Scalar::primitive( f32::from_bits(unordered_u32( scalar @@ -259,6 +270,11 @@ impl OrderedFloat { encoded_nbytes, patch_count, } = match array.ptype() { + PType::F16 => { + BlockResidualCodec::estimate_transformed(array.as_slice::(), |value| { + u64::from(ordered_u16(value.to_bits())) + }) + } PType::F32 => { BlockResidualCodec::estimate_transformed(array.as_slice::(), |value| { u64::from(ordered_u32(value.to_bits())) @@ -269,7 +285,7 @@ impl OrderedFloat { ordered_u64(value.to_bits()) }) } - ptype => vortex_bail!("OrderedFloat requires f32 or f64, got {ptype}"), + ptype => vortex_bail!("OrderedFloat requires f16, f32, or f64, got {ptype}"), }; let validity_nbytes = validity_to_child(&array.validity()?, array.len()) .map(|validity| validity.nbytes()) @@ -280,8 +296,8 @@ impl OrderedFloat { /// Construct an ordered float array from an unsigned child. pub fn try_new(encoded: ArrayRef, float_ptype: PType) -> VortexResult { vortex_ensure!( - matches!(float_ptype, PType::F32 | PType::F64), - "OrderedFloat requires f32 or f64" + matches!(float_ptype, PType::F16 | PType::F32 | PType::F64), + "OrderedFloat requires f16, f32, or f64" ); let dtype = DType::Primitive(float_ptype, encoded.dtype().nullability()); let len = encoded.len(); @@ -295,6 +311,20 @@ impl OrderedFloat { pub fn from_primitive(array: ArrayView<'_, Primitive>) -> VortexResult { let validity = array.validity()?; match array.ptype() { + PType::F16 => Self::try_new( + PrimitiveArray::new( + Buffer::from( + array + .as_slice::() + .iter() + .map(|value| ordered_u16(value.to_bits())) + .collect::>(), + ), + validity, + ) + .into_array(), + PType::F16, + ), PType::F32 => Self::try_new( PrimitiveArray::new( Buffer::from( @@ -323,7 +353,7 @@ impl OrderedFloat { .into_array(), PType::F64, ), - ptype => vortex_bail!("OrderedFloat requires f32 or f64, got {ptype}"), + ptype => vortex_bail!("OrderedFloat requires f16, f32, or f64, got {ptype}"), } } } @@ -334,6 +364,16 @@ fn decode_primitive( ) -> VortexResult { let encoded = array.encoded().clone().execute::(ctx)?; Ok(match array.dtype().as_ptype() { + PType::F16 => PrimitiveArray::new( + Buffer::from( + encoded + .as_slice::() + .iter() + .map(|&value| f16::from_bits(unordered_u16(value))) + .collect::>(), + ), + encoded.validity()?, + ), PType::F32 => PrimitiveArray::new( Buffer::from( encoded @@ -360,9 +400,26 @@ fn decode_primitive( fn ordered_ptype(ptype: PType) -> VortexResult { match ptype { + PType::F16 => Ok(PType::U16), PType::F32 => Ok(PType::U32), PType::F64 => Ok(PType::U64), - _ => vortex_bail!("OrderedFloat requires f32 or f64, got {ptype}"), + _ => vortex_bail!("OrderedFloat requires f16, f32, or f64, got {ptype}"), + } +} + +fn ordered_u16(bits: u16) -> u16 { + if bits & (1_u16 << 15) == 0 { + bits ^ (1_u16 << 15) + } else { + !bits + } +} + +fn unordered_u16(value: u16) -> u16 { + if value & (1_u16 << 15) == 0 { + !value + } else { + value ^ (1_u16 << 15) } } @@ -408,6 +465,7 @@ mod tests { use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::dtype::PType; + use vortex_array::dtype::half::f16; use vortex_array::serde::SerializeOptions; use vortex_array::serde::SerializedArray; use vortex_array::validity::Validity; @@ -421,6 +479,41 @@ mod tests { use crate::BlockResidual; use crate::BlockResidualArrayExt; + #[test] + fn roundtrip_f16_special_values() -> VortexResult<()> { + let values = [ + f16::from_bits(0xfc00), + f16::from_f32(-1.0), + f16::NEG_ZERO, + f16::ZERO, + f16::from_f32(1.0), + f16::INFINITY, + f16::from_bits(0x7e42), + ]; + let primitive = PrimitiveArray::from_iter(values); + let estimate = OrderedFloat::estimate_block_residual(primitive.as_view())?; + let ordered = OrderedFloat::from_primitive(primitive.as_view())?; + let residuals = BlockResidual::from_primitive(ordered.encoded().as_::())?; + assert_eq!(estimate.nbytes(), residuals.nbytes()); + assert_eq!(estimate.patch_count(), residuals.patch_positions().len()); + let encoded = OrderedFloat::try_new(residuals.into_array(), PType::F16)?; + let session = array_session(); + crate::initialize(&session); + let decoded = encoded + .into_array() + .execute::(&mut session.create_execution_ctx())?; + + assert_eq!( + decoded + .as_slice::() + .iter() + .map(|value| value.to_bits()) + .collect::>(), + values.map(f16::to_bits) + ); + Ok(()) + } + #[test] fn roundtrip_special_values() -> VortexResult<()> { let primitive = PrimitiveArray::from_iter([ @@ -444,6 +537,46 @@ mod tests { Ok(()) } + #[test] + fn roundtrip_f32_special_values() -> VortexResult<()> { + let values = [ + f32::NEG_INFINITY, + -1.0, + -0.0, + 0.0, + 1.0, + f32::INFINITY, + f32::from_bits(0x7fc0_0042), + ]; + let primitive = PrimitiveArray::from_iter(values); + let encoded = OrderedFloat::from_primitive(primitive.as_view())?; + let session = array_session(); + crate::initialize(&session); + let decoded = encoded + .into_array() + .execute::(&mut session.create_execution_ctx())?; + + assert_eq!( + decoded + .as_slice::() + .iter() + .map(|value| value.to_bits()) + .collect::>(), + values.map(f32::to_bits) + ); + Ok(()) + } + + #[test] + fn rejects_invalid_logical_and_child_types() { + let u32_child = PrimitiveArray::from_iter([0_u32, 1, 2]).into_array(); + assert!(OrderedFloat::try_new(u32_child.clone(), PType::F64).is_err()); + assert!(OrderedFloat::try_new(u32_child, PType::U32).is_err()); + + let u64_child = PrimitiveArray::from_iter([0_u64, 1, 2]).into_array(); + assert!(OrderedFloat::try_new(u64_child, PType::F32).is_err()); + } + #[test] fn roundtrip_f32_block_residual() -> VortexResult<()> { let values = (0..2_050) diff --git a/encodings/float-quant/src/array.rs b/encodings/float-quant/src/array.rs index 9580dccc0de..908cecc2f6a 100644 --- a/encodings/float-quant/src/array.rs +++ b/encodings/float-quant/src/array.rs @@ -25,6 +25,7 @@ use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability::NonNullable; use vortex_array::dtype::PType; +use vortex_array::dtype::half::f16; use vortex_array::scalar::Scalar; use vortex_array::serde::ArrayChildren; use vortex_array::vtable::OperationsVTable; @@ -243,6 +244,29 @@ impl OperationsVTable for FloatQuant { } let k = array.data().k; Ok(match PType::try_from(array.dtype())? { + PType::F16 => Scalar::primitive( + join_f16( + primary + .as_primitive() + .typed_value::() + .vortex_expect("validated primary scalar"), + array + .secondary() + .map(|secondary| { + secondary + .execute_scalar(index, ctx)? + .as_primitive() + .typed_value::() + .ok_or_else(|| { + vortex_error::vortex_err!("validated secondary scalar is null") + }) + }) + .transpose()? + .unwrap_or(0), + k, + ), + array.dtype().nullability(), + ), PType::F32 => Scalar::primitive( join_f32( primary @@ -329,6 +353,18 @@ impl FloatQuant { pub fn from_primitive(array: ArrayView<'_, Primitive>, k: u8) -> VortexResult { let validity = array.validity()?; match array.ptype() { + PType::F16 => { + let (primary, secondary) = split_f16(array.as_slice::(), k)?; + Self::try_new( + PrimitiveArray::new(Buffer::from(primary), validity).into_array(), + Some( + PrimitiveArray::new(Buffer::from(secondary), NonNullable.into()) + .into_array(), + ), + PType::F16, + k, + ) + } PType::F32 => { let (primary, secondary) = split_f32(array.as_slice::(), k)?; Self::try_new( @@ -353,7 +389,7 @@ impl FloatQuant { k, ) } - ptype => vortex_bail!("FloatQuant requires f32 or f64, got {ptype}"), + ptype => vortex_bail!("FloatQuant requires f16, f32, or f64, got {ptype}"), } } @@ -364,6 +400,15 @@ impl FloatQuant { ) -> VortexResult { let validity = array.validity()?; match array.ptype() { + PType::F16 => { + let primary = split_primary_f16(array.as_slice::(), k)?; + Self::try_new( + PrimitiveArray::new(Buffer::from(primary), validity).into_array(), + None, + PType::F16, + k, + ) + } PType::F32 => { let primary = split_primary_f32(array.as_slice::(), k)?; Self::try_new( @@ -382,7 +427,7 @@ impl FloatQuant { k, ) } - ptype => vortex_bail!("FloatQuant requires f32 or f64, got {ptype}"), + ptype => vortex_bail!("FloatQuant requires f16, f32, or f64, got {ptype}"), } } @@ -394,6 +439,11 @@ impl FloatQuant { ) -> VortexResult { let validity = array.validity()?; match array.ptype() { + PType::F16 => { + let primary_min = u16::try_from(primary_min)?; + let primary = split_primary_for_f16(array.as_slice::(), k, primary_min)?; + Ok(PrimitiveArray::new(Buffer::from(primary), validity)) + } PType::F32 => { let primary_min = u32::try_from(primary_min)?; let primary = split_primary_for_f32(array.as_slice::(), k, primary_min)?; @@ -403,7 +453,7 @@ impl FloatQuant { let primary = split_primary_for_f64(array.as_slice::(), k, primary_min)?; Ok(PrimitiveArray::new(Buffer::from(primary), validity)) } - ptype => vortex_bail!("FloatQuant requires f32 or f64, got {ptype}"), + ptype => vortex_bail!("FloatQuant requires f16, f32, or f64, got {ptype}"), } } } @@ -429,6 +479,7 @@ pub fn estimate_k(array: ArrayView<'_, Primitive>) -> Option { /// Analyze a canonical float array for a FloatQuant split. pub fn analyze_float_quant(array: ArrayView<'_, Primitive>) -> Option { match array.ptype() { + PType::F16 => analyze_f16(array.as_slice::()), PType::F32 => analyze_f32(array.as_slice::()), PType::F64 => analyze_f64(array.as_slice::()), _ => None, @@ -482,6 +533,26 @@ fn analyze_bits( }) } +fn analyze_f16(values: &[f16]) -> Option { + let mut minimum = u16::MAX; + let mut maximum = u16::MIN; + let mut low_bits_or = 0_u16; + for value in values { + let bits = value.to_bits(); + let ordered = ordered_u16(bits); + minimum = minimum.min(ordered); + maximum = maximum.max(ordered); + low_bits_or |= bits; + } + analyze_bits( + u64::from(low_bits_or), + 10, + values.len(), + u64::from(minimum), + u64::from(maximum), + ) +} + fn analyze_f32(values: &[f32]) -> Option { let mut minimum = u32::MAX; let mut maximum = u32::MIN; @@ -518,17 +589,27 @@ fn analyze_f64(values: &[f64]) -> Option { fn latent_ptype(ptype: PType) -> VortexResult { match ptype { + PType::F16 => Ok(PType::U16), PType::F32 => Ok(PType::U32), PType::F64 => Ok(PType::U64), - _ => vortex_bail!("FloatQuant requires f32 or f64, got {ptype}"), + _ => vortex_bail!("FloatQuant requires f16, f32, or f64, got {ptype}"), } } fn precision_bits(ptype: PType) -> VortexResult { match ptype { + PType::F16 => Ok(10), PType::F32 => Ok(23), PType::F64 => Ok(52), - _ => vortex_bail!("FloatQuant requires f32 or f64, got {ptype}"), + _ => vortex_bail!("FloatQuant requires f16, f32, or f64, got {ptype}"), + } +} + +fn ordered_u16(bits: u16) -> u16 { + if bits & (1_u16 << 15) == 0 { + bits ^ (1_u16 << 15) + } else { + !bits } } @@ -548,6 +629,25 @@ fn ordered_u64(bits: u64) -> u64 { } } +fn split_f16(values: &[f16], k: u8) -> VortexResult<(Vec, Vec)> { + vortex_ensure!(k > 0 && k <= 10, "FloatQuant f16 k must be in 1..=10"); + let low_mask = (1_u16 << k) - 1; + let mut primary = Vec::with_capacity(values.len()); + let mut secondary = Vec::with_capacity(values.len()); + for &value in values { + let bits = value.to_bits(); + let ordered = ordered_u16(bits); + primary.push(ordered >> k); + let low = ordered & low_mask; + secondary.push(if bits & (1_u16 << 15) == 0 { + low + } else { + low_mask - low + }); + } + Ok((primary, secondary)) +} + fn split_f32(values: &[f32], k: u8) -> VortexResult<(Vec, Vec)> { vortex_ensure!(k > 0 && k <= 23, "FloatQuant f32 k must be in 1..=23"); let low_mask = (1_u32 << k) - 1; @@ -599,6 +699,19 @@ fn split_primary_f32(values: &[f32], k: u8) -> VortexResult> { .collect()) } +fn split_primary_f16(values: &[f16], k: u8) -> VortexResult> { + vortex_ensure!(k > 0 && k <= 10, "FloatQuant f16 k must be in 1..=10"); + let low_mask = (1_u16 << k) - 1; + vortex_ensure!( + values.iter().all(|value| value.to_bits() & low_mask == 0), + "FloatQuant constant secondary requires zero low bits" + ); + Ok(values + .iter() + .map(|value| ordered_u16(value.to_bits()) >> k) + .collect()) +} + fn split_primary_f64(values: &[f64], k: u8) -> VortexResult> { vortex_ensure!(k > 0 && k <= 52, "FloatQuant f64 k must be in 1..=52"); let low_mask = (1_u64 << k) - 1; @@ -625,6 +738,19 @@ fn split_primary_for_f32(values: &[f32], k: u8, primary_min: u32) -> VortexResul .collect()) } +fn split_primary_for_f16(values: &[f16], k: u8, primary_min: u16) -> VortexResult> { + vortex_ensure!(k > 0 && k <= 10, "FloatQuant f16 k must be in 1..=10"); + let low_mask = (1_u16 << k) - 1; + vortex_ensure!( + values.iter().all(|value| value.to_bits() & low_mask == 0), + "FloatQuant constant secondary requires zero low bits" + ); + Ok(values + .iter() + .map(|value| (ordered_u16(value.to_bits()) >> k) - primary_min) + .collect()) +} + fn split_primary_for_f64(values: &[f64], k: u8, primary_min: u64) -> VortexResult> { vortex_ensure!(k > 0 && k <= 52, "FloatQuant f64 k must be in 1..=52"); let low_mask = (1_u64 << k) - 1; @@ -638,6 +764,23 @@ fn split_primary_for_f64(values: &[f64], k: u8, primary_min: u64) -> VortexResul .collect()) } +fn join_f16(primary: u16, secondary: u16, k: u8) -> f16 { + let low_mask = (1_u16 << k) - 1; + let sign_cutoff = (1_u16 << 15) >> k; + let low = if primary >= sign_cutoff { + secondary + } else { + low_mask.wrapping_sub(secondary) + }; + let ordered = (primary << k).wrapping_add(low); + let bits = if ordered & (1_u16 << 15) == 0 { + !ordered + } else { + ordered ^ (1_u16 << 15) + }; + f16::from_bits(bits) +} + fn join_f32(primary: u32, secondary: u32, k: u8) -> f32 { let low_mask = (1_u32 << k) - 1; let sign_cutoff = (1_u32 << 31) >> k; @@ -685,6 +828,19 @@ fn join_zero_f32(primary: u32, k: u8) -> f32 { f32::from_bits(bits) } +fn join_zero_f16(primary: u16, k: u8) -> f16 { + let low_mask = (1_u16 << k) - 1; + let sign_cutoff = (1_u16 << 15) >> k; + let low = if primary >= sign_cutoff { 0 } else { low_mask }; + let ordered = (primary << k).wrapping_add(low); + let bits = if ordered & (1_u16 << 15) == 0 { + !ordered + } else { + ordered ^ (1_u16 << 15) + }; + f16::from_bits(bits) +} + fn join_zero_f64(primary: u64, k: u8) -> f64 { let low_mask = (1_u64 << k) - 1; let sign_cutoff = (1_u64 << 63) >> k; @@ -711,6 +867,13 @@ fn decode( let k = array.data().k; let Some(secondary) = array.secondary() else { return Ok(match PType::try_from(array.dtype())? { + PType::F16 => PrimitiveArray::new( + primary + .into_buffer::() + .map_each_in_place(|primary| join_zero_f16(primary, k)) + .freeze(), + validity, + ), PType::F32 => PrimitiveArray::new( primary .into_buffer::() @@ -730,6 +893,19 @@ fn decode( }; let secondary = secondary.clone().execute::(ctx)?; Ok(match PType::try_from(array.dtype())? { + PType::F16 => { + let secondary_values = secondary.as_slice::(); + let mut index = 0; + let values = primary + .into_buffer::() + .map_each_in_place(|primary| { + let value = join_f16(primary, secondary_values[index], k); + index += 1; + value + }) + .freeze(); + PrimitiveArray::new(values, validity) + } PType::F32 => { let secondary_values = secondary.as_slice::(); let mut index = 0; @@ -783,6 +959,19 @@ fn decode_fastlanes_pair(array: ArrayView<'_, FloatQuant>) -> VortexResult { + let reference = primary_for + .reference_scalar() + .as_primitive() + .typed_value::() + .vortex_expect("validated f16 primary reference"); + PrimitiveArray::new( + unpack_pair_map::(primary, secondary, |primary, secondary| { + join_f16(primary.wrapping_add(reference), secondary, k) + })?, + validity, + ) + } PType::F32 => { let reference = primary_for .reference_scalar() @@ -839,6 +1028,34 @@ mod tests { session }); + #[test] + fn f16_bit_patterns_roundtrip() -> VortexResult<()> { + let values = [ + f16::from_bits(0xfc00), + f16::from_f32(-1.5), + f16::NEG_ZERO, + f16::ZERO, + f16::from_f32(1.5), + f16::INFINITY, + f16::from_bits(0x7e34), + f16::from_bits(0xfe56), + ]; + let array = PrimitiveArray::from_iter(values); + let encoded = FloatQuant::from_primitive(array.as_view(), 5)?; + let decoded = encoded + .into_array() + .execute::(&mut SESSION.create_execution_ctx())?; + assert_eq!( + decoded + .as_slice::() + .iter() + .map(|value| value.to_bits()) + .collect::>(), + values.map(f16::to_bits) + ); + Ok(()) + } + #[test] fn fixed_tree_analysis_accounts_for_secondary_width() -> VortexResult<()> { let values = PrimitiveArray::from_iter((0_u32..4096).map(|index| { @@ -892,6 +1109,47 @@ mod tests { Ok(()) } + #[test] + fn f32_bit_patterns_roundtrip() -> VortexResult<()> { + let values = [ + f32::NEG_INFINITY, + -1.5, + -0.0, + 0.0, + 1.5, + f32::INFINITY, + f32::from_bits(0x7fc0_1234), + f32::from_bits(0xffc0_5678), + ]; + let array = PrimitiveArray::from_iter(values); + let encoded = FloatQuant::from_primitive(array.as_view(), 8)?; + let decoded = encoded + .into_array() + .execute::(&mut SESSION.create_execution_ctx())?; + assert_eq!( + decoded + .as_slice::() + .iter() + .map(|value| value.to_bits()) + .collect::>(), + values.map(f32::to_bits) + ); + Ok(()) + } + + #[test] + fn rejects_invalid_split_shapes() { + let primary = PrimitiveArray::from_iter([0_u32, 1, 2]).into_array(); + assert!(FloatQuant::try_new(primary.clone(), None, PType::F32, 0).is_err()); + assert!(FloatQuant::try_new(primary.clone(), None, PType::F32, 24).is_err()); + assert!(FloatQuant::try_new(primary.clone(), None, PType::U32, 8).is_err()); + + let wrong_ptype = PrimitiveArray::from_iter([0_u64, 1, 2]).into_array(); + assert!(FloatQuant::try_new(primary.clone(), Some(wrong_ptype), PType::F32, 8).is_err()); + let nullable = PrimitiveArray::from_option_iter([Some(0_u32), None, Some(2)]).into_array(); + assert!(FloatQuant::try_new(primary, Some(nullable), PType::F32, 8).is_err()); + } + #[test] fn nullable_slice_and_scalar_access() -> VortexResult<()> { let array = PrimitiveArray::new( diff --git a/encodings/int-mult/src/array.rs b/encodings/int-mult/src/array.rs index 5c91d7fb114..4239a576c24 100644 --- a/encodings/int-mult/src/array.rs +++ b/encodings/int-mult/src/array.rs @@ -574,7 +574,7 @@ mod tests { #[case(vec![i16::MIN, -8, -7, -1, 0, 1, 7, 8, i16::MAX], 7)] #[case(vec![i32::MIN, -8, -7, -1, 0, 1, 7, 8, i32::MAX], 7)] #[case(vec![i64::MIN, -8, -7, -1, 0, 1, 7, 8, i64::MAX], 7)] - fn round_trip_unsigned(#[case] values: Vec, #[case] base: u64) -> VortexResult<()> + fn round_trip_integer_ptype(#[case] values: Vec, #[case] base: u64) -> VortexResult<()> where T: NativePType + Copy, { @@ -598,6 +598,19 @@ mod tests { Ok(()) } + #[test] + fn rejects_invalid_child_shapes() { + let primary = PrimitiveArray::from_iter([1_u32, 2, 3]).into_array(); + let wrong_ptype = PrimitiveArray::from_iter([1_u16, 2, 3]).into_array(); + assert!(IntMult::try_new(primary.clone(), wrong_ptype, 10).is_err()); + + let wrong_length = PrimitiveArray::from_iter([1_u32, 2]).into_array(); + assert!(IntMult::try_new(primary.clone(), wrong_length, 10).is_err()); + + let nullable = PrimitiveArray::from_option_iter([Some(1_u32), None, Some(3)]).into_array(); + assert!(IntMult::try_new(primary, nullable, 10).is_err()); + } + #[test] fn base_one_adds_generic_children() -> VortexResult<()> { let primary = PrimitiveArray::from_iter([100_u32, 200, 300]).into_array(); diff --git a/vortex-btrblocks/src/schemes/float/float_quant.rs b/vortex-btrblocks/src/schemes/float/float_quant.rs index 4079703bfd6..792141be8fd 100644 --- a/vortex-btrblocks/src/schemes/float/float_quant.rs +++ b/vortex-btrblocks/src/schemes/float/float_quant.rs @@ -12,6 +12,7 @@ use vortex_array::VTable; use vortex_array::arrays::Primitive; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::PType; +use vortex_array::dtype::half::f16; use vortex_array::scalar::Scalar; use vortex_array::validity::Validity; use vortex_compressor::scheme::CompressionEstimate; @@ -100,6 +101,39 @@ fn encode_float_quant( analysis: FloatQuantAnalysis, ) -> VortexResult { let (primary_packed, secondary_packed, latent_ptype, reference) = match primitive.ptype() { + PType::F16 => { + let primary_min = u16::try_from(analysis.primary_min)?; + let values = primitive.as_slice::(); + let (primary, secondary) = if analysis.secondary_bit_width == 0 { + ( + bitpack_primitive_map(values, analysis.primary_bit_width, |value| { + (ordered_u16(value.to_bits()) >> analysis.k) - primary_min + }), + None, + ) + } else { + let low_mask = (1_u16 << analysis.k) - 1; + let (primary, secondary) = bitpack_primitive_map_pair( + values, + analysis.primary_bit_width, + analysis.secondary_bit_width, + |value| { + let bits = value.to_bits(); + ( + (ordered_u16(bits) >> analysis.k) - primary_min, + bits & low_mask, + ) + }, + ); + (primary, Some(secondary)) + }; + ( + primary.into_byte_buffer(), + secondary.map(|packed| packed.into_byte_buffer()), + PType::U16, + Scalar::from(primary_min), + ) + } PType::F32 => { let primary_min = u32::try_from(analysis.primary_min)?; let values = primitive.as_slice::(); @@ -201,6 +235,15 @@ fn encode_float_quant( .into_array()) } +#[inline] +fn ordered_u16(bits: u16) -> u16 { + if bits & (1_u16 << 15) == 0 { + bits ^ (1_u16 << 15) + } else { + !bits + } +} + #[inline] fn ordered_u32(bits: u32) -> u32 { if bits & (1_u32 << 31) == 0 { diff --git a/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs index 90196d1acf7..6150763d8cf 100644 --- a/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs @@ -16,6 +16,7 @@ use vortex_array::assert_arrays_eq; use vortex_array::builders::ArrayBuilder; use vortex_array::builders::PrimitiveBuilder; use vortex_array::dtype::Nullability; +use vortex_array::dtype::half::f16; use vortex_array::validity::Validity; use vortex_block_residual::BlockResidual; use vortex_block_residual::OrderedFloat; @@ -30,13 +31,48 @@ use vortex_session::VortexSession; use crate::BtrBlocksCompressor; #[cfg(feature = "unstable_encodings")] use crate::BtrBlocksCompressorBuilder; +use crate::CascadingCompressor; #[cfg(feature = "unstable_encodings")] use crate::SchemeExt; +use crate::schemes::float::FloatQuantScheme; #[cfg(feature = "unstable_encodings")] use crate::schemes::integer::DeltaScheme; static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); +#[test] +fn test_quantized_f16_uses_float_quant() -> VortexResult<()> { + let values = (0_u16..16_384) + .map(|index| f16::from_bits(0x3c00 | (index.wrapping_mul(7_919) & 0x03f0))) + .collect::>(); + let array = PrimitiveArray::from_iter(values).into_array(); + let mut ctx = SESSION.create_execution_ctx(); + let compressed = BtrBlocksCompressor::default().compress(&array, &mut ctx)?; + + assert!(compressed.is::()); + assert_arrays_eq!(compressed, array, &mut ctx); + Ok(()) +} + +#[test] +fn test_f16_secondary_uses_float_quant() -> VortexResult<()> { + let values = (0_u16..16_384) + .map(|index| { + let high_mantissa = index.wrapping_mul(7_919) & 0x03f0; + f16::from_bits(0x3c00 | high_mantissa | (index & 1)) + }) + .collect::>(); + let array = PrimitiveArray::from_iter(values).into_array(); + let compressor = CascadingCompressor::new(vec![&FloatQuantScheme]); + let mut ctx = SESSION.create_execution_ctx(); + let compressed = compressor.compress(&array, &mut ctx)?; + + assert!(compressed.is::()); + assert!(compressed.as_::().secondary().is_some()); + assert_arrays_eq!(compressed, array, &mut ctx); + Ok(()) +} + #[test] fn test_constant_compressed() -> VortexResult<()> { let values: Vec = vec![42.5; 100]; diff --git a/vortex-btrblocks/tests/snapshots/golden__default__int_monotone_jitter.snap b/vortex-btrblocks/tests/snapshots/golden__default__int_monotone_jitter.snap index 4800aae289d..a29fba7b5a4 100644 --- a/vortex-btrblocks/tests/snapshots/golden__default__int_monotone_jitter.snap +++ b/vortex-btrblocks/tests/snapshots/golden__default__int_monotone_jitter.snap @@ -3,7 +3,5 @@ source: vortex-btrblocks/tests/golden.rs expression: rendered --- input: u64, len=16384, nbytes=131072 -root: fastlanes.for(u64, len=16384) nbytes=49152 - metadata: reference: 1700000001036u64 - encoded: fastlanes.bitpacked(u64, len=16384) nbytes=49152 - metadata: bit_width: 24, offset: 0 +root: vortex.block_residual(u64, len=16384) nbytes=41324 + metadata: blocks: 16, slice: 0..16384 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__string_fsst_structured.snap b/vortex-btrblocks/tests/snapshots/golden__default__string_fsst_structured.snap index 85e3422cb1b..ddbd6548e12 100644 --- a/vortex-btrblocks/tests/snapshots/golden__default__string_fsst_structured.snap +++ b/vortex-btrblocks/tests/snapshots/golden__default__string_fsst_structured.snap @@ -13,21 +13,3 @@ root: vortex.fsst(utf8, len=16384) nbytes=141400 metadata: scalar: 23u8 codes_offsets: vortex.block_residual(u32, len=16385) nbytes=27010 metadata: blocks: 17, slice: 0..16385 - bases: vortex.primitive(u64, len=17) nbytes=136 - metadata: ptype: u64 - residual_widths: vortex.primitive(u8, len=17) nbytes=17 - metadata: ptype: u8 - high_widths: vortex.primitive(u8, len=17) nbytes=17 - metadata: ptype: u8 - residual_starts: vortex.primitive(u32, len=18) nbytes=72 - metadata: ptype: u32 - patch_starts: vortex.primitive(u32, len=18) nbytes=72 - metadata: ptype: u32 - high_starts: vortex.primitive(u32, len=18) nbytes=72 - metadata: ptype: u32 - residual_words: vortex.primitive(u64, len=3328) nbytes=26624 - metadata: ptype: u64 - patch_positions: vortex.primitive(u16, len=0) nbytes=0 - metadata: ptype: u16 - patch_highs: vortex.primitive(u8, len=0) nbytes=0 - metadata: ptype: u8 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__temporal_timestamp_micros.snap b/vortex-btrblocks/tests/snapshots/golden__default__temporal_timestamp_micros.snap index caa6e265676..bc08ade5b96 100644 --- a/vortex-btrblocks/tests/snapshots/golden__default__temporal_timestamp_micros.snap +++ b/vortex-btrblocks/tests/snapshots/golden__default__temporal_timestamp_micros.snap @@ -3,37 +3,7 @@ source: vortex-btrblocks/tests/golden.rs expression: rendered --- input: vortex.timestamp[µs, tz=UTC](i64), len=16384, nbytes=131072 -root: vortex.datetimeparts(vortex.timestamp[µs, tz=UTC](i64), len=16384) nbytes=60983 +root: vortex.ext(vortex.timestamp[µs, tz=UTC](i64), len=16384) nbytes=59756 metadata: - days: vortex.dict(u16, len=16384) nbytes=10 - metadata: all_values_referenced: true - codes: vortex.runend(u8, len=16384) nbytes=6 - metadata: offset: 0 - ends: vortex.primitive(u16, len=2) nbytes=4 - metadata: ptype: u16 - values: vortex.primitive(u8, len=2) nbytes=2 - metadata: ptype: u8 - values: vortex.primitive(u16, len=2) nbytes=4 - metadata: ptype: u16 - seconds: vortex.block_residual(u32, len=16384) nbytes=20013 + storage: vortex.block_residual(i64, len=16384) nbytes=59756 metadata: blocks: 16, slice: 0..16384 - bases: vortex.primitive(u64, len=16) nbytes=128 - metadata: ptype: u64 - residual_widths: vortex.primitive(u8, len=16) nbytes=16 - metadata: ptype: u8 - high_widths: vortex.primitive(u8, len=16) nbytes=16 - metadata: ptype: u8 - residual_starts: vortex.primitive(u32, len=17) nbytes=68 - metadata: ptype: u32 - patch_starts: vortex.primitive(u32, len=17) nbytes=68 - metadata: ptype: u32 - high_starts: vortex.primitive(u32, len=17) nbytes=68 - metadata: ptype: u32 - residual_words: vortex.primitive(u64, len=2432) nbytes=19456 - metadata: ptype: u64 - patch_positions: vortex.primitive(u16, len=47) nbytes=94 - metadata: ptype: u16 - patch_highs: vortex.primitive(u8, len=99) nbytes=99 - metadata: ptype: u8 - subseconds: fastlanes.bitpacked(u32, len=16384) nbytes=40960 - metadata: bit_width: 20, offset: 0 diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index 323fcf3044b..17f9b1c4371 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -28,7 +28,9 @@ use vortex::array::builders::dict::dict_encode; use vortex::array::builtins::ArrayBuiltins; use vortex::array::dtype::Nullability; use vortex::compressor::BtrBlocksCompressorBuilder; +use vortex::dtype::NativePType; use vortex::dtype::PType; +use vortex::dtype::half::f16; use vortex::encodings::alp::RDEncoder; use vortex::encodings::alp::RDEncoderExt; use vortex::encodings::alp::alp_encode; @@ -141,6 +143,19 @@ fn setup_quantized_f32_array() -> PrimitiveArray { })) } +fn setup_quantized_f16_array() -> PrimitiveArray { + PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + let mantissa = (index.wrapping_mul(7_919) as u16) & 0x03f0; + f16::from_bits(0x3c00 | mantissa) + })) +} + +fn setup_general_f16_array() -> PrimitiveArray { + PrimitiveArray::from_iter( + (0..FLOAT_CODEC_NUM_VALUES).map(|index| f16::from_bits(index.wrapping_mul(7_919) as u16)), + ) +} + fn setup_general_f32_array() -> PrimitiveArray { PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { let mantissa = index.wrapping_mul(7_919) as u32 & 0x007f_ffff; @@ -231,6 +246,14 @@ fn setup_ordered_f32_array() -> PrimitiveArray { })) } +fn setup_ordered_f16_array() -> PrimitiveArray { + PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + let block = (index / 1_024) % 8; + let residual = index.wrapping_mul(7_919) % 64; + f16::from_bits(0x3c00 + (block * 64 + residual) as u16) + })) +} + fn setup_block_local_i16_array() -> PrimitiveArray { PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { let block = (index / 1_024) % 128; @@ -239,6 +262,36 @@ fn setup_block_local_i16_array() -> PrimitiveArray { })) } +fn setup_block_local_integer_array() -> PrimitiveArray { + match T::PTYPE { + PType::U8 => PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + let block = (index / 1_024) % 16; + let residual = index.wrapping_mul(2_654_435_761) % 16; + (block * 16 + residual) as u8 + })), + PType::U16 => PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + let block = (index / 1_024) % 256; + let residual = index.wrapping_mul(2_654_435_761) % 256; + (block * 256 + residual) as u16 + })), + PType::U32 => setup_block_local_u32_array(), + PType::U64 => setup_block_local_u64_array(), + PType::I8 => PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + let block = (index / 1_024) % 16; + let residual = index.wrapping_mul(2_654_435_761) % 16; + ((block * 16 + residual) as i16 - 128) as i8 + })), + PType::I16 => setup_block_local_i16_array(), + PType::I32 => setup_block_local_i32_array(), + PType::I64 => PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + let block = index / 1_024; + let residual = index.wrapping_mul(2_654_435_761) % 1_024; + (block as i64 - 1_000) * 1_000_000_000_000 + residual as i64 + })), + ptype => unreachable!("unsupported block residual benchmark type {ptype}"), + } +} + fn setup_int_mult_i32_array() -> PrimitiveArray { PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { let primary = (index as i32).wrapping_mul(7_919) % 1_000_000; @@ -255,6 +308,41 @@ fn setup_int_mult_u64_array() -> PrimitiveArray { })) } +fn setup_int_mult_integer_array() -> PrimitiveArray { + match T::PTYPE { + PType::U8 => PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + let primary = index.wrapping_mul(79) % 20; + (primary * 10 + index % 10) as u8 + })), + PType::U16 => PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + let primary = index.wrapping_mul(7_919) % 6_000; + (primary * 10 + index % 10) as u16 + })), + PType::U32 => PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + let primary = index.wrapping_mul(7_919) % 100_000_000; + (primary * 10 + index % 10) as u32 + })), + PType::U64 => setup_int_mult_u64_array(), + PType::I8 => PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + let primary = (index.wrapping_mul(79) % 20) as i16 - 10; + let secondary = (index % 10) as i16 - 5; + (primary * 10 + secondary) as i8 + })), + PType::I16 => PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + let primary = (index.wrapping_mul(7_919) % 3_000) as i32 - 1_500; + let secondary = (index % 10) as i32 - 5; + (primary * 10 + secondary) as i16 + })), + PType::I32 => setup_int_mult_i32_array(), + PType::I64 => PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + let primary = (index.wrapping_mul(7_919) % 1_000_000_000) as i64; + let secondary = (index % 10) as i64 - 5; + primary * 10 + secondary + })), + ptype => unreachable!("unsupported IntMult benchmark type {ptype}"), + } +} + fn setup_ranged_u64_array() -> PrimitiveArray { PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { let cluster = index % 10; @@ -364,6 +452,17 @@ fn encode_float_quant_nonzero_secondary_tree(array: &PrimitiveArray) -> vortex:: .into_array() } +fn encode_float_quant_scheme_tree(array: &PrimitiveArray) -> vortex::array::ArrayRef { + BtrBlocksCompressorBuilder::empty() + .with_new_scheme(&FloatQuantScheme) + .build() + .compress( + &array.clone().into_array(), + &mut SESSION.create_execution_ctx(), + ) + .unwrap() +} + fn encode_prior_default(array: &PrimitiveArray) -> vortex::array::ArrayRef { BtrBlocksCompressorBuilder::default() .exclude_schemes([ @@ -653,29 +752,29 @@ fn bench_ordered_float_scalar_at_f64(bencher: Bencher) { .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); } -#[divan::bench(name = "ordered_float_compress_f32")] -fn bench_ordered_float_compress_f32(bencher: Bencher) { - let float_array = setup_ordered_f32_array(); +#[divan::bench(name = "ordered_float_compress_f16")] +fn bench_ordered_float_compress_f16(bencher: Bencher) { + let float_array = setup_ordered_f16_array(); - with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 2) .with_inputs(|| &float_array) .bench_refs(|array| OrderedFloat::from_primitive(array.as_view()).unwrap()); } -#[divan::bench(name = "ordered_float_decompress_f32")] -fn bench_ordered_float_decompress_f32(bencher: Bencher) { - let encoded = OrderedFloat::from_primitive(setup_ordered_f32_array().as_view()) +#[divan::bench(name = "ordered_float_decompress_f16")] +fn bench_ordered_float_decompress_f16(bencher: Bencher) { + let encoded = OrderedFloat::from_primitive(setup_ordered_f16_array().as_view()) .unwrap() .into_array(); - with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 2) .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); } -#[divan::bench(name = "ordered_float_scalar_at_f32")] -fn bench_ordered_float_scalar_at_f32(bencher: Bencher) { - let encoded = OrderedFloat::from_primitive(setup_ordered_f32_array().as_view()) +#[divan::bench(name = "ordered_float_scalar_at_f16")] +fn bench_ordered_float_scalar_at_f16(bencher: Bencher) { + let encoded = OrderedFloat::from_primitive(setup_ordered_f16_array().as_view()) .unwrap() .into_array(); let next_index = AtomicUsize::new(0); @@ -691,18 +790,18 @@ fn bench_ordered_float_scalar_at_f32(bencher: Bencher) { .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); } -#[divan::bench(name = "int_mult_split_compress_i32")] -fn bench_int_mult_split_compress_i32(bencher: Bencher) { - let array = setup_int_mult_i32_array(); +#[divan::bench(name = "ordered_float_compress_f32")] +fn bench_ordered_float_compress_f32(bencher: Bencher) { + let float_array = setup_ordered_f32_array(); with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) - .with_inputs(|| &array) - .bench_refs(|array| IntMult::from_primitive(array.as_view(), 10).unwrap()); + .with_inputs(|| &float_array) + .bench_refs(|array| OrderedFloat::from_primitive(array.as_view()).unwrap()); } -#[divan::bench(name = "int_mult_split_decompress_i32")] -fn bench_int_mult_split_decompress_i32(bencher: Bencher) { - let encoded = IntMult::from_primitive(setup_int_mult_i32_array().as_view(), 10) +#[divan::bench(name = "ordered_float_decompress_f32")] +fn bench_ordered_float_decompress_f32(bencher: Bencher) { + let encoded = OrderedFloat::from_primitive(setup_ordered_f32_array().as_view()) .unwrap() .into_array(); @@ -711,9 +810,9 @@ fn bench_int_mult_split_decompress_i32(bencher: Bencher) { .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); } -#[divan::bench(name = "int_mult_split_scalar_at_i32")] -fn bench_int_mult_split_scalar_at_i32(bencher: Bencher) { - let encoded = IntMult::from_primitive(setup_int_mult_i32_array().as_view(), 10) +#[divan::bench(name = "ordered_float_scalar_at_f32")] +fn bench_ordered_float_scalar_at_f32(bencher: Bencher) { + let encoded = OrderedFloat::from_primitive(setup_ordered_f32_array().as_view()) .unwrap() .into_array(); let next_index = AtomicUsize::new(0); @@ -729,29 +828,31 @@ fn bench_int_mult_split_scalar_at_i32(bencher: Bencher) { .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); } -#[divan::bench(name = "int_mult_split_compress_u64")] -fn bench_int_mult_split_compress_u64(bencher: Bencher) { - let array = setup_int_mult_u64_array(); +#[divan::bench(types = [u8, u16, u32, u64, i8, i16, i32, i64])] +fn int_mult_split_compress(bencher: Bencher) { + let array = setup_int_mult_integer_array::(); + let byte_width = u64::try_from(T::PTYPE.byte_width()).unwrap(); - with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * byte_width) .with_inputs(|| &array) .bench_refs(|array| IntMult::from_primitive(array.as_view(), 10).unwrap()); } -#[divan::bench(name = "int_mult_split_decompress_u64")] -fn bench_int_mult_split_decompress_u64(bencher: Bencher) { - let encoded = IntMult::from_primitive(setup_int_mult_u64_array().as_view(), 10) +#[divan::bench(types = [u8, u16, u32, u64, i8, i16, i32, i64])] +fn int_mult_split_decompress(bencher: Bencher) { + let encoded = IntMult::from_primitive(setup_int_mult_integer_array::().as_view(), 10) .unwrap() .into_array(); + let byte_width = u64::try_from(T::PTYPE.byte_width()).unwrap(); - with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * byte_width) .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); } -#[divan::bench(name = "int_mult_split_scalar_at_u64")] -fn bench_int_mult_split_scalar_at_u64(bencher: Bencher) { - let encoded = IntMult::from_primitive(setup_int_mult_u64_array().as_view(), 10) +#[divan::bench(types = [u8, u16, u32, u64, i8, i16, i32, i64])] +fn int_mult_split_scalar_at(bencher: Bencher) { + let encoded = IntMult::from_primitive(setup_int_mult_integer_array::().as_view(), 10) .unwrap() .into_array(); let next_index = AtomicUsize::new(0); @@ -824,29 +925,31 @@ fn bench_block_residual_decompress_u64(bencher: Bencher) { .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); } -#[divan::bench(name = "block_local_block_residual_compress_u64")] -fn bench_block_local_block_residual_compress_u64(bencher: Bencher) { - let array = setup_block_local_u64_array(); +#[divan::bench(types = [u8, u16, u32, u64, i8, i16, i32, i64])] +fn block_local_block_residual_compress(bencher: Bencher) { + let array = setup_block_local_integer_array::(); + let byte_width = u64::try_from(T::PTYPE.byte_width()).unwrap(); - with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * byte_width) .with_inputs(|| &array) .bench_refs(|array| BlockResidual::from_primitive(array.as_view()).unwrap()); } -#[divan::bench(name = "block_local_block_residual_decompress_u64")] -fn bench_block_local_block_residual_decompress_u64(bencher: Bencher) { - let encoded = BlockResidual::from_primitive(setup_block_local_u64_array().as_view()) +#[divan::bench(types = [u8, u16, u32, u64, i8, i16, i32, i64])] +fn block_local_block_residual_decompress(bencher: Bencher) { + let encoded = BlockResidual::from_primitive(setup_block_local_integer_array::().as_view()) .unwrap() .into_array(); + let byte_width = u64::try_from(T::PTYPE.byte_width()).unwrap(); - with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * byte_width) .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); } -#[divan::bench(name = "block_local_block_residual_scalar_at_u64")] -fn bench_block_local_block_residual_scalar_at_u64(bencher: Bencher) { - let encoded = BlockResidual::from_primitive(setup_block_local_u64_array().as_view()) +#[divan::bench(types = [u8, u16, u32, u64, i8, i16, i32, i64])] +fn block_local_block_residual_scalar_at(bencher: Bencher) { + let encoded = BlockResidual::from_primitive(setup_block_local_integer_array::().as_view()) .unwrap() .into_array(); let next_index = AtomicUsize::new(0); @@ -896,44 +999,6 @@ fn bench_block_local_for_bitpacked_scalar_at_u64(bencher: Bencher) { .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); } -#[divan::bench(name = "block_local_block_residual_compress_u32")] -fn bench_block_local_block_residual_compress_u32(bencher: Bencher) { - let array = setup_block_local_u32_array(); - - with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) - .with_inputs(|| &array) - .bench_refs(|array| BlockResidual::from_primitive(array.as_view()).unwrap()); -} - -#[divan::bench(name = "block_local_block_residual_decompress_u32")] -fn bench_block_local_block_residual_decompress_u32(bencher: Bencher) { - let encoded = BlockResidual::from_primitive(setup_block_local_u32_array().as_view()) - .unwrap() - .into_array(); - - with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) - .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) - .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); -} - -#[divan::bench(name = "block_local_block_residual_scalar_at_u32")] -fn bench_block_local_block_residual_scalar_at_u32(bencher: Bencher) { - let encoded = BlockResidual::from_primitive(setup_block_local_u32_array().as_view()) - .unwrap() - .into_array(); - let next_index = AtomicUsize::new(0); - - bencher - .with_inputs(|| { - ( - &encoded, - SESSION.create_execution_ctx(), - next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), - ) - }) - .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); -} - #[divan::bench(name = "block_local_for_bitpacked_compress_u32")] fn bench_block_local_for_bitpacked_compress_u32(bencher: Bencher) { let array = setup_block_local_u32_array(); @@ -968,44 +1033,6 @@ fn bench_block_local_for_bitpacked_scalar_at_u32(bencher: Bencher) { .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); } -#[divan::bench(name = "block_local_block_residual_compress_i32")] -fn bench_block_local_block_residual_compress_i32(bencher: Bencher) { - let array = setup_block_local_i32_array(); - - with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) - .with_inputs(|| &array) - .bench_refs(|array| BlockResidual::from_primitive(array.as_view()).unwrap()); -} - -#[divan::bench(name = "block_local_block_residual_decompress_i32")] -fn bench_block_local_block_residual_decompress_i32(bencher: Bencher) { - let encoded = BlockResidual::from_primitive(setup_block_local_i32_array().as_view()) - .unwrap() - .into_array(); - - with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 4) - .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) - .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); -} - -#[divan::bench(name = "block_local_block_residual_scalar_at_i32")] -fn bench_block_local_block_residual_scalar_at_i32(bencher: Bencher) { - let encoded = BlockResidual::from_primitive(setup_block_local_i32_array().as_view()) - .unwrap() - .into_array(); - let next_index = AtomicUsize::new(0); - - bencher - .with_inputs(|| { - ( - &encoded, - SESSION.create_execution_ctx(), - next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), - ) - }) - .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); -} - #[divan::bench(name = "block_local_for_bitpacked_compress_i32")] fn bench_block_local_for_bitpacked_compress_i32(bencher: Bencher) { let array = setup_block_local_i32_array(); @@ -1105,44 +1132,6 @@ fn patch_density_default_decompress_u32(bencher: Bencher, stride: u64) { .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); } -#[divan::bench(name = "block_local_block_residual_compress_i16")] -fn bench_block_local_block_residual_compress_i16(bencher: Bencher) { - let array = setup_block_local_i16_array(); - - with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 2) - .with_inputs(|| &array) - .bench_refs(|array| BlockResidual::from_primitive(array.as_view()).unwrap()); -} - -#[divan::bench(name = "block_local_block_residual_decompress_i16")] -fn bench_block_local_block_residual_decompress_i16(bencher: Bencher) { - let encoded = BlockResidual::from_primitive(setup_block_local_i16_array().as_view()) - .unwrap() - .into_array(); - - with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 2) - .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) - .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); -} - -#[divan::bench(name = "block_local_block_residual_scalar_at_i16")] -fn bench_block_local_block_residual_scalar_at_i16(bencher: Bencher) { - let encoded = BlockResidual::from_primitive(setup_block_local_i16_array().as_view()) - .unwrap() - .into_array(); - let next_index = AtomicUsize::new(0); - - bencher - .with_inputs(|| { - ( - &encoded, - SESSION.create_execution_ctx(), - next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), - ) - }) - .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); -} - #[divan::bench(name = "block_local_for_bitpacked_compress_i16")] fn bench_block_local_for_bitpacked_compress_i16(bencher: Bencher) { let array = setup_block_local_i16_array(); @@ -1261,6 +1250,40 @@ fn bench_ordered_block_residual_prior_default_scalar_at_f64(bencher: Bencher) { .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); } +#[divan::bench(name = "ordered_block_residual_compress_f16")] +fn bench_ordered_block_residual_compress_f16(bencher: Bencher) { + let float_array = setup_ordered_f16_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 2) + .with_inputs(|| &float_array) + .bench_refs(|array| encode_ordered_block_residual(array)); +} + +#[divan::bench(name = "ordered_block_residual_decompress_f16")] +fn bench_ordered_block_residual_decompress_f16(bencher: Bencher) { + let encoded = encode_ordered_block_residual(&setup_ordered_f16_array()); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 2) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "ordered_block_residual_scalar_at_f16")] +fn bench_ordered_block_residual_scalar_at_f16(bencher: Bencher) { + let encoded = encode_ordered_block_residual(&setup_ordered_f16_array()); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + #[divan::bench(name = "ordered_block_residual_compress_f32")] fn bench_ordered_block_residual_compress_f32(bencher: Bencher) { let float_array = setup_ordered_f32_array(); @@ -1312,6 +1335,141 @@ fn bench_ordered_block_residual_scheme_compress_f64(bencher: Bencher) { .bench_values(|(array, mut ctx)| compressor.compress(&array, &mut ctx).unwrap()); } +#[divan::bench(name = "float_quant_split_compress_f16")] +fn bench_float_quant_split_compress_f16(bencher: Bencher) { + let float_array = setup_quantized_f16_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 2) + .with_inputs(|| &float_array) + .bench_refs(|array| { + FloatQuant::from_primitive_constant_secondary(array.as_view(), 4).unwrap() + }); +} + +#[divan::bench(name = "float_quant_split_decompress_f16")] +fn bench_float_quant_split_decompress_f16(bencher: Bencher) { + let encoded = + FloatQuant::from_primitive_constant_secondary(setup_quantized_f16_array().as_view(), 4) + .unwrap() + .into_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 2) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "float_quant_split_scalar_at_f16")] +fn bench_float_quant_split_scalar_at_f16(bencher: Bencher) { + let encoded = + FloatQuant::from_primitive_constant_secondary(setup_quantized_f16_array().as_view(), 4) + .unwrap() + .into_array(); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + +#[divan::bench(name = "float_quant_scheme_compress_f16")] +fn bench_float_quant_scheme_compress_f16(bencher: Bencher) { + let compressor = BtrBlocksCompressorBuilder::empty() + .with_new_scheme(&FloatQuantScheme) + .build(); + bench_compressor(bencher, setup_quantized_f16_array(), compressor); +} + +#[divan::bench(name = "float_quant_tree_decompress_f16")] +fn bench_float_quant_tree_decompress_f16(bencher: Bencher) { + let encoded = encode_float_quant_scheme_tree(&setup_quantized_f16_array()); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 2) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "float_quant_tree_scalar_at_f16")] +fn bench_float_quant_tree_scalar_at_f16(bencher: Bencher) { + let encoded = encode_float_quant_scheme_tree(&setup_quantized_f16_array()); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + +#[divan::bench(name = "float_quant_prior_default_compress_f16")] +fn bench_float_quant_prior_default_compress_f16(bencher: Bencher) { + let compressor = BtrBlocksCompressorBuilder::default() + .exclude_schemes([ + FloatQuantScheme.id(), + OrderedBlockResidualScheme.id(), + BlockResidualScheme.id(), + ]) + .build(); + bench_compressor(bencher, setup_quantized_f16_array(), compressor); +} + +#[divan::bench(name = "float_quant_proposed_default_compress_f16")] +fn bench_float_quant_proposed_default_compress_f16(bencher: Bencher) { + bench_compressor( + bencher, + setup_quantized_f16_array(), + BtrBlocksCompressorBuilder::default().build(), + ); +} + +#[divan::bench(name = "float_quant_prior_default_decompress_f16")] +fn bench_float_quant_prior_default_decompress_f16(bencher: Bencher) { + let encoded = encode_prior_default(&setup_quantized_f16_array()); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 2) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "float_quant_proposed_default_decompress_f16")] +fn bench_float_quant_proposed_default_decompress_f16(bencher: Bencher) { + let encoded = encode_proposed_default(&setup_quantized_f16_array()); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 2) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(name = "float_quant_prior_default_reject_f16")] +fn bench_float_quant_prior_default_reject_f16(bencher: Bencher) { + let compressor = BtrBlocksCompressorBuilder::default() + .exclude_schemes([ + FloatQuantScheme.id(), + OrderedBlockResidualScheme.id(), + BlockResidualScheme.id(), + ]) + .build(); + bench_compressor(bencher, setup_general_f16_array(), compressor); +} + +#[divan::bench(name = "float_quant_proposed_default_reject_f16")] +fn bench_float_quant_proposed_default_reject_f16(bencher: Bencher) { + bench_compressor( + bencher, + setup_general_f16_array(), + BtrBlocksCompressorBuilder::default().build(), + ); +} + #[divan::bench(name = "float_quant_split_compress_f64")] fn bench_float_quant_split_compress_f64(bencher: Bencher) { let float_array = setup_widened_f32_array(); From 9ed1d7170956dd2b9dc247ec1ce046a5b450fed3 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 05:50:55 +0100 Subject: [PATCH 43/78] bench: test decomposed fixed-bin trees Signed-off-by: Will Manning --- Cargo.lock | 1 + docs/plans/native-pcodec.md | 60 ++- vortex-btrblocks/Cargo.toml | 1 + .../examples/float_compressor_bench.rs | 442 +++++++++++++++++- 4 files changed, 490 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 58ea90d2576..e69e0fbd6cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9794,6 +9794,7 @@ dependencies = [ "vortex-fastlanes", "vortex-float-quant", "vortex-fsst", + "vortex-int-mult", "vortex-mask", "vortex-onpair", "vortex-pco", diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 80b06ff7f45..c983600c6b9 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -1282,6 +1282,45 @@ They do not establish a Default win because the range benchmark lacks incumbent The next experiment must compare complete trees on the real Pco gap columns. +### Decomposed fixed-bin real-data results + +The benchmark now builds `IntMult(base=1, Dict(BitPacked(codes), starts), offsets)` on four real float columns. + +It tests `BlockResidual` and `FoR(BitPacked)` as the offset child. + +The GloVe benchmark reader flattens numeric `List`, `LargeList`, and `FixedSizeList` columns. + +| Input | Default bytes | Compact bytes | Decomposed BR bytes | Decomposed FoR bytes | Default decode MB/s | Decomposed BR decode MB/s | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Euro subjectivity | 14,234,326 | 6,860,457 | 13,000,391 | 14,006,288 | 16,915 | 4,217 | +| Food volume | 7,529,728 | 5,368,140 | 7,223,283 | 10,504,832 | 24,104 | 10,060 | +| GloVe embeddings | 6,274,834 | 5,287,510 | 7,105,298 | 8,160,260 | 17,388 | 7,125 | +| CMS payment | 11,064,308 | 9,288,800 | 11,073,385 | 12,735,216 | 22,542 | 6,570 | + +The `BlockResidual` form saves 8.7 percent on Euro and 4.1 percent on Food against Default. + +It grows GloVe by 13.2 percent and matches Default size on CMS payment. + +Its decode throughput falls by 58 to 75 percent against Default on all four columns. + +The `FoR(BitPacked)` form encodes and decodes faster than the `BlockResidual` form. + +Its global offset range loses more size on every input. + +The offset stream interleaves values from bins with different widths. + +A generic offset child then pays for the widest offsets in each local block. + +The fused prototype avoids this cost because each value uses the width for its selected bin. + +The tested composition therefore does not pass the Default evidence bar. + +A competitive fixed-bin design requires a reusable primitive for code-dependent widths or grouped offsets with rank data. + +Earlier grouped and checkpoint prototypes did not pass the Default decode gate. + +Fixed bins remain paused while the pure IntMult experiment proceeds. + ### Nullable and Delta-heavy fixed-bin cases The optimized nullable prototype stores only valid values in the range stream. @@ -1601,18 +1640,15 @@ The nonzero-secondary FloatQuant implementation and focused validation are compl Complete these next experiments in order: -1. Measure the decomposed fixed-bin tree on CMS, Food, Euro2016, GloVe, and other no-Delta Pco wins. -2. Compare complete tree size and throughput against the displaced Default tree and Compact Pco. -3. Compare `BlockResidual` and `FoR(BitPacked)` offset children. Use recursive compression as an oracle for other trees. -4. Prototype pure IntMult on the exact GloVe and CMS ALP children. -5. Compress both IntMult children independently through normal integer schemes. -6. Test BlockResidual as a general compression candidate for patch positions. -7. Add a specialized range scheme only after a complete tree passes the column gates. -8. Profile another direct narrow BlockResidual decode optimization for `i16`. -9. Classify more float columns where Pco beats ALP, ALP-RD, and the new schemes. -10. Validate rejected-candidate analysis cost after the candidate set stabilizes. -11. Prefer cheap rejection tests when they preserve the best candidate model. -12. Require stronger corpus evidence when a useful scheme needs costly analysis. +1. Prototype pure IntMult on the exact GloVe and CMS ALP children. +2. Compress both IntMult children independently through normal integer schemes. +3. Test BlockResidual as a general compression candidate for patch positions. +4. Add a specialized range scheme only after a complete tree passes the column gates. +5. Profile another direct narrow BlockResidual decode optimization for `i16`. +6. Classify more float columns where Pco beats ALP, ALP-RD, and the new schemes. +7. Validate rejected-candidate analysis cost after the candidate set stabilizes. +8. Prefer cheap rejection tests when they preserve the best candidate model. +9. Require stronger corpus evidence when a useful scheme needs costly analysis. Use packed bin codes and primitive bin starts unless evidence supports more complexity. diff --git a/vortex-btrblocks/Cargo.toml b/vortex-btrblocks/Cargo.toml index 308cac2c98d..c9f281300a6 100644 --- a/vortex-btrblocks/Cargo.toml +++ b/vortex-btrblocks/Cargo.toml @@ -55,6 +55,7 @@ tpchgen = { workspace = true } tpchgen-arrow = { workspace = true } vortex-array = { workspace = true, features = ["_test-harness"] } vortex-arrow = { workspace = true } +vortex-int-mult = { workspace = true } vortex-mask = { workspace = true } vortex-session = { workspace = true } diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index c9212fd2e0d..8b2d5cd36db 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -14,6 +14,9 @@ use std::time::Duration; use std::time::Instant; use arrow_array::ArrayRef as ArrowArrayRef; +use arrow_array::FixedSizeListArray; +use arrow_array::LargeListArray; +use arrow_array::ListArray as ArrowListArray; use arrow_schema::DataType; use arrow_select::concat::concat; use parquet::arrow::ProjectionMask; @@ -40,6 +43,7 @@ use vortex_array::RecursiveCanonical; use vortex_array::VortexSessionExecute; use vortex_array::array_session; use vortex_array::arrays::ChunkedArray; +use vortex_array::arrays::DictArray; use vortex_array::arrays::ListArray; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; @@ -50,6 +54,7 @@ use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::PType; use vortex_array::extension::datetime::TimeUnit; +use vortex_array::match_each_integer_ptype; use vortex_array::validity::Validity; use vortex_arrow::ArrowSessionExt; use vortex_block_residual::BlockResidual; @@ -65,6 +70,13 @@ use vortex_btrblocks::schemes::integer::BlockResidualScheme; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_err; +use vortex_fastlanes::BitPacked; +use vortex_fastlanes::FoR; +use vortex_fastlanes::FoRArrayExt; +use vortex_fastlanes::FoRArraySlotsExt; +use vortex_fastlanes::bitpack_compress::bitpack_encode_unchecked; +use vortex_int_mult::IntMult; +use vortex_range_packed::RangeDecomposition; use vortex_range_packed::RangePacked; use vortex_range_packed::RangePackedCodec; use vortex_session::VortexSession; @@ -371,8 +383,14 @@ fn read_parquet_numeric( .iter() .enumerate() .filter_map(|(index, field)| { + let data_type = match field.data_type() { + DataType::List(field) + | DataType::LargeList(field) + | DataType::FixedSizeList(field, _) => field.data_type(), + data_type => data_type, + }; matches!( - field.data_type(), + data_type, DataType::Int16 | DataType::Int32 | DataType::Int64 @@ -424,6 +442,45 @@ fn read_parquet_numeric( .collect::>(); let combined = concat(&chunk_refs) .map_err(|error| vortex_err!("cannot concatenate {}: {error}", field.name()))?; + let (combined, field, name) = match field.data_type() { + DataType::FixedSizeList(value_field, _) => { + let list = combined + .as_any() + .downcast_ref::() + .ok_or_else(|| vortex_err!("{} is not a fixed-size list", field.name()))?; + let values = list.values(); + ( + values.slice(0, values.len().min(row_count)), + Arc::clone(value_field), + format!("{}.values", field.name()), + ) + } + DataType::List(value_field) => { + let list = combined + .as_any() + .downcast_ref::() + .ok_or_else(|| vortex_err!("{} is not a list", field.name()))?; + let values = list.values(); + ( + values.slice(0, values.len().min(row_count)), + Arc::clone(value_field), + format!("{}.values", field.name()), + ) + } + DataType::LargeList(value_field) => { + let list = combined + .as_any() + .downcast_ref::() + .ok_or_else(|| vortex_err!("{} is not a large list", field.name()))?; + let values = list.values(); + ( + values.slice(0, values.len().min(row_count)), + Arc::clone(value_field), + format!("{}.values", field.name()), + ) + } + _ => (combined, Arc::clone(field), field.name().to_string()), + }; let array = session.arrow().from_arrow_array(combined, field.as_ref())?; let array = if matches!(field.data_type(), DataType::Timestamp(_, _)) { array.cast(DType::Primitive(PType::I64, array.dtype().nullability()))? @@ -431,7 +488,7 @@ fn read_parquet_numeric( array }; let primitive = array.execute::(&mut session.create_execution_ctx())?; - Ok(column(field.name(), primitive)) + Ok(column(name, primitive)) }) .collect() } @@ -1234,6 +1291,283 @@ fn codec_decode_iterations() -> VortexResult { .map(|iterations| iterations.unwrap_or(20)) } +#[derive(Clone, Copy)] +enum FixedBinOffsetTree { + BlockResidual, + FoRBitPacked, +} + +impl FixedBinOffsetTree { + fn label(self) -> &'static str { + match self { + Self::BlockResidual => "block-residual", + Self::FoRBitPacked => "for-bitpacked", + } + } +} + +fn decomposed_fixed_bin_integer_tree( + primitive: vortex_array::ArrayView<'_, Primitive>, + offset_tree: FixedBinOffsetTree, + session: &VortexSession, +) -> VortexResult> { + vortex_ensure!(primitive.ptype().is_int(), "fixed bins require integers"); + let primitive = fill_integer_nulls_with_first_valid( + primitive.into_owned(), + &mut session.create_execution_ctx(), + )?; + let decomposition = RangeDecomposition::encode(&ordered_integer_values(&primitive)?)?; + if decomposition.bin_starts().is_empty() + || !offsets_fit_ptype(decomposition.offsets(), primitive.ptype()) + { + return Ok(None); + } + + let codes = PrimitiveArray::new(decomposition.codes().to_vec(), primitive.validity()?); + // SAFETY: The decomposition computes the exact code width. + let codes = + unsafe { bitpack_encode_unchecked(codes, decomposition.code_width()) }?.into_array(); + let (starts, offsets) = range_components(&decomposition, primitive.ptype())?; + let references = DictArray::try_new(codes, starts)?.into_array(); + let offsets = match offset_tree { + FixedBinOffsetTree::BlockResidual => { + BlockResidual::from_primitive(offsets.as_view())?.into_array() + } + FixedBinOffsetTree::FoRBitPacked => { + let minimum = decomposition.offsets().iter().copied().min().unwrap_or(0); + let maximum = decomposition.offsets().iter().copied().max().unwrap_or(0); + let bit_width = u8::try_from(u64::BITS - (maximum - minimum).leading_zeros())?; + let mut ctx = session.create_execution_ctx(); + let encoded = FoR::encode(offsets, &mut ctx)?; + let packed = BitPacked::encode(encoded.encoded(), bit_width, &mut ctx)?; + FoR::try_new(packed.into_array(), encoded.reference_scalar().clone())?.into_array() + } + }; + Ok(Some(IntMult::try_new(references, offsets, 1)?.into_array())) +} + +fn fill_integer_nulls_with_first_valid( + primitive: PrimitiveArray, + ctx: &mut vortex_array::ExecutionCtx, +) -> VortexResult { + let validity = primitive.validity()?; + let mask = validity.execute_mask(primitive.len(), ctx)?; + if mask.all_true() { + return Ok(primitive); + } + let first_valid = mask.first(); + Ok(match_each_integer_ptype!(primitive.ptype(), |T| { + let values = primitive.as_slice::(); + let fill = first_valid.map_or_else(T::default, |index| values[index]); + PrimitiveArray::new::( + values + .iter() + .zip(mask.iter()) + .map(|(&value, valid)| if valid { value } else { fill }) + .collect::>(), + validity, + ) + })) +} + +fn ordered_integer_values(primitive: &PrimitiveArray) -> VortexResult> { + Ok(match primitive.ptype() { + PType::U8 => primitive + .as_slice::() + .iter() + .map(|&value| u64::from(value)) + .collect(), + PType::U16 => primitive + .as_slice::() + .iter() + .map(|&value| u64::from(value)) + .collect(), + PType::U32 => primitive + .as_slice::() + .iter() + .map(|&value| u64::from(value)) + .collect(), + PType::U64 => primitive.as_slice::().to_vec(), + PType::I8 => primitive + .as_slice::() + .iter() + .map(|&value| u64::from((value as u8) ^ (1_u8 << 7))) + .collect(), + PType::I16 => primitive + .as_slice::() + .iter() + .map(|&value| u64::from((value as u16) ^ (1_u16 << 15))) + .collect(), + PType::I32 => primitive + .as_slice::() + .iter() + .map(|&value| u64::from((value as u32) ^ (1_u32 << 31))) + .collect(), + PType::I64 => primitive + .as_slice::() + .iter() + .map(|&value| (value as u64) ^ (1_u64 << 63)) + .collect(), + ptype => return Err(vortex_err!("fixed bins do not support {ptype}")), + }) +} + +fn offsets_fit_ptype(offsets: &[u64], ptype: PType) -> bool { + let maximum = match ptype { + PType::U8 => u64::from(u8::MAX), + PType::U16 => u64::from(u16::MAX), + PType::U32 => u64::from(u32::MAX), + PType::U64 => u64::MAX, + PType::I8 => i8::MAX as u64, + PType::I16 => i16::MAX as u64, + PType::I32 => i32::MAX as u64, + PType::I64 => i64::MAX as u64, + _ => return false, + }; + offsets.iter().all(|&offset| offset <= maximum) +} + +fn range_components( + decomposition: &RangeDecomposition, + ptype: PType, +) -> VortexResult<(ArrayRef, PrimitiveArray)> { + let starts = decomposition.bin_starts(); + let offsets = decomposition.offsets(); + Ok(match ptype { + PType::U8 => ( + PrimitiveArray::from_iter( + starts + .iter() + .copied() + .map(u8::try_from) + .collect::, _>>()?, + ) + .into_array(), + PrimitiveArray::from_iter( + offsets + .iter() + .copied() + .map(u8::try_from) + .collect::, _>>()?, + ), + ), + PType::U16 => ( + PrimitiveArray::from_iter( + starts + .iter() + .copied() + .map(u16::try_from) + .collect::, _>>()?, + ) + .into_array(), + PrimitiveArray::from_iter( + offsets + .iter() + .copied() + .map(u16::try_from) + .collect::, _>>()?, + ), + ), + PType::U32 => ( + PrimitiveArray::from_iter( + starts + .iter() + .copied() + .map(u32::try_from) + .collect::, _>>()?, + ) + .into_array(), + PrimitiveArray::from_iter( + offsets + .iter() + .copied() + .map(u32::try_from) + .collect::, _>>()?, + ), + ), + PType::U64 => ( + PrimitiveArray::from_iter(starts.iter().copied()).into_array(), + PrimitiveArray::from_iter(offsets.iter().copied()), + ), + PType::I8 => ( + PrimitiveArray::from_iter( + starts + .iter() + .copied() + .map(|value| { + u8::try_from(value).map(|value| i8::from_le_bytes([value ^ (1_u8 << 7)])) + }) + .collect::, _>>()?, + ) + .into_array(), + PrimitiveArray::from_iter( + offsets + .iter() + .copied() + .map(i8::try_from) + .collect::, _>>()?, + ), + ), + PType::I16 => ( + PrimitiveArray::from_iter( + starts + .iter() + .copied() + .map(|value| { + u16::try_from(value) + .map(|value| i16::from_le_bytes((value ^ (1_u16 << 15)).to_le_bytes())) + }) + .collect::, _>>()?, + ) + .into_array(), + PrimitiveArray::from_iter( + offsets + .iter() + .copied() + .map(i16::try_from) + .collect::, _>>()?, + ), + ), + PType::I32 => ( + PrimitiveArray::from_iter( + starts + .iter() + .copied() + .map(|value| { + u32::try_from(value) + .map(|value| i32::from_le_bytes((value ^ (1_u32 << 31)).to_le_bytes())) + }) + .collect::, _>>()?, + ) + .into_array(), + PrimitiveArray::from_iter( + offsets + .iter() + .copied() + .map(i32::try_from) + .collect::, _>>()?, + ), + ), + PType::I64 => ( + PrimitiveArray::from_iter( + starts + .iter() + .copied() + .map(|value| i64::from_le_bytes((value ^ (1_u64 << 63)).to_le_bytes())), + ) + .into_array(), + PrimitiveArray::from_iter( + offsets + .iter() + .copied() + .map(i64::try_from) + .collect::, _>>()?, + ), + ), + ptype => return Err(vortex_err!("fixed bins do not support {ptype}")), + }) +} + fn fixed_bin_float_tree( compact: &ArrayRef, session: &VortexSession, @@ -1517,6 +1851,94 @@ fn encode_fixed_bin_float_tree( Ok(OrderedFloat::try_new(packed.into_array(), primitive.ptype())?.into_array()) } +fn encode_decomposed_fixed_bin_float_tree( + primitive: vortex_array::ArrayView<'_, Primitive>, + compact: &ArrayRef, + offset_tree: FixedBinOffsetTree, + session: &VortexSession, +) -> VortexResult> { + if compact.encoding_id().as_ref() == "vortex.alp" { + let compact_alp = compact.as_::(); + if compact_alp.encoded().encoding_id().as_ref() != "vortex.pco" { + return Ok(None); + } + let alp = alp_encode(primitive, None, &mut session.create_execution_ctx())?; + let Some(encoded) = decomposed_fixed_bin_integer_tree( + alp.encoded().as_::(), + offset_tree, + session, + )? + else { + return Ok(None); + }; + return Ok(Some( + ALP::try_new(encoded, alp.exponents(), alp.patches())?.into_array(), + )); + } + + if compact.encoding_id().as_ref() != "vortex.pco" || !compact.dtype().is_float() { + return Ok(None); + } + let primitive = fill_float_nulls_with_first_valid( + primitive.into_owned(), + &mut session.create_execution_ctx(), + )?; + let ordered = OrderedFloat::from_primitive(primitive.as_view())?; + let Some(encoded) = decomposed_fixed_bin_integer_tree( + ordered.encoded().as_::(), + offset_tree, + session, + )? + else { + return Ok(None); + }; + Ok(Some( + OrderedFloat::try_new(encoded, primitive.ptype())?.into_array(), + )) +} + +fn measure_decomposed_fixed_bin_tree( + dataset: &str, + column: &str, + expected: &ArrayRef, + compact: &ArrayRef, + offset_tree: FixedBinOffsetTree, + session: &VortexSession, +) -> VortexResult<()> { + let Some(encoded) = encode_decomposed_fixed_bin_float_tree( + expected.as_::(), + compact, + offset_tree, + session, + )? + else { + return Ok(()); + }; + let label = format!("decomposed-fixed-bin-{}", offset_tree.label()); + measure_existing_tree(dataset, column, &label, expected, &encoded, session)?; + + let mut encode_durations = Vec::with_capacity(5); + for _ in 0..5 { + let start = Instant::now(); + black_box(encode_decomposed_fixed_bin_float_tree( + expected.as_::(), + compact, + offset_tree, + session, + )?); + encode_durations.push(start.elapsed()); + } + let encode_median = percentile(&mut encode_durations, 1, 2); + let encode_throughput = expected.nbytes() as f64 / encode_median.as_secs_f64() / 1_000_000.0; + println!( + "candidate-tree-encode\t{dataset}\t{column}\t{label}\t{}\t{}\t{encode_throughput:.1}\t{}", + encoded.len(), + encoded.nbytes(), + encoding_tree(&encoded), + ); + Ok(()) +} + fn measure_existing_tree( dataset: &str, column: &str, @@ -1859,6 +2281,22 @@ fn measure_dataset( session, )?; measure_fixed_bin_tree(dataset, &column.name, &column.array, compact_array, session)?; + measure_decomposed_fixed_bin_tree( + dataset, + &column.name, + &column.array, + compact_array, + FixedBinOffsetTree::BlockResidual, + session, + )?; + measure_decomposed_fixed_bin_tree( + dataset, + &column.name, + &column.array, + compact_array, + FixedBinOffsetTree::FoRBitPacked, + session, + )?; measure_block_residual_float_tree( dataset, &column.name, From 22bc67196ba2e1a01552d6884bb97bcbfca7182e Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 05:56:12 +0100 Subject: [PATCH 44/78] bench: compare composable IntMult trees Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 69 +++- .../examples/float_compressor_bench.rs | 307 ++++++++++++++++++ 2 files changed, 366 insertions(+), 10 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index c983600c6b9..10fbbcfe205 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -1319,7 +1319,57 @@ A competitive fixed-bin design requires a reusable primitive for code-dependent Earlier grouped and checkpoint prototypes did not pass the Default decode gate. -Fixed bins remain paused while the pure IntMult experiment proceeds. +The generic fixed-bin composition remains paused for Default. + +### Pure IntMult real-data results + +The next prototype applies ALP first and splits its integer child with one global IntMult base. + +Default compresses the quotient and remainder independently through normal integer recursion. + +The benchmark tests bases 5, 10, 100, and 1,000. + +| Input | Default bytes | BlockResidual bytes | Best IntMult bytes | Default decode MB/s | Best IntMult decode MB/s | Best IntMult encode MB/s | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| GloVe embeddings | 6,274,834 | 6,298,751 | 6,665,400 at base 1,000 | 17,504 | 10,188 | 445 | +| CMS payment | 11,064,308 | 10,716,519 | 10,442,347 at base 10 | 22,715 | 10,212 | 412 | + +GloVe grows by 6.2 percent against Default. + +CMS shrinks by 5.6 percent against Default and 2.6 percent against `ALP(BlockResidual)`. + +The CMS IntMult tree decodes 55 percent slower than Default and 51 percent slower than `ALP(BlockResidual)`. + +Its encode throughput falls 77 percent against the direct `ALP(BlockResidual)` candidate. + +CMS scalar access remains close to Default, but it remains slower than the BlockResidual candidate. + +Pure IntMult with generic children therefore does not pass the Default evidence bar. + +Pco gains more from its bin and entropy backend than from the multiplication split alone on GloVe. + +### Equal-width prefix bins + +The equal-width prototype uses only `IntMult`, `Dict`, and `BitPacked`. + +The base is a power of two. A dictionary stores at most 64 observed quotients. + +The remainder uses one fixed suffix width for every value. + +This tree provides fast rejection when more than 64 quotients appear in the sample. + +| Input | Best suffix width | Candidate bytes | Default bytes | Encode MB/s | Decode MB/s | +| --- | ---: | ---: | ---: | ---: | ---: | +| GloVe embeddings | 20 | 6,909,476 | 6,274,834 | 1,348 | 7,538 | +| CMS payment | 44 | 11,984,848 | 11,064,308 | 2,487 | 8,709 | + +The encoder is fast because it builds one dictionary and two packed streams. + +The fixed suffix grows both columns by 8 to 10 percent against Default. + +The generic decoder is also more than twice as slow as Default. + +Equal-width prefix bins do not justify a Default scheme. ### Nullable and Delta-heavy fixed-bin cases @@ -1640,15 +1690,14 @@ The nonzero-secondary FloatQuant implementation and focused validation are compl Complete these next experiments in order: -1. Prototype pure IntMult on the exact GloVe and CMS ALP children. -2. Compress both IntMult children independently through normal integer schemes. -3. Test BlockResidual as a general compression candidate for patch positions. -4. Add a specialized range scheme only after a complete tree passes the column gates. -5. Profile another direct narrow BlockResidual decode optimization for `i16`. -6. Classify more float columns where Pco beats ALP, ALP-RD, and the new schemes. -7. Validate rejected-candidate analysis cost after the candidate set stabilizes. -8. Prefer cheap rejection tests when they preserve the best candidate model. -9. Require stronger corpus evidence when a useful scheme needs costly analysis. +1. Test BlockResidual as a general compression candidate for patch positions. +2. Profile another direct narrow BlockResidual decode optimization for `i16`. +3. Classify more float columns where Pco beats ALP, ALP-RD, and the new schemes. +4. Define a bounded small-alphabet experiment from the remaining no-Delta gaps. +5. Add a specialized range scheme only after a complete tree passes the column gates. +6. Validate rejected-candidate analysis cost after the candidate set stabilizes. +7. Prefer cheap rejection tests when they preserve the best candidate model. +8. Require stronger corpus evidence when a useful scheme needs costly analysis. Use packed bin codes and primitive bin starts unless evidence supports more complexity. diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index 8b2d5cd36db..63f4ee53344 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -1346,6 +1346,181 @@ fn decomposed_fixed_bin_integer_tree( Ok(Some(IntMult::try_new(references, offsets, 1)?.into_array())) } +fn int_mult_integer_tree( + primitive: vortex_array::ArrayView<'_, Primitive>, + base: u64, + session: &VortexSession, +) -> VortexResult { + let validity = primitive.validity()?; + let (primary, secondary) = match primitive.ptype() { + PType::I32 => { + let base = i32::try_from(base)?; + let values = primitive.as_slice::(); + ( + PrimitiveArray::new( + values + .iter() + .map(|value| value.div_euclid(base)) + .collect::>(), + validity, + ), + PrimitiveArray::from_iter( + values + .iter() + .map(|value| value.rem_euclid(base)) + .collect::>(), + ), + ) + } + PType::I64 => { + let base = i64::try_from(base)?; + let values = primitive.as_slice::(); + ( + PrimitiveArray::new( + values + .iter() + .map(|value| value.div_euclid(base)) + .collect::>(), + validity, + ), + PrimitiveArray::from_iter( + values + .iter() + .map(|value| value.rem_euclid(base)) + .collect::>(), + ), + ) + } + ptype => return Err(vortex_err!("ALP IntMult does not support {ptype}")), + }; + let compressor = BtrBlocksCompressor::default(); + let primary = + compressor.compress(&primary.into_array(), &mut session.create_execution_ctx())?; + let secondary = + compressor.compress(&secondary.into_array(), &mut session.create_execution_ctx())?; + Ok(IntMult::try_new(primary, secondary, base)?.into_array()) +} + +fn prefix_int_mult_integer_tree( + primitive: vortex_array::ArrayView<'_, Primitive>, + suffix_bits: u8, +) -> VortexResult> { + let validity = primitive.validity()?; + let base = 1_u64 << suffix_bits; + let (codes, dictionary, secondary) = match primitive.ptype() { + PType::I32 if suffix_bits < 31 => { + let base = i32::try_from(base)?; + let mut dictionary = Vec::::new(); + let mut code_by_value = HashMap::::new(); + let mut codes = Vec::with_capacity(primitive.len()); + let mut secondary = Vec::with_capacity(primitive.len()); + for &value in primitive.as_slice::() { + let quotient = value.div_euclid(base); + let code = match code_by_value.get("ient) { + Some(&code) => code, + None if dictionary.len() < 64 => { + let code = u8::try_from(dictionary.len())?; + dictionary.push(quotient); + code_by_value.insert(quotient, code); + code + } + None => return Ok(None), + }; + codes.push(code); + secondary.push(value.rem_euclid(base)); + } + ( + codes, + PrimitiveArray::from_iter(dictionary).into_array(), + PrimitiveArray::from_iter(secondary), + ) + } + PType::I64 if suffix_bits < 63 => { + let base = i64::try_from(base)?; + let mut dictionary = Vec::::new(); + let mut code_by_value = HashMap::::new(); + let mut codes = Vec::with_capacity(primitive.len()); + let mut secondary = Vec::with_capacity(primitive.len()); + for &value in primitive.as_slice::() { + let quotient = value.div_euclid(base); + let code = match code_by_value.get("ient) { + Some(&code) => code, + None if dictionary.len() < 64 => { + let code = u8::try_from(dictionary.len())?; + dictionary.push(quotient); + code_by_value.insert(quotient, code); + code + } + None => return Ok(None), + }; + codes.push(code); + secondary.push(value.rem_euclid(base)); + } + ( + codes, + PrimitiveArray::from_iter(dictionary).into_array(), + PrimitiveArray::from_iter(secondary), + ) + } + _ => return Ok(None), + }; + let code_width = + u8::try_from(u8::BITS - u8::try_from(dictionary.len().saturating_sub(1))?.leading_zeros())?; + let codes = PrimitiveArray::new(codes, validity); + // SAFETY: Every code fits the width computed from the dictionary length. + let codes = unsafe { bitpack_encode_unchecked(codes, code_width) }?.into_array(); + let primary = DictArray::try_new(codes, dictionary)?.into_array(); + // SAFETY: Euclidean remainders for a power-of-two base fit in `suffix_bits` bits. + let secondary = unsafe { bitpack_encode_unchecked(secondary, suffix_bits) }?.into_array(); + Ok(Some( + IntMult::try_new(primary, secondary, base)?.into_array(), + )) +} + +fn encode_int_mult_float_tree( + primitive: vortex_array::ArrayView<'_, Primitive>, + compact: &ArrayRef, + base: u64, + session: &VortexSession, +) -> VortexResult> { + if compact.encoding_id().as_ref() != "vortex.alp" { + return Ok(None); + } + let compact_alp = compact.as_::(); + if compact_alp.encoded().encoding_id().as_ref() != "vortex.pco" { + return Ok(None); + } + let alp = alp_encode(primitive, None, &mut session.create_execution_ctx())?; + let encoded = int_mult_integer_tree(alp.encoded().as_::(), base, session)?; + Ok(Some( + ALP::try_new(encoded, alp.exponents(), alp.patches())?.into_array(), + )) +} + +fn encode_prefix_int_mult_float_tree( + primitive: vortex_array::ArrayView<'_, Primitive>, + compact: &ArrayRef, + suffix_bits: u8, + session: &VortexSession, +) -> VortexResult> { + if compact.encoding_id().as_ref() != "vortex.alp" { + return Ok(None); + } + let compact_alp = compact.as_::(); + if compact_alp.encoded().encoding_id().as_ref() != "vortex.pco" { + return Ok(None); + } + let alp = alp_encode(primitive, None, &mut session.create_execution_ctx())?; + let Some(encoded) = + prefix_int_mult_integer_tree(alp.encoded().as_::(), suffix_bits)? + else { + return Ok(None); + }; + Ok(Some( + ALP::try_new(encoded, alp.exponents(), alp.patches())?.into_array(), + )) +} + fn fill_integer_nulls_with_first_valid( primitive: PrimitiveArray, ctx: &mut vortex_array::ExecutionCtx, @@ -1939,6 +2114,86 @@ fn measure_decomposed_fixed_bin_tree( Ok(()) } +fn measure_int_mult_float_tree( + dataset: &str, + column: &str, + expected: &ArrayRef, + compact: &ArrayRef, + base: u64, + session: &VortexSession, +) -> VortexResult<()> { + let Some(encoded) = + encode_int_mult_float_tree(expected.as_::(), compact, base, session)? + else { + return Ok(()); + }; + let label = format!("int-mult-{base}"); + measure_existing_tree(dataset, column, &label, expected, &encoded, session)?; + + let mut encode_durations = Vec::with_capacity(5); + for _ in 0..5 { + let start = Instant::now(); + black_box(encode_int_mult_float_tree( + expected.as_::(), + compact, + base, + session, + )?); + encode_durations.push(start.elapsed()); + } + let encode_median = percentile(&mut encode_durations, 1, 2); + let encode_throughput = expected.nbytes() as f64 / encode_median.as_secs_f64() / 1_000_000.0; + println!( + "candidate-tree-encode\t{dataset}\t{column}\t{label}\t{}\t{}\t{encode_throughput:.1}\t{}", + encoded.len(), + encoded.nbytes(), + encoding_tree(&encoded), + ); + Ok(()) +} + +fn measure_prefix_int_mult_float_tree( + dataset: &str, + column: &str, + expected: &ArrayRef, + compact: &ArrayRef, + suffix_bits: u8, + session: &VortexSession, +) -> VortexResult<()> { + let Some(encoded) = encode_prefix_int_mult_float_tree( + expected.as_::(), + compact, + suffix_bits, + session, + )? + else { + return Ok(()); + }; + let label = format!("prefix-int-mult-{suffix_bits}"); + measure_existing_tree(dataset, column, &label, expected, &encoded, session)?; + + let mut encode_durations = Vec::with_capacity(5); + for _ in 0..5 { + let start = Instant::now(); + black_box(encode_prefix_int_mult_float_tree( + expected.as_::(), + compact, + suffix_bits, + session, + )?); + encode_durations.push(start.elapsed()); + } + let encode_median = percentile(&mut encode_durations, 1, 2); + let encode_throughput = expected.nbytes() as f64 / encode_median.as_secs_f64() / 1_000_000.0; + println!( + "candidate-tree-encode\t{dataset}\t{column}\t{label}\t{}\t{}\t{encode_throughput:.1}\t{}", + encoded.len(), + encoded.nbytes(), + encoding_tree(&encoded), + ); + Ok(()) +} + fn measure_existing_tree( dataset: &str, column: &str, @@ -2305,6 +2560,58 @@ fn measure_dataset( session, )?; } + if std::env::var_os("VORTEX_BENCH_INT_MULT_TREE").is_some() { + if std::env::var_os("VORTEX_BENCH_FIXED_BIN").is_none() { + measure_existing_tree( + dataset, + &column.name, + "default", + &column.array, + default_array, + session, + )?; + measure_existing_tree( + dataset, + &column.name, + "compact", + &column.array, + compact_array, + session, + )?; + measure_block_residual_float_tree( + dataset, + &column.name, + &column.array, + compact_array, + session, + )?; + } + for base in [5, 10, 100, 1_000] { + measure_int_mult_float_tree( + dataset, + &column.name, + &column.array, + compact_array, + base, + session, + )?; + } + let suffix_widths: &[u8] = match primitive.ptype() { + PType::F32 => &[12, 14, 16, 18, 20, 22, 24, 26, 28], + PType::F64 => &[24, 28, 32, 36, 40, 44, 48, 52, 56], + _ => &[], + }; + for &suffix_bits in suffix_widths { + measure_prefix_int_mult_float_tree( + dataset, + &column.name, + &column.array, + compact_array, + suffix_bits, + session, + )?; + } + } } if std::env::var_os("VORTEX_BENCH_PROFILE_ONLY").is_some() { From f87ac69e41b571d5139e8465bcd2299808b21cc4 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 06:00:07 +0100 Subject: [PATCH 45/78] bench: test BlockResidual patch positions Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 47 +++++- .../examples/float_compressor_bench.rs | 153 ++++++++++++++++++ 2 files changed, 192 insertions(+), 8 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 10fbbcfe205..75beed26419 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -1371,6 +1371,38 @@ The generic decoder is also more than twice as slow as Default. Equal-width prefix bins do not justify a Default scheme. +### BlockResidual patch positions + +The patch prototype replaces sorted ALP patch indices and chunk offsets with BlockResidual when it reduces their byte size. + +It leaves patch values unchanged. + +| Input and tree | Original bytes | Candidate bytes | Original decode MB/s | Candidate decode MB/s | Original scalar ns | Candidate scalar ns | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| GloVe Default | 6,274,834 | 6,071,376 | 18,077 | 17,178 | 812 | 2,076 | +| GloVe `ALP(BlockResidual)` | 6,298,751 | 6,095,293 | 19,233 | 18,735 | 503 | 1,766 | +| CMS Default | 11,064,308 | 11,040,803 | 24,168 | 23,111 | 934 | 1,551 | +| CMS `ALP(BlockResidual)` | 10,716,519 | 10,693,014 | 20,678 | 19,982 | 656 | 1,241 | +| Euro subjectivity Default | 14,234,326 | 11,211,722 | 21,055 | 16,583 | 631 | 9,801 | + +Food volume has no ALP patches on this input. + +BlockResidual saves 3.2 percent on GloVe patch trees and 21.2 percent on Euro Default. + +CMS saves only 0.2 percent. + +Patch lookup uses binary search over the index array. + +Compressed indices require one generic scalar read for each comparison. + +This path increases scalar latency by 1.7 to 15.5 times across the measured trees. + +Direct BlockResidual patch positions therefore do not pass the random-access gate. + +A future general patch backend can retain this size opportunity through a fast sorted-search kernel or a bitmap with bounded rank data. + +Do not add a BlockResidual special case to ALP or IntMult patch handling. + ### Nullable and Delta-heavy fixed-bin cases The optimized nullable prototype stores only valid values in the range stream. @@ -1690,14 +1722,13 @@ The nonzero-secondary FloatQuant implementation and focused validation are compl Complete these next experiments in order: -1. Test BlockResidual as a general compression candidate for patch positions. -2. Profile another direct narrow BlockResidual decode optimization for `i16`. -3. Classify more float columns where Pco beats ALP, ALP-RD, and the new schemes. -4. Define a bounded small-alphabet experiment from the remaining no-Delta gaps. -5. Add a specialized range scheme only after a complete tree passes the column gates. -6. Validate rejected-candidate analysis cost after the candidate set stabilizes. -7. Prefer cheap rejection tests when they preserve the best candidate model. -8. Require stronger corpus evidence when a useful scheme needs costly analysis. +1. Profile another direct narrow BlockResidual decode optimization for `i16`. +2. Classify more float columns where Pco beats ALP, ALP-RD, and the new schemes. +3. Define a bounded small-alphabet experiment from the remaining no-Delta gaps. +4. Add a specialized range scheme only after a complete tree passes the column gates. +5. Validate rejected-candidate analysis cost after the candidate set stabilizes. +6. Prefer cheap rejection tests when they preserve the best candidate model. +7. Require stronger corpus evidence when a useful scheme needs costly analysis. Use packed bin codes and primitive bin starts unless evidence supports more complexity. diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index 63f4ee53344..66ef439b864 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -55,6 +55,7 @@ use vortex_array::dtype::DType; use vortex_array::dtype::PType; use vortex_array::extension::datetime::TimeUnit; use vortex_array::match_each_integer_ptype; +use vortex_array::patches::Patches; use vortex_array::validity::Validity; use vortex_arrow::ArrowSessionExt; use vortex_block_residual::BlockResidual; @@ -1940,6 +1941,60 @@ fn measure_block_residual_float_tree( Ok(()) } +fn measure_block_residual_patch_positions( + dataset: &str, + column: &str, + expected: &ArrayRef, + default: &ArrayRef, + compact: &ArrayRef, + session: &VortexSession, +) -> VortexResult<()> { + if let Some(encoded) = rewrite_alp_patch_positions(default, session)? { + measure_existing_tree( + dataset, + column, + "default-block-residual-patch-positions", + expected, + &encoded, + session, + )?; + } + + let Some(block_residual) = block_residual_float_tree(compact, session)? else { + return Ok(()); + }; + let Some(encoded) = rewrite_alp_patch_positions(&block_residual, session)? else { + return Ok(()); + }; + measure_existing_tree( + dataset, + column, + "block-residual-patch-positions", + expected, + &encoded, + session, + )?; + + let mut encode_durations = Vec::with_capacity(5); + for _ in 0..5 { + let start = Instant::now(); + black_box(encode_block_residual_float_tree_with_patch_positions( + expected.as_::(), + session, + )?); + encode_durations.push(start.elapsed()); + } + let encode_median = percentile(&mut encode_durations, 1, 2); + let encode_throughput = expected.nbytes() as f64 / encode_median.as_secs_f64() / 1_000_000.0; + println!( + "candidate-tree-encode\t{dataset}\t{column}\tblock-residual-patch-positions\t{}\t{}\t{encode_throughput:.1}\t{}", + encoded.len(), + encoded.nbytes(), + encoding_tree(&encoded), + ); + Ok(()) +} + fn block_residual_float_tree( compact: &ArrayRef, session: &VortexSession, @@ -1978,6 +2033,74 @@ fn block_residual_float_tree( )) } +fn block_residual_patch_positions( + patches: Patches, + session: &VortexSession, +) -> VortexResult { + let indices = patches + .indices() + .clone() + .execute::(&mut session.create_execution_ctx())?; + let encoded_indices = BlockResidual::from_primitive(indices.as_view())?.into_array(); + let indices = if encoded_indices.nbytes() < indices.nbytes() { + encoded_indices + } else { + indices.into_array() + }; + let chunk_offsets = patches + .chunk_offsets() + .as_ref() + .map(|offsets| { + let offsets = offsets + .clone() + .execute::(&mut session.create_execution_ctx())?; + let encoded = BlockResidual::from_primitive(offsets.as_view())?.into_array(); + Ok::(if encoded.nbytes() < offsets.nbytes() { + encoded + } else { + offsets.into_array() + }) + }) + .transpose()?; + Patches::new( + patches.array_len(), + patches.offset(), + indices, + patches.values().clone(), + chunk_offsets, + ) +} + +fn rewrite_alp_patch_positions( + encoded: &ArrayRef, + session: &VortexSession, +) -> VortexResult> { + if encoded.encoding_id().as_ref() != "vortex.alp" { + return Ok(None); + } + let alp = encoded.as_::(); + let Some(patches) = alp.patches() else { + return Ok(None); + }; + let patches = block_residual_patch_positions(patches, session)?; + Ok(Some( + ALP::try_new(alp.encoded().clone(), alp.exponents(), Some(patches))?.into_array(), + )) +} + +fn encode_block_residual_float_tree_with_patch_positions( + primitive: vortex_array::ArrayView<'_, Primitive>, + session: &VortexSession, +) -> VortexResult { + let alp = alp_encode(primitive, None, &mut session.create_execution_ctx())?; + let encoded = BlockResidual::from_primitive(alp.encoded().as_::())?; + let patches = alp + .patches() + .map(|patches| block_residual_patch_positions(patches, session)) + .transpose()?; + Ok(ALP::try_new(encoded.into_array(), alp.exponents(), patches)?.into_array()) +} + fn encode_block_residual_float_tree( primitive: vortex_array::ArrayView<'_, Primitive>, compact: &ArrayRef, @@ -2612,6 +2735,36 @@ fn measure_dataset( )?; } } + if std::env::var_os("VORTEX_BENCH_PATCH_POSITIONS").is_some() { + if std::env::var_os("VORTEX_BENCH_FIXED_BIN").is_none() + && std::env::var_os("VORTEX_BENCH_INT_MULT_TREE").is_none() + { + measure_existing_tree( + dataset, + &column.name, + "default", + &column.array, + default_array, + session, + )?; + measure_existing_tree( + dataset, + &column.name, + "compact", + &column.array, + compact_array, + session, + )?; + } + measure_block_residual_patch_positions( + dataset, + &column.name, + &column.array, + default_array, + compact_array, + session, + )?; + } } if std::env::var_os("VORTEX_BENCH_PROFILE_ONLY").is_some() { From 2994eb933c02f240490acbfa59080cd945bd028a Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 06:04:48 +0100 Subject: [PATCH 46/78] perf: decode unsigned BlockResidual directly Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 48 ++++++++++++++++--- .../src/block_residual_array.rs | 6 +-- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 75beed26419..3b877a9bddf 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -1403,6 +1403,41 @@ A future general patch backend can retain this size opportunity through a fast s Do not add a BlockResidual special case to ALP or IntMult patch handling. +### Direct unsigned BlockResidual decode + +The array decoder now writes unpacked unsigned values directly into the final output buffer. + +The prior path used a temporary 1,024-value block for u8, u16, and u64 values. + +| Type | Prior decode GB/s | Direct decode GB/s | Change | +| --- | ---: | ---: | ---: | +| u8 | 30.14 | 40.37 | +34.0 percent | +| u16 | 32.55 | 45.86 | +40.9 percent | +| u32 | 46.61 | 47.23 | +1.3 percent | +| u64 | 35.56 | 39.37 | +10.7 percent | + +u32 already used the direct path. + +The signed prototype unpacked into the final unsigned buffer and flipped the sign bit in place. + +That extra pass reduced i16 throughput from about 31.4 GB/s to 27.4 GB/s. + +It also reduced throughput for i8, i32, and i64. + +The implementation retains the prior signed path. + +Current i16 BlockResidual decode reaches 31.4 GB/s. The comparable `FoR(BitPacked)` tree reaches 42.0 GB/s. + +This result retains the direct i8 and i16 selector exclusions. + +The u16 result is faster than the prior narrow path, but no real selected-tree evidence supports policy removal yet. + +The faster unsigned path also improves the BlockResidual patch-position bulk decoder. + +GloVe patch decode now stays within 2 percent of Default. Euro patch decode remains 12 percent slower than Default. + +Scalar patch lookup remains the blocking cost. + ### Nullable and Delta-heavy fixed-bin cases The optimized nullable prototype stores only valid values in the range stream. @@ -1722,13 +1757,12 @@ The nonzero-secondary FloatQuant implementation and focused validation are compl Complete these next experiments in order: -1. Profile another direct narrow BlockResidual decode optimization for `i16`. -2. Classify more float columns where Pco beats ALP, ALP-RD, and the new schemes. -3. Define a bounded small-alphabet experiment from the remaining no-Delta gaps. -4. Add a specialized range scheme only after a complete tree passes the column gates. -5. Validate rejected-candidate analysis cost after the candidate set stabilizes. -6. Prefer cheap rejection tests when they preserve the best candidate model. -7. Require stronger corpus evidence when a useful scheme needs costly analysis. +1. Classify more float columns where Pco beats ALP, ALP-RD, and the new schemes. +2. Define a bounded small-alphabet experiment from the remaining no-Delta gaps. +3. Add a specialized range scheme only after a complete tree passes the column gates. +4. Validate rejected-candidate analysis cost after the candidate set stabilizes. +5. Prefer cheap rejection tests when they preserve the best candidate model. +6. Require stronger corpus evidence when a useful scheme needs costly analysis. Use packed bin codes and primitive bin starts unless evidence supports more complexity. diff --git a/encodings/block-residual/src/block_residual_array.rs b/encodings/block-residual/src/block_residual_array.rs index 266b27bed74..2068e38ffdb 100644 --- a/encodings/block-residual/src/block_residual_array.rs +++ b/encodings/block-residual/src/block_residual_array.rs @@ -1106,10 +1106,10 @@ fn decompress_array( ctx: &mut ExecutionCtx, ) -> VortexResult { match array.dtype().as_ptype() { - PType::U8 => decode_array_values::(array, ctx, |value| value), - PType::U16 => decode_array_values::(array, ctx, |value| value), + PType::U8 => decode_array_values::(array, ctx, |value| value), + PType::U16 => decode_array_values::(array, ctx, |value| value), PType::U32 => decode_array_values::(array, ctx, |value| value), - PType::U64 => decode_array_values::(array, ctx, |value| value), + PType::U64 => decode_array_values::(array, ctx, |value| value), PType::I8 => { decode_array_values::(array, ctx, |value| (value ^ (1_u8 << 7)) as i8) } From 406b1b51d4ad92efa8cbbdbfede8f2692804cf02 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 06:28:51 +0100 Subject: [PATCH 47/78] perf: rank dictionary codes by frequency Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 84 ++++++++- .../examples/float_compressor_bench.rs | 159 ++++++++++++++++++ .../src/schemes/integer/block_residual.rs | 10 -- .../schemes/integer/scheme_selection_tests.rs | 6 - vortex-btrblocks/src/trace_tests.rs | 7 +- vortex-compressor/src/builtins/dict/float.rs | 24 ++- .../src/builtins/dict/integer.rs | 20 ++- vortex-compressor/src/stats/float.rs | 22 ++- 8 files changed, 285 insertions(+), 47 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 3b877a9bddf..19af4aad724 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -1348,6 +1348,23 @@ Pure IntMult with generic children therefore does not pass the Default evidence Pco gains more from its bin and entropy backend than from the multiplication split alone on GloVe. +The CMS standard-deviation payment column provides a no-Delta IntMult target. + +Pco uses `IntMult(10)` on seven of eight chunks. The remaining chunk uses classic bins. + +| Tree | Bytes | Encode MB/s | Decode MB/s | Scalar ns | +| --- | ---: | ---: | ---: | ---: | +| Default | 10,494,404 | Not isolated | 24,593 | 866 | +| `ALP(BlockResidual)` | 10,220,895 | 2,124 | 21,889 | 551 | +| Best IntMult, base 1,000 | 10,300,986 | 530 | 10,704 | 727 | +| Compact Pco | 9,325,132 | Not isolated | 1,977 | 25,987 | + +The IntMult tree saves 1.8 percent against Default. It remains larger than the BlockResidual tree. + +Its encode throughput is four times lower than BlockResidual. Its decode throughput is more than twice as low. + +This result removes Delta as a confounder. Pure IntMult remains outside Default. + ### Equal-width prefix bins The equal-width prototype uses only `IntMult`, `Dict`, and `BitPacked`. @@ -1371,6 +1388,53 @@ The generic decoder is also more than twice as slow as Default. Equal-width prefix bins do not justify a Default scheme. +### Frequency-ranked dictionary codes + +Dictionary codes previously followed hash-table iteration order. + +The prototype assigns the lowest codes to the most frequent values. It then compresses both children through normal recursion. + +This transform uses existing `Dict`, `BitPacked`, `Sparse`, and BlockResidual arrays. It adds no array format. + +Integer statistics already store frequency counts. Float statistics now retain counts during the existing distinct-value pass. + +| Column and tree | Bytes | Decode MB/s | Scalar ns | Candidate construction MB/s | +| --- | ---: | ---: | ---: | ---: | +| Bimbo `Venta_hoy`, old Default | 3,544,154 | 26,689 | 595 | Not isolated | +| Bimbo `Venta_hoy`, ranked proposed Default | 3,197,655 | 25,490 | 577 | 1,480 | +| Bimbo `Dev_proxima`, ranked prior Default | 280,812 | 26,902 | 3,834 | Not isolated | +| Bimbo `Dev_proxima`, ranked Dict with BlockResidual descendants | 234,832 | 27,134 | 3,823 | 1,367 | + +Frequency ranking reduces `Venta_hoy` by 9.8 percent against the old Default. + +The complete proposed tree keeps decode and scalar throughput close to the old tree. + +The `Dev_proxima` gain depends on BlockResidual below the dictionary code child. + +The former ancestor exclusion prevents that useful composition. The corpus trial therefore removes the integer and float Dict exclusions. + +The scheme retains exclusions for string and binary dictionaries. + +The current eight-dataset numeric trial compares the ranked prior Default with the ranked proposed Default. + +| Dataset | Size change | Encode throughput change | Decode throughput change | +| --- | ---: | ---: | ---: | +| Arade | -3.19 percent | -5.17 percent | +1.93 percent | +| Bimbo | -3.57 percent | -0.37 percent | +0.23 percent | +| CMSprovider 1 | -2.26 percent | -6.51 percent | -6.34 percent | +| CMSprovider 2 | -3.99 percent | -6.97 percent | -9.06 percent | +| Euro2016 | -11.47 percent | -3.63 percent | +11.68 percent | +| Food | -7.14 percent | -3.91 percent | -10.18 percent | +| HashTags | -5.68 percent | +0.43 percent | +5.46 percent | +| GloVe | 0.00 percent | +0.11 percent | +1.85 percent | +| Geometric mean | -4.72 percent | -3.29 percent | -0.80 percent | + +Every dataset remains inside the 20-percent throughput gate. + +Frequency ranking needs no separate selector. Dictionary analysis already computes the required distinct values. + +Cheap rejection remains useful guidance for other schemes. A costly scheme can enter Default only with stronger corpus evidence. + ### BlockResidual patch positions The patch prototype replaces sorted ALP patch indices and chunk offsets with BlockResidual when it reduces their byte size. @@ -1741,7 +1805,12 @@ This round completed these steps: - Rejected packed multi-reference blocks after patched and patch-free comparisons. - Rejected three bounded IntMult remainder layouts on GloVe. - Rejected the dense-remainder IntMult layout on CMS Payments. +- Rejected pure IntMult on a no-Delta CMS standard-deviation column. - Rejected centered block residuals after a complete Euro subjectivity comparison. +- Assigned dictionary codes by descending value frequency. +- Added float frequency counts to the existing distinct-value statistics. +- Reopened BlockResidual descendants for integer and float dictionary codes. +- Validated the ranked Dict policy across eight numeric datasets. - Audited constructor and deserialization validation for all four production arrays. - Added round-trip tests for every supported integer and float type. - Added single-encoding benchmarks for every supported integer and float type. @@ -1749,20 +1818,21 @@ This round completed these steps: - Verified zero-secondary and nonzero-secondary `f16` FloatQuant trees. - Updated stale golden trees after the BlockResidual payload and selector changes. -The Pco mode profile and the first quotient and remainder experiments are complete. +The Pco mode profile and the quotient and remainder experiments are complete. -The composable IntMult follow-up remains incomplete. +The composable IntMult follow-up is complete. The tested IntMult trees remain outside Default. The nonzero-secondary FloatQuant implementation and focused validation are complete. Complete these next experiments in order: -1. Classify more float columns where Pco beats ALP, ALP-RD, and the new schemes. -2. Define a bounded small-alphabet experiment from the remaining no-Delta gaps. +1. Classify the remaining no-Delta ALP and raw-float gaps. +2. Define a bounded small-alphabet primitive for the remaining 24-to-49-bin columns. 3. Add a specialized range scheme only after a complete tree passes the column gates. -4. Validate rejected-candidate analysis cost after the candidate set stabilizes. -5. Prefer cheap rejection tests when they preserve the best candidate model. -6. Require stronger corpus evidence when a useful scheme needs costly analysis. +4. Validate frequency-ranked Dict across the complete file corpus. +5. Validate rejected-candidate analysis cost after the candidate set stabilizes. +6. Prefer cheap rejection tests when they preserve the best candidate model. +7. Require stronger corpus evidence when a useful scheme needs costly analysis. Use packed bin codes and primitive bin starts unless evidence supports more complexity. diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index 66ef439b864..c3a333527b6 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -43,18 +43,21 @@ use vortex_array::RecursiveCanonical; use vortex_array::VortexSessionExecute; use vortex_array::array_session; use vortex_array::arrays::ChunkedArray; +use vortex_array::arrays::Dict; use vortex_array::arrays::DictArray; use vortex_array::arrays::ListArray; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::TemporalArray; use vortex_array::arrays::VarBinViewArray; +use vortex_array::arrays::dict::DictArraySlotsExt; use vortex_array::assert_arrays_eq; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::PType; use vortex_array::extension::datetime::TimeUnit; use vortex_array::match_each_integer_ptype; +use vortex_array::match_each_unsigned_integer_ptype; use vortex_array::patches::Patches; use vortex_array::validity::Validity; use vortex_arrow::ArrowSessionExt; @@ -75,7 +78,10 @@ use vortex_fastlanes::BitPacked; use vortex_fastlanes::FoR; use vortex_fastlanes::FoRArrayExt; use vortex_fastlanes::FoRArraySlotsExt; +use vortex_fastlanes::bitpack_compress::bit_width_histogram; +use vortex_fastlanes::bitpack_compress::bitpack_encode; use vortex_fastlanes::bitpack_compress::bitpack_encode_unchecked; +use vortex_fastlanes::bitpack_compress::find_best_bit_width; use vortex_int_mult::IntMult; use vortex_range_packed::RangeDecomposition; use vortex_range_packed::RangePacked; @@ -1307,6 +1313,21 @@ impl FixedBinOffsetTree { } } +#[derive(Clone, Copy)] +enum FrequencyRankedCodeTree { + BitPacked, + Default, +} + +impl FrequencyRankedCodeTree { + fn label(self) -> &'static str { + match self { + Self::BitPacked => "bitpacked", + Self::Default => "default", + } + } +} + fn decomposed_fixed_bin_integer_tree( primitive: vortex_array::ArrayView<'_, Primitive>, offset_tree: FixedBinOffsetTree, @@ -1522,6 +1543,86 @@ fn encode_prefix_int_mult_float_tree( )) } +#[expect( + clippy::useless_conversion, + reason = "the generic unsigned-code path must reject values that exceed usize" +)] +fn frequency_ranked_dict_tree( + encoded: &ArrayRef, + code_tree: FrequencyRankedCodeTree, + session: &VortexSession, +) -> VortexResult> { + let Some(dict) = encoded.as_typed::() else { + return Ok(None); + }; + let mut ctx = session.create_execution_ctx(); + let codes = dict.codes().clone().execute::(&mut ctx)?; + let old_codes = match_each_unsigned_integer_ptype!(codes.ptype(), |T| { + codes + .as_slice::() + .iter() + .map(|&code| usize::try_from(u64::from(code))) + .collect::, _>>()? + }); + let validity = codes.validity()?; + let mask = validity.execute_mask(codes.len(), &mut ctx)?; + let mut counts = vec![0_usize; dict.values().len()]; + for (&code, valid) in old_codes.iter().zip(mask.iter()) { + if valid { + vortex_ensure!( + code < counts.len(), + "dictionary code {code} exceeds {} values", + counts.len() + ); + counts[code] += 1; + } + } + + let mut order = (0..counts.len()).collect::>(); + order.sort_unstable_by_key(|&code| (std::cmp::Reverse(counts[code]), code)); + let mut new_code_by_old = vec![0_usize; order.len()]; + for (new_code, &old_code) in order.iter().enumerate() { + new_code_by_old[old_code] = new_code; + } + let remapped = match_each_unsigned_integer_ptype!(codes.ptype(), |T| { + PrimitiveArray::new( + old_codes + .iter() + .zip(mask.iter()) + .map(|(&old_code, valid)| { + if valid { + T::try_from(new_code_by_old[old_code]) + } else { + T::try_from(0) + } + }) + .collect::, _>>()?, + validity.clone(), + ) + }); + let codes = match code_tree { + FrequencyRankedCodeTree::BitPacked => { + let bit_width_freq = bit_width_histogram(remapped.as_view(), &mut ctx)?; + let bit_width = find_best_bit_width(remapped.ptype(), &bit_width_freq)?; + bitpack_encode(&remapped, bit_width, Some(&bit_width_freq), &mut ctx)?.into_array() + } + FrequencyRankedCodeTree::Default => { + BtrBlocksCompressor::default().compress(&remapped.into_array(), &mut ctx)? + } + }; + + let order = order + .into_iter() + .map(u64::try_from) + .collect::, _>>()?; + let values = dict + .values() + .take(PrimitiveArray::from_iter(order).into_array())? + .execute::(&mut ctx)?; + let values = BtrBlocksCompressor::default().compress(&values.into_array(), &mut ctx)?; + Ok(Some(DictArray::try_new(codes, values)?.into_array())) +} + fn fill_integer_nulls_with_first_valid( primitive: PrimitiveArray, ctx: &mut vortex_array::ExecutionCtx, @@ -2317,6 +2418,57 @@ fn measure_prefix_int_mult_float_tree( Ok(()) } +fn measure_frequency_ranked_dict_tree( + dataset: &str, + column: &str, + expected: &ArrayRef, + default: &ArrayRef, + code_tree: FrequencyRankedCodeTree, + session: &VortexSession, +) -> VortexResult<()> { + let Some(encoded) = frequency_ranked_dict_tree(default, code_tree, session)? else { + return Ok(()); + }; + let label = format!("frequency-ranked-dict-{}", code_tree.label()); + measure_existing_tree(dataset, column, &label, expected, &encoded, session)?; + + let mut encode_durations = Vec::with_capacity(5); + for _ in 0..5 { + let start = Instant::now(); + black_box(frequency_ranked_dict_tree(default, code_tree, session)?); + encode_durations.push(start.elapsed()); + } + let encode_median = percentile(&mut encode_durations, 1, 2); + let encode_throughput = expected.nbytes() as f64 / encode_median.as_secs_f64() / 1_000_000.0; + println!( + "candidate-tree-encode\t{dataset}\t{column}\t{label}\t{}\t{}\t{encode_throughput:.1}\t{}", + encoded.len(), + encoded.nbytes(), + encoding_tree(&encoded), + ); + Ok(()) +} + +fn measure_frequency_ranked_dict_trees( + dataset: &str, + column: &str, + expected: &ArrayRef, + default: &ArrayRef, + session: &VortexSession, +) -> VortexResult<()> { + if std::env::var_os("VORTEX_BENCH_FREQ_DICT").is_none() { + return Ok(()); + } + measure_existing_tree(dataset, column, "default", expected, default, session)?; + for code_tree in [ + FrequencyRankedCodeTree::BitPacked, + FrequencyRankedCodeTree::Default, + ] { + measure_frequency_ranked_dict_tree(dataset, column, expected, default, code_tree, session)?; + } + Ok(()) +} + fn measure_existing_tree( dataset: &str, column: &str, @@ -2636,6 +2788,13 @@ fn measure_dataset( encoding_tree(default_array), encoding_tree(compact_array), ); + measure_frequency_ranked_dict_trees( + dataset, + &column.name, + &column.array, + default_array, + session, + )?; if compact_bytes * 10 <= default_bytes * 9 || std::env::var_os("VORTEX_BENCH_FIXED_BIN").is_some() { diff --git a/vortex-btrblocks/src/schemes/integer/block_residual.rs b/vortex-btrblocks/src/schemes/integer/block_residual.rs index 9b944f40a5d..996e67f0951 100644 --- a/vortex-btrblocks/src/schemes/integer/block_residual.rs +++ b/vortex-btrblocks/src/schemes/integer/block_residual.rs @@ -15,8 +15,6 @@ use vortex_array::match_each_integer_ptype; use vortex_block_residual::BlockResidual; use vortex_block_residual::BlockResidualEstimate; use vortex_compressor::builtins::BinaryDictScheme; -use vortex_compressor::builtins::FloatDictScheme; -use vortex_compressor::builtins::IntDictScheme; use vortex_compressor::builtins::StringDictScheme; use vortex_compressor::scheme::AncestorExclusion; use vortex_compressor::scheme::ChildSelection; @@ -60,14 +58,6 @@ impl Scheme for BlockResidualScheme { fn ancestor_exclusions(&self) -> Vec { vec![ - AncestorExclusion { - ancestor: IntDictScheme.id(), - children: ChildSelection::One(1), - }, - AncestorExclusion { - ancestor: FloatDictScheme.id(), - children: ChildSelection::One(1), - }, AncestorExclusion { ancestor: StringDictScheme.id(), children: ChildSelection::One(1), diff --git a/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs index 2ec13b6d7ea..0db2877ba78 100644 --- a/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs @@ -14,7 +14,6 @@ use vortex_array::VortexSessionExecute; use vortex_array::arrays::Constant; use vortex_array::arrays::Dict; use vortex_array::arrays::PrimitiveArray; -use vortex_array::arrays::dict::DictArraySlotsExt; use vortex_array::expr::stats::Precision; use vortex_array::expr::stats::Stat; use vortex_array::expr::stats::StatsProviderExt; @@ -257,11 +256,6 @@ fn test_dict_compressed() -> VortexResult<()> { let btr = BtrBlocksCompressor::default(); let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; assert!(compressed.is::()); - assert!( - !contains_block_residual(compressed.as_::().codes()), - "BlockResidual must not encode dictionary codes:\n{}", - compressed.display_tree() - ); Ok(()) } diff --git a/vortex-btrblocks/src/trace_tests.rs b/vortex-btrblocks/src/trace_tests.rs index d779757bc77..b26829f6a98 100644 --- a/vortex-btrblocks/src/trace_tests.rs +++ b/vortex-btrblocks/src/trace_tests.rs @@ -219,7 +219,7 @@ fn trace_scan_compare_on_compressed_shipdate() -> VortexResult<()> { /// Q6-style predicate over the quantity column: `l_quantity < 24`. /// -/// The column compresses to `decimal_byte_parts -> dict -> bitpacked/sequence`. +/// The column compresses to `decimal_byte_parts -> dict -> bitpacked/primitive`. fn quantity_predicate(column: ArrayRef, len: usize) -> VortexResult { let cutoff = Scalar::decimal( DecimalValue::I128(2400), @@ -262,11 +262,6 @@ fn trace_scan_compare_on_compressed_quantity() -> VortexResult<()> { iter 4 current=vortex.dict(bool, len=4096) builder_active=false ExecuteSlot slot=1 parent=vortex.dict(bool, len=4096) child=vortex.binary(bool, len=50) iter 5 current=vortex.binary(bool, len=50) stack_parent=vortex.dict(bool, len=4096) slot=1 builder_active=false - execute_until target=AnyCanonical root=vortex.sequence(i16, len=50) - iter 0 current=vortex.sequence(i16, len=50) builder_active=false - Done array=vortex.primitive(i16, len=50) - iter 1 current=vortex.primitive(i16, len=50) builder_active=false - return output=vortex.primitive(i16, len=50) Done array=vortex.bool(bool, len=50) iter 6 current=vortex.bool(bool, len=50) stack_parent=vortex.dict(bool, len=4096) slot=1 builder_active=false pop_frame slot=1 output=vortex.dict(bool, len=4096) diff --git a/vortex-compressor/src/builtins/dict/float.rs b/vortex-compressor/src/builtins/dict/float.rs index f962c3ff967..1de5b7ce986 100644 --- a/vortex-compressor/src/builtins/dict/float.rs +++ b/vortex-compressor/src/builtins/dict/float.rs @@ -165,7 +165,13 @@ macro_rules! typed_encode { }; let codes_validity = $source_array.validity()?; - let values: Buffer<$typ> = distinct.distinct_values().iter().map(|x| x.0).collect(); + let mut values = distinct + .distinct_values() + .iter() + .map(|(value, &count)| (value.0, count)) + .collect::>(); + values.sort_unstable_by_key(|&(_, count)| std::cmp::Reverse(count)); + let values: Buffer<$typ> = values.into_iter().map(|(value, _)| value).collect(); let max_code = values.len(); let codes = if max_code <= u8::MAX as usize { @@ -271,10 +277,10 @@ mod tests { #[test] fn test_float_dict_encode() -> VortexResult<()> { let mut ctx = vortex_array::array_session().create_execution_ctx(); - let values = buffer![1f32, 2f32, 2f32, 0f32, 1f32]; + let values = buffer![1f32, 2f32, 2f32, 0f32, 2f32]; let validity = Validity::Array(BoolArray::from_iter([true, true, true, false, true]).into_array()); - let array = PrimitiveArray::new(values, validity); + let array = PrimitiveArray::new(values, validity.clone()); let stats = FloatStats::generate_opts( &array, @@ -286,9 +292,19 @@ mod tests { let dict_array = dictionary_encode(array.as_view(), &stats)?; assert_eq!(dict_array.values().len(), 2); assert_eq!(dict_array.codes().len(), 5); + assert_arrays_eq!( + dict_array.values(), + PrimitiveArray::new(buffer![2f32, 1f32], Validity::AllValid).into_array(), + &mut ctx + ); + assert_arrays_eq!( + dict_array.codes(), + PrimitiveArray::new(buffer![1u8, 0, 0, 0, 0], validity).into_array(), + &mut ctx + ); let expected = PrimitiveArray::new( - buffer![1f32, 2f32, 2f32, 1f32, 1f32], + buffer![1f32, 2f32, 2f32, 2f32, 2f32], Validity::Array(BoolArray::from_iter([true, true, true, false, true]).into_array()), ) .into_array(); diff --git a/vortex-compressor/src/builtins/dict/integer.rs b/vortex-compressor/src/builtins/dict/integer.rs index 27a17ef94ad..27955ededde 100644 --- a/vortex-compressor/src/builtins/dict/integer.rs +++ b/vortex-compressor/src/builtins/dict/integer.rs @@ -157,7 +157,13 @@ macro_rules! typed_encode { }; let codes_validity = $source_array.validity()?; - let values: Buffer<$typ> = distinct.distinct_values().keys().map(|x| x.0).collect(); + let mut values = distinct + .distinct_values() + .iter() + .map(|(value, &count)| (value.0, count)) + .collect::>(); + values.sort_unstable_by_key(|&(_, count)| std::cmp::Reverse(count)); + let values: Buffer<$typ> = values.into_iter().map(|(value, _)| value).collect(); let max_code = values.len(); let codes = if max_code <= u8::MAX as usize { @@ -280,7 +286,7 @@ mod tests { let data = buffer![100i32, 200, 100, 0, 100]; let validity = Validity::Array(BoolArray::from_iter([true, true, true, false, true]).into_array()); - let array = PrimitiveArray::new(data, validity); + let array = PrimitiveArray::new(data, validity.clone()); let stats = IntegerStats::generate_opts( &array, @@ -292,6 +298,16 @@ mod tests { let dict_array = dictionary_encode(array.as_view(), &stats)?; assert_eq!(dict_array.values().len(), 2); assert_eq!(dict_array.codes().len(), 5); + assert_arrays_eq!( + dict_array.values(), + PrimitiveArray::new(buffer![100i32, 200], Validity::AllValid).into_array(), + &mut ctx + ); + assert_arrays_eq!( + dict_array.codes(), + PrimitiveArray::new(buffer![0u8, 1, 0, 0, 0], validity).into_array(), + &mut ctx + ); let expected = PrimitiveArray::new( buffer![100i32, 200, 100, 100, 100], diff --git a/vortex-compressor/src/stats/float.rs b/vortex-compressor/src/stats/float.rs index c505993e9e5..fa61f0c6506 100644 --- a/vortex-compressor/src/stats/float.rs +++ b/vortex-compressor/src/stats/float.rs @@ -19,22 +19,22 @@ use vortex_error::VortexResult; use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_mask::AllOr; -use vortex_utils::aliases::hash_set::HashSet; +use vortex_utils::aliases::hash_map::HashMap; use super::GenerateStatsOptions; /// Information about the distinct values in a float array. #[derive(Debug, Clone)] pub struct DistinctInfo { - /// The set of distinct float values. - distinct_values: HashSet, FxBuildHasher>, + /// The distinct float values and their occurrence counts. + distinct_values: HashMap, u32, FxBuildHasher>, /// The count of unique values. This _must_ be non-zero. distinct_count: u32, } impl DistinctInfo { - /// Returns a reference to the distinct values set. - pub fn distinct_values(&self) -> &HashSet, FxBuildHasher> { + /// Returns a reference to the distinct values map. + pub fn distinct_values(&self) -> &HashMap, u32, FxBuildHasher> { &self.distinct_values } } @@ -188,7 +188,7 @@ where average_run_length: 0, erased: TypedStats { distinct: Some(DistinctInfo { - distinct_values: HashSet::with_capacity_and_hasher(0, FxBuildHasher), + distinct_values: HashMap::with_capacity_and_hasher(0, FxBuildHasher), distinct_count: 0, }), } @@ -202,12 +202,10 @@ where .ok_or_else(|| vortex_err!("Failed to compute null_count"))?; let value_count = array.len() - null_count; - // Keep a HashMap of T, then convert the keys into PValue afterward since value is - // so much more efficient to hash and search for. let mut distinct_values = if count_distinct_values { - HashSet::with_capacity_and_hasher(array.len() / 2, FxBuildHasher) + HashMap::with_capacity_and_hasher(array.len() / 2, FxBuildHasher) } else { - HashSet::with_hasher(FxBuildHasher) + HashMap::with_hasher(FxBuildHasher) }; let validity = array @@ -227,7 +225,7 @@ where AllOr::All => { for value in first_valid_buff { if count_distinct_values { - distinct_values.insert(NativeValue(value)); + *distinct_values.entry(NativeValue(value)).or_insert(0) += 1; } if value != prev { @@ -244,7 +242,7 @@ where { if valid { if count_distinct_values { - distinct_values.insert(NativeValue(value)); + *distinct_values.entry(NativeValue(value)).or_insert(0) += 1; } if value != prev { From 4d483d42f6340649b67a0c1b97502f69b5eeef07 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 06:53:37 +0100 Subject: [PATCH 48/78] perf: accelerate range-packed float trees Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 93 ++++++++++- encodings/range-packed/src/array.rs | 151 +++++++++++------- encodings/range-packed/src/lib.rs | 54 +++++-- .../examples/float_compressor_bench.rs | 47 ++++-- 4 files changed, 258 insertions(+), 87 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 19af4aad724..6fc6761386f 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -231,7 +231,7 @@ This path avoids a full materialized array of references. ## Selection policy -Fast rejection is a strong preference for schemes in normal Default recursion. +Fast rejection is a selection advantage for schemes in normal Default recursion. A cheap rejection path reads a bounded sample and avoids child compression when the model does not fit. @@ -239,7 +239,9 @@ This path lets Default test more encodings with little throughput loss on reject Fast rejection is not a hard gate. -A scheme with costly analysis can enter Default when corpus evidence shows larger size or decode gains. +Do not use fast rejection as an admission criterion. + +A scheme with costly analysis can enter Default when stronger corpus evidence justifies its analysis cost. Specialized schemes remain useful when they bypass depth limits, recursive search, or an unfused common tree. @@ -1564,6 +1566,80 @@ It uses rank checkpoints every 256 values. Scalar access scans at most four 64-b Grouped bins improve Euro bulk decode by 43 percent. The complete Default tree still decodes about four times faster. +### Precomputed offset masks + +The serial decoder previously constructed one low-bit mask for each nonzero offset. + +The 8-lane decoder now stores one mask per bin in its 1 KiB decode table. + +For offset widths from 41 through 57 bits, each lane performs one load, one shift, and one mask operation. + +This path removes the unpredictable zero-width branch and the per-lane mask construction. + +The narrow path retains its zero-width branch because an unconditional load reduced Euro throughput. + +| Input | Fixed-bin bytes | Direct decode MB/s | Complete decode MB/s | Fused decode MB/s | Default decode MB/s | +| --- | ---: | ---: | ---: | ---: | ---: | +| Euro subjectivity | 7,270,094 | 15,932 | 5,121 | 5,965 | 17,829 to 20,600 | +| Food volume | 5,653,643 | 14,244 | 9,025 | 12,114 | 24,903 | +| CMS `LINE_SRVC_CNT` | 2,463,791 | 14,031 | 5,731 | 4,071 | 26,249 | +| CMS average payment | 10,845,743 | 15,390 | 3,545 | 3,746 | 23,861 | + +Euro direct decode previously reached 3,721 to 3,979 MB/s. + +The precomputed mask increases its direct throughput by about four times. + +The 500-iteration rerun produced 15,932 MB/s. + +Food and CMS `LINE_SRVC_CNT` use offset widths of at most 40 bits, so they do not use the new path. + +CMS average payment uses the new path, but BlockResidual gives a smaller complete tree. + +The optimized codec establishes a much higher fixed-bin decode ceiling. + +The complete dense candidate still fails the Default decode gate because validity checks and float reconstruction dominate its final path. + +An in-place reverse validity expansion reduced complete Euro decode to 3,116 MB/s. + +Its reverse bit iteration cost exceeded the allocation and copy cost that it removed. + +The implementation retains the forward expansion with a separate logical output vector. + +### Full-position null storage + +The full-position mode stores physical values at null positions instead of a dense valid-value stream. + +The validity child remains present, but the rank child is absent. + +The missing rank child identifies full-position storage without a metadata flag. + +Bulk decode then writes one value per logical position and skips validity expansion. + +Scalar access uses the logical index directly. + +| Input | Mode | Bytes | Encode MB/s | Fused decode MB/s | Scalar ns | +| --- | --- | ---: | ---: | ---: | ---: | +| Euro subjectivity | Dense | 7,270,094 | 461 | 5,965 | 232 | +| Euro subjectivity | Full-position | 8,180,506 | 654 | 15,180 | 206 | +| CMS `LINE_SRVC_CNT` | Dense | 2,463,791 | 875 | 4,071 | 465 | +| CMS `LINE_SRVC_CNT` | Full-position | 2,470,956 | 942 | 10,623 | 445 | + +The Euro result uses 200 complete-tree decode iterations. + +Default uses 14,234,326 bytes and decodes at 20,924 MB/s on the same run. + +Full-position RangePacked reduces Euro size by 42.5 percent and reduces decode throughput by 27.5 percent. + +Compact uses 6,860,457 bytes and decodes at 2,574 MB/s. + +The full-position candidate retains 82.1 percent of the Compact size gain and decodes 5.9 times faster than Compact. + +CMS `LINE_SRVC_CNT` still favors `ALP(BlockResidual)` for Default. + +That tree uses 2,725,541 bytes and decodes at 23,493 MB/s. + +The Euro candidate now warrants a complete corpus and selector-cost evaluation. + CMS uses similar offset widths across its bins. The grouped scatter path then loses more than half of serial-bin throughput. Grouped bins do not generalize. The prototype remains outside production code. @@ -1826,13 +1902,14 @@ The nonzero-secondary FloatQuant implementation and focused validation are compl Complete these next experiments in order: -1. Classify the remaining no-Delta ALP and raw-float gaps. -2. Define a bounded small-alphabet primitive for the remaining 24-to-49-bin columns. -3. Add a specialized range scheme only after a complete tree passes the column gates. -4. Validate frequency-ranked Dict across the complete file corpus. -5. Validate rejected-candidate analysis cost after the candidate set stabilizes. +1. Add full-position RangePacked to the focused Default bundle as an experimental candidate. +2. Compare specialized OrderedFloat and ALP trees with their fused decoders. +3. Add a cheap sample rejection test if it preserves the best range model. +4. Validate frequency-ranked Dict and full-position RangePacked across the complete file corpus. +5. Measure rejected-candidate analysis cost after the candidate set stabilizes. 6. Prefer cheap rejection tests when they preserve the best candidate model. -7. Require stronger corpus evidence when a useful scheme needs costly analysis. +7. Treat fast rejection as a selection advantage, not an admission criterion. +8. Require stronger corpus evidence when a useful scheme needs costly analysis. Use packed bin codes and primitive bin starts unless evidence supports more complexity. diff --git a/encodings/range-packed/src/array.rs b/encodings/range-packed/src/array.rs index 5963c110c12..333989deb26 100644 --- a/encodings/range-packed/src/array.rs +++ b/encodings/range-packed/src/array.rs @@ -79,7 +79,7 @@ pub struct RangePackedSlots { #[derive(Clone, Debug)] pub struct RangePackedData { unsliced_len: usize, - dense_len: usize, + stored_len: usize, slice_start: usize, slice_stop: usize, payload_len: usize, @@ -101,7 +101,7 @@ impl Display for RangePackedData { impl ArrayHash for RangePackedData { fn array_hash(&self, state: &mut H, _accuracy: EqMode) { self.unsliced_len.hash(state); - self.dense_len.hash(state); + self.stored_len.hash(state); self.slice_start.hash(state); self.slice_stop.hash(state); self.payload_len.hash(state); @@ -112,7 +112,7 @@ impl ArrayHash for RangePackedData { impl ArrayEq for RangePackedData { fn array_eq(&self, other: &Self, _accuracy: EqMode) -> bool { self.unsliced_len == other.unsliced_len - && self.dense_len == other.dense_len + && self.stored_len == other.stored_len && self.slice_start == other.slice_start && self.slice_stop == other.slice_stop && self.payload_len == other.payload_len @@ -181,37 +181,37 @@ impl VTable for RangePacked { ) -> VortexResult> { let metadata = RangePackedMetadata::decode(metadata)?; let unsliced_len = usize::try_from(metadata.unsliced_len)?; - let dense_len = usize::try_from(metadata.dense_len)?; + let stored_len = usize::try_from(metadata.stored_len)?; let slice_start = usize::try_from(metadata.slice_start)?; let slice_stop = slice_start .checked_add(len) .ok_or_else(|| vortex_error::vortex_err!("RangePacked slice length overflows"))?; let payload_len = usize::try_from(metadata.payload_len)?; - let bin_count = bin_count(dense_len, metadata.symbol_width)?; - let block_count = dense_len.div_ceil(MAX_BLOCK_LEN); + let bin_count = bin_count(stored_len, metadata.symbol_width)?; + let block_count = stored_len.div_ceil(MAX_BLOCK_LEN); vortex_ensure!( - matches!(children.len(), 4 | 6), - "RangePackedArray expects four or six children" + matches!(children.len(), 4..=6), + "RangePackedArray expects four, five, or six children" ); let bin_lowers = children.get(0, &primitive_dtype(PType::U64), bin_count)?; let offset_widths = children.get(1, &primitive_dtype(PType::U8), bin_count)?; let block_offsets = children.get(2, &primitive_dtype(PType::U32), block_count + 1)?; let payload = children.get(3, &primitive_dtype(PType::U8), payload_len)?; - let (validity_bits, rank_checkpoints) = if children.len() == 6 { - ( - Some(children.get(4, &DType::Bool(NonNullable), unsliced_len)?), - Some(children.get( + let validity_bits = (children.len() >= 5) + .then(|| children.get(4, &DType::Bool(NonNullable), unsliced_len)) + .transpose()?; + let rank_checkpoints = (children.len() == 6) + .then(|| { + children.get( 5, &primitive_dtype(PType::U32), unsliced_len.div_ceil(VALIDITY_CHECKPOINT_INTERVAL), - )?), - ) - } else { - (None, None) - }; + ) + }) + .transpose()?; let data = RangePackedData { unsliced_len, - dense_len, + stored_len, slice_start, slice_stop, payload_len, @@ -305,6 +305,22 @@ impl RangePacked { pub fn from_primitive( array: ArrayView<'_, Primitive>, ctx: &mut ExecutionCtx, + ) -> VortexResult { + Self::from_primitive_mode(array, ctx, false) + } + + /// Encodes every physical value, including values at null positions. + pub fn from_primitive_with_null_positions( + array: ArrayView<'_, Primitive>, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + Self::from_primitive_mode(array, ctx, true) + } + + fn from_primitive_mode( + array: ArrayView<'_, Primitive>, + ctx: &mut ExecutionCtx, + preserve_null_positions: bool, ) -> VortexResult { vortex_ensure!( array.ptype().is_int(), @@ -312,22 +328,28 @@ impl RangePacked { ); let validity = array.validity()?; let mask = validity.execute_mask(array.len(), ctx)?; - let dense_len = mask.true_count(); - let dense_ordered = ordered_primitive(array)? - .into_iter() - .zip(mask.iter()) - .filter_map(|(value, valid)| valid.then_some(value)) - .collect::>(); - let codec = RangePackedCodec::encode(&dense_ordered, MAX_BLOCK_LEN)?; - let actual_nulls = dense_len != array.len(); + let valid_count = mask.true_count(); + let actual_nulls = valid_count != array.len(); + let ordered = ordered_primitive(array)?; + let stored_ordered = if preserve_null_positions { + ordered + } else { + ordered + .into_iter() + .zip(mask.iter()) + .filter_map(|(value, valid)| valid.then_some(value)) + .collect::>() + }; + let stored_len = stored_ordered.len(); + let codec = RangePackedCodec::encode(&stored_ordered, MAX_BLOCK_LEN)?; let validity_bits = actual_nulls.then(|| BoolArray::from_iter(mask.iter()).into_array()); - let rank_checkpoints = actual_nulls - .then(|| rank_checkpoints(mask.iter(), dense_len)) + let rank_checkpoints = (actual_nulls && !preserve_null_positions) + .then(|| rank_checkpoints(mask.iter(), stored_len)) .transpose()? .map(|ranks| PrimitiveArray::from_iter(ranks).into_array()); let data = RangePackedData { unsliced_len: array.len(), - dense_len, + stored_len, slice_start: 0, slice_stop: array.len(), payload_len: codec.payload.len(), @@ -368,6 +390,9 @@ impl RangePacked { let dense_values = with_decoder(array, |decoder| { decoder.decode_mapped_range(dense_start..dense_stop, map) })?; + if array.data().stored_len == array.data().unsliced_len { + return Ok(dense_values); + } let Some(validity_bits) = array.validity_bits() else { return Ok(dense_values); }; @@ -406,8 +431,8 @@ impl RangePackedData { ); vortex_ensure!(len == self.len(), "RangePacked slice length is invalid"); vortex_ensure!( - self.dense_len <= self.unsliced_len, - "RangePacked dense length exceeds its source length" + self.stored_len <= self.unsliced_len, + "RangePacked stored length exceeds its source length" ); let bin_count = self.bin_count(); validate_primitive_child(slots.bin_lowers, PType::U64, bin_count, "bin lowers")?; @@ -415,15 +440,15 @@ impl RangePackedData { validate_primitive_child( slots.block_offsets, PType::U32, - self.dense_len.div_ceil(MAX_BLOCK_LEN) + 1, + self.stored_len.div_ceil(MAX_BLOCK_LEN) + 1, "block offsets", )?; validate_primitive_child(slots.payload, PType::U8, self.payload_len, "payload")?; vortex_ensure!( - slots.validity_bits.is_some() == slots.rank_checkpoints.is_some(), - "RangePacked validity and rank children must occur together" + slots.validity_bits.is_some() || slots.rank_checkpoints.is_none(), + "RangePacked rank checkpoints require validity" ); - if let (Some(validity_bits), Some(ranks)) = (slots.validity_bits, slots.rank_checkpoints) { + if let Some(validity_bits) = slots.validity_bits { vortex_ensure!( dtype.is_nullable(), "RangePacked validity requires a nullable dtype" @@ -434,20 +459,27 @@ impl RangePackedData { && validity_bits.len() == self.unsliced_len, "RangePacked validity child is invalid" ); - validate_primitive_child( - ranks, - PType::U32, - self.unsliced_len.div_ceil(VALIDITY_CHECKPOINT_INTERVAL), - "rank checkpoints", - )?; - validate_ranks( - validity_bits.as_::(), - ranks.as_::().as_slice::(), - self.dense_len, - )?; + if let Some(ranks) = slots.rank_checkpoints { + validate_primitive_child( + ranks, + PType::U32, + self.unsliced_len.div_ceil(VALIDITY_CHECKPOINT_INTERVAL), + "rank checkpoints", + )?; + validate_ranks( + validity_bits.as_::(), + ranks.as_::().as_slice::(), + self.stored_len, + )?; + } else { + vortex_ensure!( + self.stored_len == self.unsliced_len, + "RangePacked without rank checkpoints must store every value" + ); + } } else { vortex_ensure!( - self.dense_len == self.unsliced_len, + self.stored_len == self.unsliced_len, "RangePacked without validity must store every value" ); } @@ -464,7 +496,7 @@ impl RangePackedData { } fn bin_count(&self) -> usize { - if self.dense_len == 0 { + if self.stored_len == 0 { 0 } else { 1_usize << self.symbol_width @@ -534,7 +566,7 @@ fn validate_block_offsets(offsets: &[u32], payload_len: usize) -> VortexResult<( Ok(()) } -fn validate_ranks(bits: ArrayView<'_, Bool>, ranks: &[u32], dense_len: usize) -> VortexResult<()> { +fn validate_ranks(bits: ArrayView<'_, Bool>, ranks: &[u32], stored_len: usize) -> VortexResult<()> { let bits = bits.bit_buffer_view(); let mut rank = 0usize; for (checkpoint, &stored) in ranks.iter().enumerate() { @@ -546,7 +578,7 @@ fn validate_ranks(bits: ArrayView<'_, Bool>, ranks: &[u32], dense_len: usize) -> let stop = (start + VALIDITY_CHECKPOINT_INTERVAL).min(bits.len()); rank += bits.slice(start..stop).true_count(); } - vortex_ensure!(rank == dense_len, "RangePacked dense length is invalid"); + vortex_ensure!(rank == stored_len, "RangePacked stored length is invalid"); Ok(()) } @@ -574,11 +606,14 @@ fn is_valid(array: ArrayView<'_, RangePacked>, global_index: usize) -> bool { } fn dense_rank(array: ArrayView<'_, RangePacked>, global_index: usize) -> VortexResult { + if array.data().stored_len == array.data().unsliced_len { + return Ok(global_index); + } let Some(validity) = array.validity_bits() else { return Ok(global_index); }; if global_index == array.data().unsliced_len { - return Ok(array.data().dense_len); + return Ok(array.data().stored_len); } let checkpoint = global_index / VALIDITY_CHECKPOINT_INTERVAL; let checkpoint_start = checkpoint * VALIDITY_CHECKPOINT_INTERVAL; @@ -606,7 +641,7 @@ fn with_decoder( let block_offsets = array.block_offsets().as_::(); let payload = array.payload().as_::(); let decoder = RangePackedDecoder::try_new( - array.data().dense_len, + array.data().stored_len, MAX_BLOCK_LEN, array.data().symbol_width, BinTableView::Split { @@ -757,12 +792,12 @@ fn primitive_dtype(ptype: PType) -> DType { DType::Primitive(ptype, NonNullable) } -fn bin_count(dense_len: usize, symbol_width: u8) -> VortexResult { +fn bin_count(stored_len: usize, symbol_width: u8) -> VortexResult { vortex_ensure!( symbol_width <= 6, "RangePacked symbol width exceeds six bits" ); - Ok(if dense_len == 0 { + Ok(if stored_len == 0 { 0 } else { 1_usize << symbol_width @@ -772,7 +807,7 @@ fn bin_count(dense_len: usize, symbol_width: u8) -> VortexResult { #[derive(Clone, Copy)] struct RangePackedMetadata { unsliced_len: u64, - dense_len: u64, + stored_len: u64, slice_start: u64, payload_len: u64, symbol_width: u8, @@ -782,7 +817,7 @@ impl RangePackedMetadata { fn from_data(data: &RangePackedData) -> VortexResult { Ok(Self { unsliced_len: u64::try_from(data.unsliced_len)?, - dense_len: u64::try_from(data.dense_len)?, + stored_len: u64::try_from(data.stored_len)?, slice_start: u64::try_from(data.slice_start)?, payload_len: u64::try_from(data.payload_len)?, symbol_width: data.symbol_width, @@ -793,7 +828,7 @@ impl RangePackedMetadata { let mut bytes = Vec::with_capacity(METADATA_LEN); bytes.push(METADATA_VERSION); bytes.extend_from_slice(&self.unsliced_len.to_le_bytes()); - bytes.extend_from_slice(&self.dense_len.to_le_bytes()); + bytes.extend_from_slice(&self.stored_len.to_le_bytes()); bytes.extend_from_slice(&self.slice_start.to_le_bytes()); bytes.extend_from_slice(&self.payload_len.to_le_bytes()); bytes.push(self.symbol_width); @@ -812,7 +847,7 @@ impl RangePackedMetadata { ); Ok(Self { unsliced_len: read_u64(bytes, 1), - dense_len: read_u64(bytes, 9), + stored_len: read_u64(bytes, 9), slice_start: read_u64(bytes, 17), payload_len: read_u64(bytes, 25), symbol_width: bytes[33], diff --git a/encodings/range-packed/src/lib.rs b/encodings/range-packed/src/lib.rs index bedbe39bc57..fa6c5a709fb 100644 --- a/encodings/range-packed/src/lib.rs +++ b/encodings/range-packed/src/lib.rs @@ -465,6 +465,7 @@ impl<'a> RangePackedDecoder<'a> { struct DecodeTable { lowers: [u64; MAX_BINS], widths: [u8; MAX_BINS], + masks: [u64; MAX_BINS], bin_count: usize, } @@ -472,6 +473,7 @@ impl DecodeTable { fn new(bins: BinTableView<'_>) -> Self { let mut lowers = [0_u64; MAX_BINS]; let mut widths = [0_u8; MAX_BINS]; + let mut masks = [0_u64; MAX_BINS]; for index in 0..bins.len() { lowers[index] = bins .lower(index) @@ -479,10 +481,12 @@ impl DecodeTable { widths[index] = bins .width(index) .unwrap_or_else(|| unreachable!("validated range packed bin table")); + masks[index] = low_mask(widths[index]); } Self { lowers, widths, + masks, bin_count: bins.len(), } } @@ -564,6 +568,7 @@ where if PARALLEL { let mut widths = [0_u8; 8]; let mut lowers = [0_u64; 8]; + let mut masks = [0_u64; 8]; let mut positions = [0_usize; 8]; for lane in 0..8 { let symbol = @@ -573,11 +578,17 @@ where widths[lane] = unsafe { *table.widths.get_unchecked(symbol) }; // SAFETY: A power-of-two bin count makes every symbol code valid. lowers[lane] = unsafe { *table.lowers.get_unchecked(symbol) }; + // SAFETY: A power-of-two bin count makes every symbol code valid. + masks[lane] = unsafe { *table.masks.get_unchecked(symbol) }; positions[lane] = offset_position; offset_position += usize::from(widths[lane]); } for lane in 0..8 { - let offset = read_offset::(offsets, positions[lane], widths[lane]); + let offset = if WIDE { + read_offset::(offsets, positions[lane], widths[lane]) + } else { + read_offset_masked(offsets, positions[lane], masks[lane]) + }; // SAFETY: This lane is within a complete eight-value output batch. unsafe { values @@ -638,6 +649,15 @@ fn read_symbol(payload: &[u8], index: usize, symbol_width: u8) -> VortexResult u64 { + let byte_position = bit_position / 8; + let bits_past_byte = bit_position % 8; + // SAFETY: Offset streams include fifteen readable padding bytes. + let packed = unsafe { read_u64_unaligned(offsets.as_ptr().add(byte_position)) }; + (packed >> bits_past_byte) & mask +} + #[inline(always)] fn read_offset(offsets: &[u8], bit_position: usize, width: u8) -> u64 { if WIDE { @@ -967,14 +987,22 @@ mod tests { Ok(()) } - #[test] - fn nullable_dense_roundtrip_and_rank_boundaries() -> VortexResult<()> { + #[rstest] + #[case::dense(false)] + #[case::full(true)] + fn nullable_roundtrip_and_rank_boundaries( + #[case] preserve_null_positions: bool, + ) -> VortexResult<()> { let expected = PrimitiveArray::from_option_iter((0_i64..600).map(|value| { (!matches!(value, 0 | 1 | 63 | 64 | 255 | 256 | 257 | 511)).then_some(value - 300) })); let session = array_session(); let mut ctx = session.create_execution_ctx(); - let encoded = RangePacked::from_primitive(expected.as_view(), &mut ctx)?; + let encoded = if preserve_null_positions { + RangePacked::from_primitive_with_null_positions(expected.as_view(), &mut ctx)? + } else { + RangePacked::from_primitive(expected.as_view(), &mut ctx)? + }; assert_arrays_eq!(encoded, expected, &mut ctx); for index in [0, 1, 2, 63, 64, 65, 255, 256, 257, 258, 511, 512, 599] { let actual = encoded.execute_scalar(index, &mut ctx)?; @@ -1019,17 +1047,25 @@ mod tests { Ok(()) } - #[test] - fn serialized_nullable_slice_roundtrip() -> VortexResult<()> { + #[rstest] + #[case::dense(false)] + #[case::full(true)] + fn serialized_nullable_slice_roundtrip( + #[case] preserve_null_positions: bool, + ) -> VortexResult<()> { let expected = PrimitiveArray::from_option_iter( (0_i64..1_500).map(|value| (value % 17 != 0).then_some(value - 750)), ); let session = array_session(); initialize(&session); let mut ctx = session.create_execution_ctx(); - let encoded = RangePacked::from_primitive(expected.as_view(), &mut ctx)? - .into_array() - .slice(251..1_101)?; + let encoded = if preserve_null_positions { + RangePacked::from_primitive_with_null_positions(expected.as_view(), &mut ctx)? + } else { + RangePacked::from_primitive(expected.as_view(), &mut ctx)? + } + .into_array() + .slice(251..1_101)?; let expected = expected.into_array().slice(251..1_101)?; let dtype = encoded.dtype().clone(); let len = encoded.len(); diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index c3a333527b6..c70c903d74b 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -1298,6 +1298,18 @@ fn codec_decode_iterations() -> VortexResult { .map(|iterations| iterations.unwrap_or(20)) } +fn tree_decode_iterations() -> VortexResult { + std::env::var("VORTEX_BENCH_TREE_DECODE_ITERATIONS") + .ok() + .map(|value| { + value.parse::().map_err(|error| { + vortex_err!("invalid VORTEX_BENCH_TREE_DECODE_ITERATIONS: {error}") + }) + }) + .transpose() + .map(|iterations| iterations.unwrap_or(20)) +} + #[derive(Clone, Copy)] enum FixedBinOffsetTree { BlockResidual, @@ -1863,8 +1875,7 @@ fn fixed_bin_float_tree( .cast(alp.encoded().dtype().clone())? .execute::(&mut session.create_execution_ctx())?; let encoded = - RangePacked::from_primitive(primitive.as_view(), &mut session.create_execution_ctx())? - .into_array(); + range_packed_from_primitive(primitive.as_view(), &mut session.create_execution_ctx())?; return Ok(Some( ALP::try_new(encoded, alp.exponents(), alp.patches())?.into_array(), )); @@ -1879,15 +1890,26 @@ fn fixed_bin_float_tree( let primitive = fill_float_nulls_with_first_valid(primitive, &mut session.create_execution_ctx())?; let ordered = OrderedFloat::from_primitive(primitive.as_view())?; - let encoded = RangePacked::from_primitive( + let encoded = range_packed_from_primitive( ordered.encoded().as_::(), &mut session.create_execution_ctx(), )?; Ok(Some( - OrderedFloat::try_new(encoded.into_array(), primitive.ptype())?.into_array(), + OrderedFloat::try_new(encoded, primitive.ptype())?.into_array(), )) } +fn range_packed_from_primitive( + primitive: vortex_array::ArrayView<'_, Primitive>, + ctx: &mut vortex_array::ExecutionCtx, +) -> VortexResult { + if std::env::var_os("VORTEX_BENCH_FIXED_BIN_FULL_POSITIONS").is_some() { + Ok(RangePacked::from_primitive_with_null_positions(primitive, ctx)?.into_array()) + } else { + Ok(RangePacked::from_primitive(primitive, ctx)?.into_array()) + } +} + fn fill_float_nulls_with_first_valid( primitive: PrimitiveArray, ctx: &mut vortex_array::ExecutionCtx, @@ -1945,8 +1967,9 @@ fn measure_fixed_bin_tree( ); assert_arrays_eq!(encoded, expected, &mut session.create_execution_ctx()); - let mut decode_durations = Vec::with_capacity(20); - for _ in 0..20 { + let decode_iterations = tree_decode_iterations()?; + let mut decode_durations = Vec::with_capacity(decode_iterations); + for _ in 0..decode_iterations { let start = Instant::now(); black_box( encoded @@ -1966,8 +1989,8 @@ fn measure_fixed_bin_tree( encoding_tree(&encoded), ); - let mut fused_durations = Vec::with_capacity(20); - for _ in 0..20 { + let mut fused_durations = Vec::with_capacity(decode_iterations); + for _ in 0..decode_iterations { let start = Instant::now(); black_box(decode_fixed_bin_float_tree(&encoded, session)?); fused_durations.push(start.elapsed()); @@ -2231,11 +2254,11 @@ fn encode_fixed_bin_float_tree( ) -> VortexResult { if compact.encoding_id().as_ref() == "vortex.alp" { let alp = alp_encode(primitive, None, &mut session.create_execution_ctx())?; - let packed = RangePacked::from_primitive( + let packed = range_packed_from_primitive( alp.encoded().as_::(), &mut session.create_execution_ctx(), )?; - return Ok(ALP::try_new(packed.into_array(), alp.exponents(), alp.patches())?.into_array()); + return Ok(ALP::try_new(packed, alp.exponents(), alp.patches())?.into_array()); } let primitive = fill_float_nulls_with_first_valid( @@ -2243,11 +2266,11 @@ fn encode_fixed_bin_float_tree( &mut session.create_execution_ctx(), )?; let ordered = OrderedFloat::from_primitive(primitive.as_view())?; - let packed = RangePacked::from_primitive( + let packed = range_packed_from_primitive( ordered.encoded().as_::(), &mut session.create_execution_ctx(), )?; - Ok(OrderedFloat::try_new(packed.into_array(), primitive.ptype())?.into_array()) + Ok(OrderedFloat::try_new(packed, primitive.ptype())?.into_array()) } fn encode_decomposed_fixed_bin_float_tree( From af23b710b04ec0bf375344d6b91f709d33a7f16b Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 07:09:18 +0100 Subject: [PATCH 49/78] feat: add experimental float range packing Signed-off-by: Will Manning --- Cargo.lock | 2 + docs/plans/native-pcodec.md | 69 +++- encodings/range-packed/Cargo.toml | 2 + encodings/range-packed/src/kernel.rs | 169 ++++++++++ encodings/range-packed/src/lib.rs | 69 ++++ .../examples/float_compressor_bench.rs | 16 + vortex-btrblocks/src/schemes/float/mod.rs | 2 + .../src/schemes/float/range_packed.rs | 294 ++++++++++++++++++ 8 files changed, 615 insertions(+), 8 deletions(-) create mode 100644 encodings/range-packed/src/kernel.rs create mode 100644 vortex-btrblocks/src/schemes/float/range_packed.rs diff --git a/Cargo.lock b/Cargo.lock index e69e0fbd6cd..07fc20f700c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10568,7 +10568,9 @@ name = "vortex-range-packed" version = "0.1.0" dependencies = [ "rstest", + "vortex-alp", "vortex-array", + "vortex-block-residual", "vortex-buffer", "vortex-error", "vortex-int-mult", diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 6fc6761386f..3305db00312 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -1644,6 +1644,57 @@ CMS uses similar offset widths across its bins. The grouped scatter path then lo Grouped bins do not generalize. The prototype remains outside production code. +### Fused parent kernels and the experimental selector + +RangePacked now provides parent kernels for `OrderedFloat(RangePacked)` and `ALP(RangePacked)`. + +The session registry selects these kernels from the child encoding and the parent encoding. + +The outer arrays remain generic composable arrays. + +On Euro subjectivity, normal recursive execution reaches 14,956 MB/s. + +The manual fused benchmark reaches 15,492 MB/s on the same run. + +The registered kernel is within 3.5 percent of the manual helper. + +An experimental `FloatRangePackedScheme` remains outside `ALL_SCHEMES`. + +The scheme tests two complete trees on the existing stratified one-percent sample: + +- `OrderedFloat(RangePacked)` +- `ALP(RangePacked)` + +The full encoder builds only the smaller sampled tree. + +The selector divides the sample ratio by a provisional 1.20 decode cost factor. + +This factor does not produce the intended complete-tree choices on CMS or Food. + +The one-percent sample overstates the RangePacked advantage against BlockResidual on those columns. + +| Input | Default bytes | Candidate bytes | Default encode MB/s | Candidate encode MB/s | Default decode MB/s | Candidate decode MB/s | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Euro subjectivity | 14,234,326 | 8,180,506 | 362.4 | 376.4 | 21,795 | 14,849 | +| CMS `LINE_SRVC_CNT` | 2,725,541 | 2,436,604 | 1,054.8 | 630.5 | 24,227 | 10,354 | +| Food volume | 6,599,221 | 5,653,643 | 491.3 | 538.2 | 19,747 | 11,920 | + +Euro is the target win. + +CMS and Food do not justify their decode losses at the current size gains. + +The rejected-candidate cost is also material. + +On uniform random floats, the candidate reduces total encode throughput by 6.8 percent. + +On widened `f32` values, FloatQuant still wins, but the candidate reduces encode throughput by 18.8 percent. + +These results justify a cheap rejection test, if that test retains the Euro model. + +Fast rejection remains an advantage, not an admission rule. + +A candidate with costly analysis needs stronger corpus evidence. + ### Multi-reference block prototype The prototype stores up to four quantile references per 1,024-value block. @@ -1900,16 +1951,18 @@ The composable IntMult follow-up is complete. The tested IntMult trees remain ou The nonzero-secondary FloatQuant implementation and focused validation are complete. +The focused Default bundle now includes full-position RangePacked as an experimental candidate. + +The specialized OrderedFloat and ALP trees now use registered fused parent kernels. + Complete these next experiments in order: -1. Add full-position RangePacked to the focused Default bundle as an experimental candidate. -2. Compare specialized OrderedFloat and ALP trees with their fused decoders. -3. Add a cheap sample rejection test if it preserves the best range model. -4. Validate frequency-ranked Dict and full-position RangePacked across the complete file corpus. -5. Measure rejected-candidate analysis cost after the candidate set stabilizes. -6. Prefer cheap rejection tests when they preserve the best candidate model. -7. Treat fast rejection as a selection advantage, not an admission criterion. -8. Require stronger corpus evidence when a useful scheme needs costly analysis. +1. Add a cheap sample rejection test if it preserves the Euro range model. +2. Validate frequency-ranked Dict and full-position RangePacked across the complete file corpus. +3. Measure rejected-candidate analysis cost after the candidate set stabilizes. +4. Prefer cheap rejection tests when they preserve the best candidate model. +5. Treat fast rejection as a selection advantage, not an admission criterion. +6. Require stronger corpus evidence when a useful scheme needs costly analysis. Use packed bin codes and primitive bin starts unless evidence supports more complexity. diff --git a/encodings/range-packed/Cargo.toml b/encodings/range-packed/Cargo.toml index a1323327a58..6a18fa6da4e 100644 --- a/encodings/range-packed/Cargo.toml +++ b/encodings/range-packed/Cargo.toml @@ -14,7 +14,9 @@ rust-version = { workspace = true } version = { workspace = true } [dependencies] +vortex-alp = { workspace = true } vortex-array = { workspace = true } +vortex-block-residual = { workspace = true } vortex-buffer = { workspace = true } vortex-error = { workspace = true } vortex-int-mult = { workspace = true } diff --git a/encodings/range-packed/src/kernel.rs b/encodings/range-packed/src/kernel.rs new file mode 100644 index 00000000000..6662355ccec --- /dev/null +++ b/encodings/range-packed/src/kernel.rs @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use vortex_alp::ALP; +use vortex_alp::ALPArrayExt; +use vortex_alp::ALPFloat; +use vortex_array::ArrayRef; +use vortex_array::ArrayVTable; +use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::dtype::PType; +use vortex_array::dtype::half::f16; +use vortex_array::kernel::ExecuteParentKernel; +use vortex_array::optimizer::kernels::ArrayKernelsExt; +use vortex_block_residual::OrderedFloat; +use vortex_error::VortexResult; +use vortex_error::vortex_bail; +use vortex_session::VortexSession; + +use crate::RangePacked; + +pub(super) fn initialize(session: &VortexSession) { + let kernels = session.kernels(); + kernels.register_execute_parent_kernel( + OrderedFloat.id(), + RangePacked, + OrderedFloatRangePackedKernel, + ); + kernels.register_execute_parent_kernel(ALP.id(), RangePacked, ALPRangePackedKernel); +} + +#[derive(Debug)] +struct OrderedFloatRangePackedKernel; + +impl ExecuteParentKernel for OrderedFloatRangePackedKernel { + type Parent = OrderedFloat; + + #[expect( + clippy::cast_possible_truncation, + reason = "OrderedFloat validates the child integer width against the float width" + )] + fn execute_parent( + &self, + array: ArrayView<'_, RangePacked>, + parent: ArrayView<'_, OrderedFloat>, + child_idx: usize, + _ctx: &mut ExecutionCtx, + ) -> VortexResult> { + if child_idx != 0 { + return Ok(None); + } + + let validity = array.validity()?; + let decoded = match parent.dtype().as_ptype() { + PType::F16 => PrimitiveArray::new::( + RangePacked::decode_mapped( + array, + |value| f16::from_bits(unordered_u16(value as u16)), + f16::ZERO, + )?, + validity, + ), + PType::F32 => PrimitiveArray::new::( + RangePacked::decode_mapped( + array, + |value| f32::from_bits(unordered_u32(value as u32)), + 0.0, + )?, + validity, + ), + PType::F64 => PrimitiveArray::new::( + RangePacked::decode_mapped( + array, + |value| f64::from_bits(unordered_u64(value)), + 0.0, + )?, + validity, + ), + ptype => vortex_bail!("OrderedFloat RangePacked kernel does not support {ptype}"), + }; + Ok(Some(decoded.into_array())) + } +} + +#[derive(Debug)] +struct ALPRangePackedKernel; + +impl ExecuteParentKernel for ALPRangePackedKernel { + type Parent = ALP; + + #[expect( + clippy::cast_possible_truncation, + reason = "ALP validates the child integer width against the float width" + )] + fn execute_parent( + &self, + array: ArrayView<'_, RangePacked>, + parent: ArrayView<'_, ALP>, + child_idx: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { + if child_idx != 0 { + return Ok(None); + } + + let validity = array.validity()?; + let exponents = parent.exponents(); + let decoded = match parent.dtype().as_ptype() { + PType::F32 => PrimitiveArray::new::( + RangePacked::decode_mapped( + array, + |ordered| { + let encoded = ((ordered as u32) ^ (1_u32 << 31)) as i32; + ::decode_single(encoded, exponents) + }, + 0.0, + )?, + validity, + ), + PType::F64 => PrimitiveArray::new::( + RangePacked::decode_mapped( + array, + |ordered| { + let encoded = (ordered ^ (1_u64 << 63)) as i64; + ::decode_single(encoded, exponents) + }, + 0.0, + )?, + validity, + ), + ptype => vortex_bail!("ALP RangePacked kernel does not support {ptype}"), + }; + let decoded = if let Some(patches) = parent.patches() { + decoded.patch(&patches, ctx)? + } else { + decoded + }; + Ok(Some(decoded.into_array())) + } +} + +#[inline] +fn unordered_u16(value: u16) -> u16 { + if value & (1_u16 << 15) == 0 { + !value + } else { + value ^ (1_u16 << 15) + } +} + +#[inline] +fn unordered_u32(value: u32) -> u32 { + if value & (1_u32 << 31) == 0 { + !value + } else { + value ^ (1_u32 << 31) + } +} + +#[inline] +fn unordered_u64(value: u64) -> u64 { + if value & (1_u64 << 63) == 0 { + !value + } else { + value ^ (1_u64 << 63) + } +} diff --git a/encodings/range-packed/src/lib.rs b/encodings/range-packed/src/lib.rs index fa6c5a709fb..f8609970964 100644 --- a/encodings/range-packed/src/lib.rs +++ b/encodings/range-packed/src/lib.rs @@ -4,6 +4,7 @@ //! Fixed-bin range packing with bounded random access. mod array; +mod kernel; use std::cmp::Ordering; use std::hash::Hash; @@ -119,6 +120,7 @@ impl RangeDecomposition { /// Register the RangePacked encoding in one session. pub fn initialize(session: &VortexSession) { session.arrays().register(RangePacked); + kernel::initialize(session); } impl RangePackedCodec { @@ -923,6 +925,10 @@ impl BitWriter { #[cfg(test)] mod tests { use rstest::rstest; + use vortex_alp::ALP; + use vortex_alp::ALPArrayExt; + use vortex_alp::ALPArraySlotsExt; + use vortex_alp::alp_encode; use vortex_array::ArrayContext; use vortex_array::ArrayRef; use vortex_array::IntoArray; @@ -931,8 +937,11 @@ mod tests { use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; + use vortex_array::dtype::half::f16; use vortex_array::serde::SerializeOptions; use vortex_array::serde::SerializedArray; + use vortex_block_residual::OrderedFloat; + use vortex_block_residual::OrderedFloatArraySlotsExt; use vortex_buffer::ByteBufferMut; use vortex_error::VortexResult; use vortex_session::registry::ReadContext; @@ -976,6 +985,66 @@ mod tests { Ok(()) } + #[rstest] + #[case::f16(PrimitiveArray::from_option_iter([ + Some(f16::from_f32(-7.5)), + None, + Some(f16::from_f32(0.25)), + Some(f16::INFINITY), + ]))] + #[case::f32(PrimitiveArray::from_option_iter([ + Some(-7.5_f32), + None, + Some(0.25), + Some(f32::INFINITY), + ]))] + #[case::f64(PrimitiveArray::from_option_iter([ + Some(-7.5_f64), + None, + Some(0.25), + Some(f64::INFINITY), + ]))] + fn ordered_float_parent_kernel(#[case] expected: PrimitiveArray) -> VortexResult<()> { + let session = array_session(); + initialize(&session); + let mut ctx = session.create_execution_ctx(); + let ordered = OrderedFloat::from_primitive(expected.as_view())?; + let packed = RangePacked::from_primitive_with_null_positions( + ordered.encoded().as_::(), + &mut ctx, + )?; + let encoded = OrderedFloat::try_new(packed.into_array(), expected.ptype())?; + assert_arrays_eq!(encoded, expected, &mut ctx); + Ok(()) + } + + #[rstest] + #[case::f32(PrimitiveArray::from_option_iter([ + Some(1.25_f32), + None, + Some(17.5), + Some(f32::MAX), + ]))] + #[case::f64(PrimitiveArray::from_option_iter([ + Some(1.25_f64), + None, + Some(17.5), + Some(f64::MAX), + ]))] + fn alp_parent_kernel(#[case] expected: PrimitiveArray) -> VortexResult<()> { + let session = array_session(); + initialize(&session); + let mut ctx = session.create_execution_ctx(); + let alp = alp_encode(expected.as_view(), None, &mut ctx)?; + let packed = RangePacked::from_primitive_with_null_positions( + alp.encoded().as_::(), + &mut ctx, + )?; + let encoded = ALP::try_new(packed.into_array(), alp.exponents(), alp.patches())?; + assert_arrays_eq!(encoded, expected, &mut ctx); + Ok(()) + } + #[test] fn full_width_roundtrip() -> VortexResult<()> { let values = [0, u64::MAX, 1, u64::MAX - 1]; diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index c70c903d74b..4d3454a0c96 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -69,6 +69,7 @@ use vortex_btrblocks::BtrBlocksCompressor; use vortex_btrblocks::BtrBlocksCompressorBuilder; use vortex_btrblocks::SchemeExt; use vortex_btrblocks::schemes::float::FloatQuantScheme; +use vortex_btrblocks::schemes::float::FloatRangePackedScheme; use vortex_btrblocks::schemes::float::OrderedBlockResidualScheme; use vortex_btrblocks::schemes::integer::BlockResidualScheme; use vortex_error::VortexResult; @@ -93,6 +94,7 @@ use crate::int_mult_codec::IntMultCodec32; use crate::int_mult_codec::IntMultDenseCodec64; const DEFAULT_ROW_COUNT: usize = 2_000_000; +const RANGE_PACKED_SCHEME: FloatRangePackedScheme = FloatRangePackedScheme::new(1.20); const CALIFORNIA_COLUMNS: [&str; 9] = [ "longitude", "latitude", @@ -3028,6 +3030,19 @@ fn compressors() -> Vec<(&'static str, BtrBlocksCompressor)> { "proposed-default", BtrBlocksCompressorBuilder::default().build(), ), + ( + "range-packed-only", + BtrBlocksCompressorBuilder::default() + .exclude_schemes(new_scheme_ids) + .with_new_scheme(&RANGE_PACKED_SCHEME) + .build(), + ), + ( + "proposed-default-range-packed", + BtrBlocksCompressorBuilder::default() + .with_new_scheme(&RANGE_PACKED_SCHEME) + .build(), + ), ( "prior-compact", BtrBlocksCompressorBuilder::default() @@ -3053,6 +3068,7 @@ fn main() -> VortexResult<()> { .transpose()? .unwrap_or(DEFAULT_ROW_COUNT); let session = array_session(); + vortex_range_packed::initialize(&session); let configs = compressors(); println!("structure\tdataset\tcolumn\tptype\tconfig\tencoding\tbytes"); diff --git a/vortex-btrblocks/src/schemes/float/mod.rs b/vortex-btrblocks/src/schemes/float/mod.rs index e2f8aecb398..6b002bcf94a 100644 --- a/vortex-btrblocks/src/schemes/float/mod.rs +++ b/vortex-btrblocks/src/schemes/float/mod.rs @@ -7,6 +7,7 @@ mod alp; mod alprd; mod float_quant; mod ordered_block_residual; +mod range_packed; mod rle; mod sparse; @@ -19,6 +20,7 @@ pub use float_quant::FloatQuantScheme; pub use ordered_block_residual::OrderedBlockResidualScheme; #[cfg(feature = "pco")] pub use pco::PcoScheme; +pub use range_packed::FloatRangePackedScheme; pub use rle::FloatRLEScheme; pub use sparse::NullDominatedSparseScheme; // Re-export builtin schemes from vortex-compressor. diff --git a/vortex-btrblocks/src/schemes/float/range_packed.rs b/vortex-btrblocks/src/schemes/float/range_packed.rs new file mode 100644 index 00000000000..5df5234784c --- /dev/null +++ b/vortex-btrblocks/src/schemes/float/range_packed.rs @@ -0,0 +1,294 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Fixed-bin range packing for floating-point values. + +use vortex_alp::ALP; +use vortex_alp::ALPArrayExt; +use vortex_alp::ALPArraySlotsExt; +use vortex_alp::alp_encode; +use vortex_array::ArrayId; +use vortex_array::ArrayRef; +use vortex_array::Canonical; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::VTable; +use vortex_array::arrays::Primitive; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::match_each_float_ptype; +use vortex_block_residual::OrderedFloat; +use vortex_block_residual::OrderedFloatArraySlotsExt; +use vortex_compressor::scheme::CompressionEstimate; +use vortex_compressor::scheme::DeferredEstimate; +use vortex_compressor::scheme::EstimateScore; +use vortex_compressor::scheme::EstimateVerdict; +use vortex_error::VortexResult; +use vortex_range_packed::RangePacked; + +use crate::ArrayAndStats; +use crate::CascadingCompressor; +use crate::CompressorContext; +use crate::Scheme; +use crate::compress_patches; +use crate::schemes::sample_primitive_one_percent; + +/// The default factor accounts for RangePacked decode cost during scheme selection. +const DEFAULT_DECODE_COST_FACTOR: f64 = 1.20; + +/// Compress floats through ALP or ordered IEEE bits, then use fixed-bin range packing. +#[derive(Debug, Copy, Clone, PartialEq)] +pub struct FloatRangePackedScheme { + decode_cost_factor: f64, +} + +impl FloatRangePackedScheme { + /// Creates a scheme with the specified decode cost factor. + pub const fn new(decode_cost_factor: f64) -> Self { + Self { decode_cost_factor } + } +} + +impl Default for FloatRangePackedScheme { + fn default() -> Self { + Self::new(DEFAULT_DECODE_COST_FACTOR) + } +} + +impl Scheme for FloatRangePackedScheme { + fn scheme_name(&self) -> &'static str { + "vortex.float.range_packed" + } + + fn matches(&self, canonical: &Canonical) -> bool { + canonical.dtype().is_float() + } + + fn produced_encodings(&self) -> Vec { + vec![RangePacked.id(), OrderedFloat.id(), ALP.id()] + } + + fn num_children(&self) -> usize { + 1 + } + + fn expected_compression_ratio( + &self, + _data: &ArrayAndStats, + compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> CompressionEstimate { + if compress_ctx.is_sample() { + return CompressionEstimate::Verdict(EstimateVerdict::Skip); + } + + let decode_cost_factor = self.decode_cost_factor; + CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( + move |_compressor, data, best_so_far, _compress_ctx, exec_ctx| { + let bit_width = data.array_as_primitive().ptype().bit_width() as f64; + let maximum_adjusted_ratio = bit_width / decode_cost_factor; + if best_so_far + .and_then(EstimateScore::finite_ratio) + .is_some_and(|best| maximum_adjusted_ratio <= best) + { + return Ok(EstimateVerdict::Skip); + } + + let sample = sample_primitive_one_percent(data.array_as_primitive(), exec_ctx)?; + let sample = normalize_float_null_values(sample.as_view(), exec_ctx)?; + let before_nbytes = sample.nbytes(); + let (_, after_nbytes) = choose_transform(sample.as_view(), exec_ctx)?; + if after_nbytes == 0 { + return Ok(EstimateVerdict::Skip); + } + + let adjusted_ratio = + before_nbytes as f64 / after_nbytes as f64 / decode_cost_factor; + if adjusted_ratio <= 1.0 { + return Ok(EstimateVerdict::Skip); + } + Ok(EstimateVerdict::Ratio(adjusted_ratio)) + }, + ))) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + data: &ArrayAndStats, + _compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, + ) -> VortexResult { + let sample = sample_primitive_one_percent(data.array_as_primitive(), exec_ctx)?; + let sample = normalize_float_null_values(sample.as_view(), exec_ctx)?; + let (transform, _) = choose_transform(sample.as_view(), exec_ctx)?; + + let primitive = normalize_float_null_values(data.array_as_primitive(), exec_ctx)?; + encode_transform(primitive.as_view(), transform, exec_ctx) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +enum FloatTransform { + OrderedFloat, + Alp, +} + +fn choose_transform( + primitive: vortex_array::ArrayView<'_, Primitive>, + exec_ctx: &mut ExecutionCtx, +) -> VortexResult<(FloatTransform, u64)> { + let ordered = encode_transform(primitive, FloatTransform::OrderedFloat, exec_ctx)?; + let mut best = (FloatTransform::OrderedFloat, ordered.nbytes()); + + if primitive.ptype().is_float() && primitive.ptype().bit_width() > 16 { + let alp = encode_transform(primitive, FloatTransform::Alp, exec_ctx)?; + if alp.nbytes() < best.1 { + best = (FloatTransform::Alp, alp.nbytes()); + } + } + + Ok(best) +} + +fn encode_transform( + primitive: vortex_array::ArrayView<'_, Primitive>, + transform: FloatTransform, + exec_ctx: &mut ExecutionCtx, +) -> VortexResult { + match transform { + FloatTransform::OrderedFloat => { + let ordered = OrderedFloat::from_primitive(primitive)?; + let packed = RangePacked::from_primitive_with_null_positions( + ordered.encoded().as_::(), + exec_ctx, + )?; + Ok(OrderedFloat::try_new(packed.into_array(), primitive.ptype())?.into_array()) + } + FloatTransform::Alp => { + let alp = alp_encode(primitive, None, exec_ctx)?; + let packed = RangePacked::from_primitive_with_null_positions( + alp.encoded().as_::(), + exec_ctx, + )?; + let patches = alp + .patches() + .map(|patches| compress_patches(patches, exec_ctx)) + .transpose()?; + Ok(ALP::try_new(packed.into_array(), alp.exponents(), patches)?.into_array()) + } + } +} + +fn normalize_float_null_values( + array: vortex_array::ArrayView<'_, Primitive>, + exec_ctx: &mut ExecutionCtx, +) -> VortexResult { + let validity = array.validity()?; + if validity.definitely_no_nulls() { + return Ok(array.into_owned()); + } + + let mask = validity.execute_mask(array.len(), exec_ctx)?; + Ok(match_each_float_ptype!(array.ptype(), |T| { + let values = array.as_slice::(); + let replacement = mask + .iter() + .position(|valid| valid) + .map(|index| values[index]) + .unwrap_or_default(); + array + .into_owned() + .map_each_with_validity::( + exec_ctx, + |(value, valid)| if valid { value } else { replacement }, + )? + })) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::array_session; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::assert_arrays_eq; + use vortex_array::dtype::half::f16; + use vortex_error::VortexResult; + + use super::FloatRangePackedScheme; + use super::FloatTransform; + use super::choose_transform; + use crate::CascadingCompressor; + + const TEST_SCHEME: FloatRangePackedScheme = FloatRangePackedScheme::new(1.0); + + #[rstest] + #[case::f16(PrimitiveArray::from_iter((0_u16..4_096).map(|index| { + f16::from_bits(0x3c00 + (index.wrapping_mul(101) % 1_024)) + })))] + #[case::f32(PrimitiveArray::from_iter((0_u32..4_096).map(|index| { + f32::from_bits(0x3f80_0000 + (index.wrapping_mul(7_919) % 65_536)) + })))] + #[case::f64(PrimitiveArray::from_iter((0_u64..4_096).map(|index| { + f64::from_bits(0x3ff0_0000_0000_0000 + (index.wrapping_mul(7_919) % 65_536)) + })))] + fn ordered_float_tree_roundtrips(#[case] expected: PrimitiveArray) -> VortexResult<()> { + let session = array_session(); + vortex_range_packed::initialize(&session); + let mut ctx = session.create_execution_ctx(); + let compressed = CascadingCompressor::new(vec![&TEST_SCHEME]) + .compress(&expected.clone().into_array(), &mut ctx)?; + + assert_arrays_eq!(compressed, expected, &mut ctx); + Ok(()) + } + + #[test] + fn ordered_float_wins_for_close_bit_patterns() -> VortexResult<()> { + let expected = PrimitiveArray::from_iter((0_u64..65_536).map(|index| { + f64::from_bits(0x3ff0_0000_0000_0000 + index.wrapping_mul(7_919) % 1_024) + })); + let session = array_session(); + let mut ctx = session.create_execution_ctx(); + + let (transform, _) = choose_transform(expected.as_view(), &mut ctx)?; + + assert_eq!(transform, FloatTransform::OrderedFloat); + Ok(()) + } + + #[test] + fn alp_wins_for_decimal_values() -> VortexResult<()> { + let expected = PrimitiveArray::from_iter( + (0_u64..65_536).map(|index| (index.wrapping_mul(7_919) % 100_000) as f64 / 100.0), + ); + let session = array_session(); + let mut ctx = session.create_execution_ctx(); + + let (transform, _) = choose_transform(expected.as_view(), &mut ctx)?; + + assert_eq!(transform, FloatTransform::Alp); + Ok(()) + } + + #[test] + fn nullable_tree_stores_full_positions() -> VortexResult<()> { + let expected = PrimitiveArray::from_option_iter((0_u64..65_536).map(|index| { + (index % 17 != 0) + .then(|| f64::from_bits(0x3ff0_0000_0000_0000 + index.wrapping_mul(7_919) % 1_024)) + })); + let expected_array = expected.clone().into_array(); + let session = array_session(); + vortex_range_packed::initialize(&session); + let mut ctx = session.create_execution_ctx(); + let compressed = + CascadingCompressor::new(vec![&TEST_SCHEME]).compress(&expected_array, &mut ctx)?; + + let packed = &compressed.children()[0]; + assert!(packed.is::()); + assert_eq!(packed.children().len(), 5); + assert_arrays_eq!(compressed, expected, &mut ctx); + Ok(()) + } +} From 13b5ca8fa501f0c2cab509d91e85f6629253c754 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 07:30:24 +0100 Subject: [PATCH 50/78] perf: reject poor range-packed fits early Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 47 ++- encodings/range-packed/src/array.rs | 21 ++ encodings/range-packed/src/lib.rs | 64 ++++ .../examples/float_compressor_bench.rs | 250 ++++++------- .../src/schemes/float/range_packed.rs | 330 +++++++++++++++++- 5 files changed, 580 insertions(+), 132 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 3305db00312..186744196d0 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -1695,6 +1695,39 @@ Fast rejection remains an advantage, not an admission rule. A candidate with costly analysis needs stronger corpus evidence. +### RangePacked estimate and rejection cost + +A focused Samply profile attributed 11.79 percent of CPU time to the rejected RangePacked callback. + +Bin construction consumed almost all of that time. Exact size estimation alone did not reduce the cost. + +The estimator now fits the bins and counts exact child-buffer bytes. It does not construct the packed payload. + +A coarse model uses the existing one-percent sample. It compares fixed prefix bins with a single global range. + +The model also compares 64-value local ranges with the global range model. + +A large local advantage identifies inputs that favor BlockResidual. The selector then skips RangePacked before ALP analysis. + +The locality test retains Euro and rejects the random walk, Food, and CMS candidates. + +| Input | Default encode MB/s | Candidate encode MB/s | Write change | Selected RangePacked | +| --- | ---: | ---: | ---: | --- | +| Uniform floats | 639.5 | 629.7 | -1.5 percent | No | +| Widened `f32` values | 680.3 | 662.9 | -2.6 percent | No | +| Random walk | 581.0 | 576.2 | -0.8 percent | No | +| Nonzero-secondary FloatQuant | 677.8 | 659.8 | -2.7 percent | No | +| Quantized `f32` | 1,410.8 | 1,338.7 | -5.1 percent | No | +| Euro subjectivity | 357.8 | 359.9 | +0.6 percent | Yes | + +These focused paired results contain benchmark noise. The complete corpus will determine the final policy. + +On Euro, RangePacked uses 8,180,506 bytes instead of 14,234,326 bytes. + +Its decode throughput is 15,179 MB/s instead of 19,716 MB/s. + +Food and CMS now retain their existing BlockResidual trees. Their prior RangePacked choices lost too much decode throughput. + ### Multi-reference block prototype The prototype stores up to four quantile references per 1,024-value block. @@ -1943,6 +1976,9 @@ This round completed these steps: - Added single-encoding benchmarks for every supported integer and float type. - Added `f16` support to OrderedFloat, FloatQuant, and both Default schemes. - Verified zero-secondary and nonzero-secondary `f16` FloatQuant trees. +- Added exact RangePacked size estimates without packed payload construction. +- Added a coarse RangePacked rejection model over the existing one-percent sample. +- Added a locality rejection test that retains Euro and rejects clear BlockResidual fits. - Updated stale golden trees after the BlockResidual payload and selector changes. The Pco mode profile and the quotient and remainder experiments are complete. @@ -1957,12 +1993,11 @@ The specialized OrderedFloat and ALP trees now use registered fused parent kerne Complete these next experiments in order: -1. Add a cheap sample rejection test if it preserves the Euro range model. -2. Validate frequency-ranked Dict and full-position RangePacked across the complete file corpus. -3. Measure rejected-candidate analysis cost after the candidate set stabilizes. -4. Prefer cheap rejection tests when they preserve the best candidate model. -5. Treat fast rejection as a selection advantage, not an admission criterion. -6. Require stronger corpus evidence when a useful scheme needs costly analysis. +1. Validate frequency-ranked Dict and full-position RangePacked across the complete file corpus. +2. Measure rejected-candidate analysis cost after the candidate set stabilizes. +3. Prefer cheap rejection tests when they preserve the best candidate model. +4. Treat fast rejection as a selection advantage, not an admission criterion. +5. Require stronger corpus evidence when a useful scheme needs costly analysis. Use packed bin codes and primitive bin starts unless evidence supports more complexity. diff --git a/encodings/range-packed/src/array.rs b/encodings/range-packed/src/array.rs index 333989deb26..6125f2bf454 100644 --- a/encodings/range-packed/src/array.rs +++ b/encodings/range-packed/src/array.rs @@ -52,6 +52,7 @@ use crate::MAX_BLOCK_LEN; use crate::RangePackedCodec; use crate::RangePackedDecoder; use crate::VALIDITY_CHECKPOINT_INTERVAL; +use crate::estimate_codec_nbytes; use crate::low_mask; const METADATA_VERSION: u8 = 1; @@ -317,6 +318,26 @@ impl RangePacked { Self::from_primitive_mode(array, ctx, true) } + /// Estimates storage when every physical value remains at its logical position. + pub fn estimate_primitive_with_null_positions( + array: ArrayView<'_, Primitive>, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + vortex_ensure!( + array.ptype().is_int(), + "RangePackedArray needs integer values" + ); + let validity = array.validity()?; + let mask = validity.execute_mask(array.len(), ctx)?; + let actual_nulls = mask.true_count() != array.len(); + let ordered = ordered_primitive(array)?; + let mut nbytes = estimate_codec_nbytes(&ordered, MAX_BLOCK_LEN)?; + if actual_nulls { + nbytes += array.len().div_ceil(8); + } + Ok(u64::try_from(nbytes)?) + } + fn from_primitive_mode( array: ArrayView<'_, Primitive>, ctx: &mut ExecutionCtx, diff --git a/encodings/range-packed/src/lib.rs b/encodings/range-packed/src/lib.rs index f8609970964..d3694e30630 100644 --- a/encodings/range-packed/src/lib.rs +++ b/encodings/range-packed/src/lib.rs @@ -228,6 +228,49 @@ impl RangePackedCodec { } } +pub(crate) fn estimate_codec_nbytes(values: &[u64], block_len: usize) -> VortexResult { + vortex_ensure!( + block_len > 0 && block_len <= MAX_BLOCK_LEN, + "range packed block length must be in 1..={MAX_BLOCK_LEN}" + ); + if values.is_empty() { + return Ok(size_of::()); + } + + let (sample, minimum, maximum) = training_sample(values); + let bins = cover_domain(optimize_bins(&sample), minimum, maximum); + let symbol_width = bit_width(u64::try_from(bins.len() - 1)?); + let block_count = values.len().div_ceil(block_len); + let mut payload_len = 0usize; + for block_index in 0..block_count { + let start = block_index * block_len; + let stop = (start + block_len).min(values.len()); + let values = &values[start..stop]; + let symbol_bytes = (values.len() * usize::from(symbol_width)).div_ceil(8); + let checkpoint_bytes = values.len().div_ceil(CHECKPOINT_INTERVAL) * size_of::(); + let offset_bits = values.iter().try_fold(0usize, |total, &value| { + let symbol = bins + .partition_point(|bin| bin.lower <= value) + .saturating_sub(1); + let bin = bins[symbol]; + vortex_ensure!( + value - bin.lower <= low_mask(bin.offset_bits), + "range packed value exceeds its bin" + ); + Ok::(total + usize::from(bin.offset_bits)) + })?; + payload_len += symbol_bytes + + SYMBOL_PADDING + + checkpoint_bytes + + offset_bits.div_ceil(8) + + OFFSET_PADDING; + } + + Ok(bins.len() * (size_of::() + size_of::()) + + (block_count + 1) * size_of::() + + payload_len) +} + #[derive(Clone, Copy)] pub(crate) enum BinTableView<'a> { Interleaved(&'a [Bin]), @@ -1084,6 +1127,27 @@ mod tests { Ok(()) } + #[rstest] + #[case::nonnullable(PrimitiveArray::from_iter((0_u64..4_096).map(|value| { + (value % 8) * 1_000_000 + value.wrapping_mul(7_919) % 1_024 + })))] + #[case::nullable(PrimitiveArray::from_option_iter((0_i64..4_096).map(|value| { + (value % 17 != 0).then_some((value % 8) * 1_000_000 + value * 7_919 % 1_024) + })))] + fn full_position_estimate_matches_encoded_size( + #[case] expected: PrimitiveArray, + ) -> VortexResult<()> { + let session = array_session(); + let mut ctx = session.create_execution_ctx(); + let estimate = + RangePacked::estimate_primitive_with_null_positions(expected.as_view(), &mut ctx)?; + let encoded = + RangePacked::from_primitive_with_null_positions(expected.as_view(), &mut ctx)?; + + assert_eq!(estimate, encoded.nbytes()); + Ok(()) + } + #[rstest] #[case::u8(PrimitiveArray::from_iter([0_u8, 1, u8::MAX]).into_array())] #[case::u16(PrimitiveArray::from_iter([0_u16, 1, u16::MAX]).into_array())] diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index 4d3454a0c96..c2c55db5436 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -2746,6 +2746,10 @@ fn profile_pco_array( Ok(()) } +#[expect( + clippy::cognitive_complexity, + reason = "benchmark options share the same encoded arrays" +)] fn measure_dataset( dataset: &str, columns: &[Column], @@ -2780,95 +2784,53 @@ fn measure_dataset( } } - let prior_default = encoded - .iter() - .find(|(config, _)| *config == "prior-default") - .ok_or_else(|| vortex_err!("prior-default configuration is missing"))?; - let prior_compact = encoded - .iter() - .find(|(config, _)| *config == "prior-compact") - .ok_or_else(|| vortex_err!("prior-compact configuration is missing"))?; - for (column_index, column) in columns.iter().enumerate() { - let Some(primitive) = &column.primitive else { - continue; - }; - if !primitive.ptype().is_float() { - continue; - } - let default_array = &prior_default.1[column_index]; - let compact_array = &prior_compact.1[column_index]; - let input_bytes = primitive.len() * primitive.ptype().byte_width(); - let default_bytes = default_array.nbytes(); - let compact_bytes = compact_array.nbytes(); - let compact_savings = if default_bytes == 0 { - 0.0 - } else { - 100.0 * (1.0 - compact_bytes as f64 / default_bytes as f64) - }; - println!( - "float-column\t{dataset}\t{}\t{}\t{}\t{input_bytes}\t{default_bytes}\t{compact_bytes}\t{compact_savings:.3}\t{}\t{}", - column.name, - primitive.ptype(), - primitive.len(), - encoding_tree(default_array), - encoding_tree(compact_array), - ); - measure_frequency_ranked_dict_trees( - dataset, - &column.name, - &column.array, - default_array, - session, - )?; - if compact_bytes * 10 <= default_bytes * 9 - || std::env::var_os("VORTEX_BENCH_FIXED_BIN").is_some() - { - profile_pco_array(dataset, &column.name, "root", compact_array, session)?; - } - if std::env::var_os("VORTEX_BENCH_FIXED_BIN").is_some() { - measure_existing_tree( + if std::env::var_os("VORTEX_BENCH_SKIP_AUXILIARY_ANALYSIS").is_none() { + let prior_default = encoded + .iter() + .find(|(config, _)| *config == "prior-default") + .ok_or_else(|| vortex_err!("prior-default configuration is missing"))?; + let prior_compact = encoded + .iter() + .find(|(config, _)| *config == "prior-compact") + .ok_or_else(|| vortex_err!("prior-compact configuration is missing"))?; + for (column_index, column) in columns.iter().enumerate() { + let Some(primitive) = &column.primitive else { + continue; + }; + if !primitive.ptype().is_float() { + continue; + } + let default_array = &prior_default.1[column_index]; + let compact_array = &prior_compact.1[column_index]; + let input_bytes = primitive.len() * primitive.ptype().byte_width(); + let default_bytes = default_array.nbytes(); + let compact_bytes = compact_array.nbytes(); + let compact_savings = if default_bytes == 0 { + 0.0 + } else { + 100.0 * (1.0 - compact_bytes as f64 / default_bytes as f64) + }; + println!( + "float-column\t{dataset}\t{}\t{}\t{}\t{input_bytes}\t{default_bytes}\t{compact_bytes}\t{compact_savings:.3}\t{}\t{}", + column.name, + primitive.ptype(), + primitive.len(), + encoding_tree(default_array), + encoding_tree(compact_array), + ); + measure_frequency_ranked_dict_trees( dataset, &column.name, - "default", &column.array, default_array, session, )?; - measure_existing_tree( - dataset, - &column.name, - "compact", - &column.array, - compact_array, - session, - )?; - measure_fixed_bin_tree(dataset, &column.name, &column.array, compact_array, session)?; - measure_decomposed_fixed_bin_tree( - dataset, - &column.name, - &column.array, - compact_array, - FixedBinOffsetTree::BlockResidual, - session, - )?; - measure_decomposed_fixed_bin_tree( - dataset, - &column.name, - &column.array, - compact_array, - FixedBinOffsetTree::FoRBitPacked, - session, - )?; - measure_block_residual_float_tree( - dataset, - &column.name, - &column.array, - compact_array, - session, - )?; - } - if std::env::var_os("VORTEX_BENCH_INT_MULT_TREE").is_some() { - if std::env::var_os("VORTEX_BENCH_FIXED_BIN").is_none() { + if compact_bytes * 10 <= default_bytes * 9 + || std::env::var_os("VORTEX_BENCH_FIXED_BIN").is_some() + { + profile_pco_array(dataset, &column.name, "root", compact_array, session)?; + } + if std::env::var_os("VORTEX_BENCH_FIXED_BIN").is_some() { measure_existing_tree( dataset, &column.name, @@ -2885,69 +2847,119 @@ fn measure_dataset( compact_array, session, )?; - measure_block_residual_float_tree( + measure_fixed_bin_tree( dataset, &column.name, &column.array, compact_array, session, )?; - } - for base in [5, 10, 100, 1_000] { - measure_int_mult_float_tree( + measure_decomposed_fixed_bin_tree( dataset, &column.name, &column.array, compact_array, - base, + FixedBinOffsetTree::BlockResidual, session, )?; - } - let suffix_widths: &[u8] = match primitive.ptype() { - PType::F32 => &[12, 14, 16, 18, 20, 22, 24, 26, 28], - PType::F64 => &[24, 28, 32, 36, 40, 44, 48, 52, 56], - _ => &[], - }; - for &suffix_bits in suffix_widths { - measure_prefix_int_mult_float_tree( + measure_decomposed_fixed_bin_tree( dataset, &column.name, &column.array, compact_array, - suffix_bits, + FixedBinOffsetTree::FoRBitPacked, session, )?; - } - } - if std::env::var_os("VORTEX_BENCH_PATCH_POSITIONS").is_some() { - if std::env::var_os("VORTEX_BENCH_FIXED_BIN").is_none() - && std::env::var_os("VORTEX_BENCH_INT_MULT_TREE").is_none() - { - measure_existing_tree( + measure_block_residual_float_tree( dataset, &column.name, - "default", &column.array, - default_array, + compact_array, session, )?; - measure_existing_tree( + } + if std::env::var_os("VORTEX_BENCH_INT_MULT_TREE").is_some() { + if std::env::var_os("VORTEX_BENCH_FIXED_BIN").is_none() { + measure_existing_tree( + dataset, + &column.name, + "default", + &column.array, + default_array, + session, + )?; + measure_existing_tree( + dataset, + &column.name, + "compact", + &column.array, + compact_array, + session, + )?; + measure_block_residual_float_tree( + dataset, + &column.name, + &column.array, + compact_array, + session, + )?; + } + for base in [5, 10, 100, 1_000] { + measure_int_mult_float_tree( + dataset, + &column.name, + &column.array, + compact_array, + base, + session, + )?; + } + let suffix_widths: &[u8] = match primitive.ptype() { + PType::F32 => &[12, 14, 16, 18, 20, 22, 24, 26, 28], + PType::F64 => &[24, 28, 32, 36, 40, 44, 48, 52, 56], + _ => &[], + }; + for &suffix_bits in suffix_widths { + measure_prefix_int_mult_float_tree( + dataset, + &column.name, + &column.array, + compact_array, + suffix_bits, + session, + )?; + } + } + if std::env::var_os("VORTEX_BENCH_PATCH_POSITIONS").is_some() { + if std::env::var_os("VORTEX_BENCH_FIXED_BIN").is_none() + && std::env::var_os("VORTEX_BENCH_INT_MULT_TREE").is_none() + { + measure_existing_tree( + dataset, + &column.name, + "default", + &column.array, + default_array, + session, + )?; + measure_existing_tree( + dataset, + &column.name, + "compact", + &column.array, + compact_array, + session, + )?; + } + measure_block_residual_patch_positions( dataset, &column.name, - "compact", &column.array, + default_array, compact_array, session, )?; } - measure_block_residual_patch_positions( - dataset, - &column.name, - &column.array, - default_array, - compact_array, - session, - )?; } } @@ -3069,7 +3081,11 @@ fn main() -> VortexResult<()> { .unwrap_or(DEFAULT_ROW_COUNT); let session = array_session(); vortex_range_packed::initialize(&session); - let configs = compressors(); + let mut configs = compressors(); + if let Ok(filter) = std::env::var("VORTEX_BENCH_CONFIGS") { + configs.retain(|(name, _)| filter.split(',').any(|requested| requested == *name)); + vortex_ensure!(!configs.is_empty(), "no benchmark configuration matched"); + } println!("structure\tdataset\tcolumn\tptype\tconfig\tencoding\tbytes"); println!( diff --git a/vortex-btrblocks/src/schemes/float/range_packed.rs b/vortex-btrblocks/src/schemes/float/range_packed.rs index 5df5234784c..32745f152cd 100644 --- a/vortex-btrblocks/src/schemes/float/range_packed.rs +++ b/vortex-btrblocks/src/schemes/float/range_packed.rs @@ -15,6 +15,8 @@ use vortex_array::IntoArray; use vortex_array::VTable; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; +use vortex_array::dtype::PType; +use vortex_array::dtype::half::f16; use vortex_array::match_each_float_ptype; use vortex_block_residual::OrderedFloat; use vortex_block_residual::OrderedFloatArraySlotsExt; @@ -34,6 +36,9 @@ use crate::schemes::sample_primitive_one_percent; /// The default factor accounts for RangePacked decode cost during scheme selection. const DEFAULT_DECODE_COST_FACTOR: f64 = 1.20; +const PREFILTER_MODEL_MARGIN: f64 = 1.50; +const MIN_PREFILTER_MODEL_RATIO: f64 = 1.15; +const MAX_BLOCK_TO_RANGE_RATIO: f64 = 1.10; /// Compress floats through ALP or ordered IEEE bits, then use fixed-bin range packing. #[derive(Debug, Copy, Clone, PartialEq)] @@ -86,17 +91,25 @@ impl Scheme for FloatRangePackedScheme { move |_compressor, data, best_so_far, _compress_ctx, exec_ctx| { let bit_width = data.array_as_primitive().ptype().bit_width() as f64; let maximum_adjusted_ratio = bit_width / decode_cost_factor; - if best_so_far + let best_ratio = best_so_far .and_then(EstimateScore::finite_ratio) - .is_some_and(|best| maximum_adjusted_ratio <= best) - { + .unwrap_or(1.0); + if maximum_adjusted_ratio <= best_ratio { return Ok(EstimateVerdict::Skip); } let sample = sample_primitive_one_percent(data.array_as_primitive(), exec_ctx)?; let sample = normalize_float_null_values(sample.as_view(), exec_ctx)?; let before_nbytes = sample.nbytes(); - let (_, after_nbytes) = choose_transform(sample.as_view(), exec_ctx)?; + let Some((_, after_nbytes)) = choose_transform_for_estimate( + sample.as_view(), + best_ratio, + decode_cost_factor, + exec_ctx, + )? + else { + return Ok(EstimateVerdict::Skip); + }; if after_nbytes == 0 { return Ok(EstimateVerdict::Skip); } @@ -133,23 +146,242 @@ enum FloatTransform { Alp, } +fn choose_transform_for_estimate( + primitive: vortex_array::ArrayView<'_, Primitive>, + best_ratio: f64, + decode_cost_factor: f64, + exec_ctx: &mut ExecutionCtx, +) -> VortexResult> { + let ordered_model = coarse_ordered_float_model(primitive); + if model_prefers_blocks(ordered_model) { + return Ok(None); + } + let mut best = if prefilter_passes(ordered_model, best_ratio, decode_cost_factor) { + Some(( + FloatTransform::OrderedFloat, + estimate_transform(primitive, FloatTransform::OrderedFloat, exec_ctx)?, + )) + } else { + None + }; + + if primitive.ptype().bit_width() > 16 { + let alp = alp_encode(primitive, None, exec_ctx)?; + let model = coarse_integer_model(alp.encoded().as_::()); + if prefilter_passes(model, best_ratio, decode_cost_factor) { + let nbytes = estimate_transform(primitive, FloatTransform::Alp, exec_ctx)?; + if best.is_none_or(|(_, best_nbytes)| nbytes < best_nbytes) { + best = Some((FloatTransform::Alp, nbytes)); + } + } + } + + Ok(best) +} + +fn prefilter_passes(model: CoarseModel, best_ratio: f64, decode_cost_factor: f64) -> bool { + model.range_ratio >= MIN_PREFILTER_MODEL_RATIO + && model.range_ratio / decode_cost_factor * PREFILTER_MODEL_MARGIN > best_ratio + && !model_prefers_blocks(model) +} + +fn model_prefers_blocks(model: CoarseModel) -> bool { + model.block_ratio > model.range_ratio * MAX_BLOCK_TO_RANGE_RATIO +} + +#[derive(Clone, Copy, Debug)] +struct CoarseModel { + range_ratio: f64, + block_ratio: f64, +} + +fn coarse_ordered_float_model(primitive: vortex_array::ArrayView<'_, Primitive>) -> CoarseModel { + let values: Vec = match primitive.ptype() { + PType::F16 => primitive + .as_slice::() + .iter() + .map(|value| u64::from(ordered_u16(value.to_bits()))) + .collect(), + PType::F32 => primitive + .as_slice::() + .iter() + .map(|value| u64::from(ordered_u32(value.to_bits()))) + .collect(), + PType::F64 => primitive + .as_slice::() + .iter() + .map(|value| ordered_u64(value.to_bits())) + .collect(), + _ => { + return CoarseModel { + range_ratio: 0.0, + block_ratio: 0.0, + }; + } + }; + coarse_model(&values, primitive.ptype().bit_width()) +} + +fn coarse_integer_model(primitive: vortex_array::ArrayView<'_, Primitive>) -> CoarseModel { + let values: Vec = match primitive.ptype() { + PType::I32 => primitive + .as_slice::() + .iter() + .map(|&value| u64::from((value as u32) ^ (1_u32 << 31))) + .collect(), + PType::I64 => primitive + .as_slice::() + .iter() + .map(|&value| (value as u64) ^ (1_u64 << 63)) + .collect(), + _ => { + return CoarseModel { + range_ratio: 0.0, + block_ratio: 0.0, + }; + } + }; + coarse_model(&values, primitive.ptype().bit_width()) +} + +fn coarse_model(values: &[u64], source_width: usize) -> CoarseModel { + CoarseModel { + range_ratio: coarse_prefix_ratio(values, source_width), + block_ratio: coarse_block_ratio(values, source_width), + } +} + +fn coarse_prefix_ratio(values: &[u64], source_width: usize) -> f64 { + let Some((&minimum, &maximum)) = values.iter().min().zip(values.iter().max()) else { + return 0.0; + }; + let span_width = bit_width(maximum - minimum); + let mut best_bits = values.len() * usize::from(span_width); + for code_width in 1_u8..=6 { + if code_width > span_width { + break; + } + let bucket_count = 1usize << code_width; + let shift = span_width - code_width; + let mut counts = [0usize; 64]; + let mut minima = [u64::MAX; 64]; + let mut maxima = [0_u64; 64]; + for &value in values { + let relative = value - minimum; + let bucket = usize::try_from(relative >> shift).unwrap_or(bucket_count - 1); + let bucket = bucket.min(bucket_count - 1); + counts[bucket] += 1; + minima[bucket] = minima[bucket].min(value); + maxima[bucket] = maxima[bucket].max(value); + } + let offset_bits = (0..bucket_count) + .filter(|&bucket| counts[bucket] != 0) + .map(|bucket| counts[bucket] * usize::from(bit_width(maxima[bucket] - minima[bucket]))) + .sum::(); + best_bits = best_bits.min(values.len() * usize::from(code_width) + offset_bits); + } + + if best_bits == 0 { + f64::INFINITY + } else { + values.len() as f64 * source_width as f64 / best_bits as f64 + } +} + +fn coarse_block_ratio(values: &[u64], source_width: usize) -> f64 { + let block_bits = values + .chunks(64) + .map(|block| { + let minimum = block.iter().copied().min().unwrap_or_default(); + let maximum = block.iter().copied().max().unwrap_or_default(); + block.len() * usize::from(bit_width(maximum - minimum)) + }) + .sum::(); + if block_bits == 0 { + f64::INFINITY + } else { + values.len() as f64 * source_width as f64 / block_bits as f64 + } +} + +fn bit_width(value: u64) -> u8 { + u8::try_from(u64::BITS - value.leading_zeros()).unwrap_or(u8::MAX) +} + +fn ordered_u16(bits: u16) -> u16 { + if bits & (1_u16 << 15) == 0 { + bits ^ (1_u16 << 15) + } else { + !bits + } +} + +fn ordered_u32(bits: u32) -> u32 { + if bits & (1_u32 << 31) == 0 { + bits ^ (1_u32 << 31) + } else { + !bits + } +} + +fn ordered_u64(bits: u64) -> u64 { + if bits & (1_u64 << 63) == 0 { + bits ^ (1_u64 << 63) + } else { + !bits + } +} + fn choose_transform( primitive: vortex_array::ArrayView<'_, Primitive>, exec_ctx: &mut ExecutionCtx, ) -> VortexResult<(FloatTransform, u64)> { - let ordered = encode_transform(primitive, FloatTransform::OrderedFloat, exec_ctx)?; - let mut best = (FloatTransform::OrderedFloat, ordered.nbytes()); + let ordered_nbytes = estimate_transform(primitive, FloatTransform::OrderedFloat, exec_ctx)?; + let mut best = (FloatTransform::OrderedFloat, ordered_nbytes); if primitive.ptype().is_float() && primitive.ptype().bit_width() > 16 { - let alp = encode_transform(primitive, FloatTransform::Alp, exec_ctx)?; - if alp.nbytes() < best.1 { - best = (FloatTransform::Alp, alp.nbytes()); + let alp_nbytes = estimate_transform(primitive, FloatTransform::Alp, exec_ctx)?; + if alp_nbytes < best.1 { + best = (FloatTransform::Alp, alp_nbytes); } } Ok(best) } +fn estimate_transform( + primitive: vortex_array::ArrayView<'_, Primitive>, + transform: FloatTransform, + exec_ctx: &mut ExecutionCtx, +) -> VortexResult { + match transform { + FloatTransform::OrderedFloat => { + let ordered = OrderedFloat::from_primitive(primitive)?; + RangePacked::estimate_primitive_with_null_positions( + ordered.encoded().as_::(), + exec_ctx, + ) + } + FloatTransform::Alp => { + let alp = alp_encode(primitive, None, exec_ctx)?; + let packed_nbytes = RangePacked::estimate_primitive_with_null_positions( + alp.encoded().as_::(), + exec_ctx, + )?; + let patches = alp + .patches() + .map(|patches| compress_patches(patches, exec_ctx)) + .transpose()?; + let patch_nbytes = patches.as_ref().map_or(0, |patches| { + patches.indices().nbytes() + + patches.values().nbytes() + + patches.chunk_offsets().as_ref().map_or(0, ArrayRef::nbytes) + }); + Ok(packed_nbytes + patch_nbytes) + } + } +} + fn encode_transform( primitive: vortex_array::ArrayView<'_, Primitive>, transform: FloatTransform, @@ -219,6 +451,9 @@ mod tests { use super::FloatRangePackedScheme; use super::FloatTransform; use super::choose_transform; + use super::choose_transform_for_estimate; + use super::encode_transform; + use super::estimate_transform; use crate::CascadingCompressor; const TEST_SCHEME: FloatRangePackedScheme = FloatRangePackedScheme::new(1.0); @@ -272,6 +507,83 @@ mod tests { Ok(()) } + #[rstest] + #[case::ordered(FloatTransform::OrderedFloat)] + #[case::alp(FloatTransform::Alp)] + fn transform_estimate_matches_encoded_size( + #[case] transform: FloatTransform, + ) -> VortexResult<()> { + let expected = PrimitiveArray::from_option_iter((0_u64..65_536).map(|index| { + (index % 19 != 0).then(|| { + let value = (index.wrapping_mul(7_919) % 100_000) as f64 / 100.0; + if index % 101 == 0 { + f64::from_bits(value.to_bits() | 1) + } else { + value + } + }) + })); + let session = array_session(); + let mut ctx = session.create_execution_ctx(); + + let estimate = estimate_transform(expected.as_view(), transform, &mut ctx)?; + let encoded = encode_transform(expected.as_view(), transform, &mut ctx)?; + + assert_eq!(estimate, encoded.nbytes()); + Ok(()) + } + + #[test] + fn prefilter_rejects_uniform_float_bits() -> VortexResult<()> { + let mut state = 0x4d59_5df4_d0f3_3173_u64; + let expected = PrimitiveArray::from_iter((0..65_536).map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + let unit = ((state >> 11) as f64 + 0.5) / (1_u64 << 53) as f64; + unit * 2_000_000.0 - 1_000_000.0 + })); + let session = array_session(); + let mut ctx = session.create_execution_ctx(); + + let candidate = choose_transform_for_estimate(expected.as_view(), 1.13, 1.20, &mut ctx)?; + + assert!(candidate.is_none()); + Ok(()) + } + + #[test] + fn prefilter_retains_clustered_float_bits() -> VortexResult<()> { + let expected = PrimitiveArray::from_iter((0_u64..65_536).map(|index| { + let cluster = index % 8; + let residual = index.wrapping_mul(7_919) % 1_024; + f64::from_bits(0x3ff0_0000_0000_0000 + cluster * 0x1_0000_0000 + residual) + })); + let session = array_session(); + let mut ctx = session.create_execution_ctx(); + + let candidate = choose_transform_for_estimate(expected.as_view(), 1.36, 1.20, &mut ctx)?; + + assert!(candidate.is_some()); + Ok(()) + } + + #[test] + fn prefilter_rejects_block_local_float_bits() -> VortexResult<()> { + let expected = PrimitiveArray::from_iter((0_u64..65_536).map(|index| { + let block = index / 64; + let residual = index.wrapping_mul(7_919) % 1_024; + f64::from_bits(0x3ff0_0000_0000_0000 + block * 0x10_0000 + residual) + })); + let session = array_session(); + let mut ctx = session.create_execution_ctx(); + + let candidate = choose_transform_for_estimate(expected.as_view(), 1.0, 1.20, &mut ctx)?; + + assert!(candidate.is_none()); + Ok(()) + } + #[test] fn nullable_tree_stores_full_positions() -> VortexResult<()> { let expected = PrimitiveArray::from_option_iter((0_u64..65_536).map(|index| { From 24f9ac26a8c69ae3ad89582f33a1096765157631 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 07:34:02 +0100 Subject: [PATCH 51/78] docs: record broad range-packed results Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 186744196d0..4bff8cbf577 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -1728,6 +1728,43 @@ Its decode throughput is 15,179 MB/s instead of 19,716 MB/s. Food and CMS now retain their existing BlockResidual trees. Their prior RangePacked choices lost too much decode throughput. +### Broad RangePacked selector pass + +The next pass covered seven Public BI files and the GloVe file. + +Only Euro and HashTags selected RangePacked. The other six files retained every existing tree. + +| Dataset | Default bytes | Candidate bytes | Default encode MB/s | Candidate encode MB/s | Default decode MB/s | Candidate decode MB/s | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Arade | 26,078,866 | 26,078,866 | 549.8 | 539.6 | 21,853 | 23,772 | +| Bimbo | 10,573,121 | 10,573,121 | 767.8 | 764.8 | 11,181 | 11,342 | +| CMS provider 1 | 70,490,567 | 70,490,567 | 492.0 | 480.3 | 16,387 | 16,394 | +| CMS provider 2 | 70,640,453 | 70,640,453 | 483.9 | 475.0 | 15,641 | 15,623 | +| Euro2016 | 39,571,884 | 33,518,064 | 506.8 | 493.7 | 20,356 | 18,969 | +| Food | 12,297,725 | 12,297,725 | 542.7 | 541.8 | 19,633 | 19,650 | +| HashTags | 21,283,222 | 20,147,808 | 709.8 | 631.6 | 22,303 | 22,207 | +| GloVe | 6,274,834 | 6,274,834 | 363.0 | 342.8 | 19,341 | 19,440 | + +The geometric mean gives 2.7 percent less size and 3.3 percent less write throughput. + +The geometric-mean read throughput increases by 0.4 percent. This small change is within benchmark noise. + +HashTags `interaction#received_at` is the second selected column. + +RangePacked uses 2,128,525 bytes instead of 3,263,939 bytes. It reduces size by 34.8 percent. + +Its focused decode throughput is 14,658 MB/s instead of 15,678 MB/s. + +Its focused write throughput is 231 MB/s instead of 1,254 MB/s. + +The direct RangePacked tree encoder reaches 660 MB/s. Repeated scheme analysis accounts for part of the complete write cost. + +This HashTags trade needs the final threshold review. It does not yet justify default inclusion. + +The ALP RangePacked branch selected no column in this pass. Its rejected analysis contributes to the GloVe and synthetic costs. + +The next experiment will measure an OrderedFloat-only selector before removal or retention of the ALP branch. + ### Multi-reference block prototype The prototype stores up to four quantile references per 1,024-value block. From 06ded430372b104b2b44a61658921d53eef7ab7c Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 07:36:00 +0100 Subject: [PATCH 52/78] docs: record ordered range-packed experiment Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 4bff8cbf577..7706a9f83c9 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -1763,7 +1763,26 @@ This HashTags trade needs the final threshold review. It does not yet justify de The ALP RangePacked branch selected no column in this pass. Its rejected analysis contributes to the GloVe and synthetic costs. -The next experiment will measure an OrderedFloat-only selector before removal or retention of the ALP branch. +An OrderedFloat-only estimator reduced rejected write costs on every affected control. + +| Input | General selector loss | OrderedFloat-only loss | +| --- | ---: | ---: | +| Widened `f32` | 2.6 percent | 0.5 percent | +| Nonzero-secondary FloatQuant | 2.7 percent | 1.3 percent | +| Quantized `f32` | 5.1 percent | 2.4 percent | +| GloVe | 5.6 percent | 1.3 percent | + +The same estimator increased HashTags write throughput from 231 to 294 MB/s. + +A direct OrderedFloat encoder increased it again to 379 MB/s. The Default baseline reached 1,316 MB/s. + +The direct form kept the Euro write result neutral. It also retained both selected sizes. + +The Default candidate can therefore use a specialized OrderedFloat selector. + +The ALP candidate remains experimental until independent columns justify its analysis cost. + +This split does not reject all selectors with expensive analysis. It reflects the current selection evidence for these two transforms. ### Multi-reference block prototype From 380e09cef291b048a803e5b2b1521dc22ff07a3d Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 07:37:46 +0100 Subject: [PATCH 53/78] refactor: specialize ordered range packing Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 2 +- .../examples/float_compressor_bench.rs | 4 +- vortex-btrblocks/src/schemes/float/mod.rs | 2 +- .../src/schemes/float/range_packed.rs | 225 ++++-------------- 4 files changed, 46 insertions(+), 187 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 7706a9f83c9..d79a886233b 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -1658,7 +1658,7 @@ The manual fused benchmark reaches 15,492 MB/s on the same run. The registered kernel is within 3.5 percent of the manual helper. -An experimental `FloatRangePackedScheme` remains outside `ALL_SCHEMES`. +An experimental `OrderedFloatRangePackedScheme` remains outside `ALL_SCHEMES`. The scheme tests two complete trees on the existing stratified one-percent sample: diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index c2c55db5436..4a374271e30 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -69,8 +69,8 @@ use vortex_btrblocks::BtrBlocksCompressor; use vortex_btrblocks::BtrBlocksCompressorBuilder; use vortex_btrblocks::SchemeExt; use vortex_btrblocks::schemes::float::FloatQuantScheme; -use vortex_btrblocks::schemes::float::FloatRangePackedScheme; use vortex_btrblocks::schemes::float::OrderedBlockResidualScheme; +use vortex_btrblocks::schemes::float::OrderedFloatRangePackedScheme; use vortex_btrblocks::schemes::integer::BlockResidualScheme; use vortex_error::VortexResult; use vortex_error::vortex_ensure; @@ -94,7 +94,7 @@ use crate::int_mult_codec::IntMultCodec32; use crate::int_mult_codec::IntMultDenseCodec64; const DEFAULT_ROW_COUNT: usize = 2_000_000; -const RANGE_PACKED_SCHEME: FloatRangePackedScheme = FloatRangePackedScheme::new(1.20); +const RANGE_PACKED_SCHEME: OrderedFloatRangePackedScheme = OrderedFloatRangePackedScheme::new(1.20); const CALIFORNIA_COLUMNS: [&str; 9] = [ "longitude", "latitude", diff --git a/vortex-btrblocks/src/schemes/float/mod.rs b/vortex-btrblocks/src/schemes/float/mod.rs index 6b002bcf94a..dc4cf90edf6 100644 --- a/vortex-btrblocks/src/schemes/float/mod.rs +++ b/vortex-btrblocks/src/schemes/float/mod.rs @@ -20,7 +20,7 @@ pub use float_quant::FloatQuantScheme; pub use ordered_block_residual::OrderedBlockResidualScheme; #[cfg(feature = "pco")] pub use pco::PcoScheme; -pub use range_packed::FloatRangePackedScheme; +pub use range_packed::OrderedFloatRangePackedScheme; pub use rle::FloatRLEScheme; pub use sparse::NullDominatedSparseScheme; // Re-export builtin schemes from vortex-compressor. diff --git a/vortex-btrblocks/src/schemes/float/range_packed.rs b/vortex-btrblocks/src/schemes/float/range_packed.rs index 32745f152cd..953c12b79d4 100644 --- a/vortex-btrblocks/src/schemes/float/range_packed.rs +++ b/vortex-btrblocks/src/schemes/float/range_packed.rs @@ -1,12 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! Fixed-bin range packing for floating-point values. +//! Fixed-bin range packing for ordered floating-point values. -use vortex_alp::ALP; -use vortex_alp::ALPArrayExt; -use vortex_alp::ALPArraySlotsExt; -use vortex_alp::alp_encode; use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; @@ -31,7 +27,6 @@ use crate::ArrayAndStats; use crate::CascadingCompressor; use crate::CompressorContext; use crate::Scheme; -use crate::compress_patches; use crate::schemes::sample_primitive_one_percent; /// The default factor accounts for RangePacked decode cost during scheme selection. @@ -40,28 +35,28 @@ const PREFILTER_MODEL_MARGIN: f64 = 1.50; const MIN_PREFILTER_MODEL_RATIO: f64 = 1.15; const MAX_BLOCK_TO_RANGE_RATIO: f64 = 1.10; -/// Compress floats through ALP or ordered IEEE bits, then use fixed-bin range packing. +/// Compress floats through ordered IEEE bits, then use fixed-bin range packing. #[derive(Debug, Copy, Clone, PartialEq)] -pub struct FloatRangePackedScheme { +pub struct OrderedFloatRangePackedScheme { decode_cost_factor: f64, } -impl FloatRangePackedScheme { +impl OrderedFloatRangePackedScheme { /// Creates a scheme with the specified decode cost factor. pub const fn new(decode_cost_factor: f64) -> Self { Self { decode_cost_factor } } } -impl Default for FloatRangePackedScheme { +impl Default for OrderedFloatRangePackedScheme { fn default() -> Self { Self::new(DEFAULT_DECODE_COST_FACTOR) } } -impl Scheme for FloatRangePackedScheme { +impl Scheme for OrderedFloatRangePackedScheme { fn scheme_name(&self) -> &'static str { - "vortex.float.range_packed" + "vortex.ordered_float.range_packed" } fn matches(&self, canonical: &Canonical) -> bool { @@ -69,7 +64,7 @@ impl Scheme for FloatRangePackedScheme { } fn produced_encodings(&self) -> Vec { - vec![RangePacked.id(), OrderedFloat.id(), ALP.id()] + vec![RangePacked.id(), OrderedFloat.id()] } fn num_children(&self) -> usize { @@ -101,7 +96,7 @@ impl Scheme for FloatRangePackedScheme { let sample = sample_primitive_one_percent(data.array_as_primitive(), exec_ctx)?; let sample = normalize_float_null_values(sample.as_view(), exec_ctx)?; let before_nbytes = sample.nbytes(); - let Some((_, after_nbytes)) = choose_transform_for_estimate( + let Some(after_nbytes) = estimate_ordered_float_if_promising( sample.as_view(), best_ratio, decode_cost_factor, @@ -131,52 +126,25 @@ impl Scheme for FloatRangePackedScheme { _compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - let sample = sample_primitive_one_percent(data.array_as_primitive(), exec_ctx)?; - let sample = normalize_float_null_values(sample.as_view(), exec_ctx)?; - let (transform, _) = choose_transform(sample.as_view(), exec_ctx)?; - let primitive = normalize_float_null_values(data.array_as_primitive(), exec_ctx)?; - encode_transform(primitive.as_view(), transform, exec_ctx) + encode_ordered_float(primitive.as_view(), exec_ctx) } } -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -enum FloatTransform { - OrderedFloat, - Alp, -} - -fn choose_transform_for_estimate( +fn estimate_ordered_float_if_promising( primitive: vortex_array::ArrayView<'_, Primitive>, best_ratio: f64, decode_cost_factor: f64, exec_ctx: &mut ExecutionCtx, -) -> VortexResult> { +) -> VortexResult> { let ordered_model = coarse_ordered_float_model(primitive); if model_prefers_blocks(ordered_model) { return Ok(None); } - let mut best = if prefilter_passes(ordered_model, best_ratio, decode_cost_factor) { - Some(( - FloatTransform::OrderedFloat, - estimate_transform(primitive, FloatTransform::OrderedFloat, exec_ctx)?, - )) - } else { - None - }; - - if primitive.ptype().bit_width() > 16 { - let alp = alp_encode(primitive, None, exec_ctx)?; - let model = coarse_integer_model(alp.encoded().as_::()); - if prefilter_passes(model, best_ratio, decode_cost_factor) { - let nbytes = estimate_transform(primitive, FloatTransform::Alp, exec_ctx)?; - if best.is_none_or(|(_, best_nbytes)| nbytes < best_nbytes) { - best = Some((FloatTransform::Alp, nbytes)); - } - } + if !prefilter_passes(ordered_model, best_ratio, decode_cost_factor) { + return Ok(None); } - - Ok(best) + Ok(Some(estimate_ordered_float(primitive, exec_ctx)?)) } fn prefilter_passes(model: CoarseModel, best_ratio: f64, decode_cost_factor: f64) -> bool { @@ -222,28 +190,6 @@ fn coarse_ordered_float_model(primitive: vortex_array::ArrayView<'_, Primitive>) coarse_model(&values, primitive.ptype().bit_width()) } -fn coarse_integer_model(primitive: vortex_array::ArrayView<'_, Primitive>) -> CoarseModel { - let values: Vec = match primitive.ptype() { - PType::I32 => primitive - .as_slice::() - .iter() - .map(|&value| u64::from((value as u32) ^ (1_u32 << 31))) - .collect(), - PType::I64 => primitive - .as_slice::() - .iter() - .map(|&value| (value as u64) ^ (1_u64 << 63)) - .collect(), - _ => { - return CoarseModel { - range_ratio: 0.0, - block_ratio: 0.0, - }; - } - }; - coarse_model(&values, primitive.ptype().bit_width()) -} - fn coarse_model(values: &[u64], source_width: usize) -> CoarseModel { CoarseModel { range_ratio: coarse_prefix_ratio(values, source_width), @@ -332,83 +278,27 @@ fn ordered_u64(bits: u64) -> u64 { } } -fn choose_transform( +fn estimate_ordered_float( primitive: vortex_array::ArrayView<'_, Primitive>, exec_ctx: &mut ExecutionCtx, -) -> VortexResult<(FloatTransform, u64)> { - let ordered_nbytes = estimate_transform(primitive, FloatTransform::OrderedFloat, exec_ctx)?; - let mut best = (FloatTransform::OrderedFloat, ordered_nbytes); - - if primitive.ptype().is_float() && primitive.ptype().bit_width() > 16 { - let alp_nbytes = estimate_transform(primitive, FloatTransform::Alp, exec_ctx)?; - if alp_nbytes < best.1 { - best = (FloatTransform::Alp, alp_nbytes); - } - } - - Ok(best) -} - -fn estimate_transform( - primitive: vortex_array::ArrayView<'_, Primitive>, - transform: FloatTransform, - exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - match transform { - FloatTransform::OrderedFloat => { - let ordered = OrderedFloat::from_primitive(primitive)?; - RangePacked::estimate_primitive_with_null_positions( - ordered.encoded().as_::(), - exec_ctx, - ) - } - FloatTransform::Alp => { - let alp = alp_encode(primitive, None, exec_ctx)?; - let packed_nbytes = RangePacked::estimate_primitive_with_null_positions( - alp.encoded().as_::(), - exec_ctx, - )?; - let patches = alp - .patches() - .map(|patches| compress_patches(patches, exec_ctx)) - .transpose()?; - let patch_nbytes = patches.as_ref().map_or(0, |patches| { - patches.indices().nbytes() - + patches.values().nbytes() - + patches.chunk_offsets().as_ref().map_or(0, ArrayRef::nbytes) - }); - Ok(packed_nbytes + patch_nbytes) - } - } + let ordered = OrderedFloat::from_primitive(primitive)?; + RangePacked::estimate_primitive_with_null_positions( + ordered.encoded().as_::(), + exec_ctx, + ) } -fn encode_transform( +fn encode_ordered_float( primitive: vortex_array::ArrayView<'_, Primitive>, - transform: FloatTransform, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - match transform { - FloatTransform::OrderedFloat => { - let ordered = OrderedFloat::from_primitive(primitive)?; - let packed = RangePacked::from_primitive_with_null_positions( - ordered.encoded().as_::(), - exec_ctx, - )?; - Ok(OrderedFloat::try_new(packed.into_array(), primitive.ptype())?.into_array()) - } - FloatTransform::Alp => { - let alp = alp_encode(primitive, None, exec_ctx)?; - let packed = RangePacked::from_primitive_with_null_positions( - alp.encoded().as_::(), - exec_ctx, - )?; - let patches = alp - .patches() - .map(|patches| compress_patches(patches, exec_ctx)) - .transpose()?; - Ok(ALP::try_new(packed.into_array(), alp.exponents(), patches)?.into_array()) - } - } + let ordered = OrderedFloat::from_primitive(primitive)?; + let packed = RangePacked::from_primitive_with_null_positions( + ordered.encoded().as_::(), + exec_ctx, + )?; + Ok(OrderedFloat::try_new(packed.into_array(), primitive.ptype())?.into_array()) } fn normalize_float_null_values( @@ -448,15 +338,13 @@ mod tests { use vortex_array::dtype::half::f16; use vortex_error::VortexResult; - use super::FloatRangePackedScheme; - use super::FloatTransform; - use super::choose_transform; - use super::choose_transform_for_estimate; - use super::encode_transform; - use super::estimate_transform; + use super::OrderedFloatRangePackedScheme; + use super::encode_ordered_float; + use super::estimate_ordered_float; + use super::estimate_ordered_float_if_promising; use crate::CascadingCompressor; - const TEST_SCHEME: FloatRangePackedScheme = FloatRangePackedScheme::new(1.0); + const TEST_SCHEME: OrderedFloatRangePackedScheme = OrderedFloatRangePackedScheme::new(1.0); #[rstest] #[case::f16(PrimitiveArray::from_iter((0_u16..4_096).map(|index| { @@ -480,39 +368,7 @@ mod tests { } #[test] - fn ordered_float_wins_for_close_bit_patterns() -> VortexResult<()> { - let expected = PrimitiveArray::from_iter((0_u64..65_536).map(|index| { - f64::from_bits(0x3ff0_0000_0000_0000 + index.wrapping_mul(7_919) % 1_024) - })); - let session = array_session(); - let mut ctx = session.create_execution_ctx(); - - let (transform, _) = choose_transform(expected.as_view(), &mut ctx)?; - - assert_eq!(transform, FloatTransform::OrderedFloat); - Ok(()) - } - - #[test] - fn alp_wins_for_decimal_values() -> VortexResult<()> { - let expected = PrimitiveArray::from_iter( - (0_u64..65_536).map(|index| (index.wrapping_mul(7_919) % 100_000) as f64 / 100.0), - ); - let session = array_session(); - let mut ctx = session.create_execution_ctx(); - - let (transform, _) = choose_transform(expected.as_view(), &mut ctx)?; - - assert_eq!(transform, FloatTransform::Alp); - Ok(()) - } - - #[rstest] - #[case::ordered(FloatTransform::OrderedFloat)] - #[case::alp(FloatTransform::Alp)] - fn transform_estimate_matches_encoded_size( - #[case] transform: FloatTransform, - ) -> VortexResult<()> { + fn transform_estimate_matches_encoded_size() -> VortexResult<()> { let expected = PrimitiveArray::from_option_iter((0_u64..65_536).map(|index| { (index % 19 != 0).then(|| { let value = (index.wrapping_mul(7_919) % 100_000) as f64 / 100.0; @@ -526,8 +382,8 @@ mod tests { let session = array_session(); let mut ctx = session.create_execution_ctx(); - let estimate = estimate_transform(expected.as_view(), transform, &mut ctx)?; - let encoded = encode_transform(expected.as_view(), transform, &mut ctx)?; + let estimate = estimate_ordered_float(expected.as_view(), &mut ctx)?; + let encoded = encode_ordered_float(expected.as_view(), &mut ctx)?; assert_eq!(estimate, encoded.nbytes()); Ok(()) @@ -546,7 +402,8 @@ mod tests { let session = array_session(); let mut ctx = session.create_execution_ctx(); - let candidate = choose_transform_for_estimate(expected.as_view(), 1.13, 1.20, &mut ctx)?; + let candidate = + estimate_ordered_float_if_promising(expected.as_view(), 1.13, 1.20, &mut ctx)?; assert!(candidate.is_none()); Ok(()) @@ -562,7 +419,8 @@ mod tests { let session = array_session(); let mut ctx = session.create_execution_ctx(); - let candidate = choose_transform_for_estimate(expected.as_view(), 1.36, 1.20, &mut ctx)?; + let candidate = + estimate_ordered_float_if_promising(expected.as_view(), 1.36, 1.20, &mut ctx)?; assert!(candidate.is_some()); Ok(()) @@ -578,7 +436,8 @@ mod tests { let session = array_session(); let mut ctx = session.create_execution_ctx(); - let candidate = choose_transform_for_estimate(expected.as_view(), 1.0, 1.20, &mut ctx)?; + let candidate = + estimate_ordered_float_if_promising(expected.as_view(), 1.0, 1.20, &mut ctx)?; assert!(candidate.is_none()); Ok(()) From db9ae2c5add5ec17617d919677ed8da49dcee789 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 08:47:54 +0100 Subject: [PATCH 54/78] perf: calibrate float quant selection Signed-off-by: Will Manning --- benchmarks/compress-bench/README.md | 7 ++ benchmarks/compress-bench/src/main.rs | 26 ++++--- benchmarks/compress-bench/src/vortex.rs | 47 ++++++++--- docs/plans/native-pcodec.md | 77 +++++++++++++++++-- .../examples/float_compressor_bench.rs | 27 +++++++ .../src/schemes/float/float_quant.rs | 62 +++++++++++---- vortex-compressor/src/trace.rs | 1 + 7 files changed, 205 insertions(+), 42 deletions(-) diff --git a/benchmarks/compress-bench/README.md b/benchmarks/compress-bench/README.md index 0b3c14be4b9..2274cff7d4a 100644 --- a/benchmarks/compress-bench/README.md +++ b/benchmarks/compress-bench/README.md @@ -22,6 +22,13 @@ cargo run -p compress-bench --profile release_debug -- \ --formats vortex,vortex-compact,parquet ``` +Compare a numeric scheme bundle with the same command and one of these values: + +- `--vortex-numeric-bundle prior-default` +- `--vortex-numeric-bundle block-residual` +- `--vortex-numeric-bundle current-default` +- `--vortex-numeric-bundle range-packed` + GPU decompression is opt-in and runs only the existing benchmark names allow-listed in `src/main.rs`: diff --git a/benchmarks/compress-bench/src/main.rs b/benchmarks/compress-bench/src/main.rs index 99af8db81cc..ed53474366b 100644 --- a/benchmarks/compress-bench/src/main.rs +++ b/benchmarks/compress-bench/src/main.rs @@ -11,6 +11,7 @@ use compress_bench::LanceCompressor; use compress_bench::gpu_vortex::GpuVortexCompressor; use compress_bench::parquet::ParquetCompressor; use compress_bench::vortex::VortexCompressor; +use compress_bench::vortex::VortexNumericBundle; use indicatif::ProgressBar; use itertools::Itertools; use regex::Regex; @@ -82,9 +83,9 @@ struct Args { ingest_output: Option, #[arg(long)] tracing: bool, - /// Exclude FloatQuant and block-residual schemes from Vortex compression. - #[arg(long)] - vortex_without_new_numeric: bool, + /// Select the numeric scheme bundle for Vortex compression. + #[arg(long, value_enum, default_value_t)] + vortex_numeric_bundle: VortexNumericBundle, /// Format for the primary stderr log sink. `text` is the default human-readable format; /// `json` emits one JSON object per event, suitable for piping into `jq`. #[arg(long, value_enum, default_value_t = LogFormat::Text)] @@ -116,7 +117,7 @@ async fn main() -> anyhow::Result<()> { args.display_format, args.output_path, args.ingest_output, - args.vortex_without_new_numeric, + args.vortex_numeric_bundle, ) .await } @@ -125,7 +126,7 @@ async fn main() -> anyhow::Result<()> { fn get_compressor( format: Format, gpu_decompress: bool, - vortex_without_new_numeric: bool, + vortex_numeric_bundle: VortexNumericBundle, ) -> Box { if gpu_decompress { #[cfg(feature = "cuda")] @@ -139,9 +140,12 @@ fn get_compressor( match format { Format::OnDiskVortex => Box::new(VortexCompressor::new( Format::OnDiskVortex, - !vortex_without_new_numeric, + vortex_numeric_bundle, + )), + Format::VortexCompact => Box::new(VortexCompressor::new( + Format::VortexCompact, + VortexNumericBundle::CurrentDefault, )), - Format::VortexCompact => Box::new(VortexCompressor::new(Format::VortexCompact, true)), Format::Parquet => Box::new(ParquetCompressor::new()), #[cfg(feature = "lance")] Format::Lance => Box::new(LanceCompressor), @@ -168,7 +172,7 @@ async fn run_compress( display_format: DisplayFormat, output_path: Option, ingest_output: Option, - vortex_without_new_numeric: bool, + vortex_numeric_bundle: VortexNumericBundle, ) -> anyhow::Result<()> { let targets = formats .iter() @@ -248,7 +252,7 @@ async fn run_compress( iterations, dataset_handle, gpu_decompress, - vortex_without_new_numeric, + vortex_numeric_bundle, ) .await?; measurements.push(m); @@ -292,7 +296,7 @@ async fn run_benchmark_for_dataset( iterations: usize, dataset_handle: &dyn Dataset, gpu_decompress: bool, - vortex_without_new_numeric: bool, + vortex_numeric_bundle: VortexNumericBundle, ) -> anyhow::Result<(CompressMeasurements, Vec)> { let bench_name = dataset_handle.name(); let (v3_dataset, v3_variant) = dataset_handle.v3_dataset_dims(); @@ -308,7 +312,7 @@ async fn run_benchmark_for_dataset( let mut v3_records: Vec = Vec::new(); for format in formats { - let compressor = get_compressor(*format, gpu_decompress, vortex_without_new_numeric); + let compressor = get_compressor(*format, gpu_decompress, vortex_numeric_bundle); for op in ops { let time = match op { diff --git a/benchmarks/compress-bench/src/vortex.rs b/benchmarks/compress-bench/src/vortex.rs index 40de621d1c2..0ed6cf3b4ad 100644 --- a/benchmarks/compress-bench/src/vortex.rs +++ b/benchmarks/compress-bench/src/vortex.rs @@ -10,6 +10,7 @@ use std::time::Instant; use anyhow::Result; use async_trait::async_trait; use bytes::Bytes; +use clap::ValueEnum; use futures::StreamExt; use futures::pin_mut; use vortex::array::IntoArray; @@ -31,23 +32,40 @@ use vortex_btrblocks::BtrBlocksCompressorBuilder; use vortex_btrblocks::SchemeExt; use vortex_btrblocks::schemes::float::FloatQuantScheme; use vortex_btrblocks::schemes::float::OrderedBlockResidualScheme; +use vortex_btrblocks::schemes::float::OrderedFloatRangePackedScheme; use vortex_btrblocks::schemes::integer::BlockResidualScheme; +const RANGE_PACKED_SCHEME: OrderedFloatRangePackedScheme = OrderedFloatRangePackedScheme::new(1.20); + +/// Numeric scheme bundle for the Vortex compressor benchmark. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, ValueEnum)] +pub enum VortexNumericBundle { + /// Exclude the new numeric schemes. + PriorDefault, + /// Add the integer and ordered-float BlockResidual schemes. + BlockResidual, + /// Use the current Default compressor. + #[default] + CurrentDefault, + /// Add the experimental OrderedFloat with RangePacked scheme. + RangePacked, +} + /// Compressor implementation for Vortex format. pub struct VortexCompressor { format: Format, - new_numeric_schemes: bool, + numeric_bundle: VortexNumericBundle, } impl VortexCompressor { - pub fn new(format: Format, new_numeric_schemes: bool) -> Self { + pub fn new(format: Format, numeric_bundle: VortexNumericBundle) -> Self { assert!(matches!( format, Format::OnDiskVortex | Format::VortexCompact )); Self { format, - new_numeric_schemes, + numeric_bundle, } } @@ -56,14 +74,21 @@ impl VortexCompressor { if self.format == Format::VortexCompact { return CompactionStrategy::Compact.apply_options(options); } - if self.new_numeric_schemes { - return options; - } - let compressor = BtrBlocksCompressorBuilder::default().exclude_schemes([ - FloatQuantScheme.id(), - OrderedBlockResidualScheme.id(), - BlockResidualScheme.id(), - ]); + let compressor = match self.numeric_bundle { + VortexNumericBundle::PriorDefault => BtrBlocksCompressorBuilder::default() + .exclude_schemes([ + FloatQuantScheme.id(), + OrderedBlockResidualScheme.id(), + BlockResidualScheme.id(), + ]), + VortexNumericBundle::BlockResidual => { + BtrBlocksCompressorBuilder::default().exclude_schemes([FloatQuantScheme.id()]) + } + VortexNumericBundle::CurrentDefault => BtrBlocksCompressorBuilder::default(), + VortexNumericBundle::RangePacked => { + BtrBlocksCompressorBuilder::default().with_new_scheme(&RANGE_PACKED_SCHEME) + } + }; options.with_strategy( WriteStrategyBuilder::default() .with_btrblocks_builder(compressor) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index d79a886233b..2c5018c10cd 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -19,7 +19,8 @@ Transfer that model into a native array only when it preserves these Default pro - Full decode remains close to the displaced encoding. - Compression throughput remains close to the displaced encoding. - Scalar access remains O(1) or bounded O(log N). -- Rejected candidates add little selector cost. +- Selector cost remains proportionate to measured gains. +- Cheap rejection lowers the evidence bar, but it is not required. - The size gain remains material after native array overhead. Compact can use experimental schemes during evaluation. Default selection remains the release decision. @@ -65,7 +66,7 @@ Do not add adjacent Delta, Delta-of-delta, Delta with lookback, or convolution D The branch implements `OrderedFloatArray`, `BlockResidualArray`, `FloatQuantArray`, and `IntMultArray`. -The default candidate set includes BtrBlocks schemes for the first three arrays. +The evaluation set includes BtrBlocks schemes for the first three arrays. IntMult does not have a BtrBlocks scheme yet. @@ -83,6 +84,8 @@ The retained schemes win on specific structures. They do not replace ALP or ALP- FloatQuant now passes the speed gates for zero-secondary and one-bit-secondary inputs. +The complete file corpus does not yet justify FloatQuant in Default. + BlockResidual now passes the direct speed gates for 32-bit and 64-bit integers. The GloVe result identifies a separate entropy gap inside the ALP integer child. @@ -1784,6 +1787,59 @@ The ALP candidate remains experimental until independent columns justify its ana This split does not reject all selectors with expensive analysis. It reflects the current selection evidence for these two transforms. +### Complete bundle controls + +The compression benchmark exposes four numeric bundles: + +- `prior-default` +- `block-residual` +- `current-default` +- `range-packed` + +All bundles use the same file strategy construction. + +The complete pass covered 16 datasets with three iterations per operation. + +The BlockResidual bundle produced this geometric-mean change against Prior Default: + +| Scope | Size change | Write throughput change | Read throughput change | +| --- | ---: | ---: | ---: | +| Numeric files | -2.6 percent | +0.0 percent | +2.0 percent | +| Real files | -2.5 percent | +0.0 percent | +0.9 percent | +| All files | -1.6 percent | -0.9 percent | +1.3 percent | + +This result supports BlockResidual as the primary Default bundle. + +The first FloatQuant pass exposed two selector errors. + +HashTags produced a constant FloatQuant sample and an estimated ratio of 8,193. + +Full analysis rejected the transform. The failed winner then blocked a compact Sparse tree. + +FloatQuant now rejects a constant sample. The main compressor already handles true constant arrays. + +Food selected FloatQuant from a 1.94 sample ratio. Its full ratio was 1.73. + +ALP had a 1.80 sample ratio and a 2.51 full ratio on the same chunk. + +A provisional 1.10 FloatQuant factor retained every focused FloatQuant selection test and rejected this Food choice. + +The corrected FloatQuant bundle produced this change against the BlockResidual bundle: + +| Scope | Size change | Write throughput change | Read throughput change | +| --- | ---: | ---: | ---: | +| 16-file corpus | 0.0 percent | -0.9 percent | -0.8 percent | + +Every file size matched exactly. + +No selected-tree evidence supports a read difference. Treat the measured read change as noise. + +The write result measures FloatQuant analysis without a size win in this corpus. + +FloatQuant remains a production array and a BtrBlocks candidate. + +Default inclusion now requires a real Pco-gap win with the 1.10 factor. + ### Multi-reference block prototype The prototype stores up to four quantile references per 1,024-value block. @@ -2036,6 +2092,12 @@ This round completed these steps: - Added a coarse RangePacked rejection model over the existing one-percent sample. - Added a locality rejection test that retains Euro and rejects clear BlockResidual fits. - Updated stale golden trees after the BlockResidual payload and selector changes. +- Added four numeric bundle controls to the complete compression benchmark. +- Added explicit winner result events to the compressor trace. +- Rejected constant FloatQuant samples before exact sample encoding. +- Added a provisional 1.10 FloatQuant selection factor. +- Removed the Food and HashTags FloatQuant size regressions. +- Revalidated the BlockResidual and FloatQuant bundles across 16 files. The Pco mode profile and the quotient and remainder experiments are complete. @@ -2049,11 +2111,12 @@ The specialized OrderedFloat and ALP trees now use registered fused parent kerne Complete these next experiments in order: -1. Validate frequency-ranked Dict and full-position RangePacked across the complete file corpus. -2. Measure rejected-candidate analysis cost after the candidate set stabilizes. -3. Prefer cheap rejection tests when they preserve the best candidate model. -4. Treat fast rejection as a selection advantage, not an admission criterion. -5. Require stronger corpus evidence when a useful scheme needs costly analysis. +1. Find real Pco-gap columns that retain FloatQuant with the 1.10 factor. +2. Revalidate full-position RangePacked after the FloatQuant selector corrections. +3. Measure rejected-candidate analysis cost after the candidate set stabilizes. +4. Prefer cheap rejection tests when they preserve the best candidate model. +5. Treat fast rejection as a selection advantage, not an admission criterion. +6. Require stronger corpus evidence when a useful scheme needs costly analysis. Use packed bin codes and primitive bin starts unless evidence supports more complexity. diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index 4a374271e30..44e3e7099c6 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -2784,6 +2784,27 @@ fn measure_dataset( } } + if std::env::var_os("VORTEX_BENCH_CONFIG_TREES").is_some() { + for (config, arrays) in &encoded { + for (column, array) in columns.iter().zip(arrays) { + if column + .primitive + .as_ref() + .is_some_and(|primitive| primitive.ptype().is_float()) + { + measure_existing_tree( + dataset, + &column.name, + config, + &column.array, + array, + session, + )?; + } + } + } + } + if std::env::var_os("VORTEX_BENCH_SKIP_AUXILIARY_ANALYSIS").is_none() { let prior_default = encoded .iter() @@ -3038,6 +3059,12 @@ fn compressors() -> Vec<(&'static str, BtrBlocksCompressor)> { .exclude_schemes([FloatQuantScheme.id(), OrderedBlockResidualScheme.id()]) .build(), ), + ( + "block-residual-bundle", + BtrBlocksCompressorBuilder::default() + .exclude_schemes([FloatQuantScheme.id()]) + .build(), + ), ( "proposed-default", BtrBlocksCompressorBuilder::default().build(), diff --git a/vortex-btrblocks/src/schemes/float/float_quant.rs b/vortex-btrblocks/src/schemes/float/float_quant.rs index 792141be8fd..c324db0aa9a 100644 --- a/vortex-btrblocks/src/schemes/float/float_quant.rs +++ b/vortex-btrblocks/src/schemes/float/float_quant.rs @@ -10,6 +10,7 @@ use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::VTable; use vortex_array::arrays::Primitive; +use vortex_array::arrays::PrimitiveArray; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::PType; use vortex_array::dtype::half::f16; @@ -34,6 +35,9 @@ use crate::Scheme; use crate::normalize_null_values; use crate::schemes::sample_primitive_one_percent; +// Prefer incumbents on close fits. FloatQuant sample estimates can overstate full-array gains. +const SELECTION_COST_FACTOR: f64 = 1.10; + /// FloatQuant split with a fixed frame-of-reference primary child. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct FloatQuantScheme; @@ -63,19 +67,7 @@ impl Scheme for FloatQuantScheme { CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( |_compressor, data, _best_so_far, _compress_ctx, exec_ctx| { let sample = sample_primitive_one_percent(data.array_as_primitive(), exec_ctx)?; - let Some(analysis) = analyze_float_quant(sample.as_view()) else { - return Ok(EstimateVerdict::Skip); - }; - let before_nbytes = sample.nbytes(); - let compressed = encode_float_quant(sample.as_view(), analysis)?; - let after_nbytes = compressed.nbytes(); - if after_nbytes == 0 || after_nbytes >= before_nbytes { - return Ok(EstimateVerdict::Skip); - } - - Ok(EstimateVerdict::Ratio( - before_nbytes as f64 / after_nbytes as f64, - )) + estimate_float_quant_sample(&sample) }, ))) } @@ -96,6 +88,30 @@ impl Scheme for FloatQuantScheme { } } +fn estimate_float_quant_sample(sample: &PrimitiveArray) -> VortexResult { + let Some(analysis) = analyze_float_quant(sample.as_view()) else { + return Ok(EstimateVerdict::Skip); + }; + // A constant sample does not prove that the full array is constant. + if analysis.primary_bit_width == 0 && analysis.secondary_bit_width == 0 { + return Ok(EstimateVerdict::Skip); + } + + let before_nbytes = sample.nbytes(); + let compressed = encode_float_quant(sample.as_view(), analysis)?; + let after_nbytes = compressed.nbytes(); + if after_nbytes == 0 || after_nbytes >= before_nbytes { + return Ok(EstimateVerdict::Skip); + } + + let adjusted_ratio = before_nbytes as f64 / after_nbytes as f64 / SELECTION_COST_FACTOR; + if adjusted_ratio <= 1.0 { + return Ok(EstimateVerdict::Skip); + } + + Ok(EstimateVerdict::Ratio(adjusted_ratio)) +} + fn encode_float_quant( primitive: vortex_array::ArrayView<'_, Primitive>, analysis: FloatQuantAnalysis, @@ -261,3 +277,23 @@ fn ordered_u64(bits: u64) -> u64 { !bits } } + +#[cfg(test)] +mod tests { + use vortex_array::arrays::PrimitiveArray; + use vortex_compressor::scheme::EstimateVerdict; + use vortex_error::VortexResult; + + use super::estimate_float_quant_sample; + + #[test] + fn constant_sample_is_not_evidence_for_float_quant() -> VortexResult<()> { + let sample = PrimitiveArray::from_iter(vec![1.0_f64; 1_024]); + + assert!(matches!( + estimate_float_quant_sample(&sample)?, + EstimateVerdict::Skip + )); + Ok(()) + } +} diff --git a/vortex-compressor/src/trace.rs b/vortex-compressor/src/trace.rs index ee594621199..0a7a4126cfc 100644 --- a/vortex-compressor/src/trace.rs +++ b/vortex-compressor/src/trace.rs @@ -120,6 +120,7 @@ pub(super) fn record_winner_compress_result( span.record("achieved_ratio", r); } span.record("accepted", accepted); + tracing::debug!(target: TARGET_TRACE, "winner.result"); } /// Records the final output size and, when finite, the top-level compression ratio. From d182ca929a7b51aadb540585f0e638d0cb0b9652 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 09:04:30 +0100 Subject: [PATCH 55/78] perf: reject constant range-packed samples Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 62 ++++++++++++++++++- .../src/schemes/float/range_packed.rs | 16 ++++- 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 2c5018c10cd..56c428dec2f 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -1787,6 +1787,29 @@ The ALP candidate remains experimental until independent columns justify its ana This split does not reject all selectors with expensive analysis. It reflects the current selection evidence for these two transforms. +### Corrected RangePacked bundle + +The first controlled RangePacked pass exposed another constant-sample error. + +A constant HashTags sample produced an infinite coarse range ratio. + +The exact candidate then lost to the complete incumbent, but it blocked the existing Sparse tree. + +The prefilter now rejects non-finite range ratios. + +| Dataset | Current bytes | RangePacked bytes | Size change | Write time change | Read time change | +| --- | ---: | ---: | ---: | ---: | ---: | +| Euro2016 | 159,053,012 | 153,090,172 | -3.75 percent | +1.60 percent | -1.30 percent | +| HashTags | 179,382,828 | 178,141,132 | -0.69 percent | +0.89 percent | +0.69 percent | + +The corrected 16-file geometric mean reduces size by 0.28 percent. + +The timing differences in this two-file repeat remain within benchmark noise. + +The focused Euro tree still decodes about 30 percent slower than its incumbent. + +RangePacked therefore remains an experimental Default option. + ### Complete bundle controls The compression benchmark exposes four numeric bundles: @@ -1840,6 +1863,36 @@ FloatQuant remains a production array and a BtrBlocks candidate. Default inclusion now requires a real Pco-gap win with the 1.10 factor. +### Real Pco-gap FloatQuant pass + +The current benchmark tested 33 real float columns across five inputs: + +- CMS Open Payments. +- California Housing. +- NYC Taxi. +- CMS Provider 1. +- CMS Provider 2. + +The pass compared `block-residual-bundle` with `proposed-default`. + +FloatQuant selected no column. + +Every proposed file size matched the BlockResidual bundle. + +The geometric-mean encode throughput changed by +0.12 percent. + +The geometric-mean decode throughput changed by -0.06 percent. + +Both throughput changes are benchmark noise because the selected trees are identical. + +Compact retains material size gains on many float columns in these files. + +The Pco mode profile attributes those gaps to classic bins, IntMult, and FloatMult. + +This result does not support FloatQuant in Default under the current factor. + +Retain the production array while the final candidate review decides the scheme policy. + ### Multi-reference block prototype The prototype stores up to four quantile references per 1,024-value block. @@ -2098,6 +2151,9 @@ This round completed these steps: - Added a provisional 1.10 FloatQuant selection factor. - Removed the Food and HashTags FloatQuant size regressions. - Revalidated the BlockResidual and FloatQuant bundles across 16 files. +- Rejected constant RangePacked samples before exact sample encoding. +- Revalidated RangePacked against the corrected current bundle. +- Tested FloatQuant on 33 real float columns from five Pco-gap inputs. The Pco mode profile and the quotient and remainder experiments are complete. @@ -2111,9 +2167,9 @@ The specialized OrderedFloat and ALP trees now use registered fused parent kerne Complete these next experiments in order: -1. Find real Pco-gap columns that retain FloatQuant with the 1.10 factor. -2. Revalidate full-position RangePacked after the FloatQuant selector corrections. -3. Measure rejected-candidate analysis cost after the candidate set stabilizes. +1. Decide the FloatQuant scheme policy from the no-selection corpus evidence. +2. Measure rejected-candidate analysis cost after the candidate set stabilizes. +3. Prepare final Default bundle options with selected-tree evidence. 4. Prefer cheap rejection tests when they preserve the best candidate model. 5. Treat fast rejection as a selection advantage, not an admission criterion. 6. Require stronger corpus evidence when a useful scheme needs costly analysis. diff --git a/vortex-btrblocks/src/schemes/float/range_packed.rs b/vortex-btrblocks/src/schemes/float/range_packed.rs index 953c12b79d4..f93b26ae690 100644 --- a/vortex-btrblocks/src/schemes/float/range_packed.rs +++ b/vortex-btrblocks/src/schemes/float/range_packed.rs @@ -148,7 +148,8 @@ fn estimate_ordered_float_if_promising( } fn prefilter_passes(model: CoarseModel, best_ratio: f64, decode_cost_factor: f64) -> bool { - model.range_ratio >= MIN_PREFILTER_MODEL_RATIO + model.range_ratio.is_finite() + && model.range_ratio >= MIN_PREFILTER_MODEL_RATIO && model.range_ratio / decode_cost_factor * PREFILTER_MODEL_MARGIN > best_ratio && !model_prefers_blocks(model) } @@ -409,6 +410,19 @@ mod tests { Ok(()) } + #[test] + fn prefilter_rejects_constant_sample() -> VortexResult<()> { + let expected = PrimitiveArray::from_iter(vec![1.0_f64; 1_024]); + let session = array_session(); + let mut ctx = session.create_execution_ctx(); + + let candidate = + estimate_ordered_float_if_promising(expected.as_view(), 1.0, 1.20, &mut ctx)?; + + assert!(candidate.is_none()); + Ok(()) + } + #[test] fn prefilter_retains_clustered_float_bits() -> VortexResult<()> { let expected = PrimitiveArray::from_iter((0_u64..65_536).map(|index| { From 46ad4050e32e0354365b25820604492257b84f02 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 09:12:23 +0100 Subject: [PATCH 56/78] perf: fuse packed IntMult decode Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 16 +++++ encodings/int-mult/src/array.rs | 67 +++++++++++++++++++ vortex/benches/single_encoding_throughput.rs | 69 ++++++++++++++++++++ 3 files changed, 152 insertions(+) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 56c428dec2f..97a61877809 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -376,6 +376,10 @@ These cases include the current compressed children and fused decode paths. | BlockResidual `u16` | 1.31 GB/s | 32.55 GB/s | 61 ns | | BlockResidual `u32` | 2.67 GB/s | 46.61 GB/s | 48 ns | | BlockResidual `u64` | 4.97 GB/s | 35.56 GB/s | 40 ns | +| IntMult with packed children `u8` | 0.70 GB/s | 24.81 GB/s | 125 ns | +| IntMult with packed children `u16` | 1.38 GB/s | 22.90 GB/s | 125 ns | +| IntMult with packed children `u32` | 2.64 GB/s | 22.62 GB/s | 140 ns | +| IntMult with packed children `u64` | 4.34 GB/s | 17.96 GB/s | 125 ns | | OrderedFloat with BlockResidual `f16` | 1.26 GB/s | 20.50 GB/s | 79 ns | | OrderedFloat with BlockResidual `f32` | 2.43 GB/s | 29.64 GB/s | 78 ns | | OrderedFloat with BlockResidual `f64` | 2.70 GB/s | 20.44 GB/s | 125 ns | @@ -389,6 +393,16 @@ The FloatQuant scheme includes analysis and direct tree construction. It encodes at 5.19 GB/s for `f32` and 7.07 GB/s for `f64` on selected inputs. +The packed IntMult tree uses a base of ten and patch-free BitPacked children. + +The fused pair decoder unpacks both children directly into the reconstructed output. + +It removes the full secondary buffer and the separate multiplication pass. + +The paired `u64` benchmark improved decode throughput from 13.10 GB/s to 17.73 GB/s. + +This change increased decode throughput by 35.3 percent. + The decomposed fixed-bin row uses ten separated clusters and a BlockResidual offset child. The fixed-bin result still lacks complete Default and Compact comparisons on real columns. @@ -2139,6 +2153,8 @@ This round completed these steps: - Audited constructor and deserialization validation for all four production arrays. - Added round-trip tests for every supported integer and float type. - Added single-encoding benchmarks for every supported integer and float type. +- Added packed-child IntMult benchmarks for every unsigned integer type. +- Added fused IntMult decode for aligned, patch-free BitPacked children. - Added `f16` support to OrderedFloat, FloatQuant, and both Default schemes. - Verified zero-secondary and nonzero-secondary `f16` FloatQuant trees. - Added exact RangePacked size estimates without packed payload construction. diff --git a/encodings/int-mult/src/array.rs b/encodings/int-mult/src/array.rs index 4239a576c24..7af9db4fd6c 100644 --- a/encodings/int-mult/src/array.rs +++ b/encodings/int-mult/src/array.rs @@ -45,6 +45,7 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_fastlanes::BitPacked; use vortex_fastlanes::BitPackedArrayExt; +use vortex_fastlanes::bitpack_decompress::unpack_pair_map; use vortex_session::VortexSession; use vortex_session::registry::CachedId; @@ -187,6 +188,9 @@ impl VTable for IntMult { { return Ok(ExecutionResult::done(decoded.into_array())); } + if let Some(decoded) = decode_fastlanes_pair(array.as_view())? { + return Ok(ExecutionResult::done(decoded.into_array())); + } let primary = array.primary().clone().execute::(ctx)?; let secondary = array.secondary().clone().execute::(ctx)?; Ok(ExecutionResult::done( @@ -203,6 +207,37 @@ impl VTable for IntMult { } } +fn decode_fastlanes_pair(array: ArrayView<'_, IntMult>) -> VortexResult> { + let Some(primary) = array.primary().as_opt::() else { + return Ok(None); + }; + let Some(secondary) = array.secondary().as_opt::() else { + return Ok(None); + }; + if primary.patches().is_some() + || secondary.patches().is_some() + || primary.offset() != secondary.offset() + || !array.dtype().as_ptype().is_unsigned_int() + { + return Ok(None); + } + + let validity = primary.validity()?; + let base = array.base(); + Ok(Some(vortex_array::match_each_unsigned_integer_ptype!( + array.dtype().as_ptype(), + |T| { + let base = T::try_from(base).vortex_expect("validated IntMult base"); + PrimitiveArray::new( + unpack_pair_map::(primary, secondary, |primary, secondary| { + T::wrapping_mul_add(base, primary, secondary) + })?, + validity, + ) + } + ))) +} + fn decode_dict_add( array: ArrayView<'_, IntMult>, ctx: &mut ExecutionCtx, @@ -660,6 +695,38 @@ mod tests { Ok(()) } + #[test] + fn multiplies_bitpacked_pair_with_validity_and_slice() -> VortexResult<()> { + let input = PrimitiveArray::from_option_iter( + (0..2_050_u32) + .map(|index| (index != 1_024).then_some((index % 1_000) * 10 + index % 10)), + ); + let split = IntMult::from_primitive(input.as_view(), 10)?; + let mut ctx = array_session().create_execution_ctx(); + let primary = split + .primary() + .clone() + .execute::(&mut ctx)?; + let secondary = split + .secondary() + .clone() + .execute::(&mut ctx)?; + // SAFETY: Ten bits represent every quotient in the test input. + let primary = unsafe { bitpack_encode_unchecked(primary, 10) }?.into_array(); + // SAFETY: Four bits represent every remainder in the test input. + let secondary = unsafe { bitpack_encode_unchecked(secondary, 4) }?.into_array(); + let encoded = IntMult::try_new(primary, secondary, 10)?; + + assert!(decode_fastlanes_pair(encoded.as_view())?.is_some()); + assert_arrays_eq!(encoded, input, &mut ctx); + assert_arrays_eq!( + encoded.into_array().slice(1_000..1_050)?, + input.into_array().slice(1_000..1_050)?, + &mut ctx + ); + Ok(()) + } + #[test] fn preserves_validity() -> VortexResult<()> { let input = PrimitiveArray::new( diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index 17f9b1c4371..a18e1836e50 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -51,6 +51,7 @@ use vortex::encodings::float_quant::analyze_float_quant; use vortex::encodings::fsst::fsst_compress; use vortex::encodings::fsst::fsst_train_compressor; use vortex::encodings::int_mult::IntMult; +use vortex::encodings::int_mult::IntMultArraySlotsExt; use vortex::encodings::pco::Pco; use vortex::encodings::range_packed::RangeDecomposition; use vortex::encodings::runend::RunEnd; @@ -350,6 +351,38 @@ fn setup_ranged_u64_array() -> PrimitiveArray { })) } +fn encode_int_mult_packed_tree(array: &PrimitiveArray) -> vortex::array::ArrayRef { + let split = IntMult::from_primitive(array.as_view(), 10).unwrap(); + let primary = split + .primary() + .clone() + .execute::(&mut SESSION.create_execution_ctx()) + .unwrap(); + let secondary = split + .secondary() + .clone() + .execute::(&mut SESSION.create_execution_ctx()) + .unwrap(); + let primary_width = match T::PTYPE { + PType::U8 => 5, + PType::U16 => 13, + PType::U32 => 27, + PType::U64 => 30, + ptype => unreachable!("unsupported packed IntMult benchmark type {ptype}"), + }; + // SAFETY: The benchmark generator bounds the quotient to the selected width. + let primary = unsafe { bitpack_encode_unchecked(primary, primary_width) } + .unwrap() + .into_array(); + // SAFETY: The benchmark generator bounds the remainder below ten. + let secondary = unsafe { bitpack_encode_unchecked(secondary, 4) } + .unwrap() + .into_array(); + IntMult::try_new(primary, secondary, 10) + .unwrap() + .into_array() +} + fn encode_decomposed_range_tree(array: &PrimitiveArray) -> vortex::array::ArrayRef { let decomposition = RangeDecomposition::encode(array.as_slice::()).unwrap(); let code_width = decomposition.code_width(); @@ -868,6 +901,42 @@ fn int_mult_split_scalar_at(bencher: Bencher) { .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); } +#[divan::bench(types = [u8, u16, u32, u64])] +fn int_mult_packed_tree_compress(bencher: Bencher) { + let array = setup_int_mult_integer_array::(); + let byte_width = u64::try_from(T::PTYPE.byte_width()).unwrap(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * byte_width) + .with_inputs(|| &array) + .bench_refs(|array| encode_int_mult_packed_tree::(array)); +} + +#[divan::bench(types = [u8, u16, u32, u64])] +fn int_mult_packed_tree_decompress(bencher: Bencher) { + let encoded = encode_int_mult_packed_tree::(&setup_int_mult_integer_array::()); + let byte_width = u64::try_from(T::PTYPE.byte_width()).unwrap(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * byte_width) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); +} + +#[divan::bench(types = [u8, u16, u32, u64])] +fn int_mult_packed_tree_scalar_at(bencher: Bencher) { + let encoded = encode_int_mult_packed_tree::(&setup_int_mult_integer_array::()); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); +} + #[divan::bench(name = "decomposed_range_compress_u64")] fn bench_decomposed_range_compress_u64(bencher: Bencher) { let array = setup_ranged_u64_array(); From 26c406e36b8d6ecc9b108e202449ae2eb538448c Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 10:08:51 +0100 Subject: [PATCH 57/78] feat: decompose fixed-bin float compression Signed-off-by: Will Manning --- Cargo.lock | 2 + docs/plans/native-pcodec.md | 68 ++++++ encodings/block-residual/Cargo.toml | 2 + .../block-residual/src/ordered_float_array.rs | 218 ++++++++++++++++-- encodings/int-mult/src/array.rs | 129 ++++++++--- vortex-btrblocks/Cargo.toml | 2 +- .../src/schemes/float/range_packed.rs | 99 ++++++-- vortex/src/editions/core/v2026_08.rs | 1 + 8 files changed, 447 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 07fc20f700c..b3b44456e9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9757,10 +9757,12 @@ name = "vortex-block-residual" version = "0.1.0" dependencies = [ "fastlanes", + "num-traits", "rstest", "vortex-array", "vortex-buffer", "vortex-error", + "vortex-int-mult", "vortex-session", ] diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 97a61877809..51998971cd9 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -1824,6 +1824,74 @@ The focused Euro tree still decodes about 30 percent slower than its incumbent. RangePacked therefore remains an experimental Default option. +### Decomposed fixed-bin bundle + +The storage prototype now uses only composable arrays: + +`OrderedFloat(IntMult(Dict(BitPacked(codes), starts), BlockResidual(offsets)))` + +The scheme permits 1 through 64 bins. + +The `IntMult` base is one, so decode uses addition without multiplication. + +The exact sample estimator constructs this fixed child tree. + +The writer therefore avoids generic integer recursion and its depth limit. + +The first decomposed HashTags tree used 1,991,606 bytes. + +The incumbent ALP-RD tree used 3,263,939 bytes. + +This change reduced the column size by 39.0 percent. + +The generic child execution reached 4,476 MB/s. + +A nullable dictionary fast path reached 8,383 MB/s after it removed validity work from normal payloads. + +An outer in-place decoder now preserves direct BlockResidual unpack. + +It combines bin addition and ordered-float restoration in one pass. + +The final decomposed tree reached 10,148 to 10,389 MB/s across two focused runs. + +The direct fused RangePacked prototype reached 14,658 MB/s. + +The ALP-RD incumbent reached 15,578 to 16,060 MB/s. + +Scalar access used 312 to 323 ns for the decomposed tree and 150 to 158 ns for ALP-RD. + +The decomposed tree retains bounded random access, but the constant factor remains material. + +The matched 16-dataset compression pass produced these geometric-mean changes: + +| Metric | Change versus Current Default | +| --- | ---: | +| File size | -0.056 percent | +| Write time | +1.172 percent | +| Read time | -0.268 percent | + +HashTags file size fell from 179,382,860 bytes to 178,180,812 bytes. + +Its matched write time fell by 0.82 percent, and its matched read time fell by 2.38 percent. + +Those complete-file timing changes conflict with the slower selected column. + +Treat them as aggregate noise until a larger repeated pass confirms them. + +The first two million rows selected the new tree only for HashTags `interaction#received_at`. + +Full files also changed size for Bimbo, CMSprovider, Euro2016, and Taxi. + +This difference requires a chunk-level selected-tree trace before threshold calibration. + +The current broad size gain does not justify Default inclusion by itself. + +IntMult now belongs to the August 2026 core edition, so files can serialize the decomposed tree. + +Fast rejection remains a cost advantage, not an admission rule. + +A scheme without fast rejection can enter Default when corpus evidence justifies its analysis cost. + ### Complete bundle controls The compression benchmark exposes four numeric bundles: diff --git a/encodings/block-residual/Cargo.toml b/encodings/block-residual/Cargo.toml index e0bb52093d0..bd3b06ba582 100644 --- a/encodings/block-residual/Cargo.toml +++ b/encodings/block-residual/Cargo.toml @@ -18,9 +18,11 @@ workspace = true [dependencies] fastlanes = { workspace = true } +num-traits = { workspace = true } vortex-array = { workspace = true } vortex-buffer = { workspace = true } vortex-error = { workspace = true } +vortex-int-mult = { workspace = true } vortex-session = { workspace = true } [dev-dependencies] diff --git a/encodings/block-residual/src/ordered_float_array.rs b/encodings/block-residual/src/ordered_float_array.rs index d1682c32b2d..c26a0b1ff05 100644 --- a/encodings/block-residual/src/ordered_float_array.rs +++ b/encodings/block-residual/src/ordered_float_array.rs @@ -6,6 +6,7 @@ use std::fmt::Formatter; use std::hash::Hasher; use std::ops::Range; +use num_traits::AsPrimitive; use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; @@ -19,14 +20,18 @@ use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::TypedArrayRef; use vortex_array::array_slots; +use vortex_array::arrays::Dict; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::dict::DictArraySlotsExt; use vortex_array::arrays::slice::SliceReduce; use vortex_array::arrays::slice::SliceReduceAdaptor; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; +use vortex_array::dtype::NativePType; use vortex_array::dtype::PType; use vortex_array::dtype::half::f16; +use vortex_array::match_each_integer_ptype; use vortex_array::optimizer::rules::ParentRuleSet; use vortex_array::scalar::Scalar; use vortex_array::serde::ArrayChildren; @@ -41,6 +46,9 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; +use vortex_int_mult::IntMult; +use vortex_int_mult::IntMultArrayExt; +use vortex_int_mult::IntMultArraySlotsExt; use vortex_session::VortexSession; use vortex_session::registry::CachedId; @@ -174,7 +182,13 @@ impl VTable for OrderedFloat { } fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { - let decoded = if let Some(block_residual) = array.encoded().as_typed::() { + let decoded = if let Some(int_mult) = array.encoded().as_typed::() { + if let Some(decoded) = decompress_ordered_int_mult(array.as_view(), int_mult, ctx)? { + decoded + } else { + decode_primitive(array.as_view(), ctx)? + } + } else if let Some(block_residual) = array.encoded().as_typed::() { match array.dtype().as_ptype() { PType::F16 => decode_primitive(array.as_view(), ctx)?, PType::F32 => decompress_ordered_f32(block_residual, ctx)?, @@ -196,6 +210,137 @@ impl VTable for OrderedFloat { } } +fn decompress_ordered_int_mult( + array: ArrayView<'_, OrderedFloat>, + int_mult: ArrayView<'_, IntMult>, + ctx: &mut ExecutionCtx, +) -> VortexResult> { + if int_mult.base() != 1 { + return Ok(None); + } + let Some(dict) = int_mult.primary().as_typed::() else { + return Ok(None); + }; + let Some(offsets) = int_mult.secondary().as_typed::() else { + return Ok(None); + }; + let starts = dict.values().clone().execute::(ctx)?; + if !starts.validity()?.definitely_no_nulls() || starts.ptype() != offsets.dtype().as_ptype() { + return Ok(None); + } + let codes = dict.codes().clone().execute::(ctx)?; + let validity = codes.validity()?; + let offsets = int_mult + .secondary() + .clone() + .execute::(ctx)?; + + decode_ordered_int_mult_parts( + array.dtype().as_ptype(), + starts, + offsets, + codes, + validity, + ctx, + ) +} + +fn decode_ordered_int_mult_parts( + float_ptype: PType, + starts: PrimitiveArray, + offsets: PrimitiveArray, + codes: PrimitiveArray, + validity: vortex_array::validity::Validity, + ctx: &mut ExecutionCtx, +) -> VortexResult> { + match_each_integer_ptype!(codes.ptype(), |C| { + decode_ordered_int_mult_codes::( + float_ptype, + starts, + offsets, + codes.as_slice::(), + validity, + ctx, + ) + }) +} + +fn decode_ordered_int_mult_codes( + float_ptype: PType, + starts: PrimitiveArray, + offsets: PrimitiveArray, + codes: &[C], + validity: vortex_array::validity::Validity, + ctx: &mut ExecutionCtx, +) -> VortexResult> +where + C: NativePType + AsPrimitive, +{ + match float_ptype { + PType::F16 if starts.ptype() == PType::U16 => { + decode_ordered_u16(starts.as_slice::(), offsets, codes, validity, ctx).map(Some) + } + PType::F32 if starts.ptype() == PType::U32 => { + decode_ordered_u32(starts.as_slice::(), offsets, codes, validity, ctx).map(Some) + } + PType::F64 if starts.ptype() == PType::U64 => { + decode_ordered_u64(starts.as_slice::(), offsets, codes, validity, ctx).map(Some) + } + _ => Ok(None), + } +} + +macro_rules! decode_ordered_words { + ($name:ident, $word:ty, $float:ty, $unordered:ident) => { + fn $name( + starts: &[$word], + offsets: PrimitiveArray, + codes: &[C], + validity: vortex_array::validity::Validity, + ctx: &mut ExecutionCtx, + ) -> VortexResult + where + C: NativePType + AsPrimitive, + { + let mut offsets = offsets.into_buffer_mut::<$word>(); + let mut invalid_codes = Vec::new(); + for (index, (offset, code)) in offsets.iter_mut().zip(codes).enumerate() { + let code = >::as_(*code); + let Some(start) = starts.get(code) else { + invalid_codes.push(index); + continue; + }; + *offset = $unordered(start.wrapping_add(*offset)); + } + validate_invalid_code_payloads(&validity, offsets.len(), &invalid_codes, ctx)?; + // SAFETY: Every integer bit pattern represents a valid float value of equal width. + let values = unsafe { offsets.transmute::<$float>() }.freeze(); + Ok(PrimitiveArray::new(values, validity)) + } + }; +} + +decode_ordered_words!(decode_ordered_u16, u16, f16, unordered_u16); +decode_ordered_words!(decode_ordered_u32, u32, f32, unordered_u32); +decode_ordered_words!(decode_ordered_u64, u64, f64, unordered_u64); + +fn validate_invalid_code_payloads( + validity: &vortex_array::validity::Validity, + len: usize, + invalid_codes: &[usize], + ctx: &mut ExecutionCtx, +) -> VortexResult<()> { + if !invalid_codes.is_empty() { + let mask = validity.execute_mask(len, ctx)?; + for &index in invalid_codes { + if mask.value(index) { + vortex_bail!("OrderedFloat IntMult dictionary code is invalid"); + } + } + } + Ok(()) +} + impl OperationsVTable for OrderedFloat { fn scalar_at( array: ArrayView<'_, OrderedFloat>, @@ -363,36 +508,28 @@ fn decode_primitive( ctx: &mut ExecutionCtx, ) -> VortexResult { let encoded = array.encoded().clone().execute::(ctx)?; + let validity = encoded.validity()?; Ok(match array.dtype().as_ptype() { PType::F16 => PrimitiveArray::new( - Buffer::from( - encoded - .as_slice::() - .iter() - .map(|&value| f16::from_bits(unordered_u16(value))) - .collect::>(), - ), - encoded.validity()?, + encoded + .into_buffer::() + .map_each_in_place(|value| f16::from_bits(unordered_u16(value))) + .freeze(), + validity, ), PType::F32 => PrimitiveArray::new( - Buffer::from( - encoded - .as_slice::() - .iter() - .map(|&value| f32::from_bits(unordered_u32(value))) - .collect::>(), - ), - encoded.validity()?, + encoded + .into_buffer::() + .map_each_in_place(|value| f32::from_bits(unordered_u32(value))) + .freeze(), + validity, ), PType::F64 => PrimitiveArray::new( - Buffer::from( - encoded - .as_slice::() - .iter() - .map(|&value| f64::from_bits(unordered_u64(value))) - .collect::>(), - ), - encoded.validity()?, + encoded + .into_buffer::() + .map_each_in_place(|value| f64::from_bits(unordered_u64(value))) + .freeze(), + validity, ), ptype => vortex_panic!("unsupported OrderedFloat ptype {ptype}"), }) @@ -461,6 +598,7 @@ mod tests { use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::array_session; + use vortex_array::arrays::DictArray; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -472,6 +610,7 @@ mod tests { use vortex_buffer::Buffer; use vortex_buffer::ByteBufferMut; use vortex_error::VortexResult; + use vortex_int_mult::IntMult; use vortex_session::registry::ReadContext; use super::OrderedFloat; @@ -612,6 +751,35 @@ mod tests { Ok(()) } + #[test] + fn roundtrip_nullable_int_mult_dictionary_block_residual() -> VortexResult<()> { + let primitive = PrimitiveArray::new( + Buffer::from(vec![1.0_f64, 0.0, 2.5, -3.0]), + Validity::from_iter([true, false, true, true]), + ); + let ordered = OrderedFloat::from_primitive(primitive.as_view())?; + let ordered_primitive = ordered.encoded().as_::(); + let ordered_values = ordered_primitive.as_slice::(); + let starts = + PrimitiveArray::from_iter([ordered_values[0], ordered_values[2], ordered_values[3]]); + let codes = PrimitiveArray::new( + Buffer::from(vec![0_u8, 9, 1, 2]), + Validity::from_iter([true, false, true, true]), + ); + let references = DictArray::try_new(codes.into_array(), starts.into_array())?; + let offsets = + BlockResidual::from_primitive(PrimitiveArray::from_iter([0_u64; 4]).as_view())?; + let int_mult = IntMult::try_new(references.into_array(), offsets.into_array(), 1)?; + let encoded = OrderedFloat::try_new(int_mult.into_array(), PType::F64)?; + let session = array_session(); + crate::initialize(&session); + vortex_int_mult::initialize(&session); + let mut ctx = session.create_execution_ctx(); + + assert_arrays_eq!(encoded, primitive, &mut ctx); + Ok(()) + } + #[test] fn nullable_serialized_slice_roundtrip() -> VortexResult<()> { let primitive = PrimitiveArray::new( diff --git a/encodings/int-mult/src/array.rs b/encodings/int-mult/src/array.rs index 7af9db4fd6c..606ab51cb4b 100644 --- a/encodings/int-mult/src/array.rs +++ b/encodings/int-mult/src/array.rs @@ -253,16 +253,14 @@ fn decode_dict_add( if let Some(codes) = dict.codes().as_typed::() { let validity = BitPackedArrayExt::validity(&codes); - if codes.patches().is_none() && validity.definitely_no_nulls() { - return decode_bitpacked_dict(starts, secondary, codes, validity).map(Some); + if codes.patches().is_none() { + return decode_bitpacked_dict(starts, secondary, codes, validity, ctx).map(Some); } } let codes = dict.codes().clone().execute::(ctx)?; - if !codes.validity()?.definitely_no_nulls() { - return Ok(None); - } - decode_primitive_dict(starts, secondary, codes).map(Some) + let validity = codes.validity()?; + decode_primitive_dict(starts, secondary, codes, validity, ctx).map(Some) } fn decode_bitpacked_dict( @@ -270,10 +268,13 @@ fn decode_bitpacked_dict( secondary: PrimitiveArray, codes: ArrayView<'_, BitPacked>, validity: vortex_array::validity::Validity, + ctx: &mut ExecutionCtx, ) -> VortexResult { match_each_integer_ptype!(secondary.ptype(), |T| { let starts = starts.as_slice::(); - let output = add_bitpacked_codes(starts, secondary.into_buffer_mut::(), codes)?; + let (output, invalid_codes) = + add_bitpacked_codes(starts, secondary.into_buffer_mut::(), codes)?; + validate_invalid_codes(&validity, output.len(), &invalid_codes, ctx)?; VortexResult::Ok(PrimitiveArray::new(output, validity)) }) } @@ -282,16 +283,18 @@ fn decode_primitive_dict( starts: PrimitiveArray, secondary: PrimitiveArray, codes: PrimitiveArray, + validity: vortex_array::validity::Validity, + ctx: &mut ExecutionCtx, ) -> VortexResult { - let validity = codes.validity()?; match_each_integer_ptype!(secondary.ptype(), |T| { let starts = starts.as_slice::(); match_each_integer_ptype!(codes.ptype(), |C| { - let output = add_primitive_codes( + let (output, invalid_codes) = add_primitive_codes( starts, secondary.into_buffer_mut::(), codes.as_slice::(), )?; + validate_invalid_codes(&validity, output.len(), &invalid_codes, ctx)?; VortexResult::Ok(PrimitiveArray::new(output, validity)) }) }) @@ -301,55 +304,95 @@ fn add_primitive_codes( starts: &[T], mut output: BufferMut, codes: &[C], -) -> VortexResult> +) -> VortexResult<(BufferMut, Vec)> where T: NativePType + WrappingMulAdd, C: NativePType + AsPrimitive, { - for (value, code) in output.as_mut_slice().iter_mut().zip(codes) { - let code: usize = code.as_(); + let mut invalid_codes = Vec::new(); + for (index, (value, code)) in output.as_mut_slice().iter_mut().zip(codes).enumerate() { + let code: usize = (*code).as_(); let Some(start) = starts.get(code) else { - vortex_bail!("IntMult dictionary code is invalid"); + invalid_codes.push(index); + continue; }; *value = ::wrapping_add(*start, *value); } - Ok(output) + Ok((output, invalid_codes)) +} + +fn validate_invalid_codes( + validity: &vortex_array::validity::Validity, + len: usize, + invalid_codes: &[usize], + ctx: &mut ExecutionCtx, +) -> VortexResult<()> { + if !invalid_codes.is_empty() { + let mask = validity.execute_mask(len, ctx)?; + for &index in invalid_codes { + if mask.value(index) { + vortex_bail!("IntMult dictionary code is invalid"); + } + } + } + Ok(()) } fn add_bitpacked_codes( starts: &[T], mut output: BufferMut, codes: ArrayView<'_, BitPacked>, -) -> VortexResult> +) -> VortexResult<(BufferMut, Vec)> where T: NativePType + WrappingMulAdd, { - vortex_ensure!(!starts.is_empty(), "IntMult dictionary values are empty"); + let mut invalid_codes = Vec::new(); if codes.bit_width() == 0 { - for value in output.as_mut_slice() { - *value = ::wrapping_add(starts[0], *value); + if let Some(start) = starts.first() { + for value in output.as_mut_slice() { + *value = ::wrapping_add(*start, *value); + } + } else { + invalid_codes.extend(0..output.len()); } - return Ok(output); + return Ok((output, invalid_codes)); } match_each_integer_ptype!(codes.dtype().as_ptype(), |C| { - let mut invalid_code = false; let mut chunks = codes.unpacked_chunks::()?; chunks.for_each_unpacked_chunk(|codes, range| { - for (value, code) in output.as_mut_slice()[range].iter_mut().zip(codes) { - let code: usize = code.as_(); - let Some(start) = starts.get(code) else { - invalid_code = true; - continue; - }; - *value = ::wrapping_add(*start, *value); - } + add_dictionary_starts( + starts, + &mut output.as_mut_slice()[range.clone()], + codes, + range.start, + &mut invalid_codes, + ); }); - vortex_ensure!(!invalid_code, "IntMult dictionary code is invalid"); - VortexResult::Ok(output) + VortexResult::Ok((output, invalid_codes)) }) } +fn add_dictionary_starts( + starts: &[T], + output: &mut [T], + codes: &[C], + offset: usize, + invalid_codes: &mut Vec, +) where + T: NativePType + WrappingMulAdd, + C: NativePType + AsPrimitive, +{ + for (index, (value, code)) in output.iter_mut().zip(codes).enumerate() { + let code: usize = (*code).as_(); + let Some(start) = starts.get(code) else { + invalid_codes.push(offset + index); + continue; + }; + *value = ::wrapping_add(*start, *value); + } +} + impl OperationsVTable for IntMult { fn scalar_at( array: ArrayView<'_, IntMult>, @@ -677,6 +720,32 @@ mod tests { Ok(()) } + #[test] + fn base_one_adds_nullable_bitpacked_dictionary_references() -> VortexResult<()> { + let codes = PrimitiveArray::new( + buffer![0_u8, 3, 1, 0, 3], + Validity::from_iter([true, false, true, true, false]), + ); + // SAFETY: Two bits represent every code payload, including invalid null payloads. + let codes = unsafe { bitpack_encode_unchecked(codes, 2) }?.into_array(); + let starts = PrimitiveArray::from_iter([100_u32, 1_000]).into_array(); + let references = DictArray::try_new(codes, starts)?.into_array(); + let offsets = PrimitiveArray::from_iter([1_u32, 2, 3, 4, 5]).into_array(); + let encoded = IntMult::try_new(references, offsets, 1)?; + let expected = + PrimitiveArray::from_option_iter([Some(101_u32), None, Some(1_003), Some(104), None]); + let mut ctx = array_session().create_execution_ctx(); + + assert!(decode_dict_add(encoded.as_view(), &mut ctx)?.is_some()); + assert_arrays_eq!(encoded.clone(), expected, &mut ctx); + assert_arrays_eq!( + encoded.into_array().slice(1..4)?, + expected.into_array().slice(1..4)?, + &mut ctx + ); + Ok(()) + } + #[test] fn base_one_adds_zero_width_dictionary_references() -> VortexResult<()> { let codes = PrimitiveArray::from_iter([0_u8; 3]); diff --git a/vortex-btrblocks/Cargo.toml b/vortex-btrblocks/Cargo.toml index c9f281300a6..1bdcd5f6db9 100644 --- a/vortex-btrblocks/Cargo.toml +++ b/vortex-btrblocks/Cargo.toml @@ -32,6 +32,7 @@ vortex-fastlanes = { workspace = true } vortex-float-quant = { workspace = true } vortex-range-packed = { workspace = true } vortex-fsst = { workspace = true } +vortex-int-mult = { workspace = true } vortex-onpair = { workspace = true, optional = true } vortex-pco = { workspace = true, optional = true } vortex-runend = { workspace = true } @@ -55,7 +56,6 @@ tpchgen = { workspace = true } tpchgen-arrow = { workspace = true } vortex-array = { workspace = true, features = ["_test-harness"] } vortex-arrow = { workspace = true } -vortex-int-mult = { workspace = true } vortex-mask = { workspace = true } vortex-session = { workspace = true } diff --git a/vortex-btrblocks/src/schemes/float/range_packed.rs b/vortex-btrblocks/src/schemes/float/range_packed.rs index f93b26ae690..66001ccf2fb 100644 --- a/vortex-btrblocks/src/schemes/float/range_packed.rs +++ b/vortex-btrblocks/src/schemes/float/range_packed.rs @@ -9,11 +9,14 @@ use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::VTable; +use vortex_array::arrays::Dict; +use vortex_array::arrays::DictArray; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::PType; use vortex_array::dtype::half::f16; use vortex_array::match_each_float_ptype; +use vortex_block_residual::BlockResidual; use vortex_block_residual::OrderedFloat; use vortex_block_residual::OrderedFloatArraySlotsExt; use vortex_compressor::scheme::CompressionEstimate; @@ -21,7 +24,10 @@ use vortex_compressor::scheme::DeferredEstimate; use vortex_compressor::scheme::EstimateScore; use vortex_compressor::scheme::EstimateVerdict; use vortex_error::VortexResult; -use vortex_range_packed::RangePacked; +use vortex_fastlanes::BitPacked; +use vortex_fastlanes::bitpack_compress::bitpack_encode_unchecked; +use vortex_int_mult::IntMult; +use vortex_range_packed::RangeDecomposition; use crate::ArrayAndStats; use crate::CascadingCompressor; @@ -64,7 +70,13 @@ impl Scheme for OrderedFloatRangePackedScheme { } fn produced_encodings(&self) -> Vec { - vec![RangePacked.id(), OrderedFloat.id()] + vec![ + OrderedFloat.id(), + IntMult.id(), + Dict.id(), + BitPacked.id(), + BlockResidual.id(), + ] } fn num_children(&self) -> usize { @@ -283,23 +295,70 @@ fn estimate_ordered_float( primitive: vortex_array::ArrayView<'_, Primitive>, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - let ordered = OrderedFloat::from_primitive(primitive)?; - RangePacked::estimate_primitive_with_null_positions( - ordered.encoded().as_::(), - exec_ctx, - ) + Ok(encode_ordered_float(primitive, exec_ctx)?.nbytes()) } fn encode_ordered_float( primitive: vortex_array::ArrayView<'_, Primitive>, - exec_ctx: &mut ExecutionCtx, + _exec_ctx: &mut ExecutionCtx, ) -> VortexResult { let ordered = OrderedFloat::from_primitive(primitive)?; - let packed = RangePacked::from_primitive_with_null_positions( - ordered.encoded().as_::(), - exec_ctx, - )?; - Ok(OrderedFloat::try_new(packed.into_array(), primitive.ptype())?.into_array()) + let ordered_values = ordered.encoded().as_::(); + if ordered_values.is_empty() { + return Ok(ordered.into_array()); + } + + let decomposition = match ordered_values.ptype() { + PType::U16 => RangeDecomposition::encode( + &ordered_values + .as_slice::() + .iter() + .map(|&value| u64::from(value)) + .collect::>(), + )?, + PType::U32 => RangeDecomposition::encode( + &ordered_values + .as_slice::() + .iter() + .map(|&value| u64::from(value)) + .collect::>(), + )?, + PType::U64 => RangeDecomposition::encode(ordered_values.as_slice::())?, + ptype => vortex_error::vortex_bail!( + "range decomposition requires unsigned integers, got {ptype}" + ), + }; + let code_width = decomposition.code_width(); + let codes = PrimitiveArray::new(decomposition.codes().to_vec(), ordered_values.validity()?); + // SAFETY: The decomposition computes the exact code width. + let codes = unsafe { bitpack_encode_unchecked(codes, code_width) }?.into_array(); + let starts = narrow_unsigned(decomposition.bin_starts(), ordered_values.ptype())?.into_array(); + let references = DictArray::try_new(codes, starts)?.into_array(); + let offsets = narrow_unsigned(decomposition.offsets(), ordered_values.ptype())?; + let offsets = BlockResidual::from_primitive(offsets.as_view())?.into_array(); + let reconstructed = IntMult::try_new(references, offsets, 1)?; + Ok(OrderedFloat::try_new(reconstructed.into_array(), primitive.ptype())?.into_array()) +} + +fn narrow_unsigned(values: &[u64], ptype: PType) -> VortexResult { + Ok(match ptype { + PType::U16 => PrimitiveArray::from_iter( + values + .iter() + .map(|&value| u16::try_from(value)) + .collect::, _>>()?, + ), + PType::U32 => PrimitiveArray::from_iter( + values + .iter() + .map(|&value| u32::try_from(value)) + .collect::, _>>()?, + ), + PType::U64 => PrimitiveArray::from_iter(values.iter().copied()), + _ => vortex_error::vortex_bail!( + "range decomposition requires unsigned integers, got {ptype}" + ), + }) } fn normalize_float_null_values( @@ -359,7 +418,6 @@ mod tests { })))] fn ordered_float_tree_roundtrips(#[case] expected: PrimitiveArray) -> VortexResult<()> { let session = array_session(); - vortex_range_packed::initialize(&session); let mut ctx = session.create_execution_ctx(); let compressed = CascadingCompressor::new(vec![&TEST_SCHEME]) .compress(&expected.clone().into_array(), &mut ctx)?; @@ -465,14 +523,19 @@ mod tests { })); let expected_array = expected.clone().into_array(); let session = array_session(); - vortex_range_packed::initialize(&session); let mut ctx = session.create_execution_ctx(); let compressed = CascadingCompressor::new(vec![&TEST_SCHEME]).compress(&expected_array, &mut ctx)?; - let packed = &compressed.children()[0]; - assert!(packed.is::()); - assert_eq!(packed.children().len(), 5); + let int_mult = &compressed.children()[0]; + assert!(int_mult.is::()); + let references = &int_mult.children()[0]; + let offsets = &int_mult.children()[1]; + assert!(references.is::()); + assert!(references.children()[0].is::()); + assert!(offsets.is::()); + assert_eq!(references.len(), expected.len()); + assert_eq!(offsets.len(), expected.len()); assert_arrays_eq!(compressed, expected, &mut ctx); Ok(()) } diff --git a/vortex/src/editions/core/v2026_08.rs b/vortex/src/editions/core/v2026_08.rs index 5903db06a5e..6cc1a3c1c8d 100644 --- a/vortex/src/editions/core/v2026_08.rs +++ b/vortex/src/editions/core/v2026_08.rs @@ -28,6 +28,7 @@ pub static DECLARATION: EditionDeclaration = EditionDeclaration { added: &[ EditionMember::array(&"vortex.block_residual"), EditionMember::array(&"vortex.float_quant"), + EditionMember::array(&"vortex.int_mult"), EditionMember::array(&"vortex.map"), EditionMember::array(&"vortex.ordered_float"), EditionMember::array(&"vortex.range_packed"), From f3a338cc579e11a6cde9d24d8e23ab93006df277 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 10:14:19 +0100 Subject: [PATCH 58/78] bench: report compressed chunk trees Signed-off-by: Will Manning --- .../examples/float_compressor_bench.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index 44e3e7099c6..60e9d450f8a 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -42,6 +42,7 @@ use vortex_array::IntoArray; use vortex_array::RecursiveCanonical; use vortex_array::VortexSessionExecute; use vortex_array::array_session; +use vortex_array::arrays::Chunked; use vortex_array::arrays::ChunkedArray; use vortex_array::arrays::Dict; use vortex_array::arrays::DictArray; @@ -50,6 +51,7 @@ use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::TemporalArray; use vortex_array::arrays::VarBinViewArray; +use vortex_array::arrays::chunked::ChunkedArrayExt; use vortex_array::arrays::dict::DictArraySlotsExt; use vortex_array::assert_arrays_eq; use vortex_array::builtins::ArrayBuiltins; @@ -2775,6 +2777,20 @@ fn measure_dataset( encoding_tree(array), array.nbytes() ); + if std::env::var_os("VORTEX_BENCH_CHUNK_TREES").is_some() + && let Some(chunked) = array.as_typed::() + { + for (chunk_index, chunk) in chunked.iter_chunks().enumerate() { + println!( + "chunk-structure\t{dataset}\t{}\t{}\t{config}\t{chunk_index}\t{}\t{}\t{}", + column.name, + column.dtype_label, + chunk.len(), + encoding_tree(chunk), + chunk.nbytes(), + ); + } + } if matches!( *config, "integer-block-residual-only" | "ordered-block-residual-only" From 7708e83fc7ca9e90003727fb24dd78a6264ee343 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 10:34:04 +0100 Subject: [PATCH 59/78] fix: compare fused candidates with cascades Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 51 ++++++++++++++++- .../src/schemes/float/range_packed.rs | 56 ++++++++++++++++++- vortex-compressor/src/compressor/cascade.rs | 29 ++++++++++ 3 files changed, 132 insertions(+), 4 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 51998971cd9..4d052366ca8 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -1880,9 +1880,9 @@ Treat them as aggregate noise until a larger repeated pass confirms them. The first two million rows selected the new tree only for HashTags `interaction#received_at`. -Full files also changed size for Bimbo, CMSprovider, Euro2016, and Taxi. +The initial full files also changed size for Bimbo, CMSprovider, Euro2016, and Taxi. -This difference requires a chunk-level selected-tree trace before threshold calibration. +A later chunk-level trace identified completed incumbent cascades as the cause. The current broad size gain does not justify Default inclusion by itself. @@ -1892,6 +1892,50 @@ Fast rejection remains a cost advantage, not an admission rule. A scheme without fast rejection can enter Default when corpus evidence justifies its analysis cost. +### Completed incumbent cascade comparison + +The first decomposed selector compared its complete fixed tree against each incumbent root estimate. + +Several incumbents compressed their children after root selection. + +A full Bimbo chunk trace found eight false fixed-bin selections in `Dev_proxima`. + +The fixed-bin trees used 5.3 to 239.7 percent more bytes than the completed incumbent trees. + +The selector now compares each retained candidate against the incumbent sample cascade. + +The coarse model runs first. Clear losses do not pay for a second sample compression. + +This comparison preserves the current cascade depth and excludes only the fixed-bin scheme. + +The corrected Bimbo file uses 391,860,906 bytes in both configurations. + +Its matched write throughput changes from 515.6 to 509.6 MB/s. + +Its matched read throughput changes from 11,143.6 to 11,130.7 MB/s. + +The focused HashTags file retains the useful selection: + +| Metric | Current | Fixed-bin candidate | Change | +| --- | ---: | ---: | ---: | +| File size | 21,252,695 bytes | 20,718,724 bytes | -2.51 percent | +| Write throughput | 626.6 MB/s | 610.3 MB/s | -2.60 percent | +| Read throughput | 21,294.0 MB/s | 20,913.0 MB/s | -1.79 percent | + +The matched 16-dataset file pass selected the fixed-bin tree only on HashTags. + +| Metric | Geometric-mean change | +| --- | ---: | +| File size | -0.027 percent | +| Write time | +0.301 percent | +| Read time | -0.298 percent | + +The read result is benchmark noise because only one file changes its encoded tree. + +The corpus size result does not justify Default inclusion with the current decode and scalar costs. + +Fast rejection remains an advantage. It does not replace evidence from complete candidate trees. + ### Complete bundle controls The compression benchmark exposes four numeric bundles: @@ -2231,12 +2275,15 @@ This round completed these steps: - Updated stale golden trees after the BlockResidual payload and selector changes. - Added four numeric bundle controls to the complete compression benchmark. - Added explicit winner result events to the compressor trace. +- Added optional chunk-level encoding trees to the focused compressor benchmark. - Rejected constant FloatQuant samples before exact sample encoding. - Added a provisional 1.10 FloatQuant selection factor. - Removed the Food and HashTags FloatQuant size regressions. - Revalidated the BlockResidual and FloatQuant bundles across 16 files. - Rejected constant RangePacked samples before exact sample encoding. - Revalidated RangePacked against the corrected current bundle. +- Compared retained fixed-bin candidates against the completed incumbent sample cascade. +- Removed every known Bimbo, CMSprovider, Euro2016, and Taxi fixed-bin false win. - Tested FloatQuant on 33 real float columns from five Pco-gap inputs. The Pco mode profile and the quotient and remainder experiments are complete. diff --git a/vortex-btrblocks/src/schemes/float/range_packed.rs b/vortex-btrblocks/src/schemes/float/range_packed.rs index 66001ccf2fb..0cf90c045a0 100644 --- a/vortex-btrblocks/src/schemes/float/range_packed.rs +++ b/vortex-btrblocks/src/schemes/float/range_packed.rs @@ -33,6 +33,7 @@ use crate::ArrayAndStats; use crate::CascadingCompressor; use crate::CompressorContext; use crate::Scheme; +use crate::SchemeExt; use crate::schemes::sample_primitive_one_percent; /// The default factor accounts for RangePacked decode cost during scheme selection. @@ -94,8 +95,9 @@ impl Scheme for OrderedFloatRangePackedScheme { } let decode_cost_factor = self.decode_cost_factor; + let scheme_id = self.id(); CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - move |_compressor, data, best_so_far, _compress_ctx, exec_ctx| { + move |compressor, data, best_so_far, compress_ctx, exec_ctx| { let bit_width = data.array_as_primitive().ptype().bit_width() as f64; let maximum_adjusted_ratio = bit_width / decode_cost_factor; let best_ratio = best_so_far @@ -123,7 +125,18 @@ impl Scheme for OrderedFloatRangePackedScheme { let adjusted_ratio = before_nbytes as f64 / after_nbytes as f64 / decode_cost_factor; - if adjusted_ratio <= 1.0 { + if adjusted_ratio <= best_ratio { + return Ok(EstimateVerdict::Skip); + } + + let sample = sample.into_array(); + let incumbent = compressor.compress_sample_without_scheme( + &sample, + scheme_id, + compress_ctx, + exec_ctx, + )?; + if after_nbytes as f64 * decode_cost_factor >= incumbent.nbytes() as f64 { return Ok(EstimateVerdict::Skip); } Ok(EstimateVerdict::Ratio(adjusted_ratio)) @@ -402,6 +415,7 @@ mod tests { use super::encode_ordered_float; use super::estimate_ordered_float; use super::estimate_ordered_float_if_promising; + use crate::BtrBlocksCompressorBuilder; use crate::CascadingCompressor; const TEST_SCHEME: OrderedFloatRangePackedScheme = OrderedFloatRangePackedScheme::new(1.0); @@ -515,6 +529,44 @@ mod tests { Ok(()) } + #[test] + fn completed_incumbent_tree_beats_range_tree() -> VortexResult<()> { + let expected = PrimitiveArray::from_iter((0_u64..65_536).map(|index| { + let cluster = index % 8; + f64::from_bits(0x3ff0_0000_0000_0000 + cluster * 0x1_0000_0000) + })); + let expected_array = expected.clone().into_array(); + let session = array_session(); + let mut ctx = session.create_execution_ctx(); + + let retained = estimate_ordered_float_if_promising( + expected.as_view(), + 1.0, + TEST_SCHEME.decode_cost_factor, + &mut ctx, + )?; + assert!(retained.is_some()); + + let range_only = + CascadingCompressor::new(vec![&TEST_SCHEME]).compress(&expected_array, &mut ctx)?; + assert!(range_only.is::()); + assert!(range_only.children()[0].is::()); + + let selected = BtrBlocksCompressorBuilder::default() + .with_new_scheme(&TEST_SCHEME) + .build() + .compress(&expected_array, &mut ctx)?; + let selected_is_range = selected.is::() + && selected + .children() + .first() + .is_some_and(|child| child.is::()); + assert!(!selected_is_range); + assert!(selected.nbytes() < range_only.nbytes()); + assert_arrays_eq!(selected, expected, &mut ctx); + Ok(()) + } + #[test] fn nullable_tree_stores_full_positions() -> VortexResult<()> { let expected = PrimitiveArray::from_option_iter((0_u64..65_536).map(|index| { diff --git a/vortex-compressor/src/compressor/cascade.rs b/vortex-compressor/src/compressor/cascade.rs index 86d45d2c0d9..ce4e0f2f674 100644 --- a/vortex-compressor/src/compressor/cascade.rs +++ b/vortex-compressor/src/compressor/cascade.rs @@ -66,6 +66,35 @@ impl CascadingCompressor { Ok(compressed) } + /// Compresses an estimator sample after it removes one scheme from the current cascade. + /// + /// Fused schemes use this method to compare their output with the completed incumbent tree. + /// The method preserves the current cascade depth and marks the input as a sample. + /// + /// # Errors + /// + /// Returns an error if canonicalization or compression fails. + pub fn compress_sample_without_scheme( + &self, + array: &ArrayRef, + excluded: SchemeId, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, + ) -> VortexResult { + let compressor = Self { + schemes: self + .schemes + .iter() + .copied() + .filter(|scheme| scheme.id() != excluded) + .collect(), + root_exclusions: self.root_exclusions.clone(), + }; + let canonical = array.clone().execute::(exec_ctx)?.0; + let compact = canonical.compact(exec_ctx)?; + compressor.compress_canonical(compact, compress_ctx.with_sampling(), exec_ctx) + } + /// Compresses a child array produced by a cascading scheme. /// /// If the cascade budget is exhausted, the canonical array is returned as-is. Otherwise, the From dd8bf40358cea51652e46c0e526eee2daaec559c Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 10:45:33 +0100 Subject: [PATCH 60/78] bench: cover two-child FloatQuant widths Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 89 +++++++++-- .../examples/float_compressor_bench.rs | 35 ++++ vortex/benches/single_encoding_throughput.rs | 150 ++++++++++++++++++ 3 files changed, 258 insertions(+), 16 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 4d052366ca8..dfbc6cb8e59 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -349,10 +349,12 @@ These cases use canonical primitive children. They isolate each outer array tran | IntMult base-ten `u16` | 1.42 GB/s | 30.59 GB/s | 160 ns | | IntMult base-ten `u32` | 2.84 GB/s | 30.51 GB/s | 136 ns | | IntMult base-ten `u64` | 5.66 GB/s | 24.57 GB/s | 150 ns | -| FloatQuant zero-secondary `f16` | 6.84 GB/s | 31.24 GB/s | 80 ns | -| FloatQuant zero-secondary `f32` | 11.66 GB/s | 31.17 GB/s | 83 ns | -| FloatQuant zero-secondary `f64` | 19.02 GB/s | 25.15 GB/s | 83 ns | -| FloatQuant one-bit-secondary `f64` | 9.26 GB/s | 8.14 GB/s | 167 ns | +| FloatQuant zero-secondary `f16` | 7.62 GB/s | 33.02 GB/s | 83 ns | +| FloatQuant zero-secondary `f32` | 13.63 GB/s | 33.58 GB/s | 75 ns | +| FloatQuant zero-secondary `f64` | 20.90 GB/s | 26.88 GB/s | 82 ns | +| FloatQuant one-bit-secondary `f16` | 2.41 GB/s | 2.98 GB/s | 118 ns | +| FloatQuant one-bit-secondary `f32` | 4.87 GB/s | 4.57 GB/s | 136 ns | +| FloatQuant one-bit-secondary `f64` | 9.18 GB/s | 8.23 GB/s | 148 ns | IntMult encode includes quotient and remainder creation. It excludes compression of both children. @@ -383,15 +385,19 @@ These cases include the current compressed children and fused decode paths. | OrderedFloat with BlockResidual `f16` | 1.26 GB/s | 20.50 GB/s | 79 ns | | OrderedFloat with BlockResidual `f32` | 2.43 GB/s | 29.64 GB/s | 78 ns | | OrderedFloat with BlockResidual `f64` | 2.70 GB/s | 20.44 GB/s | 125 ns | -| FloatQuant with packed primary `f16` | 3.42 GB/s | 19.39 GB/s | 129 ns | -| FloatQuant with packed primary `f32` | 7.59 GB/s | 18.21 GB/s | 125 ns | -| FloatQuant with packed primary `f64` | 8.30 GB/s | 14.77 GB/s | 125 ns | -| FloatQuant with two packed children `f64` | 4.21 GB/s | 13.05 GB/s | 209 ns | +| FloatQuant with packed primary `f16` | 3.21 GB/s | 20.05 GB/s | 125 ns | +| FloatQuant with packed primary `f32` | 5.72 GB/s | 19.79 GB/s | 129 ns | +| FloatQuant with packed primary `f64` | 7.76 GB/s | 15.69 GB/s | 125 ns | +| FloatQuant with two packed children `f16` | 3.11 GB/s | 16.75 GB/s | 174 ns | +| FloatQuant with two packed children `f32` | 5.17 GB/s | 16.30 GB/s | 183 ns | +| FloatQuant with two packed children `f64` | 6.72 GB/s | 13.13 GB/s | 208 ns | | Decomposed fixed bins `u64` | 1.20 GB/s | 14.27 GB/s | 332 ns | -The FloatQuant scheme includes analysis and direct tree construction. +The FloatQuant production encode rows use the single-scheme compressor. -It encodes at 5.19 GB/s for `f32` and 7.07 GB/s for `f64` on selected inputs. +They include sample analysis and direct packing of the final child tree. + +The paired decoder remains within 18 percent of the zero-secondary decoder for every float width. The packed IntMult tree uses a base of ten and patch-free BitPacked children. @@ -499,6 +505,30 @@ Rejected analysis changed encode throughput by less than one percent on most mea Retain the two-child form in the default candidate set for final corpus calibration. +### All-width two-child FloatQuant revalidation + +The August 20 pass added two-child benchmarks for `f16` and `f32`. + +Each case changes the lowest bit for ten percent of two million quantized values. + +| Input | Incumbent bytes | Candidate bytes | Size change | Encode change | Decode change | +| --- | ---: | ---: | ---: | ---: | ---: | +| `f16` | 1,751,040 | 1,751,040 | 0.0 percent | -0.3 percent | +0.2 percent | +| `f32` | 5,752,576 | 4,001,792 | -30.4 percent | +29.6 percent | +43.1 percent | +| `f64` | 14,057,966 | 9,004,032 | -36.0 percent | +10.0 percent | +14.2 percent | + +The `f16` selector retains `Dict(BitPacked, Primitive)`. + +The `f32` and `f64` selectors choose `FloatQuant(FoR(BitPacked), BitPacked)`. + +The paired decoder stays within 18 percent of the zero-secondary decoder for all three widths. + +Paired scalar access uses 174 ns for `f16`, 183 ns for `f32`, and 208 ns for `f64`. + +The two selected complete trees improve size, encode throughput, and decode throughput. + +This evidence supports the opportunistic FloatQuant bundle despite the zero-selection real corpus pass. + ### OrderedFloat with BlockResidual on random walks | Configuration | Bytes | Encode MB/s | Decode MB/s | Scalar access ns | @@ -1987,7 +2017,7 @@ The write result measures FloatQuant analysis without a size win in this corpus. FloatQuant remains a production array and a BtrBlocks candidate. -Default inclusion now requires a real Pco-gap win with the 1.10 factor. +Default inclusion requires evidence that justifies its analysis cost. ### Real Pco-gap FloatQuant pass @@ -2015,9 +2045,33 @@ Compact retains material size gains on many float columns in these files. The Pco mode profile attributes those gaps to classic bins, IntMult, and FloatMult. -This result does not support FloatQuant in Default under the current factor. +This corpus does not expose a FloatQuant win under the current factor. + +The complete synthetic `f32` and `f64` trees produce strict size and throughput wins. + +Retain FloatQuant as an opportunistic Default candidate while the final corpus pass measures its analysis cost. + +### Default bundle options + +The evidence supports three bundle options: + +| Option | Schemes | Comparison | Size | Write throughput | Read throughput | +| --- | --- | --- | ---: | ---: | ---: | +| Conservative | BlockResidual | Prior Default | -1.6 percent | -0.9 percent | +1.3 percent | +| Opportunistic | BlockResidual and FloatQuant | Conservative | 0.0 percent | -0.9 percent | Noise | +| Size-seeking experiment | Opportunistic plus fixed bins | Opportunistic | -0.027 percent | -0.3 percent | Noise | + +The conservative option avoids FloatQuant analysis when the workload lacks a matching distribution. + +The opportunistic option accepts that analysis cost for large wins on matching `f32` and `f64` data. + +Its selected synthetic trees improve size, write throughput, and read throughput. + +The fixed-bin option lacks enough corpus benefit for Default. + +Quick rejection reduces analysis cost and strengthens a candidate. -Retain the production array while the final candidate review decides the scheme policy. +It is not an admission rule. A costly candidate requires stronger size and throughput evidence. ### Multi-reference block prototype @@ -2248,6 +2302,9 @@ This round completed these steps: - Added direct paired FastLanes packing for both FloatQuant children. - Added fused two-child FloatQuant decode and width-sweep benchmarks. - Validated selected and rejected two-child FloatQuant paths. +- Added two-child FloatQuant benchmarks for `f16`, `f32`, and `f64`. +- Confirmed strict complete-tree wins for the selected `f32` and `f64` cases. +- Confirmed that the `f16` selector retains its smaller dictionary tree. - Added Compact as a first-class format in the compression benchmark. - Compared the prior Default, proposed Default, Compact, and Parquet across all 16 local datasets. - Attributed the largest Compact numeric gaps to Pco modes at the column level. @@ -2298,9 +2355,9 @@ The specialized OrderedFloat and ALP trees now use registered fused parent kerne Complete these next experiments in order: -1. Decide the FloatQuant scheme policy from the no-selection corpus evidence. -2. Measure rejected-candidate analysis cost after the candidate set stabilizes. -3. Prepare final Default bundle options with selected-tree evidence. +1. Measure rejected-candidate analysis cost after the candidate set stabilizes. +2. Compare the conservative and opportunistic Default bundles on the final corpus. +3. Calibrate the FloatQuant factor from complete corpus and selected-tree evidence. 4. Prefer cheap rejection tests when they preserve the best candidate model. 5. Treat fast rejection as a selection advantage, not an admission criterion. 6. Require stronger corpus evidence when a useful scheme needs costly analysis. diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index 60e9d450f8a..c737f4246ab 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -57,6 +57,7 @@ use vortex_array::assert_arrays_eq; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::PType; +use vortex_array::dtype::half::f16; use vortex_array::extension::datetime::TimeUnit; use vortex_array::match_each_integer_ptype; use vortex_array::match_each_unsigned_integer_ptype; @@ -160,6 +161,32 @@ fn synthetic_datasets(row_count: usize) -> VortexResult let mantissa = (index.wrapping_mul(7_919) & 0x7fff) << 8; f32::from_bits(0x3f80_0000 | mantissa) })); + let nonzero_secondary_f32 = + PrimitiveArray::from_iter(quantized_f32.as_slice::().iter().enumerate().map( + |(index, value)| { + if index % 10 == 0 { + f32::from_bits(value.to_bits() | 1) + } else { + *value + } + }, + )); + let quantized_f16 = PrimitiveArray::from_iter((0..row_count).map(|index| { + let permuted = + u16::try_from(index.wrapping_mul(7_919) & usize::from(u16::MAX)).unwrap_or_default(); + let mantissa = permuted & 0x03f0; + f16::from_bits(0x3c00 | mantissa) + })); + let nonzero_secondary_f16 = + PrimitiveArray::from_iter(quantized_f16.as_slice::().iter().enumerate().map( + |(index, value)| { + if index % 10 == 0 { + f16::from_bits(value.to_bits() | 1) + } else { + *value + } + }, + )); let mut walk_rng = StdRng::seed_from_u64(2); let mut value = 1_000.0_f64; @@ -280,6 +307,14 @@ fn synthetic_datasets(row_count: usize) -> VortexResult "synthetic-nonzero-secondary".to_string(), vec![column("value", nonzero_secondary)], ), + ( + "synthetic-nonzero-secondary-f16".to_string(), + vec![column("value", nonzero_secondary_f16)], + ), + ( + "synthetic-nonzero-secondary-f32".to_string(), + vec![column("value", nonzero_secondary_f32)], + ), ( "synthetic-quantized-f32".to_string(), vec![column("value", quantized_f32)], diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index a18e1836e50..879b76f0c60 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -181,6 +181,32 @@ fn setup_nonzero_secondary_array() -> PrimitiveArray { ) } +fn setup_nonzero_secondary_f16_array() -> PrimitiveArray { + let quantized = setup_quantized_f16_array(); + PrimitiveArray::from_iter(quantized.as_slice::().iter().enumerate().map( + |(index, value)| { + if index % 10 == 0 { + f16::from_bits(value.to_bits() | 1) + } else { + *value + } + }, + )) +} + +fn setup_nonzero_secondary_f32_array() -> PrimitiveArray { + let quantized = setup_quantized_f32_array(); + PrimitiveArray::from_iter(quantized.as_slice::().iter().enumerate().map( + |(index, value)| { + if index % 10 == 0 { + f32::from_bits(value.to_bits() | 1) + } else { + *value + } + }, + )) +} + fn setup_secondary_width_array(width: u8) -> PrimitiveArray { let widened = setup_widened_f32_array(); let low_mask = (1_u64 << width) - 1; @@ -496,6 +522,15 @@ fn encode_float_quant_scheme_tree(array: &PrimitiveArray) -> vortex::array::Arra .unwrap() } +fn encode_float_quant_nonzero_secondary_scheme_tree( + array: &PrimitiveArray, +) -> vortex::array::ArrayRef { + let encoded = encode_float_quant_scheme_tree(array); + let float_quant = encoded.as_::(); + assert!(float_quant.secondary().is_some()); + encoded +} + fn encode_prior_default(array: &PrimitiveArray) -> vortex::array::ArrayRef { BtrBlocksCompressorBuilder::default() .exclude_schemes([ @@ -1784,6 +1819,121 @@ fn bench_float_quant_proposed_default_reject_f32(bencher: Bencher) { bench_compressor(bencher, setup_general_f32_array(), compressor); } +macro_rules! float_quant_nonzero_secondary_benches { + ( + $split_compress:ident, + $split_decompress:ident, + $split_scalar:ident, + $scheme_compress:ident, + $tree_decompress:ident, + $tree_scalar:ident, + $setup:ident, + $byte_width:expr + ) => { + #[divan::bench] + fn $split_compress(bencher: Bencher) { + let float_array = $setup(); + let k = analyze_float_quant(float_array.as_view()).unwrap().k; + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * $byte_width) + .with_inputs(|| &float_array) + .bench_refs(|array| FloatQuant::from_primitive(array.as_view(), k).unwrap()); + } + + #[divan::bench] + fn $split_decompress(bencher: Bencher) { + let float_array = $setup(); + let k = analyze_float_quant(float_array.as_view()).unwrap().k; + let encoded = FloatQuant::from_primitive(float_array.as_view(), k) + .unwrap() + .into_array(); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * $byte_width) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); + } + + #[divan::bench] + fn $split_scalar(bencher: Bencher) { + let float_array = $setup(); + let k = analyze_float_quant(float_array.as_view()).unwrap().k; + let encoded = FloatQuant::from_primitive(float_array.as_view(), k) + .unwrap() + .into_array(); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| { + array.execute_scalar(index, &mut ctx).unwrap() + }); + } + + #[divan::bench] + fn $scheme_compress(bencher: Bencher) { + let compressor = BtrBlocksCompressorBuilder::empty() + .with_new_scheme(&FloatQuantScheme) + .build(); + bench_compressor(bencher, $setup(), compressor); + } + + #[divan::bench] + fn $tree_decompress(bencher: Bencher) { + let encoded = encode_float_quant_nonzero_secondary_scheme_tree(&$setup()); + + with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * $byte_width) + .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); + } + + #[divan::bench] + fn $tree_scalar(bencher: Bencher) { + let encoded = encode_float_quant_nonzero_secondary_scheme_tree(&$setup()); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + ( + &encoded, + SESSION.create_execution_ctx(), + next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), + ) + }) + .bench_values(|(array, mut ctx, index)| { + array.execute_scalar(index, &mut ctx).unwrap() + }); + } + }; +} + +float_quant_nonzero_secondary_benches!( + float_quant_nonzero_secondary_split_compress_f16, + float_quant_nonzero_secondary_split_decompress_f16, + float_quant_nonzero_secondary_split_scalar_at_f16, + float_quant_nonzero_secondary_scheme_compress_f16, + float_quant_nonzero_secondary_tree_decompress_f16, + float_quant_nonzero_secondary_tree_scalar_at_f16, + setup_nonzero_secondary_f16_array, + 2 +); + +float_quant_nonzero_secondary_benches!( + float_quant_nonzero_secondary_split_compress_f32, + float_quant_nonzero_secondary_split_decompress_f32, + float_quant_nonzero_secondary_split_scalar_at_f32, + float_quant_nonzero_secondary_scheme_compress_f32, + float_quant_nonzero_secondary_tree_decompress_f32, + float_quant_nonzero_secondary_tree_scalar_at_f32, + setup_nonzero_secondary_f32_array, + 4 +); + #[divan::bench(name = "float_quant_nonzero_secondary_split_compress_f64")] fn bench_float_quant_nonzero_secondary_split_compress_f64(bencher: Bencher) { let float_array = setup_nonzero_secondary_array(); From ff27e2e3e0b448f1f4c11f6b320e616feeafda6e Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 11:19:10 +0100 Subject: [PATCH 61/78] perf: avoid FloatQuant sample packing Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 46 +++++++-- .../src/schemes/float/float_quant.rs | 79 ++++++++++++++- vortex/benches/single_encoding_throughput.rs | 99 +++++++++++++++++-- 3 files changed, 208 insertions(+), 16 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index dfbc6cb8e59..d479d900a24 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -529,6 +529,38 @@ The two selected complete trees improve size, encode throughput, and decode thro This evidence supports the opportunistic FloatQuant bundle despite the zero-selection real corpus pass. +### FloatQuant rejection cost + +The August 20 pass added isolated Default controls that differ only by FloatQuant. + +Each benchmark compresses two million values with 100 samples. + +| Input | Default without FloatQuant | Default with FloatQuant | Time change | +| --- | ---: | ---: | ---: | +| General `f16` | 4.383 ms | 4.348 ms | Noise | +| General `f32` | 17.84 ms | 18.03 ms | +1.1 percent | +| General `f64` | 23.62 ms | 23.43 ms | Noise | +| Near-miss `f32` | 23.77 ms | 23.78 ms | Noise | +| Near-miss `f64` | 20.67 ms | 20.58 ms | Noise | + +The near-miss inputs pass FloatQuant transform analysis but fail the adjusted ratio test. + +The estimator now computes the exact padded FastLanes buffer size and validity size from the analysis. + +It no longer packs the sample payload only to count its bytes. + +A focused test verifies the estimate against complete `f16`, `f32`, and `f64` trees. + +The test includes a nullable two-child case. + +The optimization did not produce a measurable complete-compressor change on the near-miss controls. + +Sample construction, statistics, and the other schemes dominate those complete times. + +The isolated controls place the marginal rejected cost between noise and 1.1 percent. + +The 16-file pass remains the broader result. It measured a 0.9 percent write-throughput cost. + ### OrderedFloat with BlockResidual on random walks | Configuration | Bytes | Encode MB/s | Decode MB/s | Scalar access ns | @@ -2335,6 +2367,9 @@ This round completed these steps: - Added optional chunk-level encoding trees to the focused compressor benchmark. - Rejected constant FloatQuant samples before exact sample encoding. - Added a provisional 1.10 FloatQuant selection factor. +- Replaced FloatQuant sample packing with an exact buffer-size estimate. +- Added isolated FloatQuant rejection controls for all float widths. +- Added adjusted-ratio near-miss controls for `f32` and `f64`. - Removed the Food and HashTags FloatQuant size regressions. - Revalidated the BlockResidual and FloatQuant bundles across 16 files. - Rejected constant RangePacked samples before exact sample encoding. @@ -2355,12 +2390,11 @@ The specialized OrderedFloat and ALP trees now use registered fused parent kerne Complete these next experiments in order: -1. Measure rejected-candidate analysis cost after the candidate set stabilizes. -2. Compare the conservative and opportunistic Default bundles on the final corpus. -3. Calibrate the FloatQuant factor from complete corpus and selected-tree evidence. -4. Prefer cheap rejection tests when they preserve the best candidate model. -5. Treat fast rejection as a selection advantage, not an admission criterion. -6. Require stronger corpus evidence when a useful scheme needs costly analysis. +1. Compare the conservative and opportunistic Default bundles on the final corpus. +2. Calibrate the FloatQuant factor from complete corpus and selected-tree evidence. +3. Prefer cheap rejection tests when they preserve the best candidate model. +4. Treat fast rejection as a selection advantage, not an admission criterion. +5. Require stronger corpus evidence when a useful scheme needs costly analysis. Use packed bin codes and primitive bin starts unless evidence supports more complexity. diff --git a/vortex-btrblocks/src/schemes/float/float_quant.rs b/vortex-btrblocks/src/schemes/float/float_quant.rs index c324db0aa9a..292b21f1eed 100644 --- a/vortex-btrblocks/src/schemes/float/float_quant.rs +++ b/vortex-btrblocks/src/schemes/float/float_quant.rs @@ -16,11 +16,13 @@ use vortex_array::dtype::PType; use vortex_array::dtype::half::f16; use vortex_array::scalar::Scalar; use vortex_array::validity::Validity; +use vortex_array::vtable::validity_to_child; use vortex_compressor::scheme::CompressionEstimate; use vortex_compressor::scheme::DeferredEstimate; use vortex_compressor::scheme::EstimateVerdict; use vortex_error::VortexResult; use vortex_fastlanes::BitPacked; +use vortex_fastlanes::FL_CHUNK_SIZE; use vortex_fastlanes::FoR; use vortex_fastlanes::bitpack_compress::bitpack_primitive_map; use vortex_fastlanes::bitpack_compress::bitpack_primitive_map_pair; @@ -98,8 +100,7 @@ fn estimate_float_quant_sample(sample: &PrimitiveArray) -> VortexResult= before_nbytes { return Ok(EstimateVerdict::Skip); } @@ -112,6 +113,21 @@ fn estimate_float_quant_sample(sample: &PrimitiveArray) -> VortexResult VortexResult { + let packed_chunks = u64::try_from(sample.len().div_ceil(FL_CHUNK_SIZE))?; + let bytes_per_bit = u64::try_from(FL_CHUNK_SIZE / 8)?; + let packed_bit_width = + u64::from(analysis.primary_bit_width) + u64::from(analysis.secondary_bit_width); + let packed_nbytes = packed_chunks * bytes_per_bit * packed_bit_width; + let validity_nbytes = validity_to_child(&sample.validity()?, sample.len()) + .map(|validity| validity.nbytes()) + .unwrap_or(0); + Ok(packed_nbytes + validity_nbytes) +} + fn encode_float_quant( primitive: vortex_array::ArrayView<'_, Primitive>, analysis: FloatQuantAnalysis, @@ -281,9 +297,14 @@ fn ordered_u64(bits: u64) -> u64 { #[cfg(test)] mod tests { use vortex_array::arrays::PrimitiveArray; + use vortex_array::dtype::half::f16; use vortex_compressor::scheme::EstimateVerdict; use vortex_error::VortexResult; + use vortex_error::vortex_err; + use super::analyze_float_quant; + use super::encode_float_quant; + use super::estimate_float_quant_nbytes; use super::estimate_float_quant_sample; #[test] @@ -296,4 +317,58 @@ mod tests { )); Ok(()) } + + #[test] + fn estimated_nbytes_matches_encoded_tree() -> VortexResult<()> { + let f16_values = PrimitiveArray::from_iter((0..2_050).map(|index| { + let high_bits = u16::try_from(index).unwrap_or_default().wrapping_mul(17) & 0x03f0; + let low_bit = u16::from(index % 10 == 0); + f16::from_bits(0x3c00 | high_bits | low_bit) + })); + let f32_values = PrimitiveArray::from_option_iter((0..2_050).map(|index| { + let high_bits = (index as u32).wrapping_mul(7_919) & 0x007f_ff00; + let low_bit = u32::from(index % 10 == 0); + (index % 17 != 0).then_some(f32::from_bits(0x3f80_0000 | high_bits | low_bit)) + })); + let f64_values = PrimitiveArray::from_iter((0..2_050).map(|index| { + let high_bits = ((index as u64).wrapping_mul(7_919) << 29) & 0x000f_ffff_ffff_ff00; + let low_bit = u64::from(index % 10 == 0); + f64::from_bits(0x3ff0_0000_0000_0000 | high_bits | low_bit) + })); + + for values in [f16_values, f32_values, f64_values] { + let analysis = analyze_float_quant(values.as_view()) + .ok_or_else(|| vortex_err!("FloatQuant test input did not produce an analysis"))?; + let expected = encode_float_quant(values.as_view(), analysis)?.nbytes(); + assert_eq!(estimate_float_quant_nbytes(&values, analysis)?, expected); + } + Ok(()) + } + + #[test] + fn near_miss_sample_is_rejected() -> VortexResult<()> { + let f32_values = PrimitiveArray::from_iter((0..2_048).map(|index| { + let scrambled = (index as u32).wrapping_mul(2_654_435_761); + let sign = (scrambled & 1) << 31; + let exponent = ((scrambled >> 1) % 254 + 1) << 23; + let mantissa = scrambled & 0x007f_fffc; + f32::from_bits(sign | exponent | mantissa) + })); + let f64_values = PrimitiveArray::from_iter((0..2_048).map(|index| { + let scrambled = (index as u64).wrapping_mul(0x9e37_79b9_7f4a_7c15); + let sign = (scrambled & 1) << 63; + let exponent = ((scrambled >> 1) % 2_046 + 1) << 52; + let mantissa = scrambled & 0x000f_ffff_ffff_fffc; + f64::from_bits(sign | exponent | mantissa) + })); + + for values in [f32_values, f64_values] { + assert!(analyze_float_quant(values.as_view()).is_some()); + assert!(matches!( + estimate_float_quant_sample(&values)?, + EstimateVerdict::Skip + )); + } + Ok(()) + } } diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index 879b76f0c60..7c5279394b7 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -164,6 +164,33 @@ fn setup_general_f32_array() -> PrimitiveArray { })) } +fn setup_general_f64_array() -> PrimitiveArray { + PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + let mantissa = index.wrapping_mul(0x9e37_79b9_7f4a_7c15) & 0x000f_ffff_ffff_ffff; + f64::from_bits(0x3ff0_0000_0000_0000 | mantissa) + })) +} + +fn setup_float_quant_near_miss_f32_array() -> PrimitiveArray { + PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + let scrambled = (index as u32).wrapping_mul(2_654_435_761); + let sign = (scrambled & 1) << 31; + let exponent = ((scrambled >> 1) % 254 + 1) << 23; + let mantissa = scrambled & 0x007f_fffc; + f32::from_bits(sign | exponent | mantissa) + })) +} + +fn setup_float_quant_near_miss_f64_array() -> PrimitiveArray { + PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { + let scrambled = index.wrapping_mul(0x9e37_79b9_7f4a_7c15); + let sign = (scrambled & 1) << 63; + let exponent = ((scrambled >> 1) % 2_046 + 1) << 52; + let mantissa = scrambled & 0x000f_ffff_ffff_fffc; + f64::from_bits(sign | exponent | mantissa) + })) +} + fn setup_nonzero_secondary_array() -> PrimitiveArray { let widened = setup_widened_f32_array(); PrimitiveArray::from_iter( @@ -1574,6 +1601,70 @@ fn bench_float_quant_proposed_default_reject_f16(bencher: Bencher) { ); } +macro_rules! float_quant_rejection_benches { + ($scheme_only:ident, $without_scheme:ident, $with_scheme:ident, $setup:ident) => { + #[divan::bench] + fn $scheme_only(bencher: Bencher) { + let compressor = BtrBlocksCompressorBuilder::empty() + .with_new_scheme(&FloatQuantScheme) + .build(); + bench_compressor(bencher, $setup(), compressor); + } + + #[divan::bench] + fn $without_scheme(bencher: Bencher) { + let compressor = BtrBlocksCompressorBuilder::default() + .exclude_schemes([FloatQuantScheme.id()]) + .build(); + bench_compressor(bencher, $setup(), compressor); + } + + #[divan::bench] + fn $with_scheme(bencher: Bencher) { + bench_compressor( + bencher, + $setup(), + BtrBlocksCompressorBuilder::default().build(), + ); + } + }; +} + +float_quant_rejection_benches!( + float_quant_scheme_reject_f16, + float_quant_default_without_scheme_reject_f16, + float_quant_default_with_scheme_reject_f16, + setup_general_f16_array +); + +float_quant_rejection_benches!( + float_quant_scheme_reject_f32, + float_quant_default_without_scheme_reject_f32, + float_quant_default_with_scheme_reject_f32, + setup_general_f32_array +); + +float_quant_rejection_benches!( + float_quant_scheme_reject_f64, + float_quant_default_without_scheme_reject_f64, + float_quant_default_with_scheme_reject_f64, + setup_general_f64_array +); + +float_quant_rejection_benches!( + float_quant_scheme_near_miss_f32, + float_quant_default_without_scheme_near_miss_f32, + float_quant_default_with_scheme_near_miss_f32, + setup_float_quant_near_miss_f32_array +); + +float_quant_rejection_benches!( + float_quant_scheme_near_miss_f64, + float_quant_default_without_scheme_near_miss_f64, + float_quant_default_with_scheme_near_miss_f64, + setup_float_quant_near_miss_f64_array +); + #[divan::bench(name = "float_quant_split_compress_f64")] fn bench_float_quant_split_compress_f64(bencher: Bencher) { let float_array = setup_widened_f32_array(); @@ -1793,14 +1884,6 @@ fn bench_float_quant_proposed_default_compress_f32(bencher: Bencher) { bench_compressor(bencher, setup_quantized_f32_array(), compressor); } -#[divan::bench(name = "float_quant_scheme_reject_f32")] -fn bench_float_quant_scheme_reject_f32(bencher: Bencher) { - let compressor = BtrBlocksCompressorBuilder::empty() - .with_new_scheme(&FloatQuantScheme) - .build(); - bench_compressor(bencher, setup_general_f32_array(), compressor); -} - #[divan::bench(name = "float_quant_prior_default_reject_f32")] fn bench_float_quant_prior_default_reject_f32(bencher: Bencher) { let compressor = BtrBlocksCompressorBuilder::default() From d2565c3167c7bcf9fbb820173b1bc52cdcfca893 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 11:32:59 +0100 Subject: [PATCH 62/78] feat: select BlockResidual for 16-bit integers Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 66 +++++++++++++++---- .../src/schemes/integer/block_residual.rs | 2 +- .../schemes/integer/scheme_selection_tests.rs | 42 +++++++++++- 3 files changed, 93 insertions(+), 17 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index d479d900a24..fe390a51bd4 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -78,7 +78,7 @@ The serialized tree uses `OrderedFloat(BlockResidual(...))` because the outer ar The FloatQuant candidate accepts native `f16`, `f32`, and `f64` inputs. -Direct integer BlockResidual supports every integer type. The default selector accepts only 32-bit and 64-bit inputs. +Direct integer BlockResidual supports every integer type. The default selector accepts 16-bit, 32-bit, and 64-bit inputs. The retained schemes win on specific structures. They do not replace ALP or ALP-RD across general float data. @@ -296,7 +296,9 @@ Its adjusted score includes a 1.02 decode-cost factor. `BlockResidualScheme` uses the same locality probe for integer arrays. -The default scheme accepts only 32-bit and 64-bit integers. Direct 8-bit and 16-bit candidates cannot save enough absolute space. +The default scheme accepts 16-bit, 32-bit, and 64-bit integers. + +The scheme still excludes direct 8-bit candidates. The scheme does not run inside trial compression for an outer scheme. Generic 64-row samples do not preserve 1,024-row locality. @@ -608,7 +610,35 @@ The synthetic `i16` BlockResidual tree uses 1,793,784 bytes. The FoR plus BitPac BlockResidual is 48.7 percent smaller on that input. -Its `i16` decode throughput is 24.4 percent lower. The default selector therefore excludes direct 8-bit and 16-bit candidates. +Its `i16` decode throughput is 24.4 percent lower, but it still exceeds 31 GB/s. + +The absolute decode rate makes this a useful Default trade. The selector now includes 16-bit integers. + +### 16-bit selector revalidation + +The complete synthetic control uses the production compressor without experimental Delta. + +| Configuration | Bytes | Encode MB/s | Decode MB/s | +| --- | ---: | ---: | ---: | +| Prior Default | 3,501,568 | 599.4 | 38,415 | +| Default with 16-bit BlockResidual | 1,793,269 | 653.8 | 31,600 | + +The selected tree reduces size by 48.8 percent and increases encode throughput by 9.1 percent. + +Decode throughput decreases by 17.7 percent, but it remains 31.6 GB/s. + +Three real files selected direct or nested 16-bit BlockResidual trees. + +| Dataset | 32-bit and 64-bit bundle | Bundle with 16-bit | Size change | Encode change | Decode change | +| --- | ---: | ---: | ---: | ---: | ---: | +| Bimbo | 10,573,121 | 9,671,397 | -8.53 percent | -1.01 percent | -2.20 percent | +| Food | 12,297,725 | 11,443,009 | -6.95 percent | -1.28 percent | -5.66 percent | +| HashTags | 21,283,222 | 21,254,341 | -0.14 percent | +1.78 percent | +8.81 percent | +| Geometric mean | — | — | -5.27 percent | -0.18 percent | +0.13 percent | + +The throughput changes come from adjacent runs and include ordinary benchmark noise. + +The exact size gains and high absolute decode rates support 16-bit Default eligibility. ### Narrow BlockResidual in Compact @@ -769,11 +799,9 @@ Signed integers and ordered floats still require a transform after residual deco The specialization removes the prior `u32` decode disadvantage. -The narrow integer exclusion remains valid. Native-width unpack does not close the `i16` gap. - -One width factor does not predict both signed and unsigned results. +Native-width unpack leaves a percentage gap for `i16`, but its absolute decode rate remains high. -Retain the current factor until the final corpus calibration uses selected-tree evidence. +One width factor does not predict both signed and unsigned results. The integer selector uses no width-specific factor. ### Serialized BlockResidual topology @@ -805,7 +833,9 @@ The single-encoding benchmark uses two million block-local values. | `i32` | 2.78 | 32.23 | 41 | | `i16` | 1.42 | 32.18 | 83 | -The direct 8-bit and 16-bit selector exclusion remains valid. +The 16-bit path now meets the revised size and absolute-throughput preference. + +The direct 8-bit selector exclusion remains active. The broad file benchmark uses three iterations across eight datasets. @@ -935,9 +965,9 @@ Food `volume_total_bytes` uses classic bins inside an ALP child. The column attribution separates numeric Compact gains from file-level string compression gains. -### Selector calibration miss on a Compact gap +### Resolved selector calibration miss on a Compact gap -The CMS ALP integer child exposes a current BlockResidual threshold miss. +The CMS ALP integer child exposed a historical BlockResidual threshold miss. The current Default tree uses 11,064,308 bytes and decodes at 24.98 GB/s. @@ -945,7 +975,15 @@ The current Default tree uses 11,064,308 bytes and decodes at 24.98 GB/s. The candidate is 3.1 percent smaller and 10.8 percent slower to decode. -The current 1.20 factor rejects it. Final threshold calibration must decide whether this trade belongs in Default. +The historical 1.20 factor rejected it. + +Later calibration removed the width factors. + +The current integer scheme uses a 1.05 raw-ratio floor and no decode-cost factor. + +The current compressor selects `ALP(BlockResidual)` at 10,716,519 bytes. + +This result resolves the threshold miss. The nonlinear patch cost still protects dense-patch decode. ### Patch-density sweep @@ -2308,7 +2346,7 @@ This round completed these steps: - Rejected the multi-reference residual design. - Added the outer-sample exclusion and width-specific factors. - Added the patch-count decode cost. -- Excluded direct 8-bit and 16-bit candidates. +- Initially excluded direct 8-bit and 16-bit candidates. - Excluded BlockResidual from dictionary-code children. - Added fused f32 OrderedFloat with BlockResidual decode and selection. - Added u32, i32, f32, and patch-density benchmarks. @@ -2370,6 +2408,8 @@ This round completed these steps: - Replaced FloatQuant sample packing with an exact buffer-size estimate. - Added isolated FloatQuant rejection controls for all float widths. - Added adjusted-ratio near-miss controls for `f32` and `f64`. +- Re-enabled 16-bit BlockResidual after absolute-throughput and corpus revalidation. +- Retained the direct 8-bit selector exclusion. - Removed the Food and HashTags FloatQuant size regressions. - Revalidated the BlockResidual and FloatQuant bundles across 16 files. - Rejected constant RangePacked samples before exact sample encoding. @@ -2402,7 +2442,7 @@ Let ordinary child arrays own patches and exception payloads. Add an outer OrderedFloat fused decode only if the generic composition misses the decode gate. -Retain the direct 8-bit and 16-bit selector exclusions until benchmark evidence changes them. +Retain the direct 8-bit selector exclusion until benchmark evidence changes it. Keep fused RangePacked and RangeEntropy outside Default. diff --git a/vortex-btrblocks/src/schemes/integer/block_residual.rs b/vortex-btrblocks/src/schemes/integer/block_residual.rs index 996e67f0951..0b88d0c0431 100644 --- a/vortex-btrblocks/src/schemes/integer/block_residual.rs +++ b/vortex-btrblocks/src/schemes/integer/block_residual.rs @@ -49,7 +49,7 @@ impl Scheme for BlockResidualScheme { } fn matches(&self, canonical: &Canonical) -> bool { - canonical.dtype().is_int() && canonical.dtype().as_ptype().bit_width() >= 32 + canonical.dtype().is_int() && canonical.dtype().as_ptype().bit_width() >= 16 } fn produced_encodings(&self) -> Vec { diff --git a/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs index 0db2877ba78..701052acc84 100644 --- a/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs @@ -111,21 +111,57 @@ fn test_block_residual_ignores_null_payloads() -> VortexResult<()> { } #[test] -fn test_block_residual_skips_narrow_integers() -> VortexResult<()> { - let values = (0..8_192) +fn test_block_residual_compresses_16_bit_integers() -> VortexResult<()> { + let signed_values = (0..8_192) .map(|index| { let block = index / 1_024; let residual = (index * 2_654_435_761_usize) % 32; Ok(i16::try_from(block * 1_000 + residual)?) }) .collect::>>()?; + let unsigned_values = signed_values + .iter() + .copied() + .map(u16::try_from) + .collect::, _>>()?; + #[cfg(not(feature = "unstable_encodings"))] + let compressor = BtrBlocksCompressor::default(); + #[cfg(feature = "unstable_encodings")] + let compressor = BtrBlocksCompressorBuilder::default() + .exclude_schemes([DeltaScheme::default().id()]) + .build(); + + for array in [ + PrimitiveArray::from_iter(signed_values), + PrimitiveArray::from_iter(unsigned_values), + ] { + let compressed = + compressor.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; + assert!( + contains_block_residual(&compressed), + "BlockResidual must encode this 16-bit input:\n{}", + compressed.display_tree() + ); + } + Ok(()) +} + +#[test] +fn test_block_residual_skips_8_bit_integers() -> VortexResult<()> { + let values = (0..8_192) + .map(|index| { + let block = index / 1_024; + let residual = (index * 2_654_435_761_usize) % 8; + i8::try_from(block * 16 + residual) + }) + .collect::, _>>()?; let array = PrimitiveArray::from_iter(values); let compressed = BtrBlocksCompressor::default() .compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; assert!( !contains_block_residual(&compressed), - "BlockResidual must not encode narrow integers:\n{}", + "BlockResidual must not encode 8-bit integers:\n{}", compressed.display_tree() ); Ok(()) From a34240ec3866141f4c21b7f19744837a52f9b68a Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 11:40:52 +0100 Subject: [PATCH 63/78] feat: evaluate 8-bit BlockResidual selection Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 44 ++++++++++++++----- .../examples/float_compressor_bench.rs | 18 ++++++++ .../src/schemes/integer/block_residual.rs | 2 +- .../schemes/integer/scheme_selection_tests.rs | 35 ++++++++++----- 4 files changed, 77 insertions(+), 22 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index fe390a51bd4..8ae207c0701 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -78,7 +78,7 @@ The serialized tree uses `OrderedFloat(BlockResidual(...))` because the outer ar The FloatQuant candidate accepts native `f16`, `f32`, and `f64` inputs. -Direct integer BlockResidual supports every integer type. The default selector accepts 16-bit, 32-bit, and 64-bit inputs. +Direct integer BlockResidual supports every integer type. The Default selector accepts every integer width during the current 8-bit trial. The retained schemes win on specific structures. They do not replace ALP or ALP-RD across general float data. @@ -86,7 +86,7 @@ FloatQuant now passes the speed gates for zero-secondary and one-bit-secondary i The complete file corpus does not yet justify FloatQuant in Default. -BlockResidual now passes the direct speed gates for 32-bit and 64-bit integers. +BlockResidual now passes the direct speed gates for every integer width. The GloVe result identifies a separate entropy gap inside the ALP integer child. @@ -121,7 +121,7 @@ The Default selector can exclude a supported type when measured costs do not jus | Array | Supported logical types | Default policy | | --- | --- | --- | | OrderedFloat | `f16`, `f32`, `f64` | All float types remain eligible. | -| BlockResidual | `u8`, `u16`, `u32`, `u64`, `i8`, `i16`, `i32`, `i64` | Direct selection uses 32-bit and 64-bit integers. | +| BlockResidual | `u8`, `u16`, `u32`, `u64`, `i8`, `i16`, `i32`, `i64` | All widths are eligible during the 8-bit trial. | | FloatQuant | `f16`, `f32`, `f64` | All float types remain eligible. | | IntMult | `u8`, `u16`, `u32`, `u64`, `i8`, `i16`, `i32`, `i64` | No Default scheme exists yet. | @@ -296,9 +296,7 @@ Its adjusted score includes a 1.02 decode-cost factor. `BlockResidualScheme` uses the same locality probe for integer arrays. -The default scheme accepts 16-bit, 32-bit, and 64-bit integers. - -The scheme still excludes direct 8-bit candidates. +The Default scheme accepts every integer width during the current 8-bit trial. The scheme does not run inside trial compression for an outer scheme. Generic 64-row samples do not preserve 1,024-row locality. @@ -640,6 +638,29 @@ The throughput changes come from adjacent runs and include ordinary benchmark no The exact size gains and high absolute decode rates support 16-bit Default eligibility. +### 8-bit selector trial + +The direct benchmark already showed 30.53 GB/s decode and 44 ns scalar access for `i8`. + +The complete synthetic control uses two million block-local `i8` values. + +| Configuration | Bytes | Encode MB/s | +| --- | ---: | ---: | +| Prior Default with Primitive | 2,000,000 | 820.6 | +| Default with BlockResidual | 1,043,448 | 316.3 | + +BlockResidual reduces size by 47.8 percent and decodes at 29.76 GB/s. + +The selected tree has a material encode cost because the prior Default stores the input without compression. + +A uniform `i8` control rejects BlockResidual and retains Primitive at 2,000,000 bytes. + +The rejected analysis changes encode throughput from 720.5 MB/s to 706.3 MB/s, a 2.0 percent decrease. + +The eight real numeric files contain no `i8` columns. Their sizes do not change during this trial. + +The current branch enables 8-bit selection for evaluation. Final adoption needs real `i8` coverage and complete threshold calibration. + ### Narrow BlockResidual in Compact The Compact comparison uses the same two million block-local `i16` values. @@ -832,10 +853,11 @@ The single-encoding benchmark uses two million block-local values. | `u32` | 2.79 | 46.69 | 38 | | `i32` | 2.78 | 32.23 | 41 | | `i16` | 1.42 | 32.18 | 83 | +| `i8` | 0.56 | 30.53 | 44 | The 16-bit path now meets the revised size and absolute-throughput preference. -The direct 8-bit selector exclusion remains active. +The 8-bit path also clears the decode and scalar-access gates. The broad file benchmark uses three iterations across eight datasets. @@ -1250,9 +1272,9 @@ This comparison separates float transform quality from integer child compression The CMS candidate is 5.5 percent smaller and 42.8 percent faster to decode. -The current selector rejects it because the 64-bit BlockResidual score includes a 1.20 factor. +The selector at the time rejected it because the 64-bit BlockResidual score included a 1.20 factor. -This result makes the CMS miss a final selector calibration issue. +The current selector removed that factor. The resolved calibration section records the selected 10,716,519-byte tree. The GloVe candidate is 0.4 percent larger than the current default. @@ -2409,7 +2431,7 @@ This round completed these steps: - Added isolated FloatQuant rejection controls for all float widths. - Added adjusted-ratio near-miss controls for `f32` and `f64`. - Re-enabled 16-bit BlockResidual after absolute-throughput and corpus revalidation. -- Retained the direct 8-bit selector exclusion. +- Enabled an 8-bit BlockResidual selector trial with selected and rejected synthetic controls. - Removed the Food and HashTags FloatQuant size regressions. - Revalidated the BlockResidual and FloatQuant bundles across 16 files. - Rejected constant RangePacked samples before exact sample encoding. @@ -2442,7 +2464,7 @@ Let ordinary child arrays own patches and exception payloads. Add an outer OrderedFloat fused decode only if the generic composition misses the decode gate. -Retain the direct 8-bit selector exclusion until benchmark evidence changes it. +Add real `i8` and `u8` columns to the corpus. Use them to decide 8-bit eligibility and threshold calibration. Keep fused RangePacked and RangeEntropy outside Default. diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index c737f4246ab..bca006f53f7 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -233,6 +233,16 @@ fn synthetic_datasets(row_count: usize) -> VortexResult let residual = index.wrapping_mul(2_654_435_761) % 128; i16::try_from(block * 128 + residual).unwrap_or(i16::MAX) })); + let block_local_i8 = PrimitiveArray::from_iter((0..row_count).map(|index| { + let block = (index / 1_024) % 16; + let residual = index.wrapping_mul(2_654_435_761) % 16; + let unsigned = i16::try_from(block * 16 + residual).unwrap_or(i16::MAX); + i8::try_from(unsigned - 128).unwrap_or(i8::MAX) + })); + let uniform_i8 = PrimitiveArray::from_iter((0..row_count).map(|index| { + let value = u8::try_from(index.wrapping_mul(2_654_435_761) % 256).unwrap_or(u8::MAX); + i8::from_le_bytes([value]) + })); let sparse_block_local = PrimitiveArray::from_iter((0..row_count).map(|index| { if index % 16 == 0 { let value_index = index / 16; @@ -351,6 +361,14 @@ fn synthetic_datasets(row_count: usize) -> VortexResult "synthetic-block-local-i16".to_string(), vec![column("value", block_local_i16)], ), + ( + "synthetic-block-local-i8".to_string(), + vec![column("value", block_local_i8)], + ), + ( + "synthetic-uniform-i8".to_string(), + vec![column("value", uniform_i8)], + ), ( "synthetic-sparse-block-local".to_string(), vec![column("value", sparse_block_local)], diff --git a/vortex-btrblocks/src/schemes/integer/block_residual.rs b/vortex-btrblocks/src/schemes/integer/block_residual.rs index 0b88d0c0431..c7e648e1153 100644 --- a/vortex-btrblocks/src/schemes/integer/block_residual.rs +++ b/vortex-btrblocks/src/schemes/integer/block_residual.rs @@ -49,7 +49,7 @@ impl Scheme for BlockResidualScheme { } fn matches(&self, canonical: &Canonical) -> bool { - canonical.dtype().is_int() && canonical.dtype().as_ptype().bit_width() >= 16 + canonical.dtype().is_int() } fn produced_encodings(&self) -> Vec { diff --git a/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs index 701052acc84..fd593041721 100644 --- a/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs @@ -147,23 +147,38 @@ fn test_block_residual_compresses_16_bit_integers() -> VortexResult<()> { } #[test] -fn test_block_residual_skips_8_bit_integers() -> VortexResult<()> { - let values = (0..8_192) +fn test_block_residual_compresses_8_bit_integers() -> VortexResult<()> { + let signed_values = (0..8_192) .map(|index| { let block = index / 1_024; let residual = (index * 2_654_435_761_usize) % 8; i8::try_from(block * 16 + residual) }) .collect::, _>>()?; - let array = PrimitiveArray::from_iter(values); - let compressed = BtrBlocksCompressor::default() - .compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; + let unsigned_values = signed_values + .iter() + .copied() + .map(u8::try_from) + .collect::, _>>()?; + #[cfg(not(feature = "unstable_encodings"))] + let compressor = BtrBlocksCompressor::default(); + #[cfg(feature = "unstable_encodings")] + let compressor = BtrBlocksCompressorBuilder::default() + .exclude_schemes([DeltaScheme::default().id()]) + .build(); - assert!( - !contains_block_residual(&compressed), - "BlockResidual must not encode 8-bit integers:\n{}", - compressed.display_tree() - ); + for array in [ + PrimitiveArray::from_iter(signed_values), + PrimitiveArray::from_iter(unsigned_values), + ] { + let compressed = + compressor.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; + assert!( + contains_block_residual(&compressed), + "BlockResidual must encode this 8-bit input:\n{}", + compressed.display_tree() + ); + } Ok(()) } From 28004e23587cb31a9ee71290c6019e3335d866ab Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 12:36:40 +0100 Subject: [PATCH 64/78] bench: complete numeric bundle evaluation Signed-off-by: Will Manning --- Cargo.lock | 3 + benchmarks/compress-bench/src/main.rs | 2 +- benchmarks/compress-bench/src/vortex.rs | 46 +---- benchmarks/random-access-bench/Cargo.toml | 2 + benchmarks/random-access-bench/README.md | 9 + benchmarks/random-access-bench/src/main.rs | 42 +++++ docs/plans/native-pcodec.md | 176 +++++++++++++++--- vortex-bench/Cargo.toml | 1 + vortex-bench/src/conversions.rs | 13 +- vortex-bench/src/lib.rs | 56 ++++++ .../src/schemes/float/float_quant.rs | 2 +- 11 files changed, 279 insertions(+), 73 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b3b44456e9f..802a68a9122 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7292,6 +7292,7 @@ name = "random-access-bench" version = "0.1.0" dependencies = [ "anyhow", + "async-trait", "clap", "indicatif", "lance-bench", @@ -7299,6 +7300,7 @@ dependencies = [ "rand_distr 0.6.0", "tabled", "tokio", + "vortex", "vortex-bench", ] @@ -9738,6 +9740,7 @@ dependencies = [ "uuid", "vortex", "vortex-arrow", + "vortex-btrblocks", "vortex-spatial", "vortex-tensor", "wkb", diff --git a/benchmarks/compress-bench/src/main.rs b/benchmarks/compress-bench/src/main.rs index ed53474366b..0a79238fc35 100644 --- a/benchmarks/compress-bench/src/main.rs +++ b/benchmarks/compress-bench/src/main.rs @@ -11,7 +11,6 @@ use compress_bench::LanceCompressor; use compress_bench::gpu_vortex::GpuVortexCompressor; use compress_bench::parquet::ParquetCompressor; use compress_bench::vortex::VortexCompressor; -use compress_bench::vortex::VortexNumericBundle; use indicatif::ProgressBar; use itertools::Itertools; use regex::Regex; @@ -20,6 +19,7 @@ use vortex_bench::Engine; use vortex_bench::Format; use vortex_bench::LogFormat; use vortex_bench::Target; +use vortex_bench::VortexNumericBundle; use vortex_bench::compress::CompressMeasurements; use vortex_bench::compress::CompressOp; use vortex_bench::compress::Compressor; diff --git a/benchmarks/compress-bench/src/vortex.rs b/benchmarks/compress-bench/src/vortex.rs index 0ed6cf3b4ad..545284ef176 100644 --- a/benchmarks/compress-bench/src/vortex.rs +++ b/benchmarks/compress-bench/src/vortex.rs @@ -10,7 +10,6 @@ use std::time::Instant; use anyhow::Result; use async_trait::async_trait; use bytes::Bytes; -use clap::ValueEnum; use futures::StreamExt; use futures::pin_mut; use vortex::array::IntoArray; @@ -20,36 +19,14 @@ use vortex::expr::select; use vortex::file::OpenOptionsSessionExt; use vortex::file::VortexWriteOptions; use vortex::file::WriteOptionsSessionExt; -use vortex::file::WriteStrategyBuilder; use vortex_arrow::ArrowSessionExt; use vortex_bench::CompactionStrategy; use vortex_bench::Format; use vortex_bench::SESSION; +use vortex_bench::VortexNumericBundle; use vortex_bench::compress::Compressor; use vortex_bench::compress::read_projection; use vortex_bench::conversions::parquet_to_vortex_chunks; -use vortex_btrblocks::BtrBlocksCompressorBuilder; -use vortex_btrblocks::SchemeExt; -use vortex_btrblocks::schemes::float::FloatQuantScheme; -use vortex_btrblocks::schemes::float::OrderedBlockResidualScheme; -use vortex_btrblocks::schemes::float::OrderedFloatRangePackedScheme; -use vortex_btrblocks::schemes::integer::BlockResidualScheme; - -const RANGE_PACKED_SCHEME: OrderedFloatRangePackedScheme = OrderedFloatRangePackedScheme::new(1.20); - -/// Numeric scheme bundle for the Vortex compressor benchmark. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, ValueEnum)] -pub enum VortexNumericBundle { - /// Exclude the new numeric schemes. - PriorDefault, - /// Add the integer and ordered-float BlockResidual schemes. - BlockResidual, - /// Use the current Default compressor. - #[default] - CurrentDefault, - /// Add the experimental OrderedFloat with RangePacked scheme. - RangePacked, -} /// Compressor implementation for Vortex format. pub struct VortexCompressor { @@ -74,26 +51,7 @@ impl VortexCompressor { if self.format == Format::VortexCompact { return CompactionStrategy::Compact.apply_options(options); } - let compressor = match self.numeric_bundle { - VortexNumericBundle::PriorDefault => BtrBlocksCompressorBuilder::default() - .exclude_schemes([ - FloatQuantScheme.id(), - OrderedBlockResidualScheme.id(), - BlockResidualScheme.id(), - ]), - VortexNumericBundle::BlockResidual => { - BtrBlocksCompressorBuilder::default().exclude_schemes([FloatQuantScheme.id()]) - } - VortexNumericBundle::CurrentDefault => BtrBlocksCompressorBuilder::default(), - VortexNumericBundle::RangePacked => { - BtrBlocksCompressorBuilder::default().with_new_scheme(&RANGE_PACKED_SCHEME) - } - }; - options.with_strategy( - WriteStrategyBuilder::default() - .with_btrblocks_builder(compressor) - .build(), - ) + self.numeric_bundle.apply_options(options) } } diff --git a/benchmarks/random-access-bench/Cargo.toml b/benchmarks/random-access-bench/Cargo.toml index c45c1ac2bff..37633685d0a 100644 --- a/benchmarks/random-access-bench/Cargo.toml +++ b/benchmarks/random-access-bench/Cargo.toml @@ -16,6 +16,7 @@ publish = false [dependencies] anyhow = { workspace = true } +async-trait = { workspace = true } clap = { workspace = true, features = ["derive"] } indicatif = { workspace = true } lance-bench = { path = "../lance-bench", optional = true } @@ -23,6 +24,7 @@ rand = { workspace = true } rand_distr = { workspace = true } tabled = { workspace = true } tokio = { workspace = true, features = ["full"] } +vortex = { workspace = true } vortex-bench = { workspace = true } [features] diff --git a/benchmarks/random-access-bench/README.md b/benchmarks/random-access-bench/README.md index 14949c85fcb..23f00abdb7d 100644 --- a/benchmarks/random-access-bench/README.md +++ b/benchmarks/random-access-bench/README.md @@ -20,3 +20,12 @@ reopening the file per lookup. CI drives the full matrix via ```bash cargo run -p random-access-bench --profile release_debug --features lance ``` + +Compare random access for a numeric scheme bundle with one of these values: + +- `--vortex-numeric-bundle prior-default` +- `--vortex-numeric-bundle block-residual` +- `--vortex-numeric-bundle current-default` +- `--vortex-numeric-bundle range-packed` + +Each bundle uses a separate Vortex file. Existing files remain available for repeated runs. diff --git a/benchmarks/random-access-bench/src/main.rs b/benchmarks/random-access-bench/src/main.rs index 11ef81edd3c..b6d1e9f55c8 100644 --- a/benchmarks/random-access-bench/src/main.rs +++ b/benchmarks/random-access-bench/src/main.rs @@ -4,12 +4,17 @@ use std::path::PathBuf; use anyhow::Result; +use async_trait::async_trait; use clap::Parser; use clap::ValueEnum; use random_access_bench::AccessPattern; use random_access_bench::OpenMode; use random_access_bench::RunConfig; +use vortex::file::WriteOptionsSessionExt; use vortex_bench::Format; +use vortex_bench::SESSION; +use vortex_bench::VortexNumericBundle; +use vortex_bench::conversions::write_parquet_as_vortex_with_options; use vortex_bench::datasets::feature_vectors::FeatureVectorsData; use vortex_bench::datasets::nested_lists::NestedListsData; use vortex_bench::datasets::nested_structs::NestedStructsData; @@ -42,6 +47,34 @@ impl DatasetArg { } } +struct NumericBundleDataset { + dataset: Box, + bundle: VortexNumericBundle, +} + +#[async_trait] +impl BenchDataset for NumericBundleDataset { + fn name(&self) -> &str { + self.dataset.name() + } + + fn row_count(&self) -> u64 { + self.dataset.row_count() + } + + async fn path(&self, format: Format) -> Result { + if format != Format::OnDiskVortex { + return self.dataset.path(format).await; + } + + let parquet_path = self.dataset.path(Format::Parquet).await?; + let name = self.dataset.name(); + let path = format!("random_access/{name}/{name}-{}.vortex", self.bundle.name()); + let options = self.bundle.apply_options(SESSION.write_options()); + write_parquet_as_vortex_with_options(parquet_path, &path, options).await + } +} + #[derive(Parser, Debug)] #[command(version, about, long_about = None)] struct Args { @@ -85,6 +118,9 @@ struct Args { /// Whether to reopen the file on each iteration, use a cached handle, or run both. #[arg(long, value_enum, default_value_t = OpenMode::Both)] open_mode: OpenMode, + /// Select the numeric scheme bundle for Vortex files. + #[arg(long, value_enum, default_value_t)] + vortex_numeric_bundle: VortexNumericBundle, } #[tokio::main] @@ -97,6 +133,12 @@ async fn main() -> Result<()> { .datasets .into_iter() .map(DatasetArg::into_dataset) + .map(|dataset| { + Box::new(NumericBundleDataset { + dataset, + bundle: args.vortex_numeric_bundle, + }) as Box + }) .collect(), formats: args.formats, patterns: args.patterns, diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 8ae207c0701..720459bb8e8 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -2093,7 +2093,7 @@ Food selected FloatQuant from a 1.94 sample ratio. Its full ratio was 1.73. ALP had a 1.80 sample ratio and a 2.51 full ratio on the same chunk. -A provisional 1.10 FloatQuant factor retained every focused FloatQuant selection test and rejected this Food choice. +A 1.10 FloatQuant factor retained every focused FloatQuant selection test and rejected this Food choice. The corrected FloatQuant bundle produced this change against the BlockResidual bundle: @@ -2141,25 +2141,150 @@ This corpus does not expose a FloatQuant win under the current factor. The complete synthetic `f32` and `f64` trees produce strict size and throughput wins. -Retain FloatQuant as an opportunistic Default candidate while the final corpus pass measures its analysis cost. +The complete corpus shows no selected FloatQuant tree and no stable analysis-cost signal. + +Retain FloatQuant as an opportunistic option under the calibrated 1.10 factor. + +### Final selector factors + +The integer BlockResidual scheme keeps a 1.05 raw-ratio floor and no width-specific factor. + +This policy resolves the CMS threshold miss and produces a 2.94 percent complete-corpus size gain. + +The Ordered BlockResidual scheme keeps its 1.05 raw floor and 1.02 decode factor. + +The nonlinear patch cost rejects the known dense-patch and slow HashTags trees. + +No selected-tree evidence supports a higher BlockResidual factor. + +The FloatQuant scheme keeps its 1.10 factor. + +The Food sample mismatch requires a factor above 1.078 to retain its smaller ALP tree. + +The 1.10 value adds a small margin and retains every strict synthetic `f32` and `f64` win. + +The 33-column Pco-gap pass exposes no missed real FloatQuant candidate. + +The 8-bit BlockResidual trial remains outside this conclusion because the corpus lacks real 8-bit columns. ### Default bundle options +The final numeric pass uses complete files and three iterations per operation. + +It covers Taxi, GloVe, Arade, Bimbo, CMSprovider, Euro2016, Food, and HashTags. + +The BlockResidual bundle produces these changes against Prior Default: + +| Dataset | Size | Write throughput | Read throughput | +| --- | ---: | ---: | ---: | +| Taxi | -6.35 percent | -0.81 percent | +7.23 percent | +| GloVe | 0.00 percent | -11.09 percent | +0.15 percent | +| Arade | -2.39 percent | -1.71 percent | +5.70 percent | +| Bimbo | -8.90 percent | -1.56 percent | -5.49 percent | +| CMSprovider | -7.94 percent | -1.40 percent | -8.51 percent | +| Euro2016 | -2.71 percent | +8.26 percent | -1.98 percent | +| Food | -7.79 percent | +1.95 percent | +6.08 percent | +| HashTags | -3.22 percent | +10.67 percent | -7.45 percent | +| Geometric mean | -4.96 percent | +0.34 percent | -0.71 percent | + +Total bytes decrease by 6.18 percent across the eight files. + +Total write time increases by 0.49 percent. Total read time increases by 3.39 percent. + +The GloVe write result conflicts with adjacent runs that reverse the difference. + +GloVe retains the same tree and size. Treat its write result as benchmark noise. + +The remaining eight files contain two TPC-H comment layouts and six nested wide-table cases. + +Both TPC-H files become 3.45 percent smaller through BlockResidual descendants. + +All six wide-table files retain identical sizes. + +Across all 16 files, the BlockResidual bundle produces these changes: + +| Metric | Geometric-mean change | Aggregate change | +| --- | ---: | ---: | +| Size | -2.94 percent | -5.30 percent | +| Write throughput | -1.22 percent | -0.60 percent | +| Read throughput | -1.21 percent | -2.62 percent | + The evidence supports three bundle options: -| Option | Schemes | Comparison | Size | Write throughput | Read throughput | -| --- | --- | --- | ---: | ---: | ---: | -| Conservative | BlockResidual | Prior Default | -1.6 percent | -0.9 percent | +1.3 percent | -| Opportunistic | BlockResidual and FloatQuant | Conservative | 0.0 percent | -0.9 percent | Noise | -| Size-seeking experiment | Opportunistic plus fixed bins | Opportunistic | -0.027 percent | -0.3 percent | Noise | +| Option | Schemes | Scope | Comparison | Size | Write throughput | Read throughput | +| --- | --- | --- | --- | ---: | ---: | ---: | +| Conservative | BlockResidual | 16 files | Prior Default | -2.94 percent | -1.22 percent | -1.21 percent | +| Opportunistic | BlockResidual and FloatQuant | 16 files | Conservative | 0.00 percent | Noise | Noise | +| Fixed-bin experiment | Opportunistic plus RangePacked | 8 numeric files | Opportunistic | -0.054 percent | -1.03 percent | Noise | -The conservative option avoids FloatQuant analysis when the workload lacks a matching distribution. +The conservative option pushes the size frontier without a material geometric-mean throughput loss. -The opportunistic option accepts that analysis cost for large wins on matching `f32` and `f64` data. +The opportunistic option selects no FloatQuant tree in these eight files. -Its selected synthetic trees improve size, write throughput, and read throughput. +Focused synthetic `f32` and `f64` inputs still produce strict FloatQuant size and throughput wins. -The fixed-bin option lacks enough corpus benefit for Default. +The fixed-bin option changes only HashTags `interaction#received_at`. + +Its numeric subset is 5.99 percent smaller, but the selected tree has material access costs. + +The tree decodes at about 10.1 GB/s instead of 15.6 to 16.1 GB/s. + +Scalar access takes 312 to 323 ns instead of 150 to 158 ns. + +RangePacked therefore remains outside Default. + +### Default bundle random access + +The random-access benchmark now writes one Vortex file for each numeric bundle. + +The cached-handle pass uses correlated and uniform patterns with about 100 requested rows. + +| Dataset and pattern | Prior Default latency | Current Default latency | Change | +| --- | ---: | ---: | ---: | +| Taxi correlated | 0.609 ms | 0.624 ms | +2.45 percent | +| Taxi uniform | 2.760 ms | 2.807 ms | +1.68 percent | +| Nested lists correlated | 0.154 ms | 0.148 ms | -3.95 percent | +| Nested lists uniform | 0.922 ms | 0.922 ms | -0.06 percent | +| Nested structs correlated | 0.216 ms | 0.220 ms | +1.94 percent | +| Nested structs uniform | 0.532 ms | 0.544 ms | +2.20 percent | +| Geometric mean | — | — | +0.69 percent | + +The six standard patterns remain within four percent of Prior Default. + +Taxi file size decreases by 6.35 percent. Nested-list file size decreases by 0.71 percent. + +The nested-struct files have identical sizes. + +The legacy six-index Taxi case improves from 1.587 ms to 1.325 ms. + +This small target is more sensitive to run noise than the 100-row patterns. + +BlockResidual, Current Default, and RangePacked remain within 2.5 percent on both Taxi 100-row patterns. + +The random-access suite does not include the Public BI files. + +Selected-tree scalar benchmarks remain the direct evidence for Bimbo, CMSprovider, Euro2016, Food, and HashTags. + +Add Public BI file access only if bundle adoption needs broader late-materialization evidence. + +### Compact and Parquet size constraint + +The format pass uses three iterations per operation. + +| Scope and comparison for Current Default | Size | Write time | Read time | +| --- | ---: | ---: | ---: | +| 8 numeric files versus Compact | +38.66 percent | -13.42 percent | -66.63 percent | +| 8 numeric files versus Parquet with Zstd | -1.11 percent | -65.37 percent | -96.08 percent | +| Complete 16 files versus Compact | +21.85 percent | -11.69 percent | -55.35 percent | +| Complete 16 files versus Parquet with Zstd | -1.67 percent | -34.80 percent | -85.82 percent | + +Current Default remains 1.67 percent smaller than Parquet with Zstd across the complete corpus. + +It writes 1.5 times faster and reads 7.1 times faster than Parquet with Zstd. + +Compact remains 17.9 percent smaller than Current Default across the complete corpus. + +Current Default writes 13.2 percent faster and reads 2.2 times faster than Compact. Quick rejection reduces analysis cost and strengthens a candidate. @@ -2426,7 +2551,7 @@ This round completed these steps: - Added explicit winner result events to the compressor trace. - Added optional chunk-level encoding trees to the focused compressor benchmark. - Rejected constant FloatQuant samples before exact sample encoding. -- Added a provisional 1.10 FloatQuant selection factor. +- Added and calibrated a 1.10 FloatQuant selection factor. - Replaced FloatQuant sample packing with an exact buffer-size estimate. - Added isolated FloatQuant rejection controls for all float widths. - Added adjusted-ratio near-miss controls for `f32` and `f64`. @@ -2439,6 +2564,10 @@ This round completed these steps: - Compared retained fixed-bin candidates against the completed incumbent sample cascade. - Removed every known Bimbo, CMSprovider, Euro2016, and Taxi fixed-bin false win. - Tested FloatQuant on 33 real float columns from five Pco-gap inputs. +- Revalidated the final bundles across eight numeric files and all 16 corpus files. +- Revalidated Current Default against Compact and Parquet with Zstd across all 16 files. +- Added numeric bundle controls to the random-access benchmark. +- Compared cached random access across Taxi and two nested datasets. The Pco mode profile and the quotient and remainder experiments are complete. @@ -2452,11 +2581,12 @@ The specialized OrderedFloat and ALP trees now use registered fused parent kerne Complete these next experiments in order: -1. Compare the conservative and opportunistic Default bundles on the final corpus. -2. Calibrate the FloatQuant factor from complete corpus and selected-tree evidence. -3. Prefer cheap rejection tests when they preserve the best candidate model. -4. Treat fast rejection as a selection advantage, not an admission criterion. -5. Require stronger corpus evidence when a useful scheme needs costly analysis. +1. Add real `i8` and `u8` columns to the corpus. +2. Calibrate 8-bit BlockResidual eligibility with those real columns. +3. Decide whether Public BI file random access adds useful evidence beyond scalar benchmarks. +4. Prefer cheap rejection tests when they preserve the best candidate model. +5. Treat fast rejection as a selection advantage, not an admission criterion. +6. Require stronger corpus evidence when a useful scheme needs costly analysis. Use packed bin codes and primitive bin starts unless evidence supports more complexity. @@ -2464,19 +2594,13 @@ Let ordinary child arrays own patches and exception payloads. Add an outer OrderedFloat fused decode only if the generic composition misses the decode gate. -Add real `i8` and `u8` columns to the corpus. Use them to decide 8-bit eligibility and threshold calibration. - Keep fused RangePacked and RangeEntropy outside Default. After the candidate set stabilizes, complete these final steps: -1. Run the complete compression corpus. -2. Compare the final geometric mean against Compact and Parquet with Zstd. -3. Calibrate all selector thresholds from complete corpus and selected-tree evidence. -4. Revisit the BlockResidual factors with selected-tree evidence. -5. Add a true ALP-RD child composition case if a real selected tree exposes one. -6. Add more real embedding datasets when licenses and loaders permit them. -7. Update this plan after each experiment. +1. Add a true ALP-RD child composition case if a real selected tree exposes one. +2. Add more real embedding datasets when licenses and loaders permit them. +3. Update this plan after each experiment. ## Pull request structure diff --git a/vortex-bench/Cargo.toml b/vortex-bench/Cargo.toml index 965a80c6259..4569cdf765f 100644 --- a/vortex-bench/Cargo.toml +++ b/vortex-bench/Cargo.toml @@ -28,6 +28,7 @@ vortex = { workspace = true, features = [ "zstd", ] } vortex-arrow = { workspace = true } +vortex-btrblocks = { workspace = true } vortex-spatial = { workspace = true } vortex-tensor = { workspace = true } # TODO(connor): In the future, this might be inside vortex. diff --git a/vortex-bench/src/conversions.rs b/vortex-bench/src/conversions.rs index 69cba42c6b0..ae961a07eb3 100644 --- a/vortex-bench/src/conversions.rs +++ b/vortex-bench/src/conversions.rs @@ -291,11 +291,22 @@ pub async fn write_parquet_as_vortex( parquet_path: PathBuf, vortex_path: &str, compaction: CompactionStrategy, +) -> anyhow::Result { + let write_options = compaction.apply_options(SESSION.write_options()); + write_parquet_as_vortex_with_options(parquet_path, vortex_path, write_options).await +} + +/// Convert a Parquet file to Vortex with explicit write options. +/// +/// The function skips conversion when the output file already exists. +pub async fn write_parquet_as_vortex_with_options( + parquet_path: PathBuf, + vortex_path: &str, + write_options: VortexWriteOptions, ) -> anyhow::Result { idempotent_async(vortex_path, |output_fname| async move { let mut output_file = File::create(&output_fname).await?; let data = parquet_to_vortex_chunks(parquet_path).await?; - let write_options = compaction.apply_options(SESSION.write_options()); write_options .write(&mut output_file, data.into_array().to_array_stream()) .await?; diff --git a/vortex-bench/src/lib.rs b/vortex-bench/src/lib.rs index 286d9d41bcc..49c3a8d60fa 100644 --- a/vortex-bench/src/lib.rs +++ b/vortex-bench/src/lib.rs @@ -34,6 +34,11 @@ use vortex::error::vortex_err; use vortex::file::VortexWriteOptions; use vortex::file::WriteStrategyBuilder; use vortex::utils::aliases::hash_map::HashMap; +use vortex_btrblocks::SchemeExt; +use vortex_btrblocks::schemes::float::FloatQuantScheme; +use vortex_btrblocks::schemes::float::OrderedBlockResidualScheme; +use vortex_btrblocks::schemes::float::OrderedFloatRangePackedScheme; +use vortex_btrblocks::schemes::integer::BlockResidualScheme; use crate::spatialbench::SpatialBenchBenchmark; use crate::vortex_queries::VortexBenchmark; @@ -239,6 +244,57 @@ pub enum CompactionStrategy { Default, } +const RANGE_PACKED_SCHEME: OrderedFloatRangePackedScheme = OrderedFloatRangePackedScheme::new(1.20); + +/// Numeric scheme bundles for compression benchmarks. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, ValueEnum)] +pub enum VortexNumericBundle { + /// Exclude the new numeric schemes. + PriorDefault, + /// Add the integer and ordered-float BlockResidual schemes. + BlockResidual, + /// Use the current Default compressor. + #[default] + CurrentDefault, + /// Add the experimental OrderedFloat with RangePacked scheme. + RangePacked, +} + +impl VortexNumericBundle { + /// Return the stable CLI and file-name component for this bundle. + pub fn name(self) -> &'static str { + match self { + Self::PriorDefault => "prior-default", + Self::BlockResidual => "block-residual", + Self::CurrentDefault => "current-default", + Self::RangePacked => "range-packed", + } + } + + /// Apply this numeric bundle to Vortex write options. + pub fn apply_options(self, options: VortexWriteOptions) -> VortexWriteOptions { + let compressor = match self { + Self::PriorDefault => BtrBlocksCompressorBuilder::default().exclude_schemes([ + FloatQuantScheme.id(), + OrderedBlockResidualScheme.id(), + BlockResidualScheme.id(), + ]), + Self::BlockResidual => { + BtrBlocksCompressorBuilder::default().exclude_schemes([FloatQuantScheme.id()]) + } + Self::CurrentDefault => BtrBlocksCompressorBuilder::default(), + Self::RangePacked => { + BtrBlocksCompressorBuilder::default().with_new_scheme(&RANGE_PACKED_SCHEME) + } + }; + options.with_strategy( + WriteStrategyBuilder::default() + .with_btrblocks_builder(compressor) + .build(), + ) + } +} + impl CompactionStrategy { pub fn apply_options(&self, options: VortexWriteOptions) -> VortexWriteOptions { match self { diff --git a/vortex-btrblocks/src/schemes/float/float_quant.rs b/vortex-btrblocks/src/schemes/float/float_quant.rs index 292b21f1eed..9c3befb72e3 100644 --- a/vortex-btrblocks/src/schemes/float/float_quant.rs +++ b/vortex-btrblocks/src/schemes/float/float_quant.rs @@ -37,7 +37,7 @@ use crate::Scheme; use crate::normalize_null_values; use crate::schemes::sample_primitive_one_percent; -// Prefer incumbents on close fits. FloatQuant sample estimates can overstate full-array gains. +// Food needs a factor above 1.078 to prevent its sample from displacing a smaller ALP tree. const SELECTION_COST_FACTOR: f64 = 1.10; /// FloatQuant split with a fixed frame-of-reference primary child. From 446c3238e07ff9d2d2b174d57e1be61580e96573 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 13:06:16 +0100 Subject: [PATCH 65/78] bench: validate 8-bit BlockResidual selection Signed-off-by: Will Manning --- benchmarks/compress-bench/README.md | 7 ++ benchmarks/compress-bench/src/main.rs | 83 +++++++++++++++---- docs/plans/native-pcodec.md | 75 +++++++++++++---- .../examples/float_compressor_bench.rs | 72 +++++++++++++--- .../schemes/integer/scheme_selection_tests.rs | 46 ++++++++-- vortex-btrblocks/src/trace_tests.rs | 5 ++ .../golden__unstable__list_of_int_runs.snap | 12 +-- 7 files changed, 242 insertions(+), 58 deletions(-) diff --git a/benchmarks/compress-bench/README.md b/benchmarks/compress-bench/README.md index 2274cff7d4a..31a54533a13 100644 --- a/benchmarks/compress-bench/README.md +++ b/benchmarks/compress-bench/README.md @@ -29,6 +29,13 @@ Compare a numeric scheme bundle with the same command and one of these values: - `--vortex-numeric-bundle current-default` - `--vortex-numeric-bundle range-packed` +Add a local Parquet file with `--parquet-path`. Use `--datasets` to select its file stem. + +```bash +cargo run -p compress-bench --profile release_debug -- \ + --parquet-path /tmp/input.parquet --datasets '^input$' +``` + GPU decompression is opt-in and runs only the existing benchmark names allow-listed in `src/main.rs`: diff --git a/benchmarks/compress-bench/src/main.rs b/benchmarks/compress-bench/src/main.rs index 0a79238fc35..42e61bb2034 100644 --- a/benchmarks/compress-bench/src/main.rs +++ b/benchmarks/compress-bench/src/main.rs @@ -4,6 +4,7 @@ use std::path::PathBuf; use std::time::Duration; +use async_trait::async_trait; use clap::Parser; #[cfg(feature = "lance")] use compress_bench::LanceCompressor; @@ -14,6 +15,8 @@ use compress_bench::vortex::VortexCompressor; use indicatif::ProgressBar; use itertools::Itertools; use regex::Regex; +use vortex::array::ArrayRef; +use vortex::array::ExecutionCtx; use vortex::utils::aliases::hash_map::HashMap; use vortex_bench::Engine; use vortex_bench::Format; @@ -26,6 +29,7 @@ use vortex_bench::compress::Compressor; use vortex_bench::compress::benchmark_compress; use vortex_bench::compress::benchmark_decompress; use vortex_bench::compress::calculate_ratios; +use vortex_bench::conversions::parquet_to_vortex_chunks; use vortex_bench::create_output_writer; use vortex_bench::datasets::Dataset; use vortex_bench::datasets::feature_vectors::GloveEmbeddingsData; @@ -69,6 +73,9 @@ struct Args { ops: Vec, #[arg(long)] datasets: Option, + /// Add a local Parquet file to the benchmark suite. + #[arg(long)] + parquet_path: Vec, /// Run GPU decompression for the allow-listed benchmarks. /// /// This filters the suite to GPU-supported dataset names and runs only Vortex decompression. @@ -111,6 +118,7 @@ async fn main() -> anyhow::Result<()> { run_compress( args.iterations, args.datasets.map(|d| Regex::new(&d)).transpose()?, + args.parquet_path, formats, ops, args.gpu_decompress, @@ -122,6 +130,37 @@ async fn main() -> anyhow::Result<()> { .await } +struct LocalParquetData { + name: String, + path: PathBuf, +} + +impl LocalParquetData { + fn try_new(path: PathBuf) -> anyhow::Result { + let name = path + .file_stem() + .and_then(|value| value.to_str()) + .ok_or_else(|| anyhow::anyhow!("local Parquet path has no valid file stem"))? + .to_string(); + Ok(Self { name, path }) + } +} + +#[async_trait] +impl Dataset for LocalParquetData { + fn name(&self) -> &str { + &self.name + } + + async fn to_vortex_array(&self, _ctx: &mut ExecutionCtx) -> anyhow::Result { + Ok(parquet_to_vortex_chunks(self.path.clone()).await?.into()) + } + + async fn to_parquet_path(&self) -> anyhow::Result { + Ok(self.path.clone()) + } +} + /// Get a compressor for the given format. fn get_compressor( format: Format, @@ -166,6 +205,7 @@ const DOC_PATH: &str = "benchmarks/compress-bench/README.md"; async fn run_compress( iterations: usize, datasets_filter: Option, + parquet_paths: Vec, formats: Vec, ops: Vec, gpu_decompress: bool, @@ -195,6 +235,10 @@ async fn run_compress( // Some(READ_PROJECTION_COLUMNS), // ), ]; + let local_parquet = parquet_paths + .into_iter() + .map(LocalParquetData::try_new) + .collect::>>()?; // Add an existing benchmark name here only after its CUDA-compatible compression and // decompression kernels have been verified end to end. @@ -204,28 +248,36 @@ async fn run_compress( )] let gpu_decompress_benchmarks = vec!["TPC-H l_comment canonical"]; - let datasets: Vec<&dyn Dataset> = [ - &TaxiData as &dyn Dataset, - &GloveEmbeddingsData, - PBI_DATASETS.get(Arade), - PBI_DATASETS.get(Bimbo), - PBI_DATASETS.get(CMSprovider), + let mut datasets: Vec<&dyn Dataset> = vec![&TaxiData, &GloveEmbeddingsData]; + for (name, dataset) in [ + ("Arade", Arade), + ("Bimbo", Bimbo), + ("CMSprovider", CMSprovider), // Corporations, // duckdb thinks ' is a quote character but its used as an apostrophe // CityMaxCapita, // 11th column has F, M, and U but is inferred as boolean - PBI_DATASETS.get(Euro2016), - PBI_DATASETS.get(Food), - PBI_DATASETS.get(HashTags), + ("Euro2016", Euro2016), + ("Food", Food), + ("HashTags", HashTags), // Hatred, // panic in fsst_compress_iter // TableroSistemaPenal, // Unexpected type error // YaleLanguages, // 4th column looks like integer but also contains Y - &TPCHLCommentChunked, + ] { + if datasets_filter + .as_ref() + .is_none_or(|filter| filter.is_match(name)) + { + datasets.push(PBI_DATASETS.get(dataset)); + } + } + datasets.extend([ + &TPCHLCommentChunked as &dyn Dataset, &TPCHLCommentCanonical, &DownloadableDataset::RPlace, &DownloadableDataset::AirQuality, - ] - .into_iter() - .chain(structlistofints.iter().map(|d| d as &dyn Dataset)) - .filter(|d| { + ]); + datasets.extend(structlistofints.iter().map(|d| d as &dyn Dataset)); + datasets.extend(local_parquet.iter().map(|d| d as &dyn Dataset)); + datasets.retain(|d| { if gpu_decompress && !gpu_decompress_benchmarks.contains(&d.name()) { return false; } @@ -236,8 +288,7 @@ async fn run_compress( // for pcodec. As such, we do not run in CI. d.name() != "airquality" && d.name() != "rplace" } - }) - .collect(); + }); let progress = ProgressBar::new((datasets.len() * formats.len() * ops.len()) as u64); diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 720459bb8e8..be9a9970800 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -78,7 +78,7 @@ The serialized tree uses `OrderedFloat(BlockResidual(...))` because the outer ar The FloatQuant candidate accepts native `f16`, `f32`, and `f64` inputs. -Direct integer BlockResidual supports every integer type. The Default selector accepts every integer width during the current 8-bit trial. +Direct integer BlockResidual supports every integer type. The Default selector accepts every integer width. The retained schemes win on specific structures. They do not replace ALP or ALP-RD across general float data. @@ -121,7 +121,7 @@ The Default selector can exclude a supported type when measured costs do not jus | Array | Supported logical types | Default policy | | --- | --- | --- | | OrderedFloat | `f16`, `f32`, `f64` | All float types remain eligible. | -| BlockResidual | `u8`, `u16`, `u32`, `u64`, `i8`, `i16`, `i32`, `i64` | All widths are eligible during the 8-bit trial. | +| BlockResidual | `u8`, `u16`, `u32`, `u64`, `i8`, `i16`, `i32`, `i64` | All widths are eligible. | | FloatQuant | `f16`, `f32`, `f64` | All float types remain eligible. | | IntMult | `u8`, `u16`, `u32`, `u64`, `i8`, `i16`, `i32`, `i64` | No Default scheme exists yet. | @@ -296,7 +296,7 @@ Its adjusted score includes a 1.02 decode-cost factor. `BlockResidualScheme` uses the same locality probe for integer arrays. -The Default scheme accepts every integer width during the current 8-bit trial. +The Default scheme accepts every integer width. The scheme does not run inside trial compression for an outer scheme. Generic 64-row samples do not preserve 1,024-row locality. @@ -638,7 +638,7 @@ The throughput changes come from adjacent runs and include ordinary benchmark no The exact size gains and high absolute decode rates support 16-bit Default eligibility. -### 8-bit selector trial +### 8-bit selector validation The direct benchmark already showed 30.53 GB/s decode and 44 ns scalar access for `i8`. @@ -657,9 +657,51 @@ A uniform `i8` control rejects BlockResidual and retains Primitive at 2,000,000 The rejected analysis changes encode throughput from 720.5 MB/s to 706.3 MB/s, a 2.0 percent decrease. -The eight real numeric files contain no `i8` columns. Their sizes do not change during this trial. +The original eight numeric files contain no 8-bit columns. -The current branch enables 8-bit selection for evaluation. Final adoption needs real `i8` coverage and complete threshold calibration. +A second pass quantizes 20 million real GloVe values into six `i8` and `u8` variants. + +It contains these quantizers: + +- Symmetric quantization uses the maximum absolute corpus value. The unsigned form adds 128. +- Global affine quantization maps the corpus minimum and maximum to 0 and 255. The signed form subtracts 128. +- Per-vector affine quantization uses each 200-value vector minimum and maximum. The signed form subtracts 128. + +The global affine `i8` and `u8` variants select BlockResidual. The unsigned symmetric variant also selects BlockResidual. + +The signed symmetric variant retains ZigZag with BitPacked. Both per-vector affine variants retain Primitive. + +| Selected variant | Prior bytes | BlockResidual bytes | Size change | Encode MB/s | Decode MB/s | Scalar access ns | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Global affine `i8` | 20,000,000 | 16,775,243 | -16.12 percent | 277.8 | 21,791 | 80 | +| Global affine `u8` | 20,000,000 | 16,775,243 | -16.12 percent | 278.3 | 20,745 | 82 | +| Symmetric `u8` | 20,000,000 | 15,558,250 | -22.21 percent | 294.4 | 32,870 | 78 | + +The isolated Primitive encoders reach 754 to 777 MB/s on these selected columns. + +The complete writer reduces that isolated encode gap: + +| File | Size change | Write time change | Read throughput | +| --- | ---: | ---: | ---: | +| Global affine `i8` only | -16.12 percent | +10.06 percent | 17.3 GB/s | +| Two `i8` variants | -8.06 percent | +0.21 percent | 30.7 GB/s | +| Four `u8` variants | -10.19 percent | +1.62 to +2.25 percent | 35.4 GB/s | + +The single-column result isolates the maximum measured writer cost. The multi-column results represent mixed accepted and rejected candidates. + +A patch-free boundary control uses one seven-bit local range per block. + +It reduces complete file size by 10.35 percent and increases write time by 1.43 percent. + +Its complete read throughput is 31.7 GB/s. Scalar access takes 81 ns for `u8` and 110 ns for `i8`. + +Residual widths change in whole bits. Therefore, the first patch-free 8-bit win already saves about ten percent after file overhead. + +The patch-density adjustment protects cases between those discrete widths. + +These results support Default eligibility for `i8` and `u8` with the common 1.05 floor. + +No width-specific factor is necessary. ### Narrow BlockResidual in Compact @@ -1633,9 +1675,11 @@ The implementation retains the prior signed path. Current i16 BlockResidual decode reaches 31.4 GB/s. The comparable `FoR(BitPacked)` tree reaches 42.0 GB/s. -This result retains the direct i8 and i16 selector exclusions. +At that stage, this result retained the direct i8 and i16 selector exclusions. + +At that stage, the u16 result lacked real selected-tree evidence. -The u16 result is faster than the prior narrow path, but no real selected-tree evidence supports policy removal yet. +Later absolute-throughput and complete-file results re-enabled all narrow integer widths. The faster unsigned path also improves the BlockResidual patch-position bulk decoder. @@ -2165,7 +2209,7 @@ The 1.10 value adds a small margin and retains every strict synthetic `f32` and The 33-column Pco-gap pass exposes no missed real FloatQuant candidate. -The 8-bit BlockResidual trial remains outside this conclusion because the corpus lacks real 8-bit columns. +The quantized GloVe pass supports the same factor for `i8` and `u8`. ### Default bundle options @@ -2568,6 +2612,9 @@ This round completed these steps: - Revalidated Current Default against Compact and Parquet with Zstd across all 16 files. - Added numeric bundle controls to the random-access benchmark. - Compared cached random access across Taxi and two nested datasets. +- Added `i8` and `u8` support to the focused Parquet and scalar benchmarks. +- Validated 8-bit selection with quantized real GloVe values. +- Added local Parquet inputs to the complete compression benchmark. The Pco mode profile and the quotient and remainder experiments are complete. @@ -2581,12 +2628,10 @@ The specialized OrderedFloat and ALP trees now use registered fused parent kerne Complete these next experiments in order: -1. Add real `i8` and `u8` columns to the corpus. -2. Calibrate 8-bit BlockResidual eligibility with those real columns. -3. Decide whether Public BI file random access adds useful evidence beyond scalar benchmarks. -4. Prefer cheap rejection tests when they preserve the best candidate model. -5. Treat fast rejection as a selection advantage, not an admission criterion. -6. Require stronger corpus evidence when a useful scheme needs costly analysis. +1. Decide whether Public BI file random access adds useful evidence beyond scalar benchmarks. +2. Prefer cheap rejection tests when they preserve the best candidate model. +3. Treat fast rejection as a selection advantage, not an admission criterion. +4. Require stronger corpus evidence when a useful scheme needs costly analysis. Use packed bin codes and primitive bin starts unless evidence supports more complexity. diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index bca006f53f7..c5672c6ad4d 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -455,9 +455,11 @@ fn read_parquet_numeric( }; matches!( data_type, - DataType::Int16 + DataType::Int8 + | DataType::Int16 | DataType::Int32 | DataType::Int64 + | DataType::UInt8 | DataType::UInt16 | DataType::UInt32 | DataType::UInt64 @@ -2593,6 +2595,63 @@ fn measure_scalar_access(encoded: &ArrayRef, session: &VortexSession) -> VortexR u64::MAX } else { match encoded.dtype().as_ptype() { + PType::U8 => u64::from( + scalar + .as_primitive() + .typed_value::() + .ok_or_else(|| vortex_err!("tree produced an invalid u8 scalar"))?, + ), + PType::U16 => u64::from( + scalar + .as_primitive() + .typed_value::() + .ok_or_else(|| vortex_err!("tree produced an invalid u16 scalar"))?, + ), + PType::U32 => u64::from( + scalar + .as_primitive() + .typed_value::() + .ok_or_else(|| vortex_err!("tree produced an invalid u32 scalar"))?, + ), + PType::U64 => scalar + .as_primitive() + .typed_value::() + .ok_or_else(|| vortex_err!("tree produced an invalid u64 scalar"))?, + PType::I8 => u64::from(u8::from_le_bytes( + scalar + .as_primitive() + .typed_value::() + .ok_or_else(|| vortex_err!("tree produced an invalid i8 scalar"))? + .to_le_bytes(), + )), + PType::I16 => u64::from(u16::from_le_bytes( + scalar + .as_primitive() + .typed_value::() + .ok_or_else(|| vortex_err!("tree produced an invalid i16 scalar"))? + .to_le_bytes(), + )), + PType::I32 => u64::from(u32::from_le_bytes( + scalar + .as_primitive() + .typed_value::() + .ok_or_else(|| vortex_err!("tree produced an invalid i32 scalar"))? + .to_le_bytes(), + )), + PType::I64 => u64::from_le_bytes( + scalar + .as_primitive() + .typed_value::() + .ok_or_else(|| vortex_err!("tree produced an invalid i64 scalar"))? + .to_le_bytes(), + ), + PType::F16 => u64::from( + scalar + .as_primitive() + .typed_value::() + .ok_or_else(|| vortex_err!("tree produced an invalid f16 scalar"))? + .to_bits(), + ), PType::F32 => u64::from( scalar .as_primitive() @@ -2605,11 +2664,6 @@ fn measure_scalar_access(encoded: &ArrayRef, session: &VortexSession) -> VortexR .typed_value::() .ok_or_else(|| vortex_err!("tree produced an invalid f64 scalar"))? .to_bits(), - ptype => { - return Err(vortex_err!( - "tree scalar benchmark does not support {ptype}" - )); - } } }; scalar_checksum ^= black_box(bits); @@ -2856,11 +2910,7 @@ fn measure_dataset( if std::env::var_os("VORTEX_BENCH_CONFIG_TREES").is_some() { for (config, arrays) in &encoded { for (column, array) in columns.iter().zip(arrays) { - if column - .primitive - .as_ref() - .is_some_and(|primitive| primitive.ptype().is_float()) - { + if column.primitive.is_some() { measure_existing_tree( dataset, &column.name, diff --git a/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs index fd593041721..aa1c983c30a 100644 --- a/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs @@ -148,18 +148,18 @@ fn test_block_residual_compresses_16_bit_integers() -> VortexResult<()> { #[test] fn test_block_residual_compresses_8_bit_integers() -> VortexResult<()> { - let signed_values = (0..8_192) + let unsigned_values = (0..16_384) .map(|index| { - let block = index / 1_024; - let residual = (index * 2_654_435_761_usize) % 8; - i8::try_from(block * 16 + residual) + let block = (index / 1_024) % 2; + let residual = (index * 2_654_435_761_usize) % 128; + u8::try_from(block * 128 + residual) }) .collect::, _>>()?; - let unsigned_values = signed_values + let signed_values = unsigned_values .iter() .copied() - .map(u8::try_from) - .collect::, _>>()?; + .map(|value| i8::from_le_bytes([value])) + .collect::>(); #[cfg(not(feature = "unstable_encodings"))] let compressor = BtrBlocksCompressor::default(); #[cfg(feature = "unstable_encodings")] @@ -182,6 +182,38 @@ fn test_block_residual_compresses_8_bit_integers() -> VortexResult<()> { Ok(()) } +#[test] +fn test_block_residual_rejects_uniform_8_bit_integers() -> VortexResult<()> { + let unsigned_values = (0..16_384) + .map(|index| u8::try_from((index * 2_654_435_761_usize) % 256)) + .collect::, _>>()?; + let signed_values = unsigned_values + .iter() + .copied() + .map(|value| i8::from_le_bytes([value])) + .collect::>(); + #[cfg(not(feature = "unstable_encodings"))] + let compressor = BtrBlocksCompressor::default(); + #[cfg(feature = "unstable_encodings")] + let compressor = BtrBlocksCompressorBuilder::default() + .exclude_schemes([DeltaScheme::default().id()]) + .build(); + + for array in [ + PrimitiveArray::from_iter(signed_values), + PrimitiveArray::from_iter(unsigned_values), + ] { + let compressed = + compressor.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; + assert!( + !contains_block_residual(&compressed), + "BlockResidual must reject this uniform 8-bit input:\n{}", + compressed.display_tree() + ); + } + Ok(()) +} + #[test] fn test_block_residual_rejects_dense_patches() -> VortexResult<()> { let values = (0..8_192_u32).map(|index| if index % 4 == 0 { u32::MAX - index } else { 42 }); diff --git a/vortex-btrblocks/src/trace_tests.rs b/vortex-btrblocks/src/trace_tests.rs index b26829f6a98..517aae1829f 100644 --- a/vortex-btrblocks/src/trace_tests.rs +++ b/vortex-btrblocks/src/trace_tests.rs @@ -359,6 +359,11 @@ fn trace_scan_like_on_compressed_comment() -> VortexResult<()> { insta::assert_snapshot!(executed.trace.to_string(), @" execute_until target=AnyCanonical root=vortex.like(bool, len=4096) iter 0 current=vortex.like(bool, len=4096) builder_active=false + execute_until target=AnyCanonical root=vortex.block_residual(u16, len=4097) + iter 0 current=vortex.block_residual(u16, len=4097) builder_active=false + Done array=vortex.primitive(u16, len=4097) + iter 1 current=vortex.primitive(u16, len=4097) builder_active=false + return output=vortex.primitive(u16, len=4097) child_execute_parent session[0]:execute_parent_fn slot=0 parent=vortex.like(bool, len=4096) child=vortex.fsst(utf8, len=4096) -> vortex.bool(bool, len=4096) iter 1 current=vortex.bool(bool, len=4096) builder_active=false return output=vortex.bool(bool, len=4096) diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__list_of_int_runs.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__list_of_int_runs.snap index c9554add05a..e5f7bbb6eee 100644 --- a/vortex-btrblocks/tests/snapshots/golden__unstable__list_of_int_runs.snap +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__list_of_int_runs.snap @@ -3,7 +3,7 @@ source: vortex-btrblocks/tests/golden.rs expression: rendered --- input: list(i32), len=4066, nbytes=81804 -root: vortex.list(list(i32), len=4066) nbytes=11146 +root: vortex.list(list(i32), len=4066) nbytes=10304 metadata: elements: vortex.runend(i32, len=16384) nbytes=3968 metadata: offset: 0 @@ -15,11 +15,5 @@ root: vortex.list(list(i32), len=4066) nbytes=11146 metadata: reference: -49931i32 encoded: fastlanes.bitpacked(i32, len=1020) nbytes=2176 metadata: bit_width: 17, offset: 0 - offsets: fastlanes.bitpacked(u16, len=4067) nbytes=7178 - metadata: bit_width: 14, offset: 0 - patch_indices: vortex.primitive(u16, len=1) nbytes=2 - metadata: ptype: u16 - patch_values: vortex.constant(u16, len=1) nbytes=4 - metadata: scalar: 16384u16 - patch_chunk_offsets: vortex.primitive(u8, len=4) nbytes=4 - metadata: ptype: u8 + offsets: vortex.block_residual(u16, len=4067) nbytes=6336 + metadata: blocks: 4, slice: 0..4067 From 4a393ea0ed49d3ebcc5db90757fc243cdacd0b3e Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 13:13:20 +0100 Subject: [PATCH 66/78] bench: measure 8-bit random access Signed-off-by: Will Manning --- benchmarks/compress-bench/src/main.rs | 36 +---------- benchmarks/random-access-bench/README.md | 9 +++ benchmarks/random-access-bench/src/main.rs | 37 ++++++++--- docs/plans/native-pcodec.md | 37 +++++++++-- vortex-bench/src/datasets/local_parquet.rs | 75 ++++++++++++++++++++++ vortex-bench/src/datasets/mod.rs | 1 + 6 files changed, 144 insertions(+), 51 deletions(-) create mode 100644 vortex-bench/src/datasets/local_parquet.rs diff --git a/benchmarks/compress-bench/src/main.rs b/benchmarks/compress-bench/src/main.rs index 42e61bb2034..085558b289a 100644 --- a/benchmarks/compress-bench/src/main.rs +++ b/benchmarks/compress-bench/src/main.rs @@ -4,7 +4,6 @@ use std::path::PathBuf; use std::time::Duration; -use async_trait::async_trait; use clap::Parser; #[cfg(feature = "lance")] use compress_bench::LanceCompressor; @@ -15,8 +14,6 @@ use compress_bench::vortex::VortexCompressor; use indicatif::ProgressBar; use itertools::Itertools; use regex::Regex; -use vortex::array::ArrayRef; -use vortex::array::ExecutionCtx; use vortex::utils::aliases::hash_map::HashMap; use vortex_bench::Engine; use vortex_bench::Format; @@ -29,10 +26,10 @@ use vortex_bench::compress::Compressor; use vortex_bench::compress::benchmark_compress; use vortex_bench::compress::benchmark_decompress; use vortex_bench::compress::calculate_ratios; -use vortex_bench::conversions::parquet_to_vortex_chunks; use vortex_bench::create_output_writer; use vortex_bench::datasets::Dataset; use vortex_bench::datasets::feature_vectors::GloveEmbeddingsData; +use vortex_bench::datasets::local_parquet::LocalParquetData; use vortex_bench::datasets::struct_list_of_ints::StructListOfInts; use vortex_bench::datasets::taxi_data::TaxiData; use vortex_bench::datasets::tpch_l_comment::TPCHLCommentCanonical; @@ -130,37 +127,6 @@ async fn main() -> anyhow::Result<()> { .await } -struct LocalParquetData { - name: String, - path: PathBuf, -} - -impl LocalParquetData { - fn try_new(path: PathBuf) -> anyhow::Result { - let name = path - .file_stem() - .and_then(|value| value.to_str()) - .ok_or_else(|| anyhow::anyhow!("local Parquet path has no valid file stem"))? - .to_string(); - Ok(Self { name, path }) - } -} - -#[async_trait] -impl Dataset for LocalParquetData { - fn name(&self) -> &str { - &self.name - } - - async fn to_vortex_array(&self, _ctx: &mut ExecutionCtx) -> anyhow::Result { - Ok(parquet_to_vortex_chunks(self.path.clone()).await?.into()) - } - - async fn to_parquet_path(&self) -> anyhow::Result { - Ok(self.path.clone()) - } -} - /// Get a compressor for the given format. fn get_compressor( format: Format, diff --git a/benchmarks/random-access-bench/README.md b/benchmarks/random-access-bench/README.md index 23f00abdb7d..f12c3dd9020 100644 --- a/benchmarks/random-access-bench/README.md +++ b/benchmarks/random-access-bench/README.md @@ -29,3 +29,12 @@ Compare random access for a numeric scheme bundle with one of these values: - `--vortex-numeric-bundle range-packed` Each bundle uses a separate Vortex file. Existing files remain available for repeated runs. + +Add a local Parquet file with `--parquet-path`. A local path replaces the default dataset list. + +```bash +cargo run -p random-access-bench --profile release_debug -- \ + --formats vortex --parquet-path /tmp/input.parquet +``` + +If `--datasets` also selects built-in datasets, the benchmark runs both groups. diff --git a/benchmarks/random-access-bench/src/main.rs b/benchmarks/random-access-bench/src/main.rs index b6d1e9f55c8..59d3748efc8 100644 --- a/benchmarks/random-access-bench/src/main.rs +++ b/benchmarks/random-access-bench/src/main.rs @@ -16,6 +16,7 @@ use vortex_bench::SESSION; use vortex_bench::VortexNumericBundle; use vortex_bench::conversions::write_parquet_as_vortex_with_options; use vortex_bench::datasets::feature_vectors::FeatureVectorsData; +use vortex_bench::datasets::local_parquet::LocalParquetData; use vortex_bench::datasets::nested_lists::NestedListsData; use vortex_bench::datasets::nested_structs::NestedStructsData; use vortex_bench::datasets::taxi_data::TaxiData; @@ -100,13 +101,11 @@ struct Args { #[arg(long = "ingest-jsonl")] ingest_output: Option, /// Which datasets to benchmark random access on. - #[arg( - long, - value_delimiter = ',', - value_enum, - default_values_t = vec![DatasetArg::Taxi, DatasetArg::FeatureVectors, DatasetArg::NestedLists, DatasetArg::NestedStructs] - )] + #[arg(long, value_delimiter = ',', value_enum)] datasets: Vec, + /// Add a local Parquet file to the benchmark suite. + #[arg(long)] + parquet_path: Vec, /// Which access patterns to benchmark. #[arg( long, @@ -128,11 +127,31 @@ async fn main() -> Result<()> { let args = Args::parse(); setup_logging_and_tracing(args.verbose, args.tracing)?; + let dataset_args = if args.datasets.is_empty() && args.parquet_path.is_empty() { + vec![ + DatasetArg::Taxi, + DatasetArg::FeatureVectors, + DatasetArg::NestedLists, + DatasetArg::NestedStructs, + ] + } else { + args.datasets + }; + let mut datasets = dataset_args + .into_iter() + .map(DatasetArg::into_dataset) + .collect::>(); + datasets.extend( + args.parquet_path + .into_iter() + .map(LocalParquetData::try_new) + .map(|dataset| dataset.map(|dataset| Box::new(dataset) as Box)) + .collect::>>()?, + ); + let run_config = RunConfig { - datasets: args - .datasets + datasets: datasets .into_iter() - .map(DatasetArg::into_dataset) .map(|dataset| { Box::new(NumericBundleDataset { dataset, diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index be9a9970800..69aeded5fbb 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -699,9 +699,27 @@ Residual widths change in whole bits. Therefore, the first patch-free 8-bit win The patch-density adjustment protects cases between those discrete widths. -These results support Default eligibility for `i8` and `u8` with the common 1.05 floor. +The bulk and writer results support Default eligibility for `i8` and `u8`. -No width-specific factor is necessary. +The file random-access pass adds a material selection trade: + +| Input | Size change | Prior correlated | BlockResidual correlated | Prior uniform | BlockResidual uniform | +| --- | ---: | ---: | ---: | ---: | ---: | +| Global affine `i8` | -16.12 percent | 78.6 us | 156.1 us | 497.7 us | 835.6 us | +| Symmetric `u8` | -22.16 percent | 79.5 us | 136.4 us | 490.6 us | 708.1 us | +| Seven-bit boundary | -10.35 percent | 112.7 us | 174.3 us | 785.3 us | 1,014.3 us | + +Each pattern requests about 100 rows through a cached file handle. + +The candidate increases correlated latency by 55 to 99 percent. It increases uniform latency by 29 to 68 percent. + +Every measured lookup remains below 1.1 ms. Sparse patches make the global affine case the slowest relative result. + +The current selector keeps the common 1.05 floor and no width factor. + +The final file ratios place the seven-bit boundary near a 1.12 factor. They place the global affine case near 1.20. + +The final 8-bit factor remains a policy choice because random access has priority over size. ### Narrow BlockResidual in Compact @@ -2209,7 +2227,9 @@ The 1.10 value adds a small margin and retains every strict synthetic `f32` and The 33-column Pco-gap pass exposes no missed real FloatQuant candidate. -The quantized GloVe pass supports the same factor for `i8` and `u8`. +The 8-bit bulk and writer evidence supports the same factor. + +The 8-bit file-access evidence leaves the final width factor open. ### Default bundle options @@ -2615,6 +2635,8 @@ This round completed these steps: - Added `i8` and `u8` support to the focused Parquet and scalar benchmarks. - Validated 8-bit selection with quantized real GloVe values. - Added local Parquet inputs to the complete compression benchmark. +- Added local Parquet inputs to the random-access benchmark. +- Measured 8-bit BlockResidual file access on accepted and boundary cases. The Pco mode profile and the quotient and remainder experiments are complete. @@ -2628,10 +2650,11 @@ The specialized OrderedFloat and ALP trees now use registered fused parent kerne Complete these next experiments in order: -1. Decide whether Public BI file random access adds useful evidence beyond scalar benchmarks. -2. Prefer cheap rejection tests when they preserve the best candidate model. -3. Treat fast rejection as a selection advantage, not an admission criterion. -4. Require stronger corpus evidence when a useful scheme needs costly analysis. +1. Choose the final 8-bit width factor from the size and file-access trade. +2. Decide whether Public BI file random access adds useful evidence beyond scalar benchmarks. +3. Prefer cheap rejection tests when they preserve the best candidate model. +4. Treat fast rejection as a selection advantage, not an admission criterion. +5. Require stronger corpus evidence when a useful scheme needs costly analysis. Use packed bin codes and primitive bin starts unless evidence supports more complexity. diff --git a/vortex-bench/src/datasets/local_parquet.rs b/vortex-bench/src/datasets/local_parquet.rs new file mode 100644 index 00000000000..fb3c8c983a6 --- /dev/null +++ b/vortex-bench/src/datasets/local_parquet.rs @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::fs::File; +use std::path::PathBuf; + +use anyhow::Result; +use anyhow::bail; +use async_trait::async_trait; +use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; +use vortex::array::ArrayRef; +use vortex::array::ExecutionCtx; + +use crate::Format; +use crate::conversions::parquet_to_vortex_chunks; +use crate::datasets::Dataset; +use crate::random_access::BenchDataset; + +/// A local Parquet file for compression and random-access benchmarks. +pub struct LocalParquetData { + name: String, + path: PathBuf, + row_count: u64, +} + +impl LocalParquetData { + /// Read the dataset name and row count from a local Parquet file. + pub fn try_new(path: PathBuf) -> Result { + let name = path + .file_stem() + .and_then(|value| value.to_str()) + .ok_or_else(|| anyhow::anyhow!("local Parquet path has no valid file stem"))? + .to_string(); + let reader = ParquetRecordBatchReaderBuilder::try_new(File::open(&path)?)?; + let row_count = u64::try_from(reader.metadata().file_metadata().num_rows())?; + Ok(Self { + name, + path, + row_count, + }) + } +} + +#[async_trait] +impl Dataset for LocalParquetData { + fn name(&self) -> &str { + &self.name + } + + async fn to_vortex_array(&self, _ctx: &mut ExecutionCtx) -> Result { + Ok(parquet_to_vortex_chunks(self.path.clone()).await?.into()) + } + + async fn to_parquet_path(&self) -> Result { + Ok(self.path.clone()) + } +} + +#[async_trait] +impl BenchDataset for LocalParquetData { + fn name(&self) -> &str { + &self.name + } + + fn row_count(&self) -> u64 { + self.row_count + } + + async fn path(&self, format: Format) -> Result { + if format == Format::Parquet { + return Ok(self.path.clone()); + } + bail!("local Parquet dataset does not provide {format}") + } +} diff --git a/vortex-bench/src/datasets/mod.rs b/vortex-bench/src/datasets/mod.rs index 7eb341edbb7..b53ed965324 100644 --- a/vortex-bench/src/datasets/mod.rs +++ b/vortex-bench/src/datasets/mod.rs @@ -14,6 +14,7 @@ use crate::clickbench::Flavor; pub mod data_downloads; pub mod feature_vectors; +pub mod local_parquet; pub mod nested_lists; pub mod nested_structs; pub mod struct_list_of_ints; From f1df2527e4758ba0cefbfd496d6c023376eca532 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 13:18:09 +0100 Subject: [PATCH 67/78] feat: calibrate 8-bit BlockResidual selection Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 25 ++++++------ .../src/schemes/integer/block_residual.rs | 10 ++++- .../schemes/integer/scheme_selection_tests.rs | 40 ++++++++++++++++++- 3 files changed, 59 insertions(+), 16 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 69aeded5fbb..4b7037f5dcd 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -715,11 +715,13 @@ The candidate increases correlated latency by 55 to 99 percent. It increases uni Every measured lookup remains below 1.1 ms. Sparse patches make the global affine case the slowest relative result. -The current selector keeps the common 1.05 floor and no width factor. +The selector keeps the common 1.05 raw-ratio floor. It applies a 1.12 access cost factor to 8-bit estimates. -The final file ratios place the seven-bit boundary near a 1.12 factor. They place the global affine case near 1.20. +The factor rejects the seven-bit boundary case. That case saved 10.35 percent and increased random-access latency by 29 to 55 percent. -The final 8-bit factor remains a policy choice because random access has priority over size. +The factor retains the global affine `i8` and `u8` variants. It also retains the unsigned symmetric variant. + +These retained variants save 16.12 to 22.21 percent. The measured file-access latency remains below one millisecond for about 100 requested rows. ### Narrow BlockResidual in Compact @@ -2209,7 +2211,7 @@ Retain FloatQuant as an opportunistic option under the calibrated 1.10 factor. ### Final selector factors -The integer BlockResidual scheme keeps a 1.05 raw-ratio floor and no width-specific factor. +The integer BlockResidual scheme keeps a 1.05 raw-ratio floor. It applies a 1.12 access cost factor to 8-bit estimates. This policy resolves the CMS threshold miss and produces a 2.94 percent complete-corpus size gain. @@ -2217,7 +2219,7 @@ The Ordered BlockResidual scheme keeps its 1.05 raw floor and 1.02 decode factor The nonlinear patch cost rejects the known dense-patch and slow HashTags trees. -No selected-tree evidence supports a higher BlockResidual factor. +No selected-tree evidence supports a factor for wider integers. The FloatQuant scheme keeps its 1.10 factor. @@ -2227,9 +2229,7 @@ The 1.10 value adds a small margin and retains every strict synthetic `f32` and The 33-column Pco-gap pass exposes no missed real FloatQuant candidate. -The 8-bit bulk and writer evidence supports the same factor. - -The 8-bit file-access evidence leaves the final width factor open. +The 8-bit factor rejects the weakest measured gain. It retains three real quantized GloVe variants with 16 to 22 percent savings. ### Default bundle options @@ -2650,11 +2650,10 @@ The specialized OrderedFloat and ALP trees now use registered fused parent kerne Complete these next experiments in order: -1. Choose the final 8-bit width factor from the size and file-access trade. -2. Decide whether Public BI file random access adds useful evidence beyond scalar benchmarks. -3. Prefer cheap rejection tests when they preserve the best candidate model. -4. Treat fast rejection as a selection advantage, not an admission criterion. -5. Require stronger corpus evidence when a useful scheme needs costly analysis. +1. Decide whether Public BI file random access adds useful evidence beyond scalar benchmarks. +2. Prefer cheap rejection tests when they preserve the best candidate model. +3. Treat fast rejection as a selection advantage, not an admission criterion. +4. Require stronger corpus evidence when a useful scheme needs costly analysis. Use packed bin codes and primitive bin starts unless evidence supports more complexity. diff --git a/vortex-btrblocks/src/schemes/integer/block_residual.rs b/vortex-btrblocks/src/schemes/integer/block_residual.rs index c7e648e1153..1c05ff19e33 100644 --- a/vortex-btrblocks/src/schemes/integer/block_residual.rs +++ b/vortex-btrblocks/src/schemes/integer/block_residual.rs @@ -35,6 +35,9 @@ use crate::schemes::sample_primitive_blocks; const BLOCK_LEN: usize = 1024; const ESTIMATE_BLOCKS: usize = 8; const MIN_COMPRESSION_RATIO: f64 = 1.05; +// The weakest measured 8-bit gain increased file-access latency by 29 to 55 percent. +// Require about 12 percent estimated savings for 8-bit values. +const EIGHT_BIT_ACCESS_COST_FACTOR: f64 = 1.12; // The 25-percent patch sweep needs about 80 cost bits per patch to reject its slow tree. // This quadratic slope reaches that cost at 25 percent while it keeps sparse patches cheap. const PATCH_DENSITY_COST_BITS: u64 = 320; @@ -97,7 +100,12 @@ impl Scheme for BlockResidualScheme { if ratio < MIN_COMPRESSION_RATIO { return Ok(EstimateVerdict::Skip); } - Ok(EstimateVerdict::Ratio(ratio)) + let adjusted_ratio = if sample.ptype().byte_width() == 1 { + ratio / EIGHT_BIT_ACCESS_COST_FACTOR + } else { + ratio + }; + Ok(EstimateVerdict::Ratio(adjusted_ratio)) }, ))) } diff --git a/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs index aa1c983c30a..477cafb91fd 100644 --- a/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs @@ -148,6 +148,42 @@ fn test_block_residual_compresses_16_bit_integers() -> VortexResult<()> { #[test] fn test_block_residual_compresses_8_bit_integers() -> VortexResult<()> { + let unsigned_values = (0..16_384) + .map(|index| { + let block = (index / 1_024) % 32; + let residual = (index * 2_654_435_761_usize) % 8; + u8::try_from(block * 8 + residual) + }) + .collect::, _>>()?; + let signed_values = unsigned_values + .iter() + .copied() + .map(|value| i8::from_le_bytes([value])) + .collect::>(); + #[cfg(not(feature = "unstable_encodings"))] + let compressor = BtrBlocksCompressor::default(); + #[cfg(feature = "unstable_encodings")] + let compressor = BtrBlocksCompressorBuilder::default() + .exclude_schemes([DeltaScheme::default().id()]) + .build(); + + for array in [ + PrimitiveArray::from_iter(signed_values), + PrimitiveArray::from_iter(unsigned_values), + ] { + let compressed = + compressor.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; + assert!( + contains_block_residual(&compressed), + "BlockResidual must encode this 8-bit input:\n{}", + compressed.display_tree() + ); + } + Ok(()) +} + +#[test] +fn test_block_residual_rejects_weak_8_bit_gain() -> VortexResult<()> { let unsigned_values = (0..16_384) .map(|index| { let block = (index / 1_024) % 2; @@ -174,8 +210,8 @@ fn test_block_residual_compresses_8_bit_integers() -> VortexResult<()> { let compressed = compressor.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; assert!( - contains_block_residual(&compressed), - "BlockResidual must encode this 8-bit input:\n{}", + !contains_block_residual(&compressed), + "BlockResidual must reject this weak 8-bit gain:\n{}", compressed.display_tree() ); } From 3b6eec3da9800e30b7aba72cbc95725cfa00746c Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 13:30:26 +0100 Subject: [PATCH 68/78] perf: fuse f16 ordered BlockResidual decode Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 35 +++++++++++++++---- .../src/block_residual_array.rs | 15 ++++++++ .../block-residual/src/ordered_float_array.rs | 3 +- 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 4b7037f5dcd..e6b6dd53b5f 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -382,7 +382,7 @@ These cases include the current compressed children and fused decode paths. | IntMult with packed children `u16` | 1.38 GB/s | 22.90 GB/s | 125 ns | | IntMult with packed children `u32` | 2.64 GB/s | 22.62 GB/s | 140 ns | | IntMult with packed children `u64` | 4.34 GB/s | 17.96 GB/s | 125 ns | -| OrderedFloat with BlockResidual `f16` | 1.26 GB/s | 20.50 GB/s | 79 ns | +| OrderedFloat with BlockResidual `f16` | 1.26 GB/s | 26.36 GB/s | 79 ns | | OrderedFloat with BlockResidual `f32` | 2.43 GB/s | 29.64 GB/s | 78 ns | | OrderedFloat with BlockResidual `f64` | 2.70 GB/s | 20.44 GB/s | 125 ns | | FloatQuant with packed primary `f16` | 3.21 GB/s | 20.05 GB/s | 125 ns | @@ -857,7 +857,11 @@ Compression throughput increased by 2.77 times. Decode throughput increased by 2 On general rejected `f16` values, median compression throughput decreased by 1.8 percent. -The `f16` BlockResidual tree decoded at 20.50 GB/s. Its generic inverse transform lacks a fused narrow path. +The first `f16` BlockResidual tree decoded at 20.50 GB/s. + +A later same-benchmark comparison measured 24.31 GB/s before a fused narrow transform and 26.36 GB/s after it. + +The fused path improves median decode throughput by 8.4 percent. It reconstructs residuals and float bits in one output pass. Retain `f16` eligibility. Revisit its selector factors during final corpus calibration. @@ -2325,11 +2329,28 @@ This small target is more sensitive to run noise than the 100-row patterns. BlockResidual, Current Default, and RangePacked remain within 2.5 percent on both Taxi 100-row patterns. -The random-access suite does not include the Public BI files. +The Public BI pass uses six complete files and cached handles. Each pattern requests about 100 rows. + +Two passes use five-second target windows. A ten-second CMS rerun replaces one correlated outlier. + +| Dataset | File size | Correlated latency | Uniform latency | +| --- | ---: | ---: | ---: | +| Arade | -2.39 percent | -9.74 to -0.38 percent | -0.72 to +0.86 percent | +| Bimbo | -7.54 percent | +5.88 to +17.20 percent | +7.07 to +7.85 percent | +| CMSprovider | -7.86 percent | +14.03 to +18.46 percent | +9.89 to +11.73 percent | +| Euro2016 | -2.44 percent | -0.06 to +5.27 percent | -0.44 to +0.50 percent | +| Food | -6.19 percent | -16.78 to -10.54 percent | +12.19 to +13.94 percent | +| HashTags | -2.56 percent | +7.28 to +13.38 percent | +4.66 to +9.94 percent | + +The file-size geometric mean decreases by 4.86 percent. Aggregate bytes decrease by 5.81 percent. + +Across both passes, correlated latency increases by 3.06 percent geometrically. Uniform latency increases by 6.33 percent. + +The combined latency geometric mean increases by 4.69 percent. -Selected-tree scalar benchmarks remain the direct evidence for Bimbo, CMSprovider, Euro2016, Food, and HashTags. +The Public BI pass adds useful evidence beyond scalar benchmarks. It exposes a measurable file-access trade for the conservative bundle. -Add Public BI file access only if bundle adoption needs broader late-materialization evidence. +The final threshold review must include this result. The current factor remains unchanged until that review. ### Compact and Parquet size constraint @@ -2637,6 +2658,8 @@ This round completed these steps: - Added local Parquet inputs to the complete compression benchmark. - Added local Parquet inputs to the random-access benchmark. - Measured 8-bit BlockResidual file access on accepted and boundary cases. +- Measured complete-file random access for six Public BI datasets. +- Added a fused `f16` decoder for `OrderedFloat(BlockResidual)`. The Pco mode profile and the quotient and remainder experiments are complete. @@ -2650,7 +2673,7 @@ The specialized OrderedFloat and ALP trees now use registered fused parent kerne Complete these next experiments in order: -1. Decide whether Public BI file random access adds useful evidence beyond scalar benchmarks. +1. Use the Public BI file-access evidence during the final threshold review. 2. Prefer cheap rejection tests when they preserve the best candidate model. 3. Treat fast rejection as a selection advantage, not an admission criterion. 4. Require stronger corpus evidence when a useful scheme needs costly analysis. diff --git a/encodings/block-residual/src/block_residual_array.rs b/encodings/block-residual/src/block_residual_array.rs index 2068e38ffdb..37101380ad4 100644 --- a/encodings/block-residual/src/block_residual_array.rs +++ b/encodings/block-residual/src/block_residual_array.rs @@ -28,6 +28,7 @@ use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; use vortex_array::dtype::PType; +use vortex_array::dtype::half::f16; use vortex_array::optimizer::rules::ParentRuleSet; use vortex_array::scalar::Scalar; use vortex_array::serde::ArrayChildren; @@ -1140,6 +1141,20 @@ pub(crate) fn decompress_ordered_f32( }) } +pub(crate) fn decompress_ordered_f16( + array: ArrayView<'_, BlockResidual>, + ctx: &mut ExecutionCtx, +) -> VortexResult { + decode_array_values::(array, ctx, |ordered| { + let bits = if ordered & (1_u16 << 15) == 0 { + !ordered + } else { + ordered ^ (1_u16 << 15) + }; + f16::from_bits(bits) + }) +} + pub(crate) fn decompress_ordered_f64( array: ArrayView<'_, BlockResidual>, ctx: &mut ExecutionCtx, diff --git a/encodings/block-residual/src/ordered_float_array.rs b/encodings/block-residual/src/ordered_float_array.rs index c26a0b1ff05..efcee20949f 100644 --- a/encodings/block-residual/src/ordered_float_array.rs +++ b/encodings/block-residual/src/ordered_float_array.rs @@ -55,6 +55,7 @@ use vortex_session::registry::CachedId; use crate::BlockResidual; use crate::BlockResidualCodec; use crate::BlockResidualEstimate; +use crate::block_residual_array::decompress_ordered_f16; use crate::block_residual_array::decompress_ordered_f32; use crate::block_residual_array::decompress_ordered_f64; use crate::codec::BlockResidualCodecEstimate; @@ -190,7 +191,7 @@ impl VTable for OrderedFloat { } } else if let Some(block_residual) = array.encoded().as_typed::() { match array.dtype().as_ptype() { - PType::F16 => decode_primitive(array.as_view(), ctx)?, + PType::F16 => decompress_ordered_f16(block_residual, ctx)?, PType::F32 => decompress_ordered_f32(block_residual, ctx)?, PType::F64 => decompress_ordered_f64(block_residual, ctx)?, ptype => vortex_bail!("unsupported OrderedFloat ptype {ptype}"), From f85e40bc530cbb6726bb02a3bbc1a443d26688c9 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 13:37:49 +0100 Subject: [PATCH 69/78] perf: fuse implicit FloatQuant f64 decode Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 11 ++- .../bitpacking/array/bitpack_decompress.rs | 40 +++++++++++ encodings/float-quant/src/array.rs | 67 +++++++++++++++++++ 3 files changed, 117 insertions(+), 1 deletion(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index e6b6dd53b5f..ace5b557f26 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -387,7 +387,7 @@ These cases include the current compressed children and fused decode paths. | OrderedFloat with BlockResidual `f64` | 2.70 GB/s | 20.44 GB/s | 125 ns | | FloatQuant with packed primary `f16` | 3.21 GB/s | 20.05 GB/s | 125 ns | | FloatQuant with packed primary `f32` | 5.72 GB/s | 19.79 GB/s | 129 ns | -| FloatQuant with packed primary `f64` | 7.76 GB/s | 15.69 GB/s | 125 ns | +| FloatQuant with packed primary `f64` | 7.76 GB/s | 16.19 GB/s | 125 ns | | FloatQuant with two packed children `f16` | 3.11 GB/s | 16.75 GB/s | 174 ns | | FloatQuant with two packed children `f32` | 5.17 GB/s | 16.30 GB/s | 183 ns | | FloatQuant with two packed children `f64` | 6.72 GB/s | 13.13 GB/s | 208 ns | @@ -397,6 +397,14 @@ The FloatQuant production encode rows use the single-scheme compressor. They include sample analysis and direct packing of the final child tree. +The implicit-zero `f64` decoder maps FastLanes output directly to reconstructed floats. + +A controlled five-second comparison measured 14.08 GB/s before fusion and 16.19 GB/s after it. + +The fused path improves median decode throughput by 15.0 percent. + +The same trial reduced `f16` and `f32` throughput by 2 to 3 percent. Those types retain the generic path. + The paired decoder remains within 18 percent of the zero-secondary decoder for every float width. The packed IntMult tree uses a base of ten and patch-free BitPacked children. @@ -2660,6 +2668,7 @@ This round completed these steps: - Measured 8-bit BlockResidual file access on accepted and boundary cases. - Measured complete-file random access for six Public BI datasets. - Added a fused `f16` decoder for `OrderedFloat(BlockResidual)`. +- Added a fused implicit-zero `f64` decoder for `FloatQuant(FoR(BitPacked))`. The Pco mode profile and the quotient and remainder experiments are complete. diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs index 24f0eb029e7..60141cde5e3 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs @@ -48,6 +48,22 @@ pub fn unpack_primitive_array( Ok(builder.finish_into_primitive()) } +/// Unpacks one bit-packed array and maps each value into the output primitive array. +pub fn unpack_map( + array: ArrayView<'_, BitPacked>, + ctx: &mut ExecutionCtx, + map: M, +) -> VortexResult +where + F: BitPackedUnpack, + T: NativePType, + M: Fn(F) -> T, +{ + let mut builder = PrimitiveBuilder::with_capacity(array.dtype().nullability(), array.len()); + unpack_map_into_builder::(array, &mut builder, ctx, map)?; + Ok(builder.finish_into_primitive()) +} + /// Unpacks two aligned bit-packed arrays and maps each value pair into one output buffer. /// /// This path requires equal lengths and offsets. Neither input can contain patches. @@ -324,6 +340,30 @@ mod tests { use crate::BitPackedData; use crate::bitpack_compress::bitpack_encode; + #[test] + fn unpack_map_preserves_slice_and_validity() -> VortexResult<()> { + let values = (0_u32..3073) + .map(|value| (value % 17 != 0).then_some(value & 0x3ff)) + .collect::>(); + let input = PrimitiveArray::from_option_iter(values.iter().copied()); + let mut ctx = SESSION.create_execution_ctx(); + let packed = bitpack_encode(&input, 10, None, &mut ctx)?.into_array(); + + for range in [0..3073, 3..2051, 1023..2050] { + let packed = packed.slice(range.clone())?; + let actual = unpack_map::(packed.as_::(), &mut ctx, |value| { + u64::from(value) * 17 + })?; + let expected = PrimitiveArray::from_option_iter( + values[range] + .iter() + .map(|value| value.map(|value| u64::from(value) * 17)), + ); + assert_arrays_eq!(actual, expected, &mut ctx); + } + Ok(()) + } + #[test] fn unpack_pair_map_matches_sliced_inputs() -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); diff --git a/encodings/float-quant/src/array.rs b/encodings/float-quant/src/array.rs index 908cecc2f6a..5c7cb989885 100644 --- a/encodings/float-quant/src/array.rs +++ b/encodings/float-quant/src/array.rs @@ -43,6 +43,7 @@ use vortex_fastlanes::BitPackedArrayExt; use vortex_fastlanes::FoR; use vortex_fastlanes::FoRArrayExt; use vortex_fastlanes::FoRArraySlotsExt; +use vortex_fastlanes::bitpack_decompress::unpack_map; use vortex_fastlanes::bitpack_decompress::unpack_pair_map; use vortex_session::VortexSession; use vortex_session::registry::CachedId; @@ -858,6 +859,11 @@ fn decode( array: ArrayView<'_, FloatQuant>, ctx: &mut ExecutionCtx, ) -> VortexResult { + if array.dtype().as_ptype() == PType::F64 + && let Some(decoded) = decode_fastlanes_zero(array, ctx)? + { + return Ok(decoded); + } if let Some(decoded) = decode_fastlanes_pair(array)? { return Ok(decoded); } @@ -936,6 +942,31 @@ fn decode( }) } +fn decode_fastlanes_zero( + array: ArrayView<'_, FloatQuant>, + ctx: &mut ExecutionCtx, +) -> VortexResult> { + if array.secondary().is_some() { + return Ok(None); + } + let Some(primary_for) = array.primary().as_opt::() else { + return Ok(None); + }; + let Some(primary) = primary_for.encoded().as_opt::() else { + return Ok(None); + }; + + let k = array.data().k; + let reference = primary_for + .reference_scalar() + .as_primitive() + .typed_value::() + .vortex_expect("validated f64 primary reference"); + Ok(Some(unpack_map::(primary, ctx, |primary| { + join_zero_f64(primary.wrapping_add(reference), k) + })?)) +} + fn decode_fastlanes_pair(array: ArrayView<'_, FloatQuant>) -> VortexResult> { let Some(secondary) = array .secondary() @@ -1218,6 +1249,42 @@ mod tests { assert!(FloatQuant::primary_for_primitive(f64_values.as_view(), 29, 0).is_err()); } + #[test] + fn fastlanes_zero_decode_roundtrip_and_slice() -> VortexResult<()> { + let original = PrimitiveArray::from_option_iter((0_u32..4097).map(|index| { + (index % 17 != 0).then(|| { + let mantissa = index.wrapping_mul(7_919) & 0x007f_ffff; + f64::from(f32::from_bits(0x3f80_0000 | mantissa)) + }) + })); + let analysis = analyze_float_quant(original.as_view()).vortex_expect("FloatQuant input"); + assert_eq!(analysis.secondary_bit_width, 0); + let primary = FloatQuant::primary_for_primitive( + original.as_view(), + analysis.k, + analysis.primary_min, + )?; + // SAFETY: The analysis computes the exact primary width. + let primary = unsafe { + vortex_fastlanes::bitpack_compress::bitpack_encode_unchecked( + primary, + analysis.primary_bit_width, + )? + }; + let primary = FoR::try_new(primary.into_array(), Scalar::from(analysis.primary_min))?; + let encoded = + FloatQuant::try_new(primary.into_array(), None, PType::F64, analysis.k)?.into_array(); + + let mut ctx = SESSION.create_execution_ctx(); + assert_arrays_eq!(encoded, original, &mut ctx); + assert_arrays_eq!( + encoded.slice(3..2051)?, + original.into_array().slice(3..2051)?, + &mut ctx + ); + Ok(()) + } + #[test] fn fastlanes_pair_decode_roundtrip_and_slice() -> VortexResult<()> { let original = PrimitiveArray::from_iter((0_u32..4097).map(|index| { From be6635ebba0b2e34b0fa96654af17a0076242c81 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 14:11:48 +0100 Subject: [PATCH 70/78] bench: calibrate wider BlockResidual selection Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index ace5b557f26..1150885f957 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -2221,6 +2221,31 @@ The complete corpus shows no selected FloatQuant tree and no stable analysis-cos Retain FloatQuant as an opportunistic option under the calibrated 1.10 factor. +### Wider-integer BlockResidual factor sweep + +The final sweep compares wider-integer access cost factors of 1.00, 1.02, and 1.05. + +Each compression pass uses six complete Public BI files and three iterations per operation. + +The access pass requests about 100 rows through cached file handles. Each pattern runs for five seconds. + +| Factor | Size geometric mean | Total bytes | Compression time geometric mean | Decompression time geometric mean | Correlated access | Uniform access | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| 1.02 | -0.25 percent | +0.44 percent | -0.46 percent | +2.17 percent | +3.93 percent | +1.88 percent | +| 1.05 | +0.92 percent | +1.49 percent | -0.05 percent | +5.04 percent | +1.21 percent | +0.03 percent | + +The first candidate access passes contain large Bimbo and CMS correlated outliers. + +Reverse-order repeats remove those outliers. The table uses the repeated correlated results. + +The 1.02 factor diverts enough Food trees to reduce geometric-mean size by 0.25 percent. + +It increases total bytes by 0.44 percent. It also reduces bulk decode and file-access throughput. + +The 1.05 factor increases both geometric-mean size and total bytes. It provides no stable access gain. + +Retain the 1.00 wider-integer factor. The raw ratio floor and patch cost remain sufficient. + ### Final selector factors The integer BlockResidual scheme keeps a 1.05 raw-ratio floor. It applies a 1.12 access cost factor to 8-bit estimates. @@ -2231,7 +2256,7 @@ The Ordered BlockResidual scheme keeps its 1.05 raw floor and 1.02 decode factor The nonlinear patch cost rejects the known dense-patch and slow HashTags trees. -No selected-tree evidence supports a factor for wider integers. +The 1.02 and 1.05 trials provide no evidence for a wider-integer factor. The FloatQuant scheme keeps its 1.10 factor. @@ -2669,6 +2694,7 @@ This round completed these steps: - Measured complete-file random access for six Public BI datasets. - Added a fused `f16` decoder for `OrderedFloat(BlockResidual)`. - Added a fused implicit-zero `f64` decoder for `FloatQuant(FoR(BitPacked))`. +- Rejected 1.02 and 1.05 wider-integer BlockResidual access cost factors. The Pco mode profile and the quotient and remainder experiments are complete. From 2bc86a8ff451f19c97d289ef978657a6fd1ea071 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Thu, 20 Aug 2026 15:30:24 +0100 Subject: [PATCH 71/78] perf: avoid repeated BlockResidual validation Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 38 +++++++++++++------ .../src/block_residual_array.rs | 29 ++++++++------ vortex/benches/single_encoding_throughput.rs | 18 +++++++++ 3 files changed, 62 insertions(+), 23 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 1150885f957..ca44089a090 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -2364,26 +2364,40 @@ BlockResidual, Current Default, and RangePacked remain within 2.5 percent on bot The Public BI pass uses six complete files and cached handles. Each pattern requests about 100 rows. -Two passes use five-second target windows. A ten-second CMS rerun replaces one correlated outlier. +The first pass exposed repeated BlockResidual payload validation during each slice operation. + +The source array already passed validation. The slice only changes its logical bounds. + +The slice path now constructs the narrowed array without repeated payload validation. + +The valid patch-position branch now avoids a non-inlined fallible call for each patch. + +A focused CMS correlated pass decreased from 1.127 ms to 0.964 ms after both changes. + +The Prior Default control changed from 0.991 ms to 0.954 ms across the same runs. + +The CMS gap therefore decreased from 13.7 percent to 1.1 percent. + +Two new passes use five-second target windows. The second pass uses reverse dataset order. | Dataset | File size | Correlated latency | Uniform latency | | --- | ---: | ---: | ---: | -| Arade | -2.39 percent | -9.74 to -0.38 percent | -0.72 to +0.86 percent | -| Bimbo | -7.54 percent | +5.88 to +17.20 percent | +7.07 to +7.85 percent | -| CMSprovider | -7.86 percent | +14.03 to +18.46 percent | +9.89 to +11.73 percent | -| Euro2016 | -2.44 percent | -0.06 to +5.27 percent | -0.44 to +0.50 percent | -| Food | -6.19 percent | -16.78 to -10.54 percent | +12.19 to +13.94 percent | -| HashTags | -2.56 percent | +7.28 to +13.38 percent | +4.66 to +9.94 percent | +| Arade | -2.39 percent | -15.75 to -6.11 percent | -5.11 to +2.75 percent | +| Bimbo | -7.54 percent | -8.28 to +1.96 percent | -1.09 to +7.99 percent | +| CMSprovider | -7.86 percent | -2.22 to +9.22 percent | -6.64 to +11.82 percent | +| Euro2016 | -2.44 percent | +1.55 to +2.64 percent | -1.76 to +3.54 percent | +| Food | -6.19 percent | -29.89 to -15.70 percent | -2.83 to +6.97 percent | +| HashTags | -2.56 percent | +3.23 to +6.85 percent | +8.34 to +13.26 percent | -The file-size geometric mean decreases by 4.86 percent. Aggregate bytes decrease by 5.81 percent. +The file-size geometric mean still decreases by 4.86 percent. Aggregate bytes still decrease by 5.81 percent. -Across both passes, correlated latency increases by 3.06 percent geometrically. Uniform latency increases by 6.33 percent. +Across both passes, correlated latency decreases by 5.05 percent geometrically. Uniform latency increases by 2.91 percent. -The combined latency geometric mean increases by 4.69 percent. +The combined latency geometric mean decreases by 1.15 percent. -The Public BI pass adds useful evidence beyond scalar benchmarks. It exposes a measurable file-access trade for the conservative bundle. +The individual file values remain noisy. The aggregate result removes the earlier random-access trade for the conservative bundle. -The final threshold review must include this result. The current factor remains unchanged until that review. +The selector factors remain unchanged. ### Compact and Parquet size constraint diff --git a/encodings/block-residual/src/block_residual_array.rs b/encodings/block-residual/src/block_residual_array.rs index 37101380ad4..02d7f77d322 100644 --- a/encodings/block-residual/src/block_residual_array.rs +++ b/encodings/block-residual/src/block_residual_array.rs @@ -324,12 +324,11 @@ impl ValidityVTable for BlockResidual { impl SliceReduce for BlockResidual { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { let data = array.data().slice(range); + let parts = ArrayParts::new(BlockResidual, array.dtype().clone(), data.len(), data) + .with_slots(array.slots().iter().cloned().collect()); + // SAFETY: The source array is valid. The slice only narrows its logical bounds. Ok(Some( - Array::try_from_parts( - ArrayParts::new(BlockResidual, array.dtype().clone(), data.len(), data) - .with_slots(array.slots().iter().cloned().collect()), - )? - .into_array(), + unsafe { Array::from_parts_unchecked(parts) }.into_array(), )) } } @@ -1278,17 +1277,25 @@ fn validate_patch_header( Ok(()) } +#[inline(always)] fn validate_patch_position( block_len: usize, previous_position: Option, position: u16, ) -> VortexResult<()> { - vortex_ensure!( - usize::from(position) < block_len - && previous_position.is_none_or(|previous| previous < position), - "block residual patch positions are invalid" - ); - Ok(()) + if usize::from(position) < block_len + && previous_position.is_none_or(|previous| previous < position) + { + Ok(()) + } else { + invalid_patch_position() + } +} + +#[cold] +#[inline(never)] +fn invalid_patch_position() -> VortexResult<()> { + vortex_bail!("block residual patch positions are invalid") } fn validate_offset_table( diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index 7c5279394b7..159c4ec9a59 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -1096,6 +1096,24 @@ fn block_local_block_residual_scalar_at(bencher: Bencher) { .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); } +#[divan::bench(name = "block_residual_slice_patched_u32")] +fn bench_block_residual_slice_patched_u32(bencher: Bencher) { + const SLICE_LEN: usize = 100; + + let encoded = BlockResidual::from_primitive(setup_patch_density_u32_array(16).as_view()) + .unwrap() + .into_array(); + let next_index = AtomicUsize::new(0); + + bencher + .with_inputs(|| { + let start = next_index.fetch_add(2_654_435_761, Ordering::Relaxed) + % (encoded.len() - SLICE_LEN); + (&encoded, start) + }) + .bench_values(|(array, start)| array.slice(start..start + SLICE_LEN).unwrap()); +} + #[divan::bench(name = "block_local_for_bitpacked_compress_u64")] fn bench_block_local_for_bitpacked_compress_u64(bencher: Bencher) { let array = setup_block_local_u64_array(); From 5cbc73dfe2fc5f3391cef693a9d049cd2faa3ae7 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Fri, 21 Aug 2026 01:32:17 +0100 Subject: [PATCH 72/78] bench: map fixed-bin coverage Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 94 +++++++++++++++++-- .../examples/float_compressor_bench.rs | 58 ++++++------ 2 files changed, 118 insertions(+), 34 deletions(-) diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index ca44089a090..50cdd71ed7a 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -2318,7 +2318,9 @@ The evidence supports three bundle options: | Opportunistic | BlockResidual and FloatQuant | 16 files | Conservative | 0.00 percent | Noise | Noise | | Fixed-bin experiment | Opportunistic plus RangePacked | 8 numeric files | Opportunistic | -0.054 percent | -1.03 percent | Noise | -The conservative option pushes the size frontier without a material geometric-mean throughput loss. +The opportunistic option is the leading Default bundle. + +It adds strict synthetic FloatQuant wins without a measured corpus size regression. The opportunistic option selects no FloatQuant tree in these eight files. @@ -2332,7 +2334,9 @@ The tree decodes at about 10.1 GB/s instead of 15.6 to 16.1 GB/s. Scalar access takes 312 to 323 ns instead of 150 to 158 ns. -RangePacked therefore remains outside Default. +The absolute decode rate remains valid for Default consideration. + +RangePacked remains an active experiment until the broad corpus establishes its coverage and total cost. ### Default bundle random access @@ -2399,6 +2403,72 @@ The individual file values remain noisy. The aggregate result removes the earlie The selector factors remain unchanged. +### Broad fixed-bin coverage pass + +The Pco mode scan covers 53 float columns across 12 real datasets. + +It uses 524,288 rows per large column. This limit includes two complete Pco chunks. + +Pco selects classic bins without Delta on 41 chunks. It selects classic bins with consecutive Delta on another 17 chunks. + +Four columns use direct classic bins without Delta: + +| Dataset and column | Compact gap | Maximum Pco bins | +| --- | ---: | ---: | +| HashTags `interaction#received_at` | 65.2 percent | 36 | +| Taxi `airport_fee` | 53.6 percent | 4 | +| Euro2016 `subjectivity_confidence` | 51.1 percent | 39 | +| Taxi `tips` | 45.4 percent | 17 | + +The current corpus therefore contains the direct fixed-bin distribution. + +Another 21 columns use classic bins only below an outer transform such as ALP. + +Fifteen of these columns have a Compact gap of at least ten percent. + +Twelve also use at most 64 Pco bins. Six have a Compact gap of at least 20 percent. + +The complete bundle pass uses 18 files and three iterations per operation. + +It adds the CMS Payments and Twitter Pco inputs to the standard 16-file corpus. + +| Fixed-bin change against opportunistic | Geometric mean | Aggregate | +| --- | ---: | ---: | +| Size | -0.024 percent | -0.034 percent | +| Write throughput | -0.45 percent | Noise | +| Read throughput | -0.65 percent | Noise | + +Only HashTags changes its file size. The other 17 files retain identical sizes. + +The HashTags file becomes 0.429 percent smaller. + +Its selected `interaction#received_at` tree changes from 3,263,939 bytes to 1,991,606 bytes. + +The column becomes 39.0 percent smaller. Decode throughput changes from 15.79 GB/s to 11.44 GB/s. + +Scalar access changes from 165 ns to 322 ns. + +Fresh files from the same run produce this cached file-access result: + +| Pattern | Opportunistic | Fixed-bin | Change | +| --- | ---: | ---: | ---: | +| Correlated | 1.978 ms | 2.009 ms | +1.58 percent | +| Uniform | 12.142 ms | 12.018 ms | -1.02 percent | + +The selected tree passes the absolute decode and bounded-access bar. + +The current direct scheme remains niche. Its full-suite size gain does not justify Default inclusion alone. + +The nested Pco evidence defines the next fixed-bin prototype. + +Add a pure integer scheme that constructs this tree: + +`IntMult(base=1, Dict(BitPacked(codes), starts), BlockResidual(offsets))` + +Let ALP and other outer schemes select it through normal child recursion. + +Keep the black-box RangePacked array outside Default. + ### Compact and Parquet size constraint The format pass uses three iterations per operation. @@ -2709,6 +2779,10 @@ This round completed these steps: - Added a fused `f16` decoder for `OrderedFloat(BlockResidual)`. - Added a fused implicit-zero `f64` decoder for `FloatQuant(FoR(BitPacked))`. - Rejected 1.02 and 1.05 wider-integer BlockResidual access cost factors. +- Added a fast Pco mode-only profile path. +- Profiled Pco modes across 53 float columns from 12 real datasets. +- Compared opportunistic and fixed-bin bundles across 18 complete files. +- Measured the selected HashTags fixed-bin tree and fresh cached file access. The Pco mode profile and the quotient and remainder experiments are complete. @@ -2720,12 +2794,16 @@ The focused Default bundle now includes full-position RangePacked as an experime The specialized OrderedFloat and ALP trees now use registered fused parent kernels. +Use the opportunistic bundle as the leading Default candidate. + Complete these next experiments in order: -1. Use the Public BI file-access evidence during the final threshold review. -2. Prefer cheap rejection tests when they preserve the best candidate model. -3. Treat fast rejection as a selection advantage, not an admission criterion. -4. Require stronger corpus evidence when a useful scheme needs costly analysis. +1. Prototype a pure integer fixed-bin scheme for normal child recursion. +2. Add a fast model that rejects poor integer fixed-bin fits. +3. Test the scheme on the 12 nested Pco targets with at most 64 bins. +4. Compare each selected tree against the complete incumbent child cascade. +5. Repeat the 18-file bundle and access passes. +6. Decide fixed-bin admission from coverage, total cost, and bounded random access. Use packed bin codes and primitive bin starts unless evidence supports more complexity. @@ -2733,7 +2811,9 @@ Let ordinary child arrays own patches and exception payloads. Add an outer OrderedFloat fused decode only if the generic composition misses the decode gate. -Keep fused RangePacked and RangeEntropy outside Default. +Keep the black-box RangePacked array and RangeEntropy outside Default. + +Retain decomposed fixed bins as an active Default candidate. After the candidate set stabilizes, complete these final steps: diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index c5672c6ad4d..861ff098cdf 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -2776,6 +2776,36 @@ fn profile_pco_array( .execute::(&mut ctx)?; let ordered = ordered_values(&values); let validity_bytes = array.children().iter().map(ArrayRef::nbytes).sum::(); + match values.ptype() { + PType::F32 => { + profile_pco_values(dataset, column, path, PType::F32, values.as_slice::())? + } + PType::F64 => { + profile_pco_values(dataset, column, path, PType::F64, values.as_slice::())? + } + PType::I16 => { + profile_pco_values(dataset, column, path, PType::I16, values.as_slice::())? + } + PType::I32 => { + profile_pco_values(dataset, column, path, PType::I32, values.as_slice::())? + } + PType::I64 => { + profile_pco_values(dataset, column, path, PType::I64, values.as_slice::())? + } + PType::U16 => { + profile_pco_values(dataset, column, path, PType::U16, values.as_slice::())? + } + PType::U32 => { + profile_pco_values(dataset, column, path, PType::U32, values.as_slice::())? + } + PType::U64 => { + profile_pco_values(dataset, column, path, PType::U64, values.as_slice::())? + } + ptype => return Err(vortex_err!("cannot profile Pco ptype {ptype}")), + } + if std::env::var_os("VORTEX_BENCH_PCO_MODES_ONLY").is_some() { + return Ok(()); + } if std::env::var_os("VORTEX_BENCH_FIXED_BIN").is_some() { profile_range_packed( dataset, @@ -2815,33 +2845,6 @@ fn profile_pco_array( one_reference.min(two_references).min(four_references), bitmap_patches, ); - match values.ptype() { - PType::F32 => { - profile_pco_values(dataset, column, path, PType::F32, values.as_slice::())? - } - PType::F64 => { - profile_pco_values(dataset, column, path, PType::F64, values.as_slice::())? - } - PType::I16 => { - profile_pco_values(dataset, column, path, PType::I16, values.as_slice::())? - } - PType::I32 => { - profile_pco_values(dataset, column, path, PType::I32, values.as_slice::())? - } - PType::I64 => { - profile_pco_values(dataset, column, path, PType::I64, values.as_slice::())? - } - PType::U16 => { - profile_pco_values(dataset, column, path, PType::U16, values.as_slice::())? - } - PType::U32 => { - profile_pco_values(dataset, column, path, PType::U32, values.as_slice::())? - } - PType::U64 => { - profile_pco_values(dataset, column, path, PType::U64, values.as_slice::())? - } - ptype => return Err(vortex_err!("cannot profile Pco ptype {ptype}")), - } } for (child_index, child) in array.children().iter().enumerate() { profile_pco_array( @@ -2967,6 +2970,7 @@ fn measure_dataset( )?; if compact_bytes * 10 <= default_bytes * 9 || std::env::var_os("VORTEX_BENCH_FIXED_BIN").is_some() + || std::env::var_os("VORTEX_BENCH_PCO_MODES_ONLY").is_some() { profile_pco_array(dataset, &column.name, "root", compact_array, session)?; } From b73d5a091c1e5ee3aeeee1b2f28b77ac9aee79c0 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Fri, 21 Aug 2026 02:10:19 +0100 Subject: [PATCH 73/78] bench: test integer fixed bins under ALP Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 83 +++- encodings/range-packed/src/lib.rs | 27 +- .../examples/float_compressor_bench.rs | 8 + vortex-btrblocks/src/schemes/fixed_bins.rs | 74 +++ .../src/schemes/float/range_packed.rs | 72 +-- .../src/schemes/integer/fixed_bins.rs | 446 ++++++++++++++++++ vortex-btrblocks/src/schemes/integer/mod.rs | 2 + vortex-btrblocks/src/schemes/mod.rs | 1 + 8 files changed, 628 insertions(+), 85 deletions(-) create mode 100644 vortex-btrblocks/src/schemes/fixed_bins.rs create mode 100644 vortex-btrblocks/src/schemes/integer/fixed_bins.rs diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 50cdd71ed7a..a841ea61762 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -56,7 +56,9 @@ Prototype fixed bins as a composed tree instead: The prototype permits any bin count from one through 64. -The BtrBlocks selector and final child choices remain incomplete. +The branch includes an experimental `IntegerFixedBinsScheme` for the composed tree. + +The scheme runs only below ALP. It remains outside `ALL_SCHEMES`. Do not add the fused RangePacked array to the Default or Compact selector. @@ -68,7 +70,7 @@ The branch implements `OrderedFloatArray`, `BlockResidualArray`, `FloatQuantArra The evaluation set includes BtrBlocks schemes for the first three arrays. -IntMult does not have a BtrBlocks scheme yet. +The experimental BtrBlocks scheme for IntMult applies only to ALP integer children. `OrderedFloat(BlockResidual)` now supports `f16`, `f32`, and `f64` inputs. @@ -102,7 +104,9 @@ The range decomposition prototype uses `IntMult(base=1)` as generic addition. Its fused decode path skips multiplication and avoids materialization of dictionary references. -Neither IntMult nor decomposed fixed bins participate in the Default selector yet. +IntMult participates only through explicit experimental schemes. + +Decomposed fixed bins do not participate in the Default selector. Final calibration requires the complete corpus and selected-tree evidence. @@ -2459,16 +2463,60 @@ The selected tree passes the absolute decode and bounded-access bar. The current direct scheme remains niche. Its full-suite size gain does not justify Default inclusion alone. -The nested Pco evidence defines the next fixed-bin prototype. +The nested Pco evidence motivated the pure integer fixed-bin prototype. -Add a pure integer scheme that constructs this tree: +The experiment added a pure integer scheme that constructs this tree: `IntMult(base=1, Dict(BitPacked(codes), starts), BlockResidual(offsets))` -Let ALP and other outer schemes select it through normal child recursion. +ALP selects it through normal child recursion. Keep the black-box RangePacked array outside Default. +### Pure integer fixed-bin result + +The prototype constructs this complete integer tree: + +`IntMult(base=1, Dict(BitPacked(codes), starts), BlockResidual(offsets))` + +It supports every signed and unsigned integer type. + +Signed inputs flip the sign bit before the bin fit. The tree restores starts and offsets with modular addition. + +Tests cover every integer width, nullable values, signed-domain boundaries, and composition below ALP. + +The bin trainer previously used one fixed stride. Periodic inputs could alias that stride and hide complete clusters. + +A deterministic offset within each sample stratum now covers periodic phases. One regression test covers four interleaved clusters. + +The first selector version examined every integer recursion site. + +Across nine available datasets, this version reduced write throughput by 6.85 percent geometric mean. + +An ALP-child gate reduced the write loss to 2.78 percent geometric mean. + +The focused pass used 524,288 rows per dataset. + +It covered Arade, Bimbo, both CMS Provider files, Euro2016, Food, HashTags, CMS Payments, and GloVe embeddings. + +The fixed-bin scheme selected no tree. Every encoded size matched the Opportunistic bundle exactly. + +This result agrees with the earlier forced-tree measurements. + +One generic offset child mixes residuals from bins with different local widths. + +BlockResidual then pays for local width variation and occasional cross-bin outliers. + +The Pco bin model retains one offset width per bin. The generic tree loses that advantage. + +The pure integer fixed-bin scheme therefore does not enter Default. + +Keep the implementation as an experimental reference. Do not spend more selector time on this tree without new evidence. + +The direct OrderedFloat fixed-bin scheme remains a separate experiment. + +It still reduces HashTags `interaction#received_at` by 39.0 percent. Its complete file gain remains 0.429 percent. + ### Compact and Parquet size constraint The format pass uses three iterations per operation. @@ -2783,6 +2831,9 @@ This round completed these steps: - Profiled Pco modes across 53 float columns from 12 real datasets. - Compared opportunistic and fixed-bin bundles across 18 complete files. - Measured the selected HashTags fixed-bin tree and fresh cached file access. +- Added the pure integer fixed-bin scheme for ALP child recursion. +- Fixed alias errors in the deterministic fixed-bin sample. +- Rejected the integer fixed-bin tree after a focused nine-dataset pass. The Pco mode profile and the quotient and remainder experiments are complete. @@ -2796,14 +2847,18 @@ The specialized OrderedFloat and ALP trees now use registered fused parent kerne Use the opportunistic bundle as the leading Default candidate. -Complete these next experiments in order: +The pure integer fixed-bin experiment is complete. + +It selected no real tree and reduced write throughput. Keep it outside Default. + +Use these next steps: -1. Prototype a pure integer fixed-bin scheme for normal child recursion. -2. Add a fast model that rejects poor integer fixed-bin fits. -3. Test the scheme on the 12 nested Pco targets with at most 64 bins. -4. Compare each selected tree against the complete incumbent child cascade. -5. Repeat the 18-file bundle and access passes. -6. Decide fixed-bin admission from coverage, total cost, and bounded random access. +1. Keep the Opportunistic bundle as the leading Default candidate. +2. Retain the direct OrderedFloat fixed-bin bundle as an optional experiment. +3. Revisit direct fixed bins when the corpus gains more applicable distributions. +4. Complete the final threshold review for the retained schemes. +5. Compare the retained bundle across the complete file corpus. +6. Add more real embedding datasets when licenses and loaders permit them. Use packed bin codes and primitive bin starts unless evidence supports more complexity. @@ -2813,7 +2868,7 @@ Add an outer OrderedFloat fused decode only if the generic composition misses th Keep the black-box RangePacked array and RangeEntropy outside Default. -Retain decomposed fixed bins as an active Default candidate. +Retain decomposed fixed bins only as an experimental reference. After the candidate set stabilizes, complete these final steps: diff --git a/encodings/range-packed/src/lib.rs b/encodings/range-packed/src/lib.rs index d3694e30630..47c12516608 100644 --- a/encodings/range-packed/src/lib.rs +++ b/encodings/range-packed/src/lib.rs @@ -833,7 +833,12 @@ fn training_sample(values: &[u64]) -> (Vec, u64, u64) { values.to_vec() } else { (0..TRAINING_SAMPLE_SIZE) - .map(|index| values[index * values.len() / TRAINING_SAMPLE_SIZE]) + .map(|index| { + let start = index * values.len() / TRAINING_SAMPLE_SIZE; + let stop = (index + 1) * values.len() / TRAINING_SAMPLE_SIZE; + let offset = index.wrapping_mul(2_654_435_761) % (stop - start); + values[start + offset] + }) .collect() }; sample.push(minimum); @@ -1010,6 +1015,26 @@ mod tests { Ok(()) } + #[test] + fn decomposition_sample_covers_periodic_clusters() -> VortexResult<()> { + let values = (0_u64..65_536) + .map(|index| (index % 4) * 4_000_000_000_000_000_000 + index % 1_024) + .collect::>(); + let decomposition = RangeDecomposition::encode(&values)?; + let distant_offsets = decomposition + .offsets() + .iter() + .filter(|&&offset| offset >= 1_000_000) + .count(); + + assert!(decomposition.bin_starts().len() >= 4); + assert!( + distant_offsets < values.len() / 50, + "distant offsets: {distant_offsets}" + ); + Ok(()) + } + #[test] fn clustered_roundtrip_and_scalar_access() -> VortexResult<()> { let values = (0_u64..20_000) diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs index 861ff098cdf..eca4c56a03a 100644 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ b/vortex-btrblocks/examples/float_compressor_bench.rs @@ -75,6 +75,7 @@ use vortex_btrblocks::schemes::float::FloatQuantScheme; use vortex_btrblocks::schemes::float::OrderedBlockResidualScheme; use vortex_btrblocks::schemes::float::OrderedFloatRangePackedScheme; use vortex_btrblocks::schemes::integer::BlockResidualScheme; +use vortex_btrblocks::schemes::integer::IntegerFixedBinsScheme; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_err; @@ -98,6 +99,7 @@ use crate::int_mult_codec::IntMultDenseCodec64; const DEFAULT_ROW_COUNT: usize = 2_000_000; const RANGE_PACKED_SCHEME: OrderedFloatRangePackedScheme = OrderedFloatRangePackedScheme::new(1.20); +const INTEGER_FIXED_BINS_SCHEME: IntegerFixedBinsScheme = IntegerFixedBinsScheme::new(1.20); const CALIFORNIA_COLUMNS: [&str; 9] = [ "longitude", "latitude", @@ -3205,6 +3207,12 @@ fn compressors() -> Vec<(&'static str, BtrBlocksCompressor)> { .with_new_scheme(&RANGE_PACKED_SCHEME) .build(), ), + ( + "proposed-default-integer-fixed-bins", + BtrBlocksCompressorBuilder::default() + .with_new_scheme(&INTEGER_FIXED_BINS_SCHEME) + .build(), + ), ( "prior-compact", BtrBlocksCompressorBuilder::default() diff --git a/vortex-btrblocks/src/schemes/fixed_bins.rs b/vortex-btrblocks/src/schemes/fixed_bins.rs new file mode 100644 index 00000000000..01e730e4cb8 --- /dev/null +++ b/vortex-btrblocks/src/schemes/fixed_bins.rs @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Shared coarse models for fixed-bin scheme rejection. + +#[derive(Clone, Copy, Debug)] +pub(crate) struct CoarseModel { + pub(crate) range_ratio: f64, + pub(crate) block_ratio: f64, +} + +pub(crate) fn coarse_model(values: &[u64], source_width: usize) -> CoarseModel { + CoarseModel { + range_ratio: coarse_prefix_ratio(values, source_width), + block_ratio: coarse_block_ratio(values, source_width), + } +} + +fn coarse_prefix_ratio(values: &[u64], source_width: usize) -> f64 { + let Some((&minimum, &maximum)) = values.iter().min().zip(values.iter().max()) else { + return 0.0; + }; + let span_width = bit_width(maximum - minimum); + let mut best_bits = values.len() * usize::from(span_width); + for code_width in 1_u8..=6 { + if code_width > span_width { + break; + } + let bucket_count = 1usize << code_width; + let shift = span_width - code_width; + let mut counts = [0usize; 64]; + let mut minima = [u64::MAX; 64]; + let mut maxima = [0_u64; 64]; + for &value in values { + let relative = value - minimum; + let bucket = usize::try_from(relative >> shift).unwrap_or(bucket_count - 1); + let bucket = bucket.min(bucket_count - 1); + counts[bucket] += 1; + minima[bucket] = minima[bucket].min(value); + maxima[bucket] = maxima[bucket].max(value); + } + let offset_bits = (0..bucket_count) + .filter(|&bucket| counts[bucket] != 0) + .map(|bucket| counts[bucket] * usize::from(bit_width(maxima[bucket] - minima[bucket]))) + .sum::(); + best_bits = best_bits.min(values.len() * usize::from(code_width) + offset_bits); + } + + if best_bits == 0 { + f64::INFINITY + } else { + values.len() as f64 * source_width as f64 / best_bits as f64 + } +} + +fn coarse_block_ratio(values: &[u64], source_width: usize) -> f64 { + let block_bits = values + .chunks(64) + .map(|block| { + let minimum = block.iter().copied().min().unwrap_or_default(); + let maximum = block.iter().copied().max().unwrap_or_default(); + block.len() * usize::from(bit_width(maximum - minimum)) + }) + .sum::(); + if block_bits == 0 { + f64::INFINITY + } else { + values.len() as f64 * source_width as f64 / block_bits as f64 + } +} + +fn bit_width(value: u64) -> u8 { + u8::try_from(u64::BITS - value.leading_zeros()).unwrap_or(u8::MAX) +} diff --git a/vortex-btrblocks/src/schemes/float/range_packed.rs b/vortex-btrblocks/src/schemes/float/range_packed.rs index 0cf90c045a0..7f9db4147bf 100644 --- a/vortex-btrblocks/src/schemes/float/range_packed.rs +++ b/vortex-btrblocks/src/schemes/float/range_packed.rs @@ -34,6 +34,8 @@ use crate::CascadingCompressor; use crate::CompressorContext; use crate::Scheme; use crate::SchemeExt; +use crate::schemes::fixed_bins::CoarseModel; +use crate::schemes::fixed_bins::coarse_model; use crate::schemes::sample_primitive_one_percent; /// The default factor accounts for RangePacked decode cost during scheme selection. @@ -183,12 +185,6 @@ fn model_prefers_blocks(model: CoarseModel) -> bool { model.block_ratio > model.range_ratio * MAX_BLOCK_TO_RANGE_RATIO } -#[derive(Clone, Copy, Debug)] -struct CoarseModel { - range_ratio: f64, - block_ratio: f64, -} - fn coarse_ordered_float_model(primitive: vortex_array::ArrayView<'_, Primitive>) -> CoarseModel { let values: Vec = match primitive.ptype() { PType::F16 => primitive @@ -216,70 +212,6 @@ fn coarse_ordered_float_model(primitive: vortex_array::ArrayView<'_, Primitive>) coarse_model(&values, primitive.ptype().bit_width()) } -fn coarse_model(values: &[u64], source_width: usize) -> CoarseModel { - CoarseModel { - range_ratio: coarse_prefix_ratio(values, source_width), - block_ratio: coarse_block_ratio(values, source_width), - } -} - -fn coarse_prefix_ratio(values: &[u64], source_width: usize) -> f64 { - let Some((&minimum, &maximum)) = values.iter().min().zip(values.iter().max()) else { - return 0.0; - }; - let span_width = bit_width(maximum - minimum); - let mut best_bits = values.len() * usize::from(span_width); - for code_width in 1_u8..=6 { - if code_width > span_width { - break; - } - let bucket_count = 1usize << code_width; - let shift = span_width - code_width; - let mut counts = [0usize; 64]; - let mut minima = [u64::MAX; 64]; - let mut maxima = [0_u64; 64]; - for &value in values { - let relative = value - minimum; - let bucket = usize::try_from(relative >> shift).unwrap_or(bucket_count - 1); - let bucket = bucket.min(bucket_count - 1); - counts[bucket] += 1; - minima[bucket] = minima[bucket].min(value); - maxima[bucket] = maxima[bucket].max(value); - } - let offset_bits = (0..bucket_count) - .filter(|&bucket| counts[bucket] != 0) - .map(|bucket| counts[bucket] * usize::from(bit_width(maxima[bucket] - minima[bucket]))) - .sum::(); - best_bits = best_bits.min(values.len() * usize::from(code_width) + offset_bits); - } - - if best_bits == 0 { - f64::INFINITY - } else { - values.len() as f64 * source_width as f64 / best_bits as f64 - } -} - -fn coarse_block_ratio(values: &[u64], source_width: usize) -> f64 { - let block_bits = values - .chunks(64) - .map(|block| { - let minimum = block.iter().copied().min().unwrap_or_default(); - let maximum = block.iter().copied().max().unwrap_or_default(); - block.len() * usize::from(bit_width(maximum - minimum)) - }) - .sum::(); - if block_bits == 0 { - f64::INFINITY - } else { - values.len() as f64 * source_width as f64 / block_bits as f64 - } -} - -fn bit_width(value: u64) -> u8 { - u8::try_from(u64::BITS - value.leading_zeros()).unwrap_or(u8::MAX) -} - fn ordered_u16(bits: u16) -> u16 { if bits & (1_u16 << 15) == 0 { bits ^ (1_u16 << 15) diff --git a/vortex-btrblocks/src/schemes/integer/fixed_bins.rs b/vortex-btrblocks/src/schemes/integer/fixed_bins.rs new file mode 100644 index 00000000000..68908e8381c --- /dev/null +++ b/vortex-btrblocks/src/schemes/integer/fixed_bins.rs @@ -0,0 +1,446 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Integer compression with fixed ranges and block-local offsets. + +use vortex_array::ArrayId; +use vortex_array::ArrayRef; +use vortex_array::Canonical; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::VTable; +use vortex_array::arrays::Dict; +use vortex_array::arrays::DictArray; +use vortex_array::arrays::Primitive; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::dtype::PType; +use vortex_block_residual::BlockResidual; +use vortex_compressor::scheme::CompressionEstimate; +use vortex_compressor::scheme::DeferredEstimate; +use vortex_compressor::scheme::EstimateScore; +use vortex_compressor::scheme::EstimateVerdict; +use vortex_error::VortexResult; +use vortex_fastlanes::BitPacked; +use vortex_fastlanes::bitpack_compress::bitpack_encode_unchecked; +use vortex_int_mult::IntMult; +use vortex_range_packed::RangeDecomposition; + +use crate::ArrayAndStats; +use crate::CascadingCompressor; +use crate::CompressorContext; +use crate::Scheme; +use crate::SchemeExt; +use crate::normalize_null_values; +use crate::schemes::fixed_bins::CoarseModel; +use crate::schemes::fixed_bins::coarse_model; +use crate::schemes::float::ALPScheme; +use crate::schemes::sample_primitive_one_percent; + +const DEFAULT_DECODE_COST_FACTOR: f64 = 1.20; +const PREFILTER_MODEL_MARGIN: f64 = 1.50; +const MIN_PREFILTER_MODEL_RATIO: f64 = 1.15; +const MAX_BLOCK_TO_RANGE_RATIO: f64 = 1.10; + +/// Compress integers with at most 64 range starts and block-local offsets. +#[derive(Debug, Copy, Clone, PartialEq)] +pub struct IntegerFixedBinsScheme { + decode_cost_factor: f64, +} + +impl IntegerFixedBinsScheme { + /// Creates a scheme with the specified decode cost factor. + pub const fn new(decode_cost_factor: f64) -> Self { + Self { decode_cost_factor } + } +} + +impl Default for IntegerFixedBinsScheme { + fn default() -> Self { + Self::new(DEFAULT_DECODE_COST_FACTOR) + } +} + +impl Scheme for IntegerFixedBinsScheme { + fn scheme_name(&self) -> &'static str { + "vortex.int.fixed_bins" + } + + fn matches(&self, canonical: &Canonical) -> bool { + canonical.dtype().is_int() + } + + fn produced_encodings(&self) -> Vec { + vec![IntMult.id(), Dict.id(), BitPacked.id(), BlockResidual.id()] + } + + fn num_children(&self) -> usize { + 2 + } + + fn expected_compression_ratio( + &self, + _data: &ArrayAndStats, + compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> CompressionEstimate { + let is_alp_child = compress_ctx + .cascade_history() + .last() + .is_some_and(|(scheme, child)| *scheme == ALPScheme.id() && *child == 0); + if compress_ctx.is_sample() || !is_alp_child { + return CompressionEstimate::Verdict(EstimateVerdict::Skip); + } + + let decode_cost_factor = self.decode_cost_factor; + let scheme_id = self.id(); + CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( + move |compressor, data, best_so_far, compress_ctx, exec_ctx| { + let source_width = data.array_as_primitive().ptype().bit_width() as f64; + let best_ratio = best_so_far + .and_then(EstimateScore::finite_ratio) + .unwrap_or(1.0); + if source_width / decode_cost_factor <= best_ratio { + return Ok(EstimateVerdict::Skip); + } + + let sample = sample_primitive_one_percent(data.array_as_primitive(), exec_ctx)?; + let sample = normalize_null_values(sample.as_view(), exec_ctx)?; + let ordered = ordered_integer_values(sample.as_view()); + let model = coarse_model(&ordered, sample.ptype().bit_width()); + if !prefilter_passes(model, best_ratio, decode_cost_factor) { + return Ok(EstimateVerdict::Skip); + } + + let encoded = encode_fixed_bins(sample.as_view())?; + let after_nbytes = encoded.nbytes(); + if after_nbytes == 0 { + return Ok(EstimateVerdict::Skip); + } + let adjusted_ratio = + sample.nbytes() as f64 / after_nbytes as f64 / decode_cost_factor; + if adjusted_ratio <= best_ratio { + return Ok(EstimateVerdict::Skip); + } + + let incumbent = compressor.compress_sample_without_scheme( + &sample.into_array(), + scheme_id, + compress_ctx, + exec_ctx, + )?; + if after_nbytes as f64 * decode_cost_factor >= incumbent.nbytes() as f64 { + return Ok(EstimateVerdict::Skip); + } + Ok(EstimateVerdict::Ratio(adjusted_ratio)) + }, + ))) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + data: &ArrayAndStats, + _compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, + ) -> VortexResult { + let primitive = normalize_null_values(data.array_as_primitive(), exec_ctx)?; + encode_fixed_bins(primitive.as_view()) + } +} + +fn prefilter_passes(model: CoarseModel, best_ratio: f64, decode_cost_factor: f64) -> bool { + model.range_ratio.is_finite() + && model.range_ratio >= MIN_PREFILTER_MODEL_RATIO + && model.range_ratio / decode_cost_factor * PREFILTER_MODEL_MARGIN > best_ratio + && model.block_ratio <= model.range_ratio * MAX_BLOCK_TO_RANGE_RATIO +} + +fn encode_fixed_bins(primitive: vortex_array::ArrayView<'_, Primitive>) -> VortexResult { + let ordered = ordered_integer_values(primitive); + let decomposition = RangeDecomposition::encode(&ordered)?; + let code_width = decomposition.code_width(); + let codes = PrimitiveArray::new(decomposition.codes().to_vec(), primitive.validity()?); + // SAFETY: The decomposition computes the exact code width. + let codes = unsafe { bitpack_encode_unchecked(codes, code_width) }?.into_array(); + let starts = restore_starts(decomposition.bin_starts(), primitive.ptype())?.into_array(); + let references = DictArray::try_new(codes, starts)?.into_array(); + let offsets = restore_offsets(decomposition.offsets(), primitive.ptype())?; + let offsets = BlockResidual::from_primitive(offsets.as_view())?.into_array(); + Ok(IntMult::try_new(references, offsets, 1)?.into_array()) +} + +fn ordered_integer_values(primitive: vortex_array::ArrayView<'_, Primitive>) -> Vec { + match primitive.ptype() { + PType::U8 => primitive + .as_slice::() + .iter() + .map(|&value| u64::from(value)) + .collect(), + PType::U16 => primitive + .as_slice::() + .iter() + .map(|&value| u64::from(value)) + .collect(), + PType::U32 => primitive + .as_slice::() + .iter() + .map(|&value| u64::from(value)) + .collect(), + PType::U64 => primitive.as_slice::().to_vec(), + PType::I8 => primitive + .as_slice::() + .iter() + .map(|&value| u64::from((value as u8) ^ (1 << 7))) + .collect(), + PType::I16 => primitive + .as_slice::() + .iter() + .map(|&value| u64::from((value as u16) ^ (1 << 15))) + .collect(), + PType::I32 => primitive + .as_slice::() + .iter() + .map(|&value| u64::from((value as u32) ^ (1 << 31))) + .collect(), + PType::I64 => primitive + .as_slice::() + .iter() + .map(|&value| (value as u64) ^ (1 << 63)) + .collect(), + ptype => unreachable!("fixed bins require integers, got {ptype}"), + } +} + +fn restore_starts(values: &[u64], ptype: PType) -> VortexResult { + Ok(match ptype { + PType::U8 => PrimitiveArray::from_iter( + values + .iter() + .copied() + .map(u8::try_from) + .collect::, _>>()?, + ), + PType::U16 => PrimitiveArray::from_iter( + values + .iter() + .copied() + .map(u16::try_from) + .collect::, _>>()?, + ), + PType::U32 => PrimitiveArray::from_iter( + values + .iter() + .copied() + .map(u32::try_from) + .collect::, _>>()?, + ), + PType::U64 => PrimitiveArray::from_iter(values.iter().copied()), + PType::I8 => PrimitiveArray::from_iter( + values + .iter() + .copied() + .map(u8::try_from) + .collect::, _>>()? + .into_iter() + .map(|value| i8::from_le_bytes([(value ^ (1 << 7))])), + ), + PType::I16 => PrimitiveArray::from_iter( + values + .iter() + .copied() + .map(u16::try_from) + .collect::, _>>()? + .into_iter() + .map(|value| i16::from_le_bytes((value ^ (1 << 15)).to_le_bytes())), + ), + PType::I32 => PrimitiveArray::from_iter( + values + .iter() + .copied() + .map(u32::try_from) + .collect::, _>>()? + .into_iter() + .map(|value| i32::from_le_bytes((value ^ (1 << 31)).to_le_bytes())), + ), + PType::I64 => PrimitiveArray::from_iter( + values + .iter() + .map(|&value| i64::from_le_bytes((value ^ (1 << 63)).to_le_bytes())), + ), + _ => vortex_error::vortex_bail!("fixed bins require integers, got {ptype}"), + }) +} + +fn restore_offsets(values: &[u64], ptype: PType) -> VortexResult { + Ok(match ptype { + PType::U8 => PrimitiveArray::from_iter( + values + .iter() + .copied() + .map(u8::try_from) + .collect::, _>>()?, + ), + PType::U16 => PrimitiveArray::from_iter( + values + .iter() + .copied() + .map(u16::try_from) + .collect::, _>>()?, + ), + PType::U32 => PrimitiveArray::from_iter( + values + .iter() + .copied() + .map(u32::try_from) + .collect::, _>>()?, + ), + PType::U64 => PrimitiveArray::from_iter(values.iter().copied()), + PType::I8 => PrimitiveArray::from_iter( + values + .iter() + .copied() + .map(u8::try_from) + .collect::, _>>()? + .into_iter() + .map(|value| i8::from_le_bytes([value])), + ), + PType::I16 => PrimitiveArray::from_iter( + values + .iter() + .copied() + .map(u16::try_from) + .collect::, _>>()? + .into_iter() + .map(|value| i16::from_le_bytes(value.to_le_bytes())), + ), + PType::I32 => PrimitiveArray::from_iter( + values + .iter() + .copied() + .map(u32::try_from) + .collect::, _>>()? + .into_iter() + .map(|value| i32::from_le_bytes(value.to_le_bytes())), + ), + PType::I64 => PrimitiveArray::from_iter( + values + .iter() + .map(|&value| i64::from_le_bytes(value.to_le_bytes())), + ), + _ => vortex_error::vortex_bail!("fixed bins require integers, got {ptype}"), + }) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::cast_possible_truncation)] + + use rstest::rstest; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::assert_arrays_eq; + use vortex_error::VortexResult; + + use super::IntegerFixedBinsScheme; + use super::encode_fixed_bins; + use crate::CascadingCompressor; + use crate::schemes::float::ALPScheme; + use crate::schemes::integer::BitPackingScheme; + use crate::schemes::integer::FoRScheme; + + const TEST_SCHEME: IntegerFixedBinsScheme = IntegerFixedBinsScheme::new(1.0); + + #[rstest] + #[case::u8(PrimitiveArray::from_iter((0_u16..4_096).map(|index| { + ((index % 4) * 64 + (index.wrapping_mul(37) % 8)) as u8 + })))] + #[case::u16(PrimitiveArray::from_iter((0_u32..4_096).map(|index| { + ((index % 4) * 16_000 + (index.wrapping_mul(37) % 32)) as u16 + })))] + #[case::u32(PrimitiveArray::from_iter((0_u64..4_096).map(|index| { + ((index % 4) * 1_000_000_000 + index.wrapping_mul(37) % 1_024) as u32 + })))] + #[case::u64(PrimitiveArray::from_iter((0_u64..4_096).map(|index| { + (index % 4) * 4_000_000_000_000_000_000 + index.wrapping_mul(37) % 1_024 + })))] + #[case::i8(PrimitiveArray::from_iter((0_i16..4_096).map(|index| { + (-120 + (index % 4) * 64 + (index.wrapping_mul(37) % 8)) as i8 + })))] + #[case::i16(PrimitiveArray::from_iter((0_i32..4_096).map(|index| { + (-30_000 + (index % 4) * 16_000 + (index.wrapping_mul(37) % 32)) as i16 + })))] + #[case::i32(PrimitiveArray::from_iter((0_i64..4_096).map(|index| { + (-2_000_000_000 + (index % 4) * 1_000_000_000 + index.wrapping_mul(37) % 1_024) as i32 + })))] + #[case::i64(PrimitiveArray::from_iter((0_i64..4_096).map(|index| { + [-8_000_000_000_000_000_000, -4_000_000_000_000_000_000, 0, 4_000_000_000_000_000_000] + [usize::try_from(index % 4).unwrap_or_default()] + + index.wrapping_mul(37) % 1_024 + })))] + fn fixed_bin_tree_roundtrips(#[case] expected: PrimitiveArray) -> VortexResult<()> { + let session = vortex_array::array_session(); + let mut ctx = session.create_execution_ctx(); + let compressed = encode_fixed_bins(expected.as_view())?; + + assert_arrays_eq!(compressed, expected, &mut ctx); + Ok(()) + } + + #[test] + fn nullable_signed_values_roundtrip() -> VortexResult<()> { + let expected = PrimitiveArray::from_option_iter((0_i64..65_536).map(|index| { + (index % 17 != 0).then(|| { + [ + -8_000_000_000_000_000_000, + -4_000_000_000_000_000_000, + 0, + 4_000_000_000_000_000_000, + ][usize::try_from(index % 4).unwrap_or_default()] + + index.wrapping_mul(37) % 1_024 + }) + })); + let session = vortex_array::array_session(); + let mut ctx = session.create_execution_ctx(); + let compressed = encode_fixed_bins(expected.as_view())?; + + assert_arrays_eq!(compressed, expected, &mut ctx); + Ok(()) + } + + #[test] + fn selector_composes_below_alp() -> VortexResult<()> { + let expected = PrimitiveArray::from_iter((0_i64..65_536).map(|index| { + [0_i64, 1_000_000_000, 2_000_000_000, 3_000_000_000] + [usize::try_from(index % 4).unwrap_or_default()] as f64 + + (index.wrapping_mul(37) % 1_024) as f64 + })); + let expected_array = expected.clone().into_array(); + let session = vortex_array::array_session(); + let mut ctx = session.create_execution_ctx(); + let compressed = CascadingCompressor::new(vec![ + &ALPScheme, + &TEST_SCHEME, + &FoRScheme, + &BitPackingScheme, + ]) + .compress(&expected_array, &mut ctx)?; + + assert!( + compressed.is::(), + "expected ALP, got tree:\n{}", + compressed.display_tree() + ); + let encoded = &compressed.children()[0]; + assert!( + encoded.is::(), + "expected IntMult child, got tree:\n{}", + compressed.display_tree() + ); + assert!(encoded.children()[0].is::()); + assert!(encoded.children()[0].children()[0].is::()); + assert!(encoded.children()[1].is::()); + assert_arrays_eq!(compressed, expected, &mut ctx); + Ok(()) + } +} diff --git a/vortex-btrblocks/src/schemes/integer/mod.rs b/vortex-btrblocks/src/schemes/integer/mod.rs index 43ff8bbb6cd..101f2a1ab0f 100644 --- a/vortex-btrblocks/src/schemes/integer/mod.rs +++ b/vortex-btrblocks/src/schemes/integer/mod.rs @@ -7,6 +7,7 @@ mod bitpacking; mod block_residual; #[cfg(feature = "unstable_encodings")] mod delta; +mod fixed_bins; mod for_; mod rle; mod runend; @@ -22,6 +23,7 @@ pub use block_residual::BlockResidualScheme; pub(crate) use block_residual::patch_adjusted_estimate_nbytes; #[cfg(feature = "unstable_encodings")] pub use delta::DeltaScheme; +pub use fixed_bins::IntegerFixedBinsScheme; pub use for_::FoRScheme; #[cfg(feature = "pco")] pub use pco::PcoScheme; diff --git a/vortex-btrblocks/src/schemes/mod.rs b/vortex-btrblocks/src/schemes/mod.rs index 0b9baab0b16..60938b5b15b 100644 --- a/vortex-btrblocks/src/schemes/mod.rs +++ b/vortex-btrblocks/src/schemes/mod.rs @@ -11,6 +11,7 @@ pub mod string; pub mod decimal; pub mod temporal; +pub(crate) mod fixed_bins; pub(crate) mod patches; use std::ops::Range; From d11660c5b4b6d3499e875103d3a4bd306b1681d4 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Fri, 21 Aug 2026 04:39:08 +0100 Subject: [PATCH 74/78] feat: finalize opportunistic numeric compression Signed-off-by: Will Manning --- Cargo.lock | 19 - Cargo.toml | 2 - benchmarks/compress-bench/README.md | 1 - benchmarks/compress-bench/src/main.rs | 4 +- benchmarks/random-access-bench/README.md | 1 - docs/plans/native-pcodec.md | 122 +- encodings/range-packed/Cargo.toml | 30 - encodings/range-packed/src/array.rs | 885 ----- encodings/range-packed/src/kernel.rs | 169 - encodings/range-packed/src/lib.rs | 1241 ------ vortex-bench/src/datasets/feature_vectors.rs | 25 + vortex-bench/src/lib.rs | 9 - vortex-btrblocks/Cargo.toml | 7 - .../examples/float_compressor_bench.rs | 3324 ----------------- .../float_compressor_bench/int_mult_codec.rs | 1057 ------ vortex-btrblocks/src/schemes/fixed_bins.rs | 74 - vortex-btrblocks/src/schemes/float/mod.rs | 2 - .../src/schemes/float/range_packed.rs | 526 --- .../src/schemes/integer/block_residual.rs | 8 +- .../src/schemes/integer/fixed_bins.rs | 446 --- vortex-btrblocks/src/schemes/integer/mod.rs | 2 - vortex-btrblocks/src/schemes/mod.rs | 1 - .../golden__default__list_of_int_runs.snap | 12 +- vortex-file/Cargo.toml | 1 - vortex-file/src/lib.rs | 1 - vortex/Cargo.toml | 1 - vortex/benches/single_encoding_throughput.rs | 62 - vortex/src/editions/core/v2026_08.rs | 1 - vortex/src/lib.rs | 5 - 29 files changed, 121 insertions(+), 7917 deletions(-) delete mode 100644 encodings/range-packed/Cargo.toml delete mode 100644 encodings/range-packed/src/array.rs delete mode 100644 encodings/range-packed/src/kernel.rs delete mode 100644 encodings/range-packed/src/lib.rs delete mode 100644 vortex-btrblocks/examples/float_compressor_bench.rs delete mode 100644 vortex-btrblocks/examples/float_compressor_bench/int_mult_codec.rs delete mode 100644 vortex-btrblocks/src/schemes/fixed_bins.rs delete mode 100644 vortex-btrblocks/src/schemes/float/range_packed.rs delete mode 100644 vortex-btrblocks/src/schemes/integer/fixed_bins.rs diff --git a/Cargo.lock b/Cargo.lock index 802a68a9122..9efe0098e44 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9560,7 +9560,6 @@ dependencies = [ "vortex-metrics", "vortex-pco", "vortex-proto", - "vortex-range-packed", "vortex-runend", "vortex-scan", "vortex-sequence", @@ -9777,7 +9776,6 @@ dependencies = [ "arrow-schema 58.4.0", "arrow-select 58.4.0", "codspeed-divan-compat", - "fastlanes", "insta", "itertools 0.14.0", "parquet 58.4.0", @@ -9799,11 +9797,9 @@ dependencies = [ "vortex-fastlanes", "vortex-float-quant", "vortex-fsst", - "vortex-int-mult", "vortex-mask", "vortex-onpair", "vortex-pco", - "vortex-range-packed", "vortex-runend", "vortex-sequence", "vortex-session", @@ -10201,7 +10197,6 @@ dependencies = [ "vortex-metrics", "vortex-onpair", "vortex-pco", - "vortex-range-packed", "vortex-runend", "vortex-scan", "vortex-sequence", @@ -10568,20 +10563,6 @@ dependencies = [ "vortex-python-abi", ] -[[package]] -name = "vortex-range-packed" -version = "0.1.0" -dependencies = [ - "rstest", - "vortex-alp", - "vortex-array", - "vortex-block-residual", - "vortex-buffer", - "vortex-error", - "vortex-int-mult", - "vortex-session", -] - [[package]] name = "vortex-row" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 7dd52ad12d5..18d2f2b9718 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,7 +65,6 @@ members = [ "encodings/block-residual", "encodings/float-quant", "encodings/int-mult", - "encodings/range-packed", # Benchmarks "benchmarks/bench-support", "benchmarks/lance-bench", @@ -328,7 +327,6 @@ vortex-pco = { version = "0.1.0", path = "./encodings/pco", default-features = f vortex-block-residual = { version = "0.1.0", path = "./encodings/block-residual", default-features = false } vortex-float-quant = { version = "0.1.0", path = "./encodings/float-quant", default-features = false } vortex-int-mult = { version = "0.1.0", path = "./encodings/int-mult", default-features = false } -vortex-range-packed = { version = "0.1.0", path = "./encodings/range-packed", default-features = false } vortex-proto = { version = "0.1.0", path = "./vortex-proto", default-features = false } vortex-row = { version = "0.1.0", path = "./vortex-row", default-features = false } vortex-runend = { version = "0.1.0", path = "./encodings/runend", default-features = false } diff --git a/benchmarks/compress-bench/README.md b/benchmarks/compress-bench/README.md index 31a54533a13..f1e5d7e904e 100644 --- a/benchmarks/compress-bench/README.md +++ b/benchmarks/compress-bench/README.md @@ -27,7 +27,6 @@ Compare a numeric scheme bundle with the same command and one of these values: - `--vortex-numeric-bundle prior-default` - `--vortex-numeric-bundle block-residual` - `--vortex-numeric-bundle current-default` -- `--vortex-numeric-bundle range-packed` Add a local Parquet file with `--parquet-path`. Use `--datasets` to select its file stem. diff --git a/benchmarks/compress-bench/src/main.rs b/benchmarks/compress-bench/src/main.rs index 085558b289a..0fdc9eb1110 100644 --- a/benchmarks/compress-bench/src/main.rs +++ b/benchmarks/compress-bench/src/main.rs @@ -29,6 +29,7 @@ use vortex_bench::compress::calculate_ratios; use vortex_bench::create_output_writer; use vortex_bench::datasets::Dataset; use vortex_bench::datasets::feature_vectors::GloveEmbeddingsData; +use vortex_bench::datasets::feature_vectors::OpenAiEmbeddingsData; use vortex_bench::datasets::local_parquet::LocalParquetData; use vortex_bench::datasets::struct_list_of_ints::StructListOfInts; use vortex_bench::datasets::taxi_data::TaxiData; @@ -214,7 +215,8 @@ async fn run_compress( )] let gpu_decompress_benchmarks = vec!["TPC-H l_comment canonical"]; - let mut datasets: Vec<&dyn Dataset> = vec![&TaxiData, &GloveEmbeddingsData]; + let mut datasets: Vec<&dyn Dataset> = + vec![&TaxiData, &GloveEmbeddingsData, &OpenAiEmbeddingsData]; for (name, dataset) in [ ("Arade", Arade), ("Bimbo", Bimbo), diff --git a/benchmarks/random-access-bench/README.md b/benchmarks/random-access-bench/README.md index f12c3dd9020..d9faaaf1dbe 100644 --- a/benchmarks/random-access-bench/README.md +++ b/benchmarks/random-access-bench/README.md @@ -26,7 +26,6 @@ Compare random access for a numeric scheme bundle with one of these values: - `--vortex-numeric-bundle prior-default` - `--vortex-numeric-bundle block-residual` - `--vortex-numeric-bundle current-default` -- `--vortex-numeric-bundle range-packed` Each bundle uses a separate Vortex file. Existing files remain available for repeated runs. diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index a841ea61762..b8aed1e1736 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -46,19 +46,21 @@ Remove `RangeEntropyArray`, `RangeEntropyScheme`, and `BitSplitCodec` from the f The `wm/pcodec-entropy-experiments` branch preserves the complete entropy and bit-split prototypes. -Keep the fused `RangePackedArray` and its manual benchmark as experimental work. +Remove `RangePackedArray` and its manual benchmark from the focused branch. -Prototype fixed bins as a composed tree instead: +The Git history and this plan preserve the fixed-bin experiment. + +The fixed-bin experiment used this composed tree: - `Dict(BitPacked(bin_codes), bin_starts)` reconstructs one reference per value. - `BlockResidual(offsets)` stores each distance from the selected reference. - `IntMult(base=1, references, offsets)` adds both components. -The prototype permits any bin count from one through 64. +The prototype accepted any bin count from one through 64. -The branch includes an experimental `IntegerFixedBinsScheme` for the composed tree. +The experiment also tested an `IntegerFixedBinsScheme` below ALP. -The scheme runs only below ALP. It remains outside `ALL_SCHEMES`. +The complete corpus did not justify either scheme. The focused branch no longer contains either scheme. Do not add the fused RangePacked array to the Default or Compact selector. @@ -70,8 +72,6 @@ The branch implements `OrderedFloatArray`, `BlockResidualArray`, `FloatQuantArra The evaluation set includes BtrBlocks schemes for the first three arrays. -The experimental BtrBlocks scheme for IntMult applies only to ALP integer children. - `OrderedFloat(BlockResidual)` now supports `f16`, `f32`, and `f64` inputs. The float scheme applies `OrderedFloat` first, then `BlockResidual` to the unsigned child. @@ -86,7 +86,9 @@ The retained schemes win on specific structures. They do not replace ALP or ALP- FloatQuant now passes the speed gates for zero-secondary and one-bit-secondary inputs. -The complete file corpus does not yet justify FloatQuant in Default. +The complete file corpus justifies FloatQuant in Default. + +OpenAI-on-C4 f64 embeddings select zero-secondary FloatQuant and shrink by 40.5 percent. BlockResidual now passes the direct speed gates for every integer width. @@ -94,25 +96,27 @@ The GloVe result identifies a separate entropy gap inside the ALP integer child. The previously tested quotient and remainder trees do not close that gap. -The current selector factors and nonlinear patch cost remain provisional. +The final calibration retains the 1.05 BlockResidual ratio floor. + +It retains the 1.02 ordered-float factor, 1.10 FloatQuant factor, 1.12 8-bit factor, and nonlinear patch cost. `IntMultArray` owns only the quotient and remainder transform. Its two children remain generic arrays. Child compression owns patches and other integer models. -The range decomposition prototype uses `IntMult(base=1)` as generic addition. +The range decomposition prototype used `IntMult(base=1)` as generic addition. -Its fused decode path skips multiplication and avoids materialization of dictionary references. +Its fused decode path skipped multiplication and avoided dictionary reference materialization. -IntMult participates only through explicit experimental schemes. +IntMult has no Default scheme. Decomposed fixed bins do not participate in the Default selector. -Final calibration requires the complete corpus and selected-tree evidence. +The final calibration includes the complete corpus and selected-tree evidence. -The current branch recovers a small share of Compact's aggregate numeric advantage. +The Opportunistic bundle cuts aggregate size by 11.7 percent against Prior Default. -The remaining work targets repeated Compact mechanisms with native, bounded-access trees. +Future work targets repeated Compact mechanisms with native, bounded-access trees. ## Array support and validation @@ -2841,40 +2845,84 @@ The composable IntMult follow-up is complete. The tested IntMult trees remain ou The nonzero-secondary FloatQuant implementation and focused validation are complete. -The focused Default bundle now includes full-position RangePacked as an experimental candidate. - The specialized OrderedFloat and ALP trees now use registered fused parent kernels. -Use the opportunistic bundle as the leading Default candidate. - The pure integer fixed-bin experiment is complete. -It selected no real tree and reduced write throughput. Keep it outside Default. +It selected no integer tree and reduced write throughput. The focused branch no longer contains either fixed-bin scheme. + +## Final calibration + +The final pass compared 17 datasets with five timed iterations per operation. + +The corpus included GloVe `f32` embeddings and OpenAI-on-C4 `f64` embeddings. + +| Bundle or format | Aggregate bytes | Encode time | Decode time | +| --- | ---: | ---: | ---: | +| Prior Default | 3,025,406,960 | 10.928 seconds | 0.424 seconds | +| Opportunistic | 2,672,321,904 | 10.487 seconds | 0.422 seconds | +| Compact | 2,027,179,944 | 11.655 seconds | 1.082 seconds | +| Parquet with Zstd | 2,718,861,135 | 24.830 seconds | 5.635 seconds | + +Against Prior Default, Opportunistic reduced aggregate size by 11.67 percent. + +It reduced aggregate encode time by 4.04 percent and aggregate decode time by 0.56 percent. + +The geometric means improved by 5.69 percent for size, 10.49 percent for encode time, and 4.99 percent for decode time. -Use these next steps: +Opportunistic produced 1.71 percent fewer aggregate bytes than Parquet with Zstd. -1. Keep the Opportunistic bundle as the leading Default candidate. -2. Retain the direct OrderedFloat fixed-bin bundle as an optional experiment. -3. Revisit direct fixed bins when the corpus gains more applicable distributions. -4. Complete the final threshold review for the retained schemes. -5. Compare the retained bundle across the complete file corpus. -6. Add more real embedding datasets when licenses and loaders permit them. +Its geometric-mean size ratio against Parquet with Zstd was 0.980. -Use packed bin codes and primitive bin starts unless evidence supports more complexity. +The focused cached-access corpus included five tabular datasets and the OpenAI vector dataset. -Let ordinary child arrays own patches and exception payloads. +Opportunistic reduced geometric-mean scalar latency by 10.97 percent and aggregate scalar latency by 10.20 percent. -Add an outer OrderedFloat fused decode only if the generic composition misses the decode gate. +OpenAI-on-C4 selected `FloatQuant(FoR(BitPacked))`. -Keep the black-box RangePacked array and RangeEntropy outside Default. +That tree reduced size by 40.5 percent and improved bulk decode time against Prior Default. -Retain decomposed fixed bins only as an experimental reference. +GloVe retained its prior tree. Its result still identifies an entropy gap inside the ALP integer child. -After the candidate set stabilizes, complete these final steps: +The fixed-bin bundle reduced aggregate size by only 0.025 percent against Opportunistic. -1. Add a true ALP-RD child composition case if a real selected tree exposes one. -2. Add more real embedding datasets when licenses and loaders permit them. -3. Update this plan after each experiment. +It changed only the HashTags file in the complete corpus, where it reduced file size by 0.43 percent. + +The selected HashTags column was 35.0 percent smaller, but that narrow win did not justify the scheme cost. + +## Final production decision + +Use the Opportunistic bundle in Default. + +The bundle contains these schemes: + +- `BlockResidualScheme` for integers. +- `OrderedBlockResidualScheme` for floating-point values. +- `FloatQuantScheme` for floating-point values. + +Keep `IntMultArray` as a composable primitive without a Default scheme. + +Retain these selector controls: + +- A 1.05 minimum ratio for integer BlockResidual. +- A 1.05 minimum ratio and 1.02 decode factor for ordered BlockResidual. +- A 1.10 FloatQuant factor. +- A 1.12 factor for 8-bit BlockResidual. +- The nonlinear BlockResidual patch cost. +- The one-block rejection for integer BlockResidual. + +Remove RangePacked and both fixed-bin schemes from the focused branch. + +Keep RangeEntropy, BitSplit, and alternate residual models on the experimental branch. + +Use ordinary child arrays for patches and exception payloads. + +Test these items in future research: + +1. Test nonzero-secondary FloatQuant on more real distributions. +2. Add more real float datasets where Pco beats ALP and ALP-RD. +3. Test a true ALP-RD child composition when a real selected tree exposes one. +4. Revisit fixed bins only when the corpus includes more applicable distributions. ## Pull request structure @@ -2884,8 +2932,6 @@ Prepare a separate stack for `FloatQuantArray` and `FloatQuantScheme`. Prepare a separate primitive-array change for `IntMultArray`. -Prepare a separate BtrBlocks change for decomposed fixed bins only after real-data validation. - Keep entropy, bit-split, and alternate residual models on `wm/pcodec-entropy-experiments`. ## References diff --git a/encodings/range-packed/Cargo.toml b/encodings/range-packed/Cargo.toml deleted file mode 100644 index 6a18fa6da4e..00000000000 --- a/encodings/range-packed/Cargo.toml +++ /dev/null @@ -1,30 +0,0 @@ -[package] -name = "vortex-range-packed" -authors = { workspace = true } -categories = { workspace = true } -description = "Vortex fixed-bin range-packed integer array encoding" -edition = { workspace = true } -homepage = { workspace = true } -include = { workspace = true } -keywords = { workspace = true } -license = { workspace = true } -readme = { workspace = true } -repository = { workspace = true } -rust-version = { workspace = true } -version = { workspace = true } - -[dependencies] -vortex-alp = { workspace = true } -vortex-array = { workspace = true } -vortex-block-residual = { workspace = true } -vortex-buffer = { workspace = true } -vortex-error = { workspace = true } -vortex-int-mult = { workspace = true } -vortex-session = { workspace = true } - -[dev-dependencies] -rstest = { workspace = true } -vortex-array = { workspace = true, features = ["_test-harness"] } - -[lints] -workspace = true diff --git a/encodings/range-packed/src/array.rs b/encodings/range-packed/src/array.rs deleted file mode 100644 index 6125f2bf454..00000000000 --- a/encodings/range-packed/src/array.rs +++ /dev/null @@ -1,885 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::fmt::Display; -use std::fmt::Formatter; -use std::hash::Hash; -use std::hash::Hasher; -use std::ops::Range; - -use vortex_array::Array; -use vortex_array::ArrayEq; -use vortex_array::ArrayHash; -use vortex_array::ArrayId; -use vortex_array::ArrayParts; -use vortex_array::ArrayRef; -use vortex_array::ArrayView; -use vortex_array::EqMode; -use vortex_array::ExecutionCtx; -use vortex_array::ExecutionResult; -use vortex_array::IntoArray; -use vortex_array::TypedArrayRef; -use vortex_array::array_slots; -use vortex_array::arrays::Bool; -use vortex_array::arrays::BoolArray; -use vortex_array::arrays::Primitive; -use vortex_array::arrays::PrimitiveArray; -use vortex_array::arrays::bool::BoolArrayExt; -use vortex_array::arrays::slice::SliceReduce; -use vortex_array::arrays::slice::SliceReduceAdaptor; -use vortex_array::buffer::BufferHandle; -use vortex_array::dtype::DType; -use vortex_array::dtype::Nullability; -use vortex_array::dtype::Nullability::NonNullable; -use vortex_array::dtype::PType; -use vortex_array::optimizer::rules::ParentRuleSet; -use vortex_array::scalar::Scalar; -use vortex_array::serde::ArrayChildren; -use vortex_array::validity::Validity; -use vortex_array::vtable::OperationsVTable; -use vortex_array::vtable::VTable; -use vortex_array::vtable::ValidityVTable; -use vortex_array::vtable::child_to_validity; -use vortex_error::VortexResult; -use vortex_error::vortex_bail; -use vortex_error::vortex_ensure; -use vortex_error::vortex_panic; -use vortex_session::VortexSession; -use vortex_session::registry::CachedId; - -use crate::BinTableView; -use crate::MAX_BLOCK_LEN; -use crate::RangePackedCodec; -use crate::RangePackedDecoder; -use crate::VALIDITY_CHECKPOINT_INTERVAL; -use crate::estimate_codec_nbytes; -use crate::low_mask; - -const METADATA_VERSION: u8 = 1; -const METADATA_LEN: usize = 34; - -/// Fixed-bin range packing with bounded scalar access. -pub type RangePackedArray = Array; - -#[array_slots(RangePacked)] -pub struct RangePackedSlots { - #[slot(0)] - pub bin_lowers: ArrayRef, - #[slot(1)] - pub offset_widths: ArrayRef, - #[slot(2)] - pub block_offsets: ArrayRef, - #[slot(3)] - pub payload: ArrayRef, - #[slot(4)] - pub validity_bits: Option, - #[slot(5)] - pub rank_checkpoints: Option, -} - -#[derive(Clone, Debug)] -pub struct RangePackedData { - unsliced_len: usize, - stored_len: usize, - slice_start: usize, - slice_stop: usize, - payload_len: usize, - symbol_width: u8, -} - -impl Display for RangePackedData { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "bins: {}, slice: {}..{}", - self.bin_count(), - self.slice_start, - self.slice_stop - ) - } -} - -impl ArrayHash for RangePackedData { - fn array_hash(&self, state: &mut H, _accuracy: EqMode) { - self.unsliced_len.hash(state); - self.stored_len.hash(state); - self.slice_start.hash(state); - self.slice_stop.hash(state); - self.payload_len.hash(state); - self.symbol_width.hash(state); - } -} - -impl ArrayEq for RangePackedData { - fn array_eq(&self, other: &Self, _accuracy: EqMode) -> bool { - self.unsliced_len == other.unsliced_len - && self.stored_len == other.stored_len - && self.slice_start == other.slice_start - && self.slice_stop == other.slice_stop - && self.payload_len == other.payload_len - && self.symbol_width == other.symbol_width - } -} - -#[derive(Clone, Debug)] -pub struct RangePacked; - -impl VTable for RangePacked { - type TypedArrayData = RangePackedData; - type OperationsVTable = Self; - type ValidityVTable = Self; - - fn id(&self) -> ArrayId { - static ID: CachedId = CachedId::new("vortex.range_packed"); - *ID - } - - fn validate( - &self, - data: &Self::TypedArrayData, - dtype: &DType, - len: usize, - slots: &[Option], - ) -> VortexResult<()> { - data.validate(dtype, len, RangePackedSlotsView::from_slots(slots)) - } - - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { - 0 - } - - fn buffer(_array: ArrayView<'_, Self>, index: usize) -> BufferHandle { - vortex_panic!("RangePackedArray buffer index {index} is invalid") - } - - fn buffer_name(_array: ArrayView<'_, Self>, index: usize) -> Option { - vortex_panic!("RangePackedArray buffer index {index} is invalid") - } - - fn with_buffers( - &self, - array: ArrayView<'_, Self>, - buffers: &[BufferHandle], - ) -> VortexResult> { - vortex_array::vtable::with_empty_buffers(self, array, buffers) - } - - fn serialize( - array: ArrayView<'_, Self>, - _session: &VortexSession, - ) -> VortexResult>> { - Ok(Some(RangePackedMetadata::from_data(array.data())?.encode())) - } - - fn deserialize( - &self, - dtype: &DType, - len: usize, - metadata: &[u8], - _buffers: &[BufferHandle], - children: &dyn ArrayChildren, - _session: &VortexSession, - ) -> VortexResult> { - let metadata = RangePackedMetadata::decode(metadata)?; - let unsliced_len = usize::try_from(metadata.unsliced_len)?; - let stored_len = usize::try_from(metadata.stored_len)?; - let slice_start = usize::try_from(metadata.slice_start)?; - let slice_stop = slice_start - .checked_add(len) - .ok_or_else(|| vortex_error::vortex_err!("RangePacked slice length overflows"))?; - let payload_len = usize::try_from(metadata.payload_len)?; - let bin_count = bin_count(stored_len, metadata.symbol_width)?; - let block_count = stored_len.div_ceil(MAX_BLOCK_LEN); - vortex_ensure!( - matches!(children.len(), 4..=6), - "RangePackedArray expects four, five, or six children" - ); - let bin_lowers = children.get(0, &primitive_dtype(PType::U64), bin_count)?; - let offset_widths = children.get(1, &primitive_dtype(PType::U8), bin_count)?; - let block_offsets = children.get(2, &primitive_dtype(PType::U32), block_count + 1)?; - let payload = children.get(3, &primitive_dtype(PType::U8), payload_len)?; - let validity_bits = (children.len() >= 5) - .then(|| children.get(4, &DType::Bool(NonNullable), unsliced_len)) - .transpose()?; - let rank_checkpoints = (children.len() == 6) - .then(|| { - children.get( - 5, - &primitive_dtype(PType::U32), - unsliced_len.div_ceil(VALIDITY_CHECKPOINT_INTERVAL), - ) - }) - .transpose()?; - let data = RangePackedData { - unsliced_len, - stored_len, - slice_start, - slice_stop, - payload_len, - symbol_width: metadata.symbol_width, - }; - let slots = RangePackedSlots { - bin_lowers, - offset_widths, - block_offsets, - payload, - validity_bits, - rank_checkpoints, - }; - Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots.into_slots())) - } - - fn slot_name(_array: ArrayView<'_, Self>, index: usize) -> String { - RangePackedSlots::NAMES[index].to_string() - } - - fn execute(array: Array, _ctx: &mut ExecutionCtx) -> VortexResult { - Ok(ExecutionResult::done( - decode_primitive(array.as_view())?.into_array(), - )) - } - - fn reduce_parent( - array: ArrayView<'_, Self>, - parent: &ArrayRef, - child_idx: usize, - ) -> VortexResult> { - RULES.evaluate(array, parent, child_idx) - } -} - -impl OperationsVTable for RangePacked { - fn scalar_at( - array: ArrayView<'_, RangePacked>, - index: usize, - _ctx: &mut ExecutionCtx, - ) -> VortexResult { - let global_index = array.data().slice_start + index; - if !is_valid(array, global_index) { - return Ok(Scalar::null(array.dtype().clone())); - } - let dense_index = dense_rank(array, global_index)?; - ordered_scalar( - with_decoder(array, |decoder| decoder.scalar_at(dense_index))?, - array.dtype().as_ptype(), - array.dtype().nullability(), - ) - } -} - -impl ValidityVTable for RangePacked { - fn validity(array: ArrayView<'_, RangePacked>) -> VortexResult { - array - .unsliced_validity() - .slice(array.data().slice_start..array.data().slice_stop) - } -} - -impl SliceReduce for RangePacked { - fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { - let data = array.data().slice(range); - Ok(Some( - Array::try_from_parts( - ArrayParts::new(RangePacked, array.dtype().clone(), data.len(), data) - .with_slots(array.slots().iter().cloned().collect()), - )? - .into_array(), - )) - } -} - -static RULES: ParentRuleSet = - ParentRuleSet::new(&[ParentRuleSet::lift(&SliceReduceAdaptor(RangePacked))]); - -pub trait RangePackedArrayExt: TypedArrayRef + RangePackedArraySlotsExt { - fn unsliced_validity(&self) -> Validity { - child_to_validity( - self.as_ref().slots()[RangePackedSlots::VALIDITY_BITS].as_ref(), - self.as_ref().dtype().nullability(), - ) - } -} - -impl> RangePackedArrayExt for T {} - -impl RangePacked { - pub fn from_primitive( - array: ArrayView<'_, Primitive>, - ctx: &mut ExecutionCtx, - ) -> VortexResult { - Self::from_primitive_mode(array, ctx, false) - } - - /// Encodes every physical value, including values at null positions. - pub fn from_primitive_with_null_positions( - array: ArrayView<'_, Primitive>, - ctx: &mut ExecutionCtx, - ) -> VortexResult { - Self::from_primitive_mode(array, ctx, true) - } - - /// Estimates storage when every physical value remains at its logical position. - pub fn estimate_primitive_with_null_positions( - array: ArrayView<'_, Primitive>, - ctx: &mut ExecutionCtx, - ) -> VortexResult { - vortex_ensure!( - array.ptype().is_int(), - "RangePackedArray needs integer values" - ); - let validity = array.validity()?; - let mask = validity.execute_mask(array.len(), ctx)?; - let actual_nulls = mask.true_count() != array.len(); - let ordered = ordered_primitive(array)?; - let mut nbytes = estimate_codec_nbytes(&ordered, MAX_BLOCK_LEN)?; - if actual_nulls { - nbytes += array.len().div_ceil(8); - } - Ok(u64::try_from(nbytes)?) - } - - fn from_primitive_mode( - array: ArrayView<'_, Primitive>, - ctx: &mut ExecutionCtx, - preserve_null_positions: bool, - ) -> VortexResult { - vortex_ensure!( - array.ptype().is_int(), - "RangePackedArray needs integer values" - ); - let validity = array.validity()?; - let mask = validity.execute_mask(array.len(), ctx)?; - let valid_count = mask.true_count(); - let actual_nulls = valid_count != array.len(); - let ordered = ordered_primitive(array)?; - let stored_ordered = if preserve_null_positions { - ordered - } else { - ordered - .into_iter() - .zip(mask.iter()) - .filter_map(|(value, valid)| valid.then_some(value)) - .collect::>() - }; - let stored_len = stored_ordered.len(); - let codec = RangePackedCodec::encode(&stored_ordered, MAX_BLOCK_LEN)?; - let validity_bits = actual_nulls.then(|| BoolArray::from_iter(mask.iter()).into_array()); - let rank_checkpoints = (actual_nulls && !preserve_null_positions) - .then(|| rank_checkpoints(mask.iter(), stored_len)) - .transpose()? - .map(|ranks| PrimitiveArray::from_iter(ranks).into_array()); - let data = RangePackedData { - unsliced_len: array.len(), - stored_len, - slice_start: 0, - slice_stop: array.len(), - payload_len: codec.payload.len(), - symbol_width: codec.symbol_width, - }; - let slots = RangePackedSlots { - bin_lowers: PrimitiveArray::from_iter(codec.bins.iter().map(|bin| bin.lower)) - .into_array(), - offset_widths: PrimitiveArray::from_iter(codec.bins.iter().map(|bin| bin.offset_bits)) - .into_array(), - block_offsets: PrimitiveArray::from_iter(codec.block_offsets).into_array(), - payload: PrimitiveArray::from_iter(codec.payload).into_array(), - validity_bits, - rank_checkpoints, - }; - Array::try_from_parts( - ArrayParts::new( - RangePacked, - DType::Primitive(array.ptype(), validity.nullability()), - array.len(), - data, - ) - .with_slots(slots.into_slots()), - ) - } - - pub fn decode_mapped( - array: ArrayView<'_, RangePacked>, - map: F, - null_value: T, - ) -> VortexResult> - where - T: Copy, - F: FnMut(u64) -> T, - { - let dense_start = dense_rank(array, array.data().slice_start)?; - let dense_stop = dense_rank(array, array.data().slice_stop)?; - let dense_values = with_decoder(array, |decoder| { - decoder.decode_mapped_range(dense_start..dense_stop, map) - })?; - if array.data().stored_len == array.data().unsliced_len { - return Ok(dense_values); - } - let Some(validity_bits) = array.validity_bits() else { - return Ok(dense_values); - }; - let validity_bits = validity_bits.as_::(); - let bits = validity_bits.bit_buffer_view(); - let mut dense = dense_values.into_iter(); - let mut values = Vec::with_capacity(array.len()); - for valid in bits - .slice(array.data().slice_start..array.data().slice_stop) - .iter() - { - values.push(if valid { - dense - .next() - .ok_or_else(|| vortex_error::vortex_err!("dense values are too short"))? - } else { - null_value - }); - } - vortex_ensure!(dense.next().is_none(), "dense values are too long"); - Ok(values) - } -} - -impl RangePackedData { - fn validate( - &self, - dtype: &DType, - len: usize, - slots: RangePackedSlotsView<'_>, - ) -> VortexResult<()> { - vortex_ensure!(dtype.is_int(), "RangePackedArray requires an integer dtype"); - vortex_ensure!( - self.slice_start <= self.slice_stop && self.slice_stop <= self.unsliced_len, - "RangePacked slice exceeds its source length" - ); - vortex_ensure!(len == self.len(), "RangePacked slice length is invalid"); - vortex_ensure!( - self.stored_len <= self.unsliced_len, - "RangePacked stored length exceeds its source length" - ); - let bin_count = self.bin_count(); - validate_primitive_child(slots.bin_lowers, PType::U64, bin_count, "bin lowers")?; - validate_primitive_child(slots.offset_widths, PType::U8, bin_count, "offset widths")?; - validate_primitive_child( - slots.block_offsets, - PType::U32, - self.stored_len.div_ceil(MAX_BLOCK_LEN) + 1, - "block offsets", - )?; - validate_primitive_child(slots.payload, PType::U8, self.payload_len, "payload")?; - vortex_ensure!( - slots.validity_bits.is_some() || slots.rank_checkpoints.is_none(), - "RangePacked rank checkpoints require validity" - ); - if let Some(validity_bits) = slots.validity_bits { - vortex_ensure!( - dtype.is_nullable(), - "RangePacked validity requires a nullable dtype" - ); - vortex_ensure!( - validity_bits.is::() - && validity_bits.dtype() == &DType::Bool(NonNullable) - && validity_bits.len() == self.unsliced_len, - "RangePacked validity child is invalid" - ); - if let Some(ranks) = slots.rank_checkpoints { - validate_primitive_child( - ranks, - PType::U32, - self.unsliced_len.div_ceil(VALIDITY_CHECKPOINT_INTERVAL), - "rank checkpoints", - )?; - validate_ranks( - validity_bits.as_::(), - ranks.as_::().as_slice::(), - self.stored_len, - )?; - } else { - vortex_ensure!( - self.stored_len == self.unsliced_len, - "RangePacked without rank checkpoints must store every value" - ); - } - } else { - vortex_ensure!( - self.stored_len == self.unsliced_len, - "RangePacked without validity must store every value" - ); - } - validate_bins( - slots.bin_lowers.as_::().as_slice::(), - slots.offset_widths.as_::().as_slice::(), - dtype.as_ptype(), - )?; - validate_block_offsets( - slots.block_offsets.as_::().as_slice::(), - self.payload_len, - )?; - Ok(()) - } - - fn bin_count(&self) -> usize { - if self.stored_len == 0 { - 0 - } else { - 1_usize << self.symbol_width - } - } - - fn len(&self) -> usize { - self.slice_stop - self.slice_start - } - - fn slice(&self, range: Range) -> Self { - Self { - slice_start: self.slice_start + range.start, - slice_stop: self.slice_start + range.end, - ..self.clone() - } - } -} - -fn validate_primitive_child( - child: &ArrayRef, - ptype: PType, - len: usize, - name: &str, -) -> VortexResult<()> { - vortex_ensure!( - child.is::() && child.dtype() == &primitive_dtype(ptype) && child.len() == len, - "RangePacked {name} child is invalid" - ); - Ok(()) -} - -fn validate_bins(lowers: &[u64], widths: &[u8], ptype: PType) -> VortexResult<()> { - let word_width = u8::try_from(ptype.bit_width())?; - for (index, (&lower, &width)) in lowers.iter().zip(widths).enumerate() { - vortex_ensure!(width <= word_width, "RangePacked offset width is invalid"); - if word_width < 64 { - vortex_ensure!( - lower <= low_mask(word_width), - "RangePacked bin lower exceeds its integer width" - ); - } - if let Some(&next) = lowers.get(index + 1) { - vortex_ensure!(next > lower, "RangePacked bin lowers are not ordered"); - vortex_ensure!( - next - lower - 1 <= low_mask(width), - "RangePacked bin does not cover the next boundary" - ); - } - } - Ok(()) -} - -fn validate_block_offsets(offsets: &[u32], payload_len: usize) -> VortexResult<()> { - vortex_ensure!( - offsets.first() == Some(&0), - "RangePacked offsets must start at zero" - ); - vortex_ensure!( - offsets.last().copied().map(usize::try_from).transpose()? == Some(payload_len), - "RangePacked offsets must end at the payload length" - ); - vortex_ensure!( - offsets.windows(2).all(|pair| pair[0] <= pair[1]), - "RangePacked offsets are not ordered" - ); - Ok(()) -} - -fn validate_ranks(bits: ArrayView<'_, Bool>, ranks: &[u32], stored_len: usize) -> VortexResult<()> { - let bits = bits.bit_buffer_view(); - let mut rank = 0usize; - for (checkpoint, &stored) in ranks.iter().enumerate() { - vortex_ensure!( - usize::try_from(stored)? == rank, - "RangePacked rank checkpoint is invalid" - ); - let start = checkpoint * VALIDITY_CHECKPOINT_INTERVAL; - let stop = (start + VALIDITY_CHECKPOINT_INTERVAL).min(bits.len()); - rank += bits.slice(start..stop).true_count(); - } - vortex_ensure!(rank == stored_len, "RangePacked stored length is invalid"); - Ok(()) -} - -fn rank_checkpoints( - validity: impl Iterator, - valid_count: usize, -) -> VortexResult> { - let mut ranks = Vec::new(); - let mut rank = 0usize; - for (index, valid) in validity.enumerate() { - if index.is_multiple_of(VALIDITY_CHECKPOINT_INTERVAL) { - ranks.push(u32::try_from(rank)?); - } - rank += usize::from(valid); - } - vortex_ensure!(rank == valid_count, "RangePacked validity count differs"); - Ok(ranks) -} - -#[inline] -fn is_valid(array: ArrayView<'_, RangePacked>, global_index: usize) -> bool { - array - .validity_bits() - .is_none_or(|validity| validity.as_::().bit_buffer_view().value(global_index)) -} - -fn dense_rank(array: ArrayView<'_, RangePacked>, global_index: usize) -> VortexResult { - if array.data().stored_len == array.data().unsliced_len { - return Ok(global_index); - } - let Some(validity) = array.validity_bits() else { - return Ok(global_index); - }; - if global_index == array.data().unsliced_len { - return Ok(array.data().stored_len); - } - let checkpoint = global_index / VALIDITY_CHECKPOINT_INTERVAL; - let checkpoint_start = checkpoint * VALIDITY_CHECKPOINT_INTERVAL; - let base = usize::try_from( - array - .rank_checkpoints() - .ok_or_else(|| vortex_error::vortex_err!("RangePacked rank child is missing"))? - .as_::() - .as_slice::()[checkpoint], - )?; - Ok(base - + validity - .as_::() - .bit_buffer_view() - .slice(checkpoint_start..global_index) - .true_count()) -} - -fn with_decoder( - array: ArrayView<'_, RangePacked>, - function: impl FnOnce(RangePackedDecoder<'_>) -> VortexResult, -) -> VortexResult { - let bin_lowers = array.bin_lowers().as_::(); - let offset_widths = array.offset_widths().as_::(); - let block_offsets = array.block_offsets().as_::(); - let payload = array.payload().as_::(); - let decoder = RangePackedDecoder::try_new( - array.data().stored_len, - MAX_BLOCK_LEN, - array.data().symbol_width, - BinTableView::Split { - lowers: bin_lowers.as_slice::(), - widths: offset_widths.as_slice::(), - }, - block_offsets.as_slice::(), - payload.as_slice::(), - )?; - function(decoder) -} - -fn ordered_primitive(array: ArrayView<'_, Primitive>) -> VortexResult> { - Ok(match array.ptype() { - PType::U8 => array - .as_slice::() - .iter() - .map(|&value| u64::from(value)) - .collect(), - PType::U16 => array - .as_slice::() - .iter() - .map(|&value| u64::from(value)) - .collect(), - PType::U32 => array - .as_slice::() - .iter() - .map(|&value| u64::from(value)) - .collect(), - PType::U64 => array.as_slice::().to_vec(), - PType::I8 => array - .as_slice::() - .iter() - .map(|&value| u64::from((value as u8) ^ (1_u8 << 7))) - .collect(), - PType::I16 => array - .as_slice::() - .iter() - .map(|&value| u64::from((value as u16) ^ (1_u16 << 15))) - .collect(), - PType::I32 => array - .as_slice::() - .iter() - .map(|&value| u64::from((value as u32) ^ (1_u32 << 31))) - .collect(), - PType::I64 => array - .as_slice::() - .iter() - .map(|&value| (value as u64) ^ (1_u64 << 63)) - .collect(), - ptype => vortex_bail!("RangePackedArray does not support {ptype}"), - }) -} - -fn decode_primitive(array: ArrayView<'_, RangePacked>) -> VortexResult { - let ordered = RangePacked::decode_mapped(array, |value| value, 0)?; - let validity = array.validity()?; - Ok(match array.dtype().as_ptype() { - PType::U8 => PrimitiveArray::new::( - ordered - .into_iter() - .map(u8::try_from) - .collect::, _>>()?, - validity, - ), - PType::U16 => PrimitiveArray::new::( - ordered - .into_iter() - .map(u16::try_from) - .collect::, _>>()?, - validity, - ), - PType::U32 => PrimitiveArray::new::( - ordered - .into_iter() - .map(u32::try_from) - .collect::, _>>()?, - validity, - ), - PType::U64 => PrimitiveArray::new::(ordered, validity), - PType::I8 => PrimitiveArray::new::( - ordered - .into_iter() - .map(|value| { - u8::try_from(value).map(|value| i8::from_le_bytes([value ^ (1_u8 << 7)])) - }) - .collect::, _>>()?, - validity, - ), - PType::I16 => PrimitiveArray::new::( - ordered - .into_iter() - .map(|value| { - u16::try_from(value) - .map(|value| i16::from_le_bytes((value ^ (1_u16 << 15)).to_le_bytes())) - }) - .collect::, _>>()?, - validity, - ), - PType::I32 => PrimitiveArray::new::( - ordered - .into_iter() - .map(|value| { - u32::try_from(value) - .map(|value| i32::from_le_bytes((value ^ (1_u32 << 31)).to_le_bytes())) - }) - .collect::, _>>()?, - validity, - ), - PType::I64 => PrimitiveArray::new::( - ordered - .into_iter() - .map(|value| i64::from_le_bytes((value ^ (1_u64 << 63)).to_le_bytes())) - .collect::>(), - validity, - ), - ptype => vortex_bail!("RangePackedArray does not support {ptype}"), - }) -} - -fn ordered_scalar(value: u64, ptype: PType, nullability: Nullability) -> VortexResult { - Ok(match ptype { - PType::U8 => Scalar::primitive(u8::try_from(value)?, nullability), - PType::U16 => Scalar::primitive(u16::try_from(value)?, nullability), - PType::U32 => Scalar::primitive(u32::try_from(value)?, nullability), - PType::U64 => Scalar::primitive(value, nullability), - PType::I8 => Scalar::primitive( - i8::from_le_bytes([u8::try_from(value)? ^ (1_u8 << 7)]), - nullability, - ), - PType::I16 => Scalar::primitive( - i16::from_le_bytes((u16::try_from(value)? ^ (1_u16 << 15)).to_le_bytes()), - nullability, - ), - PType::I32 => Scalar::primitive( - i32::from_le_bytes((u32::try_from(value)? ^ (1_u32 << 31)).to_le_bytes()), - nullability, - ), - PType::I64 => Scalar::primitive( - i64::from_le_bytes((value ^ (1_u64 << 63)).to_le_bytes()), - nullability, - ), - ptype => vortex_bail!("RangePackedArray does not support {ptype}"), - }) -} - -fn primitive_dtype(ptype: PType) -> DType { - DType::Primitive(ptype, NonNullable) -} - -fn bin_count(stored_len: usize, symbol_width: u8) -> VortexResult { - vortex_ensure!( - symbol_width <= 6, - "RangePacked symbol width exceeds six bits" - ); - Ok(if stored_len == 0 { - 0 - } else { - 1_usize << symbol_width - }) -} - -#[derive(Clone, Copy)] -struct RangePackedMetadata { - unsliced_len: u64, - stored_len: u64, - slice_start: u64, - payload_len: u64, - symbol_width: u8, -} - -impl RangePackedMetadata { - fn from_data(data: &RangePackedData) -> VortexResult { - Ok(Self { - unsliced_len: u64::try_from(data.unsliced_len)?, - stored_len: u64::try_from(data.stored_len)?, - slice_start: u64::try_from(data.slice_start)?, - payload_len: u64::try_from(data.payload_len)?, - symbol_width: data.symbol_width, - }) - } - - fn encode(self) -> Vec { - let mut bytes = Vec::with_capacity(METADATA_LEN); - bytes.push(METADATA_VERSION); - bytes.extend_from_slice(&self.unsliced_len.to_le_bytes()); - bytes.extend_from_slice(&self.stored_len.to_le_bytes()); - bytes.extend_from_slice(&self.slice_start.to_le_bytes()); - bytes.extend_from_slice(&self.payload_len.to_le_bytes()); - bytes.push(self.symbol_width); - bytes - } - - fn decode(bytes: &[u8]) -> VortexResult { - vortex_ensure!( - bytes.len() == METADATA_LEN, - "RangePacked metadata requires {METADATA_LEN} bytes" - ); - vortex_ensure!( - bytes[0] == METADATA_VERSION, - "unsupported RangePacked metadata version {}", - bytes[0] - ); - Ok(Self { - unsliced_len: read_u64(bytes, 1), - stored_len: read_u64(bytes, 9), - slice_start: read_u64(bytes, 17), - payload_len: read_u64(bytes, 25), - symbol_width: bytes[33], - }) - } -} - -fn read_u64(bytes: &[u8], offset: usize) -> u64 { - u64::from_le_bytes( - bytes[offset..offset + size_of::()] - .try_into() - .unwrap_or_else(|_| unreachable!("validated RangePacked metadata length")), - ) -} diff --git a/encodings/range-packed/src/kernel.rs b/encodings/range-packed/src/kernel.rs deleted file mode 100644 index 6662355ccec..00000000000 --- a/encodings/range-packed/src/kernel.rs +++ /dev/null @@ -1,169 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use vortex_alp::ALP; -use vortex_alp::ALPArrayExt; -use vortex_alp::ALPFloat; -use vortex_array::ArrayRef; -use vortex_array::ArrayVTable; -use vortex_array::ArrayView; -use vortex_array::ExecutionCtx; -use vortex_array::IntoArray; -use vortex_array::arrays::PrimitiveArray; -use vortex_array::dtype::PType; -use vortex_array::dtype::half::f16; -use vortex_array::kernel::ExecuteParentKernel; -use vortex_array::optimizer::kernels::ArrayKernelsExt; -use vortex_block_residual::OrderedFloat; -use vortex_error::VortexResult; -use vortex_error::vortex_bail; -use vortex_session::VortexSession; - -use crate::RangePacked; - -pub(super) fn initialize(session: &VortexSession) { - let kernels = session.kernels(); - kernels.register_execute_parent_kernel( - OrderedFloat.id(), - RangePacked, - OrderedFloatRangePackedKernel, - ); - kernels.register_execute_parent_kernel(ALP.id(), RangePacked, ALPRangePackedKernel); -} - -#[derive(Debug)] -struct OrderedFloatRangePackedKernel; - -impl ExecuteParentKernel for OrderedFloatRangePackedKernel { - type Parent = OrderedFloat; - - #[expect( - clippy::cast_possible_truncation, - reason = "OrderedFloat validates the child integer width against the float width" - )] - fn execute_parent( - &self, - array: ArrayView<'_, RangePacked>, - parent: ArrayView<'_, OrderedFloat>, - child_idx: usize, - _ctx: &mut ExecutionCtx, - ) -> VortexResult> { - if child_idx != 0 { - return Ok(None); - } - - let validity = array.validity()?; - let decoded = match parent.dtype().as_ptype() { - PType::F16 => PrimitiveArray::new::( - RangePacked::decode_mapped( - array, - |value| f16::from_bits(unordered_u16(value as u16)), - f16::ZERO, - )?, - validity, - ), - PType::F32 => PrimitiveArray::new::( - RangePacked::decode_mapped( - array, - |value| f32::from_bits(unordered_u32(value as u32)), - 0.0, - )?, - validity, - ), - PType::F64 => PrimitiveArray::new::( - RangePacked::decode_mapped( - array, - |value| f64::from_bits(unordered_u64(value)), - 0.0, - )?, - validity, - ), - ptype => vortex_bail!("OrderedFloat RangePacked kernel does not support {ptype}"), - }; - Ok(Some(decoded.into_array())) - } -} - -#[derive(Debug)] -struct ALPRangePackedKernel; - -impl ExecuteParentKernel for ALPRangePackedKernel { - type Parent = ALP; - - #[expect( - clippy::cast_possible_truncation, - reason = "ALP validates the child integer width against the float width" - )] - fn execute_parent( - &self, - array: ArrayView<'_, RangePacked>, - parent: ArrayView<'_, ALP>, - child_idx: usize, - ctx: &mut ExecutionCtx, - ) -> VortexResult> { - if child_idx != 0 { - return Ok(None); - } - - let validity = array.validity()?; - let exponents = parent.exponents(); - let decoded = match parent.dtype().as_ptype() { - PType::F32 => PrimitiveArray::new::( - RangePacked::decode_mapped( - array, - |ordered| { - let encoded = ((ordered as u32) ^ (1_u32 << 31)) as i32; - ::decode_single(encoded, exponents) - }, - 0.0, - )?, - validity, - ), - PType::F64 => PrimitiveArray::new::( - RangePacked::decode_mapped( - array, - |ordered| { - let encoded = (ordered ^ (1_u64 << 63)) as i64; - ::decode_single(encoded, exponents) - }, - 0.0, - )?, - validity, - ), - ptype => vortex_bail!("ALP RangePacked kernel does not support {ptype}"), - }; - let decoded = if let Some(patches) = parent.patches() { - decoded.patch(&patches, ctx)? - } else { - decoded - }; - Ok(Some(decoded.into_array())) - } -} - -#[inline] -fn unordered_u16(value: u16) -> u16 { - if value & (1_u16 << 15) == 0 { - !value - } else { - value ^ (1_u16 << 15) - } -} - -#[inline] -fn unordered_u32(value: u32) -> u32 { - if value & (1_u32 << 31) == 0 { - !value - } else { - value ^ (1_u32 << 31) - } -} - -#[inline] -fn unordered_u64(value: u64) -> u64 { - if value & (1_u64 << 63) == 0 { - !value - } else { - value ^ (1_u64 << 63) - } -} diff --git a/encodings/range-packed/src/lib.rs b/encodings/range-packed/src/lib.rs deleted file mode 100644 index 47c12516608..00000000000 --- a/encodings/range-packed/src/lib.rs +++ /dev/null @@ -1,1241 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! Fixed-bin range packing with bounded random access. - -mod array; -mod kernel; - -use std::cmp::Ordering; -use std::hash::Hash; -use std::mem::MaybeUninit; -use std::ops::Range; - -pub use array::*; -use vortex_array::IntoArray; -use vortex_array::arrays::DictArray; -use vortex_array::arrays::PrimitiveArray; -use vortex_array::session::ArraySessionExt; -use vortex_array::validity::Validity; -use vortex_error::VortexResult; -use vortex_error::vortex_ensure; -use vortex_int_mult::IntMult; -use vortex_int_mult::IntMultArray; -use vortex_session::VortexSession; - -const MAX_BINS: usize = 64; -const SPLIT_CANDIDATES: usize = 64; -const TRAINING_SAMPLE_SIZE: usize = 8_192; -const BIN_TABLE_COST_BITS: f64 = 128.0; -const SYMBOL_PADDING: usize = 8; -const OFFSET_PADDING: usize = 15; -const CHECKPOINT_INTERVAL: usize = 32; -const VALIDITY_CHECKPOINT_INTERVAL: usize = 256; -const MAX_BLOCK_LEN: usize = 1_024; - -/// Fixed-width range identifiers with variable-width offsets and scalar checkpoints. -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub struct RangePackedCodec { - pub(crate) len: usize, - pub(crate) block_len: usize, - pub(crate) symbol_width: u8, - pub(crate) bins: Vec, - pub(crate) block_offsets: Vec, - pub(crate) payload: Vec, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub(crate) struct Bin { - pub(crate) lower: u64, - pub(crate) offset_bits: u8, -} - -/// A fixed-bin split before compression of its component arrays. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RangeDecomposition { - bin_starts: Vec, - codes: Vec, - offsets: Vec, -} - -impl RangeDecomposition { - /// Fit at most 64 bins and split values into bin codes and offsets. - pub fn encode(values: &[u64]) -> VortexResult { - if values.is_empty() { - return Ok(Self { - bin_starts: Vec::new(), - codes: Vec::new(), - offsets: Vec::new(), - }); - } - - let (sample, minimum, maximum) = training_sample(values); - let bins = cover_domain(optimize_bins_arbitrary(&sample), minimum, maximum); - let codes = assign_bins(values, &bins)?; - let offsets = values - .iter() - .zip(&codes) - .map(|(&value, &code)| value - bins[usize::from(code)].lower) - .collect(); - Ok(Self { - bin_starts: bins.iter().map(|bin| bin.lower).collect(), - codes, - offsets, - }) - } - - /// Return the selected bin starts. - pub fn bin_starts(&self) -> &[u64] { - &self.bin_starts - } - - /// Return the per-value bin codes. - pub fn codes(&self) -> &[u8] { - &self.codes - } - - /// Return each value's offset from its selected bin start. - pub fn offsets(&self) -> &[u64] { - &self.offsets - } - - /// Return the number of bits required for each fixed-width code. - pub fn code_width(&self) -> u8 { - self.bin_starts - .len() - .checked_sub(1) - .map_or(0, |maximum| bit_width(maximum as u64)) - } - - /// Compose the raw components from a dictionary and an IntMult array. - pub fn into_array(self, validity: Validity) -> VortexResult { - let codes = PrimitiveArray::new(self.codes, validity).into_array(); - let starts = PrimitiveArray::from_iter(self.bin_starts).into_array(); - let references = DictArray::try_new(codes, starts)?.into_array(); - let offsets = PrimitiveArray::from_iter(self.offsets).into_array(); - IntMult::try_new(references, offsets, 1) - } -} - -/// Register the RangePacked encoding in one session. -pub fn initialize(session: &VortexSession) { - session.arrays().register(RangePacked); - kernel::initialize(session); -} - -impl RangePackedCodec { - pub fn encode(values: &[u64], block_len: usize) -> VortexResult { - vortex_ensure!( - block_len > 0 && block_len <= MAX_BLOCK_LEN, - "range packed block length must be in 1..={MAX_BLOCK_LEN}" - ); - if values.is_empty() { - return Ok(Self { - len: 0, - block_len, - symbol_width: 0, - bins: Vec::new(), - block_offsets: vec![0], - payload: Vec::new(), - }); - } - - let (sample, minimum, maximum) = training_sample(values); - let bins = cover_domain(optimize_bins(&sample), minimum, maximum); - vortex_ensure!( - bins.len().is_power_of_two(), - "range packed bin count must be a power of two" - ); - let symbols = assign_bins(values, &bins)?; - let symbol_width = bit_width(u64::try_from(bins.len() - 1)?); - let block_count = values.len().div_ceil(block_len); - let mut block_offsets = Vec::with_capacity(block_count + 1); - let mut payload = Vec::new(); - block_offsets.push(0); - for block_index in 0..block_count { - let start = block_index * block_len; - let stop = (start + block_len).min(values.len()); - encode_block( - &values[start..stop], - &symbols[start..stop], - &bins, - symbol_width, - &mut payload, - )?; - block_offsets.push(u32::try_from(payload.len())?); - } - - Ok(Self { - len: values.len(), - block_len, - symbol_width, - bins, - block_offsets, - payload, - }) - } - - pub fn decode(&self) -> VortexResult> { - self.decoder()?.decode_mapped(|value| value) - } - - pub fn decode_mapped(&self, map: F) -> VortexResult> - where - F: FnMut(u64) -> T, - { - self.decoder()?.decode_mapped(map) - } - - pub fn scalar_at(&self, index: usize) -> VortexResult { - self.decoder()?.scalar_at(index) - } - - pub fn encoded_size(&self) -> usize { - self.bins.len() * (size_of::() + size_of::()) - + self.block_offsets.len() * size_of::() - + self.payload.len() - } - - pub fn bin_count(&self) -> usize { - self.bins.len() - } - - pub fn max_offset_bits(&self) -> u8 { - self.bins - .iter() - .map(|bin| bin.offset_bits) - .max() - .unwrap_or_default() - } - - pub fn offset_widths(&self) -> String { - self.bins - .iter() - .map(|bin| bin.offset_bits.to_string()) - .collect::>() - .join(",") - } - - fn decoder(&self) -> VortexResult> { - RangePackedDecoder::try_new( - self.len, - self.block_len, - self.symbol_width, - BinTableView::Interleaved(&self.bins), - &self.block_offsets, - &self.payload, - ) - } -} - -pub(crate) fn estimate_codec_nbytes(values: &[u64], block_len: usize) -> VortexResult { - vortex_ensure!( - block_len > 0 && block_len <= MAX_BLOCK_LEN, - "range packed block length must be in 1..={MAX_BLOCK_LEN}" - ); - if values.is_empty() { - return Ok(size_of::()); - } - - let (sample, minimum, maximum) = training_sample(values); - let bins = cover_domain(optimize_bins(&sample), minimum, maximum); - let symbol_width = bit_width(u64::try_from(bins.len() - 1)?); - let block_count = values.len().div_ceil(block_len); - let mut payload_len = 0usize; - for block_index in 0..block_count { - let start = block_index * block_len; - let stop = (start + block_len).min(values.len()); - let values = &values[start..stop]; - let symbol_bytes = (values.len() * usize::from(symbol_width)).div_ceil(8); - let checkpoint_bytes = values.len().div_ceil(CHECKPOINT_INTERVAL) * size_of::(); - let offset_bits = values.iter().try_fold(0usize, |total, &value| { - let symbol = bins - .partition_point(|bin| bin.lower <= value) - .saturating_sub(1); - let bin = bins[symbol]; - vortex_ensure!( - value - bin.lower <= low_mask(bin.offset_bits), - "range packed value exceeds its bin" - ); - Ok::(total + usize::from(bin.offset_bits)) - })?; - payload_len += symbol_bytes - + SYMBOL_PADDING - + checkpoint_bytes - + offset_bits.div_ceil(8) - + OFFSET_PADDING; - } - - Ok(bins.len() * (size_of::() + size_of::()) - + (block_count + 1) * size_of::() - + payload_len) -} - -#[derive(Clone, Copy)] -pub(crate) enum BinTableView<'a> { - Interleaved(&'a [Bin]), - Split { lowers: &'a [u64], widths: &'a [u8] }, -} - -impl BinTableView<'_> { - fn len(self) -> usize { - match self { - Self::Interleaved(bins) => bins.len(), - Self::Split { lowers, .. } => lowers.len(), - } - } - - fn lower(self, index: usize) -> Option { - match self { - Self::Interleaved(bins) => bins.get(index).map(|bin| bin.lower), - Self::Split { lowers, .. } => lowers.get(index).copied(), - } - } - - fn width(self, index: usize) -> Option { - match self { - Self::Interleaved(bins) => bins.get(index).map(|bin| bin.offset_bits), - Self::Split { widths, .. } => widths.get(index).copied(), - } - } - - fn max_width(self) -> u8 { - (0..self.len()) - .map(|index| self.width(index).unwrap_or_default()) - .max() - .unwrap_or_default() - } -} - -pub(crate) struct RangePackedDecoder<'a> { - len: usize, - block_len: usize, - symbol_width: u8, - bins: BinTableView<'a>, - block_offsets: &'a [u32], - payload: &'a [u8], -} - -impl<'a> RangePackedDecoder<'a> { - pub(crate) fn try_new( - len: usize, - block_len: usize, - symbol_width: u8, - bins: BinTableView<'a>, - block_offsets: &'a [u32], - payload: &'a [u8], - ) -> VortexResult { - vortex_ensure!( - block_len > 0 && block_len <= MAX_BLOCK_LEN, - "range packed block length is invalid" - ); - let expected_bins = if len == 0 { - 0 - } else { - vortex_ensure!( - symbol_width <= 6, - "range packed symbol width exceeds six bits" - ); - 1_usize << symbol_width - }; - vortex_ensure!( - bins.len() == expected_bins, - "range packed bin count is invalid" - ); - if let BinTableView::Split { lowers, widths } = bins { - vortex_ensure!( - lowers.len() == widths.len(), - "range packed split bin table lengths differ" - ); - } - vortex_ensure!( - block_offsets.len() == len.div_ceil(block_len) + 1, - "range packed block offset count is invalid" - ); - vortex_ensure!( - block_offsets.first() == Some(&0) - && block_offsets - .last() - .copied() - .map(usize::try_from) - .transpose()? - == Some(payload.len()), - "range packed payload boundaries are invalid" - ); - Ok(Self { - len, - block_len, - symbol_width, - bins, - block_offsets, - payload, - }) - } - - fn decode_mapped(&self, map: F) -> VortexResult> - where - F: FnMut(u64) -> T, - { - self.decode_mapped_range(0..self.len, map) - } - - pub(crate) fn decode_mapped_range( - &self, - range: Range, - map: F, - ) -> VortexResult> - where - F: FnMut(u64) -> T, - { - vortex_ensure!( - range.start <= range.end && range.end <= self.len, - "range packed decode range exceeds array length" - ); - let max_offset_bits = self.bins.max_width(); - if max_offset_bits <= 40 { - self.decode_mapped_range_with_width::(range, map) - } else if max_offset_bits <= 57 { - self.decode_mapped_range_with_width::(range, map) - } else { - self.decode_mapped_range_with_width::(range, map) - } - } - - fn decode_mapped_range_with_width( - &self, - range: Range, - mut map: F, - ) -> VortexResult> - where - F: FnMut(u64) -> T, - { - let mut values = Vec::with_capacity(range.len()); - if range.is_empty() { - return Ok(values); - } - let table = DecodeTable::new(self.bins); - let first_block = range.start / self.block_len; - let last_block = range.end.div_ceil(self.block_len); - for block_index in first_block..last_block { - let block_start = block_index * self.block_len; - let value_count = self.block_value_count(block_index); - let local_start = range.start.saturating_sub(block_start); - let local_stop = (range.end - block_start).min(value_count); - let payload = self.block_payload(block_index)?; - if local_start == 0 && local_stop == value_count { - let output_start = values.len(); - decode_block::( - payload, - value_count, - self.symbol_width, - &table, - &mut values.spare_capacity_mut()[..value_count], - &mut map, - )?; - // SAFETY: decode_block initialized each output slot. - unsafe { values.set_len(output_start + value_count) }; - continue; - } - - let mut block_values = Vec::with_capacity(value_count); - decode_block::( - payload, - value_count, - self.symbol_width, - &table, - &mut block_values.spare_capacity_mut()[..value_count], - &mut map, - )?; - // SAFETY: decode_block initialized each output slot. - unsafe { block_values.set_len(value_count) }; - values.extend(block_values.drain(local_start..local_stop)); - } - Ok(values) - } - - pub(crate) fn scalar_at(&self, index: usize) -> VortexResult { - vortex_ensure!(index < self.len, "range packed index exceeds array length"); - let block_index = index / self.block_len; - let index_in_block = index % self.block_len; - let value_count = self.block_value_count(block_index); - let payload = self.block_payload(block_index)?; - let layout = BlockLayout::new(value_count, self.symbol_width, payload.len())?; - let symbol = read_symbol(payload, index_in_block, self.symbol_width)?; - let lower = self - .bins - .lower(symbol) - .ok_or_else(|| vortex_error::vortex_err!("range packed symbol exceeds bin table"))?; - let width = self - .bins - .width(symbol) - .ok_or_else(|| vortex_error::vortex_err!("range packed symbol exceeds bin table"))?; - let checkpoint_index = index_in_block / CHECKPOINT_INTERVAL; - let checkpoint_start = layout.checkpoints_start + checkpoint_index * size_of::(); - let checkpoint_bytes = payload - .get(checkpoint_start..checkpoint_start + size_of::()) - .ok_or_else(|| vortex_error::vortex_err!("range packed checkpoint is missing"))?; - let checkpoint = u16::from_le_bytes([checkpoint_bytes[0], checkpoint_bytes[1]]); - let mut offset_position = usize::from(checkpoint); - let scan_start = checkpoint_index * CHECKPOINT_INTERVAL; - for preceding in scan_start..index_in_block { - let preceding_symbol = read_symbol(payload, preceding, self.symbol_width)?; - offset_position += usize::from(self.bins.width(preceding_symbol).ok_or_else(|| { - vortex_error::vortex_err!("range packed symbol exceeds bin table") - })?); - } - let offset = read_bits_padded(&payload[layout.offsets_start..], offset_position, width); - Ok(lower.wrapping_add(offset)) - } - - fn block_value_count(&self, block_index: usize) -> usize { - (self.len - block_index * self.block_len).min(self.block_len) - } - - fn block_payload(&self, block_index: usize) -> VortexResult<&'a [u8]> { - let start = - usize::try_from(*self.block_offsets.get(block_index).ok_or_else(|| { - vortex_error::vortex_err!("range packed block offset is missing") - })?)?; - let stop = - usize::try_from(*self.block_offsets.get(block_index + 1).ok_or_else(|| { - vortex_error::vortex_err!("range packed block offset is missing") - })?)?; - vortex_ensure!(start <= stop, "range packed block offsets are not ordered"); - self.payload - .get(start..stop) - .ok_or_else(|| vortex_error::vortex_err!("range packed block exceeds its payload")) - } -} - -struct DecodeTable { - lowers: [u64; MAX_BINS], - widths: [u8; MAX_BINS], - masks: [u64; MAX_BINS], - bin_count: usize, -} - -impl DecodeTable { - fn new(bins: BinTableView<'_>) -> Self { - let mut lowers = [0_u64; MAX_BINS]; - let mut widths = [0_u8; MAX_BINS]; - let mut masks = [0_u64; MAX_BINS]; - for index in 0..bins.len() { - lowers[index] = bins - .lower(index) - .unwrap_or_else(|| unreachable!("validated range packed bin table")); - widths[index] = bins - .width(index) - .unwrap_or_else(|| unreachable!("validated range packed bin table")); - masks[index] = low_mask(widths[index]); - } - Self { - lowers, - widths, - masks, - bin_count: bins.len(), - } - } -} - -struct BlockLayout { - checkpoints_start: usize, - offsets_start: usize, -} - -impl BlockLayout { - fn new(value_count: usize, symbol_width: u8, payload_len: usize) -> VortexResult { - let symbol_bytes = (value_count * usize::from(symbol_width)).div_ceil(8); - let checkpoints_start = symbol_bytes + SYMBOL_PADDING; - let checkpoint_bytes = value_count.div_ceil(CHECKPOINT_INTERVAL) * size_of::(); - let offsets_start = checkpoints_start + checkpoint_bytes; - vortex_ensure!( - payload_len >= offsets_start + OFFSET_PADDING, - "range packed block is too short" - ); - Ok(Self { - checkpoints_start, - offsets_start, - }) - } -} - -fn encode_block( - values: &[u64], - symbols: &[u8], - bins: &[Bin], - symbol_width: u8, - payload: &mut Vec, -) -> VortexResult<()> { - let mut symbol_writer = BitWriter::with_capacity(values.len()); - let mut offset_writer = BitWriter::with_capacity(size_of_val(values)); - let mut checkpoints = Vec::with_capacity(values.len().div_ceil(CHECKPOINT_INTERVAL)); - for (index, (&value, &symbol)) in values.iter().zip(symbols).enumerate() { - if index.is_multiple_of(CHECKPOINT_INTERVAL) { - checkpoints.push(u16::try_from(offset_writer.bit_len())?); - } - symbol_writer.write(u64::from(symbol), symbol_width); - let bin = bins[usize::from(symbol)]; - offset_writer.write(value - bin.lower, bin.offset_bits); - } - payload.extend_from_slice(&symbol_writer.finish()); - payload.extend_from_slice(&[0; SYMBOL_PADDING]); - for checkpoint in checkpoints { - payload.extend_from_slice(&checkpoint.to_le_bytes()); - } - payload.extend_from_slice(&offset_writer.finish()); - payload.extend_from_slice(&[0; OFFSET_PADDING]); - Ok(()) -} - -#[expect( - clippy::cast_possible_truncation, - reason = "the symbol mask never exceeds 63" -)] -fn decode_block( - payload: &[u8], - value_count: usize, - symbol_width: u8, - table: &DecodeTable, - values: &mut [MaybeUninit], - map: &mut F, -) -> VortexResult<()> -where - F: FnMut(u64) -> T, -{ - let layout = BlockLayout::new(value_count, symbol_width, payload.len())?; - let offsets = &payload[layout.offsets_start..]; - let mut offset_position = 0usize; - let full_len = value_count / 8 * 8; - let symbol_mask = low_mask(symbol_width); - for base in (0..full_len).step_by(8) { - let symbol_position = base * usize::from(symbol_width); - let packed = read_bits_padded(payload, symbol_position, symbol_width * 8); - if PARALLEL { - let mut widths = [0_u8; 8]; - let mut lowers = [0_u64; 8]; - let mut masks = [0_u64; 8]; - let mut positions = [0_usize; 8]; - for lane in 0..8 { - let symbol = - ((packed >> (lane * usize::from(symbol_width))) & symbol_mask) as usize; - debug_assert!(symbol < table.bin_count); - // SAFETY: A power-of-two bin count makes every symbol code valid. - widths[lane] = unsafe { *table.widths.get_unchecked(symbol) }; - // SAFETY: A power-of-two bin count makes every symbol code valid. - lowers[lane] = unsafe { *table.lowers.get_unchecked(symbol) }; - // SAFETY: A power-of-two bin count makes every symbol code valid. - masks[lane] = unsafe { *table.masks.get_unchecked(symbol) }; - positions[lane] = offset_position; - offset_position += usize::from(widths[lane]); - } - for lane in 0..8 { - let offset = if WIDE { - read_offset::(offsets, positions[lane], widths[lane]) - } else { - read_offset_masked(offsets, positions[lane], masks[lane]) - }; - // SAFETY: This lane is within a complete eight-value output batch. - unsafe { - values - .get_unchecked_mut(base + lane) - .write(map(lowers[lane].wrapping_add(offset))) - }; - } - } else { - for lane in 0..8 { - let symbol = - ((packed >> (lane * usize::from(symbol_width))) & symbol_mask) as usize; - debug_assert!(symbol < table.bin_count); - // SAFETY: A power-of-two bin count makes every symbol code valid. - let width = unsafe { *table.widths.get_unchecked(symbol) }; - let offset = read_offset::(offsets, offset_position, width); - offset_position += usize::from(width); - // SAFETY: A power-of-two bin count makes every symbol code valid. - let lower = unsafe { *table.lowers.get_unchecked(symbol) }; - // SAFETY: This lane is within a complete eight-value output batch. - unsafe { - values - .get_unchecked_mut(base + lane) - .write(map(lower.wrapping_add(offset))) - }; - } - } - } - for index in full_len..value_count { - let symbol = read_symbol(payload, index, symbol_width)?; - debug_assert!(symbol < table.bin_count); - // SAFETY: A power-of-two bin count makes every symbol code valid. - let width = unsafe { *table.widths.get_unchecked(symbol) }; - let offset = read_offset::(offsets, offset_position, width); - offset_position += usize::from(width); - // SAFETY: A power-of-two bin count makes every symbol code valid. - let lower = unsafe { *table.lowers.get_unchecked(symbol) }; - // SAFETY: The trailing index is less than value_count. - unsafe { - values - .get_unchecked_mut(index) - .write(map(lower.wrapping_add(offset))) - }; - } - let offset_bytes = offset_position.div_ceil(8); - vortex_ensure!( - offsets.len() == offset_bytes + OFFSET_PADDING, - "range packed offset stream has unused bytes" - ); - Ok(()) -} - -fn read_symbol(payload: &[u8], index: usize, symbol_width: u8) -> VortexResult { - usize::try_from(read_bits_padded( - payload, - index * usize::from(symbol_width), - symbol_width, - )) - .map_err(Into::into) -} - -#[inline(always)] -fn read_offset_masked(offsets: &[u8], bit_position: usize, mask: u64) -> u64 { - let byte_position = bit_position / 8; - let bits_past_byte = bit_position % 8; - // SAFETY: Offset streams include fifteen readable padding bytes. - let packed = unsafe { read_u64_unaligned(offsets.as_ptr().add(byte_position)) }; - (packed >> bits_past_byte) & mask -} - -#[inline(always)] -fn read_offset(offsets: &[u8], bit_position: usize, width: u8) -> u64 { - if WIDE { - read_bits_padded(offsets, bit_position, width) - } else if width == 0 { - 0 - } else { - let byte_position = bit_position / 8; - let bits_past_byte = bit_position % 8; - // SAFETY: Offset streams include fifteen readable padding bytes. - let packed = unsafe { read_u64_unaligned(offsets.as_ptr().add(byte_position)) }; - (packed >> bits_past_byte) & low_mask(width) - } -} - -fn read_bits_padded(bytes: &[u8], bit_position: usize, width: u8) -> u64 { - if width == 0 { - return 0; - } - let byte_position = bit_position / 8; - let bits_past_byte = bit_position % 8; - // SAFETY: Every encoded stream includes at least eight readable padding bytes. - let first = unsafe { read_u64_unaligned(bytes.as_ptr().add(byte_position)) }; - if usize::from(width) <= 64 - bits_past_byte { - (first >> bits_past_byte) & low_mask(width) - } else { - // SAFETY: Offset streams include fifteen readable padding bytes. - let second = unsafe { read_u64_unaligned(bytes.as_ptr().add(byte_position + 7)) }; - let processed = 56 - bits_past_byte; - ((first >> bits_past_byte) | (second << processed)) & low_mask(width) - } -} - -unsafe fn read_u64_unaligned(pointer: *const u8) -> u64 { - // SAFETY: The caller provides eight readable bytes at the pointer. - u64::from_le(unsafe { pointer.cast::().read_unaligned() }) -} - -fn assign_bins(values: &[u64], bins: &[Bin]) -> VortexResult> { - values - .iter() - .map(|&value| { - let symbol = bins - .partition_point(|bin| bin.lower <= value) - .saturating_sub(1); - let bin = bins[symbol]; - vortex_ensure!( - value - bin.lower <= low_mask(bin.offset_bits), - "range packed value exceeds its bin" - ); - Ok(u8::try_from(symbol)?) - }) - .collect() -} - -fn optimize_bins(sorted: &[u64]) -> Vec { - optimize_bins_with_count_policy(sorted, usize::is_power_of_two) -} - -fn optimize_bins_arbitrary(sorted: &[u64]) -> Vec { - optimize_bins_with_count_policy(sorted, |_| true) -} - -fn optimize_bins_with_count_policy( - sorted: &[u64], - allowed_count: impl Fn(usize) -> bool, -) -> Vec { - let mut segments = Vec::with_capacity(MAX_BINS); - segments.push(0..sorted.len()); - let mut best_segments = segments.clone(); - let mut best_cost = partition_cost(sorted, &segments); - - while segments.len() < MAX_BINS { - let best = segments - .iter() - .enumerate() - .filter_map(|(segment_index, segment)| { - best_split(sorted, segment).map(|split| (segment_index, split)) - }) - .max_by(|(_, left), (_, right)| { - left.gain - .partial_cmp(&right.gain) - .unwrap_or(Ordering::Equal) - }); - let Some((segment_index, split)) = best else { - break; - }; - if split.gain <= 0.0 { - break; - } - let segment = segments.remove(segment_index); - segments.push(segment.start..split.at); - segments.push(split.at..segment.end); - segments.sort_unstable_by_key(|segment| segment.start); - let cost = partition_cost(sorted, &segments); - if allowed_count(segments.len()) && cost < best_cost { - best_cost = cost; - best_segments.clone_from(&segments); - } - } - - best_segments - .into_iter() - .map(|segment| Bin::from_values(sorted[segment.start], sorted[segment.end - 1])) - .collect() -} - -fn partition_cost(sorted: &[u64], segments: &[Range]) -> f64 { - let residual_cost = segments - .iter() - .map(|segment| { - segment.len() as f64 - * f64::from(bit_width(sorted[segment.end - 1] - sorted[segment.start])) - }) - .sum::(); - let symbol_cost = sorted.len() as f64 * f64::from(bit_width((segments.len() - 1) as u64)); - residual_cost + symbol_cost + segments.len() as f64 * BIN_TABLE_COST_BITS -} - -fn training_sample(values: &[u64]) -> (Vec, u64, u64) { - let mut minimum = u64::MAX; - let mut maximum = 0; - for &value in values { - minimum = minimum.min(value); - maximum = maximum.max(value); - } - let mut sample = if values.len() <= TRAINING_SAMPLE_SIZE { - values.to_vec() - } else { - (0..TRAINING_SAMPLE_SIZE) - .map(|index| { - let start = index * values.len() / TRAINING_SAMPLE_SIZE; - let stop = (index + 1) * values.len() / TRAINING_SAMPLE_SIZE; - let offset = index.wrapping_mul(2_654_435_761) % (stop - start); - values[start + offset] - }) - .collect() - }; - sample.push(minimum); - sample.push(maximum); - sample.sort_unstable(); - (sample, minimum, maximum) -} - -fn cover_domain(mut bins: Vec, minimum: u64, maximum: u64) -> Vec { - bins[0].lower = minimum; - for index in 0..bins.len() - 1 { - let upper = bins[index + 1].lower - 1; - bins[index].offset_bits = bit_width(upper - bins[index].lower); - } - let last = bins.len() - 1; - bins[last].offset_bits = bit_width(maximum - bins[last].lower); - bins -} - -#[derive(Clone, Copy)] -struct Split { - at: usize, - gain: f64, -} - -fn best_split(sorted: &[u64], segment: &Range) -> Option { - if segment.len() < 2 || sorted[segment.start] == sorted[segment.end - 1] { - return None; - } - let whole_cost = segment.len() as f64 - * f64::from(bit_width(sorted[segment.end - 1] - sorted[segment.start])); - let mut best = None; - for candidate in 1..SPLIT_CANDIDATES { - let mut at = segment.start + segment.len() * candidate / SPLIT_CANDIDATES; - if at <= segment.start || at >= segment.end { - continue; - } - while at < segment.end && sorted[at - 1] == sorted[at] { - at += 1; - } - if at >= segment.end { - continue; - } - let left_cost = (at - segment.start) as f64 - * f64::from(bit_width(sorted[at - 1] - sorted[segment.start])); - let right_cost = - (segment.end - at) as f64 * f64::from(bit_width(sorted[segment.end - 1] - sorted[at])); - let split = Split { - at, - gain: whole_cost - left_cost - right_cost, - }; - if best.is_none_or(|current: Split| split.gain > current.gain) { - best = Some(split); - } - } - best -} - -impl Bin { - fn from_values(lower: u64, upper: u64) -> Self { - Self { - lower, - offset_bits: bit_width(upper - lower), - } - } -} - -fn bit_width(value: u64) -> u8 { - u8::try_from(u64::BITS - value.leading_zeros()).unwrap_or(64) -} - -fn low_mask(bits: u8) -> u64 { - match bits { - 0 => 0, - 64 => u64::MAX, - _ => (1_u64 << bits) - 1, - } -} - -struct BitWriter { - bytes: Vec, - pending: u64, - pending_bits: u8, - bit_len: usize, -} - -impl BitWriter { - fn with_capacity(capacity: usize) -> Self { - Self { - bytes: Vec::with_capacity(capacity), - pending: 0, - pending_bits: 0, - bit_len: 0, - } - } - - fn bit_len(&self) -> usize { - self.bit_len - } - - fn write(&mut self, value: u64, width: u8) { - self.bit_len += usize::from(width); - if width == 0 { - return; - } - let available = 64 - self.pending_bits; - self.pending |= value << self.pending_bits; - if width < available { - self.pending_bits += width; - return; - } - self.bytes.extend_from_slice(&self.pending.to_le_bytes()); - let remaining = width - available; - self.pending = if remaining == 0 { - 0 - } else { - value >> available - }; - self.pending_bits = remaining; - } - - fn finish(mut self) -> Vec { - if self.pending_bits > 0 { - let byte_count = usize::from(self.pending_bits).div_ceil(8); - self.bytes - .extend_from_slice(&self.pending.to_le_bytes()[..byte_count]); - } - self.bytes - } -} - -#[cfg(test)] -mod tests { - use rstest::rstest; - use vortex_alp::ALP; - use vortex_alp::ALPArrayExt; - use vortex_alp::ALPArraySlotsExt; - use vortex_alp::alp_encode; - use vortex_array::ArrayContext; - use vortex_array::ArrayRef; - use vortex_array::IntoArray; - use vortex_array::VortexSessionExecute; - use vortex_array::array_session; - use vortex_array::arrays::Primitive; - use vortex_array::arrays::PrimitiveArray; - use vortex_array::assert_arrays_eq; - use vortex_array::dtype::half::f16; - use vortex_array::serde::SerializeOptions; - use vortex_array::serde::SerializedArray; - use vortex_block_residual::OrderedFloat; - use vortex_block_residual::OrderedFloatArraySlotsExt; - use vortex_buffer::ByteBufferMut; - use vortex_error::VortexResult; - use vortex_session::registry::ReadContext; - - use super::RangeDecomposition; - use super::RangePacked; - use super::RangePackedCodec; - use super::initialize; - - #[test] - fn decomposition_uses_arbitrary_bin_count() -> VortexResult<()> { - let values = (0_u64..10) - .flat_map(|cluster| (0_u64..1_000).map(move |_| cluster * 1_000_000_000)) - .collect::>(); - let decomposition = RangeDecomposition::encode(&values)?; - assert_eq!(decomposition.bin_starts().len(), 10); - assert_eq!(decomposition.code_width(), 4); - assert_arrays_eq!( - decomposition.into_array(vortex_array::validity::Validity::NonNullable)?, - PrimitiveArray::from_iter(values), - &mut array_session().create_execution_ctx() - ); - Ok(()) - } - - #[test] - fn decomposition_sample_covers_periodic_clusters() -> VortexResult<()> { - let values = (0_u64..65_536) - .map(|index| (index % 4) * 4_000_000_000_000_000_000 + index % 1_024) - .collect::>(); - let decomposition = RangeDecomposition::encode(&values)?; - let distant_offsets = decomposition - .offsets() - .iter() - .filter(|&&offset| offset >= 1_000_000) - .count(); - - assert!(decomposition.bin_starts().len() >= 4); - assert!( - distant_offsets < values.len() / 50, - "distant offsets: {distant_offsets}" - ); - Ok(()) - } - - #[test] - fn clustered_roundtrip_and_scalar_access() -> VortexResult<()> { - let values = (0_u64..20_000) - .map(|index| match index % 4 { - 0 => index % 17, - 1 => 1_000_000 + index % 31, - 2 => u64::MAX - index % 13, - _ => 1_000 + index % 7, - }) - .collect::>(); - let encoded = RangePackedCodec::encode(&values, 1_024)?; - assert_eq!(encoded.decode()?, values); - for index in [0, 1, 31, 32, 33, 1_023, 1_024, 19_999] { - assert_eq!(encoded.scalar_at(index)?, values[index]); - } - Ok(()) - } - - #[rstest] - #[case::f16(PrimitiveArray::from_option_iter([ - Some(f16::from_f32(-7.5)), - None, - Some(f16::from_f32(0.25)), - Some(f16::INFINITY), - ]))] - #[case::f32(PrimitiveArray::from_option_iter([ - Some(-7.5_f32), - None, - Some(0.25), - Some(f32::INFINITY), - ]))] - #[case::f64(PrimitiveArray::from_option_iter([ - Some(-7.5_f64), - None, - Some(0.25), - Some(f64::INFINITY), - ]))] - fn ordered_float_parent_kernel(#[case] expected: PrimitiveArray) -> VortexResult<()> { - let session = array_session(); - initialize(&session); - let mut ctx = session.create_execution_ctx(); - let ordered = OrderedFloat::from_primitive(expected.as_view())?; - let packed = RangePacked::from_primitive_with_null_positions( - ordered.encoded().as_::(), - &mut ctx, - )?; - let encoded = OrderedFloat::try_new(packed.into_array(), expected.ptype())?; - assert_arrays_eq!(encoded, expected, &mut ctx); - Ok(()) - } - - #[rstest] - #[case::f32(PrimitiveArray::from_option_iter([ - Some(1.25_f32), - None, - Some(17.5), - Some(f32::MAX), - ]))] - #[case::f64(PrimitiveArray::from_option_iter([ - Some(1.25_f64), - None, - Some(17.5), - Some(f64::MAX), - ]))] - fn alp_parent_kernel(#[case] expected: PrimitiveArray) -> VortexResult<()> { - let session = array_session(); - initialize(&session); - let mut ctx = session.create_execution_ctx(); - let alp = alp_encode(expected.as_view(), None, &mut ctx)?; - let packed = RangePacked::from_primitive_with_null_positions( - alp.encoded().as_::(), - &mut ctx, - )?; - let encoded = ALP::try_new(packed.into_array(), alp.exponents(), alp.patches())?; - assert_arrays_eq!(encoded, expected, &mut ctx); - Ok(()) - } - - #[test] - fn full_width_roundtrip() -> VortexResult<()> { - let values = [0, u64::MAX, 1, u64::MAX - 1]; - let encoded = RangePackedCodec::encode(&values, 4)?; - assert_eq!(encoded.decode()?, values); - for (index, value) in values.into_iter().enumerate() { - assert_eq!(encoded.scalar_at(index)?, value); - } - Ok(()) - } - - #[rstest] - #[case::dense(false)] - #[case::full(true)] - fn nullable_roundtrip_and_rank_boundaries( - #[case] preserve_null_positions: bool, - ) -> VortexResult<()> { - let expected = PrimitiveArray::from_option_iter((0_i64..600).map(|value| { - (!matches!(value, 0 | 1 | 63 | 64 | 255 | 256 | 257 | 511)).then_some(value - 300) - })); - let session = array_session(); - let mut ctx = session.create_execution_ctx(); - let encoded = if preserve_null_positions { - RangePacked::from_primitive_with_null_positions(expected.as_view(), &mut ctx)? - } else { - RangePacked::from_primitive(expected.as_view(), &mut ctx)? - }; - assert_arrays_eq!(encoded, expected, &mut ctx); - for index in [0, 1, 2, 63, 64, 65, 255, 256, 257, 258, 511, 512, 599] { - let actual = encoded.execute_scalar(index, &mut ctx)?; - let expected_scalar = expected - .clone() - .into_array() - .execute_scalar(index, &mut ctx)?; - assert_eq!(actual, expected_scalar); - } - Ok(()) - } - - #[rstest] - #[case::nonnullable(PrimitiveArray::from_iter((0_u64..4_096).map(|value| { - (value % 8) * 1_000_000 + value.wrapping_mul(7_919) % 1_024 - })))] - #[case::nullable(PrimitiveArray::from_option_iter((0_i64..4_096).map(|value| { - (value % 17 != 0).then_some((value % 8) * 1_000_000 + value * 7_919 % 1_024) - })))] - fn full_position_estimate_matches_encoded_size( - #[case] expected: PrimitiveArray, - ) -> VortexResult<()> { - let session = array_session(); - let mut ctx = session.create_execution_ctx(); - let estimate = - RangePacked::estimate_primitive_with_null_positions(expected.as_view(), &mut ctx)?; - let encoded = - RangePacked::from_primitive_with_null_positions(expected.as_view(), &mut ctx)?; - - assert_eq!(estimate, encoded.nbytes()); - Ok(()) - } - - #[rstest] - #[case::u8(PrimitiveArray::from_iter([0_u8, 1, u8::MAX]).into_array())] - #[case::u16(PrimitiveArray::from_iter([0_u16, 1, u16::MAX]).into_array())] - #[case::u32(PrimitiveArray::from_iter([0_u32, 1, u32::MAX]).into_array())] - #[case::u64(PrimitiveArray::from_iter([0_u64, 1, u64::MAX]).into_array())] - #[case::i8(PrimitiveArray::from_iter([i8::MIN, -1, 0, i8::MAX]).into_array())] - #[case::i16(PrimitiveArray::from_iter([i16::MIN, -1, 0, i16::MAX]).into_array())] - #[case::i32(PrimitiveArray::from_iter([i32::MIN, -1, 0, i32::MAX]).into_array())] - #[case::i64(PrimitiveArray::from_iter([i64::MIN, -1, 0, i64::MAX]).into_array())] - fn integer_ptype_roundtrip(#[case] expected: ArrayRef) -> VortexResult<()> { - let session = array_session(); - let mut ctx = session.create_execution_ctx(); - let encoded = RangePacked::from_primitive(expected.as_::(), &mut ctx)?; - assert_arrays_eq!(encoded, expected, &mut ctx); - Ok(()) - } - - #[test] - fn nullable_slice_crosses_rank_and_value_blocks() -> VortexResult<()> { - let expected = PrimitiveArray::from_option_iter((0_i32..2_500).map(|value| { - (!matches!(value, 255 | 256 | 257 | 1_023 | 1_024 | 1_025)).then_some(value - 1_250) - })); - let session = array_session(); - let mut ctx = session.create_execution_ctx(); - let encoded = RangePacked::from_primitive(expected.as_view(), &mut ctx)? - .into_array() - .slice(253..1_030)?; - let expected = expected.into_array().slice(253..1_030)?; - assert_arrays_eq!(encoded, expected, &mut ctx); - Ok(()) - } - - #[rstest] - #[case::dense(false)] - #[case::full(true)] - fn serialized_nullable_slice_roundtrip( - #[case] preserve_null_positions: bool, - ) -> VortexResult<()> { - let expected = PrimitiveArray::from_option_iter( - (0_i64..1_500).map(|value| (value % 17 != 0).then_some(value - 750)), - ); - let session = array_session(); - initialize(&session); - let mut ctx = session.create_execution_ctx(); - let encoded = if preserve_null_positions { - RangePacked::from_primitive_with_null_positions(expected.as_view(), &mut ctx)? - } else { - RangePacked::from_primitive(expected.as_view(), &mut ctx)? - } - .into_array() - .slice(251..1_101)?; - let expected = expected.into_array().slice(251..1_101)?; - let dtype = encoded.dtype().clone(); - let len = encoded.len(); - let array_ctx = ArrayContext::empty(); - let serialized = encoded.serialize(&array_ctx, &session, &SerializeOptions::default())?; - let mut bytes = ByteBufferMut::empty(); - for buffer in serialized { - bytes.extend_from_slice(buffer.as_ref()); - } - let parts = SerializedArray::try_from(bytes.freeze())?; - let decoded = parts.decode(&dtype, len, &ReadContext::new(array_ctx.to_ids()), &session)?; - assert_arrays_eq!(decoded, expected, &mut ctx); - Ok(()) - } -} diff --git a/vortex-bench/src/datasets/feature_vectors.rs b/vortex-bench/src/datasets/feature_vectors.rs index 87e91a0573c..2ae9e5b7701 100644 --- a/vortex-bench/src/datasets/feature_vectors.rs +++ b/vortex-bench/src/datasets/feature_vectors.rs @@ -44,6 +44,9 @@ pub struct FeatureVectorsData; /// A real f32 embedding dataset from the GloVe vector benchmark corpus. pub struct GloveEmbeddingsData; +/// A real f64 embedding dataset from the OpenAI-on-C4 vector benchmark corpus. +pub struct OpenAiEmbeddingsData; + #[async_trait] impl Dataset for GloveEmbeddingsData { fn name(&self) -> &str { @@ -66,6 +69,28 @@ impl Dataset for GloveEmbeddingsData { } } +#[async_trait] +impl Dataset for OpenAiEmbeddingsData { + fn name(&self) -> &str { + "openai-c4-embeddings-50k" + } + + async fn to_vortex_array(&self, _ctx: &mut ExecutionCtx) -> Result { + Ok(parquet_to_vortex_chunks(self.to_parquet_path().await?) + .await? + .into()) + } + + async fn to_parquet_path(&self) -> Result { + let paths = download(VectorDataset::OpenaiSmall50k, TrainLayout::Single).await?; + paths + .train_files + .into_iter() + .next() + .ok_or_else(|| anyhow::anyhow!("OpenAI embedding train file is missing")) + } +} + #[async_trait] impl BenchDataset for FeatureVectorsData { fn name(&self) -> &str { diff --git a/vortex-bench/src/lib.rs b/vortex-bench/src/lib.rs index 49c3a8d60fa..196991ede46 100644 --- a/vortex-bench/src/lib.rs +++ b/vortex-bench/src/lib.rs @@ -37,7 +37,6 @@ use vortex::utils::aliases::hash_map::HashMap; use vortex_btrblocks::SchemeExt; use vortex_btrblocks::schemes::float::FloatQuantScheme; use vortex_btrblocks::schemes::float::OrderedBlockResidualScheme; -use vortex_btrblocks::schemes::float::OrderedFloatRangePackedScheme; use vortex_btrblocks::schemes::integer::BlockResidualScheme; use crate::spatialbench::SpatialBenchBenchmark; @@ -244,8 +243,6 @@ pub enum CompactionStrategy { Default, } -const RANGE_PACKED_SCHEME: OrderedFloatRangePackedScheme = OrderedFloatRangePackedScheme::new(1.20); - /// Numeric scheme bundles for compression benchmarks. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, ValueEnum)] pub enum VortexNumericBundle { @@ -256,8 +253,6 @@ pub enum VortexNumericBundle { /// Use the current Default compressor. #[default] CurrentDefault, - /// Add the experimental OrderedFloat with RangePacked scheme. - RangePacked, } impl VortexNumericBundle { @@ -267,7 +262,6 @@ impl VortexNumericBundle { Self::PriorDefault => "prior-default", Self::BlockResidual => "block-residual", Self::CurrentDefault => "current-default", - Self::RangePacked => "range-packed", } } @@ -283,9 +277,6 @@ impl VortexNumericBundle { BtrBlocksCompressorBuilder::default().exclude_schemes([FloatQuantScheme.id()]) } Self::CurrentDefault => BtrBlocksCompressorBuilder::default(), - Self::RangePacked => { - BtrBlocksCompressorBuilder::default().with_new_scheme(&RANGE_PACKED_SCHEME) - } }; options.with_strategy( WriteStrategyBuilder::default() diff --git a/vortex-btrblocks/Cargo.toml b/vortex-btrblocks/Cargo.toml index 1bdcd5f6db9..f517a92d942 100644 --- a/vortex-btrblocks/Cargo.toml +++ b/vortex-btrblocks/Cargo.toml @@ -30,9 +30,7 @@ vortex-decimal-byte-parts = { workspace = true } vortex-error = { workspace = true } vortex-fastlanes = { workspace = true } vortex-float-quant = { workspace = true } -vortex-range-packed = { workspace = true } vortex-fsst = { workspace = true } -vortex-int-mult = { workspace = true } vortex-onpair = { workspace = true, optional = true } vortex-pco = { workspace = true, optional = true } vortex-runend = { workspace = true } @@ -47,7 +45,6 @@ arrow-array = { workspace = true } arrow-schema = { workspace = true } arrow-select = { workspace = true } divan = { workspace = true } -fastlanes = { workspace = true } insta = { workspace = true } parquet = { workspace = true } rstest = { workspace = true } @@ -81,7 +78,3 @@ test = false [[example]] name = "pco_probe" required-features = ["pco"] - -[[example]] -name = "float_compressor_bench" -required-features = ["pco", "zstd"] diff --git a/vortex-btrblocks/examples/float_compressor_bench.rs b/vortex-btrblocks/examples/float_compressor_bench.rs deleted file mode 100644 index eca4c56a03a..00000000000 --- a/vortex-btrblocks/examples/float_compressor_bench.rs +++ /dev/null @@ -1,3324 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#[path = "float_compressor_bench/int_mult_codec.rs"] -mod int_mult_codec; - -use std::fs::File; -use std::hint::black_box; -use std::io::BufRead; -use std::io::BufReader; -use std::path::Path; -use std::sync::Arc; -use std::time::Duration; -use std::time::Instant; - -use arrow_array::ArrayRef as ArrowArrayRef; -use arrow_array::FixedSizeListArray; -use arrow_array::LargeListArray; -use arrow_array::ListArray as ArrowListArray; -use arrow_schema::DataType; -use arrow_select::concat::concat; -use parquet::arrow::ProjectionMask; -use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; -use pco::ChunkConfig; -use pco::data_types::Number; -use pco::metadata::ChunkLatentVarMeta; -use pco::metadata::DeltaEncoding; -use pco::metadata::DynBins; -use pco::metadata::DynLatent; -use pco::metadata::Mode; -use pco::wrapped::FileCompressor; -use rand::RngExt; -use rand::SeedableRng; -use rand::rngs::StdRng; -use vortex_alp::ALP; -use vortex_alp::ALPArrayExt; -use vortex_alp::ALPArraySlotsExt; -use vortex_alp::ALPFloat; -use vortex_alp::alp_encode; -use vortex_array::ArrayRef; -use vortex_array::IntoArray; -use vortex_array::RecursiveCanonical; -use vortex_array::VortexSessionExecute; -use vortex_array::array_session; -use vortex_array::arrays::Chunked; -use vortex_array::arrays::ChunkedArray; -use vortex_array::arrays::Dict; -use vortex_array::arrays::DictArray; -use vortex_array::arrays::ListArray; -use vortex_array::arrays::Primitive; -use vortex_array::arrays::PrimitiveArray; -use vortex_array::arrays::TemporalArray; -use vortex_array::arrays::VarBinViewArray; -use vortex_array::arrays::chunked::ChunkedArrayExt; -use vortex_array::arrays::dict::DictArraySlotsExt; -use vortex_array::assert_arrays_eq; -use vortex_array::builtins::ArrayBuiltins; -use vortex_array::dtype::DType; -use vortex_array::dtype::PType; -use vortex_array::dtype::half::f16; -use vortex_array::extension::datetime::TimeUnit; -use vortex_array::match_each_integer_ptype; -use vortex_array::match_each_unsigned_integer_ptype; -use vortex_array::patches::Patches; -use vortex_array::validity::Validity; -use vortex_arrow::ArrowSessionExt; -use vortex_block_residual::BlockResidual; -use vortex_block_residual::BlockResidualArrayExt; -use vortex_block_residual::OrderedFloat; -use vortex_block_residual::OrderedFloatArraySlotsExt; -use vortex_btrblocks::BtrBlocksCompressor; -use vortex_btrblocks::BtrBlocksCompressorBuilder; -use vortex_btrblocks::SchemeExt; -use vortex_btrblocks::schemes::float::FloatQuantScheme; -use vortex_btrblocks::schemes::float::OrderedBlockResidualScheme; -use vortex_btrblocks::schemes::float::OrderedFloatRangePackedScheme; -use vortex_btrblocks::schemes::integer::BlockResidualScheme; -use vortex_btrblocks::schemes::integer::IntegerFixedBinsScheme; -use vortex_error::VortexResult; -use vortex_error::vortex_ensure; -use vortex_error::vortex_err; -use vortex_fastlanes::BitPacked; -use vortex_fastlanes::FoR; -use vortex_fastlanes::FoRArrayExt; -use vortex_fastlanes::FoRArraySlotsExt; -use vortex_fastlanes::bitpack_compress::bit_width_histogram; -use vortex_fastlanes::bitpack_compress::bitpack_encode; -use vortex_fastlanes::bitpack_compress::bitpack_encode_unchecked; -use vortex_fastlanes::bitpack_compress::find_best_bit_width; -use vortex_int_mult::IntMult; -use vortex_range_packed::RangeDecomposition; -use vortex_range_packed::RangePacked; -use vortex_range_packed::RangePackedCodec; -use vortex_session::VortexSession; -use vortex_utils::aliases::hash_map::HashMap; - -use crate::int_mult_codec::IntMultCodec32; -use crate::int_mult_codec::IntMultDenseCodec64; - -const DEFAULT_ROW_COUNT: usize = 2_000_000; -const RANGE_PACKED_SCHEME: OrderedFloatRangePackedScheme = OrderedFloatRangePackedScheme::new(1.20); -const INTEGER_FIXED_BINS_SCHEME: IntegerFixedBinsScheme = IntegerFixedBinsScheme::new(1.20); -const CALIFORNIA_COLUMNS: [&str; 9] = [ - "longitude", - "latitude", - "housingMedianAge", - "totalRooms", - "totalBedrooms", - "population", - "households", - "medianIncome", - "medianHouseValue", -]; - -struct Column { - name: String, - primitive: Option, - array: ArrayRef, - input_bytes: u64, - dtype_label: String, -} - -fn column(name: impl Into, primitive: PrimitiveArray) -> Column { - let array = primitive.clone().into_array(); - let input_bytes = array.nbytes(); - let dtype_label = primitive.ptype().to_string(); - Column { - name: name.into(), - primitive: Some(primitive), - array, - input_bytes, - dtype_label, - } -} - -fn array_column(name: impl Into, array: ArrayRef) -> Column { - Column { - name: name.into(), - primitive: None, - input_bytes: array.nbytes(), - dtype_label: array.dtype().to_string(), - array, - } -} - -fn synthetic_datasets(row_count: usize) -> VortexResult)>> { - let mut widened_rng = StdRng::seed_from_u64(1); - let widened_f32 = PrimitiveArray::from_iter((0..row_count).map(|index| { - let trend = (index % 10_000) as f32 * 0.001; - f64::from(trend + widened_rng.random_range(-1.0_f32..1.0)) - })); - let nonzero_secondary = - PrimitiveArray::from_iter(widened_f32.as_slice::().iter().enumerate().map( - |(index, value)| { - if index % 10 == 0 { - f64::from_bits(value.to_bits() | 1) - } else { - *value - } - }, - )); - let quantized_f32 = PrimitiveArray::from_iter((0_u32..).take(row_count).map(|index| { - let mantissa = (index.wrapping_mul(7_919) & 0x7fff) << 8; - f32::from_bits(0x3f80_0000 | mantissa) - })); - let nonzero_secondary_f32 = - PrimitiveArray::from_iter(quantized_f32.as_slice::().iter().enumerate().map( - |(index, value)| { - if index % 10 == 0 { - f32::from_bits(value.to_bits() | 1) - } else { - *value - } - }, - )); - let quantized_f16 = PrimitiveArray::from_iter((0..row_count).map(|index| { - let permuted = - u16::try_from(index.wrapping_mul(7_919) & usize::from(u16::MAX)).unwrap_or_default(); - let mantissa = permuted & 0x03f0; - f16::from_bits(0x3c00 | mantissa) - })); - let nonzero_secondary_f16 = - PrimitiveArray::from_iter(quantized_f16.as_slice::().iter().enumerate().map( - |(index, value)| { - if index % 10 == 0 { - f16::from_bits(value.to_bits() | 1) - } else { - *value - } - }, - )); - - let mut walk_rng = StdRng::seed_from_u64(2); - let mut value = 1_000.0_f64; - let random_walk = PrimitiveArray::from_iter((0..row_count).map(|_| { - value += walk_rng.random_range(-0.01_f64..0.01); - value - })); - - let mut uniform_rng = StdRng::seed_from_u64(3); - let uniform = PrimitiveArray::from_iter( - (0..row_count).map(|_| uniform_rng.random_range(-1_000_000.0_f64..1_000_000.0)), - ); - - let integer_valued = PrimitiveArray::from_iter((0..row_count).map(|index| { - let permuted = (index as u64).wrapping_mul(2_654_435_761) % 1_000_000; - permuted as f64 - 500_000.0 - })); - let block_local_integer_valued = PrimitiveArray::from_iter((0..row_count).map(|index| { - let block = index / 1_024; - let residual = index.wrapping_mul(2_654_435_761) % 1_024; - (block * 1_000_000 + residual) as f64 - })); - let block_local_ordered_float = PrimitiveArray::from_iter((0..row_count).map(|index| { - let block = index / 1_024; - let residual = index.wrapping_mul(2_654_435_761) % 1_024; - let bits = 0x3ff0_0000_0000_0000_u64 - + u64::try_from(block).unwrap_or(u64::MAX) * 0x1_0000 - + u64::try_from(residual).unwrap_or(u64::MAX); - f64::from_bits(bits) - })); - - let patch_density_4 = PrimitiveArray::from_iter((0..row_count).map(|index| { - if index % 4 == 0 { - u32::MAX - u32::try_from(index).unwrap_or(u32::MAX) - } else { - 42 - } - })); - let patch_density_1 = PrimitiveArray::from_iter( - (0..row_count).map(|index| u32::MAX - u32::try_from(index).unwrap_or(u32::MAX)), - ); - let block_local_i16 = PrimitiveArray::from_iter((0..row_count).map(|index| { - let block = (index / 1_024) % 128; - let residual = index.wrapping_mul(2_654_435_761) % 128; - i16::try_from(block * 128 + residual).unwrap_or(i16::MAX) - })); - let block_local_i8 = PrimitiveArray::from_iter((0..row_count).map(|index| { - let block = (index / 1_024) % 16; - let residual = index.wrapping_mul(2_654_435_761) % 16; - let unsigned = i16::try_from(block * 16 + residual).unwrap_or(i16::MAX); - i8::try_from(unsigned - 128).unwrap_or(i8::MAX) - })); - let uniform_i8 = PrimitiveArray::from_iter((0..row_count).map(|index| { - let value = u8::try_from(index.wrapping_mul(2_654_435_761) % 256).unwrap_or(u8::MAX); - i8::from_le_bytes([value]) - })); - let sparse_block_local = PrimitiveArray::from_iter((0..row_count).map(|index| { - if index % 16 == 0 { - let value_index = index / 16; - let block = value_index / 1_024; - let residual = value_index.wrapping_mul(2_654_435_761) % 1_024; - u64::try_from(block).unwrap_or(u64::MAX) * 1_000_000_000_000 - + u64::try_from(residual).unwrap_or(u64::MAX) - } else { - 42 - } - })); - let runend_block_local = PrimitiveArray::from_iter((0..row_count).map(|index| { - let value_index = index / 16; - let block = value_index / 1_024; - let residual = value_index.wrapping_mul(2_654_435_761) % 1_024; - u64::try_from(block).unwrap_or(u64::MAX) * 1_000_000_000_000 - + u64::try_from(residual).unwrap_or(u64::MAX) - })); - - let mut temporal_rng = StdRng::seed_from_u64(4); - let mut timestamp = 1_700_000_000_000_000_i64; - let timestamp_values = PrimitiveArray::from_iter((0..row_count).map(|_| { - timestamp += temporal_rng.random_range(1_000_i64..1_000_000); - timestamp - })); - let temporal = TemporalArray::new_timestamp( - timestamp_values.into_array(), - TimeUnit::Microseconds, - Some(Arc::from("UTC")), - ) - .into_array(); - - let list_elements = PrimitiveArray::from_iter((0..row_count).map(|index| { - let block = index / 1_024; - let residual = index.wrapping_mul(2_654_435_761) % 1_024; - u64::try_from(block).unwrap_or(u64::MAX) * 1_000_000_000_000 - + u64::try_from(residual).unwrap_or(u64::MAX) - })); - let mut list_offsets = Vec::with_capacity(row_count.div_ceil(8) + 1); - list_offsets.push(0_u32); - let mut list_offset = 0usize; - while list_offset < row_count { - list_offset = (list_offset + 8).min(row_count); - list_offsets.push(u32::try_from(list_offset).unwrap_or(u32::MAX)); - } - let list = ListArray::try_new( - list_elements.into_array(), - PrimitiveArray::from_iter(list_offsets).into_array(), - Validity::NonNullable, - )? - .into_array(); - - let string_count = row_count.min(500_000); - let strings = (0..string_count) - .map(|index| { - format!( - "user{:06}@example{}.com", - index.wrapping_mul(2_654_435_761) % 1_000_000, - index % 100 - ) - }) - .collect::>(); - let fsst_strings = - VarBinViewArray::from_iter_str(strings.iter().map(String::as_str)).into_array(); - - Ok(vec![ - ( - "synthetic-widened-f32".to_string(), - vec![column("value", widened_f32)], - ), - ( - "synthetic-nonzero-secondary".to_string(), - vec![column("value", nonzero_secondary)], - ), - ( - "synthetic-nonzero-secondary-f16".to_string(), - vec![column("value", nonzero_secondary_f16)], - ), - ( - "synthetic-nonzero-secondary-f32".to_string(), - vec![column("value", nonzero_secondary_f32)], - ), - ( - "synthetic-quantized-f32".to_string(), - vec![column("value", quantized_f32)], - ), - ( - "synthetic-random-walk".to_string(), - vec![column("value", random_walk)], - ), - ( - "synthetic-uniform".to_string(), - vec![column("value", uniform)], - ), - ( - "synthetic-integer-valued".to_string(), - vec![column("value", integer_valued)], - ), - ( - "synthetic-alp-block-local".to_string(), - vec![column("value", block_local_integer_valued)], - ), - ( - "synthetic-ordered-float-block-local".to_string(), - vec![column("value", block_local_ordered_float)], - ), - ( - "synthetic-patch-density-4".to_string(), - vec![column("value", patch_density_4)], - ), - ( - "synthetic-patch-density-1".to_string(), - vec![column("value", patch_density_1)], - ), - ( - "synthetic-block-local-i16".to_string(), - vec![column("value", block_local_i16)], - ), - ( - "synthetic-block-local-i8".to_string(), - vec![column("value", block_local_i8)], - ), - ( - "synthetic-uniform-i8".to_string(), - vec![column("value", uniform_i8)], - ), - ( - "synthetic-sparse-block-local".to_string(), - vec![column("value", sparse_block_local)], - ), - ( - "synthetic-runend-block-local".to_string(), - vec![column("value", runend_block_local)], - ), - ( - "synthetic-temporal-parent".to_string(), - vec![array_column("value", temporal)], - ), - ( - "synthetic-list-parent".to_string(), - vec![array_column("value", list)], - ), - ( - "synthetic-fsst-parent".to_string(), - vec![array_column("value", fsst_strings)], - ), - ]) -} - -fn read_california(path: &Path, row_count: usize) -> VortexResult> { - let reader = BufReader::new(File::open(path)?); - let mut values = std::array::from_fn::<_, 9, _>(|_| Vec::::new()); - for (line_index, line) in reader.lines().enumerate() { - let line = line?; - let fields = line.split(',').collect::>(); - vortex_ensure!( - fields.len() == CALIFORNIA_COLUMNS.len(), - "line {} contains {} fields instead of {}", - line_index + 1, - fields.len(), - CALIFORNIA_COLUMNS.len() - ); - for (column_index, field) in fields.into_iter().enumerate() { - values[column_index].push(field.parse::().map_err(|error| { - vortex_err!( - "cannot parse line {} column {} as f32: {}", - line_index + 1, - column_index + 1, - error - ) - })?); - } - } - let repeat_short_input = std::env::var_os("VORTEX_BENCH_REPEAT_SHORT").is_some(); - for values in &mut values { - vortex_ensure!(!values.is_empty(), "California Housing input is empty"); - values.truncate(row_count); - if repeat_short_input { - while values.len() < row_count { - let copy_len = (row_count - values.len()).min(values.len()); - values.extend_from_within(..copy_len); - } - } - } - Ok(CALIFORNIA_COLUMNS - .into_iter() - .zip(values) - .map(|(name, values)| column(name, PrimitiveArray::from_iter(values))) - .collect()) -} - -fn read_parquet_numeric( - path: &Path, - row_count: usize, - session: &VortexSession, -) -> VortexResult> { - let builder = ParquetRecordBatchReaderBuilder::try_new(File::open(path)?) - .map_err(|error| vortex_err!("cannot read Parquet metadata: {error}"))?; - let schema = Arc::clone(builder.schema()); - let selected = schema - .fields() - .iter() - .enumerate() - .filter_map(|(index, field)| { - let data_type = match field.data_type() { - DataType::List(field) - | DataType::LargeList(field) - | DataType::FixedSizeList(field, _) => field.data_type(), - data_type => data_type, - }; - matches!( - data_type, - DataType::Int8 - | DataType::Int16 - | DataType::Int32 - | DataType::Int64 - | DataType::UInt8 - | DataType::UInt16 - | DataType::UInt32 - | DataType::UInt64 - | DataType::Float32 - | DataType::Float64 - | DataType::Timestamp(_, _) - ) - .then_some(index) - }) - .collect::>(); - vortex_ensure!( - !selected.is_empty(), - "Parquet file contains no numeric columns" - ); - let mask = ProjectionMask::roots(builder.parquet_schema(), selected.iter().copied()); - let reader = builder - .with_projection(mask) - .with_batch_size(65_536) - .build() - .map_err(|error| vortex_err!("cannot build Parquet reader: {error}"))?; - let mut chunks = (0..selected.len()) - .map(|_| Vec::::new()) - .collect::>(); - let mut rows_read = 0usize; - for batch in reader { - let batch = batch.map_err(|error| vortex_err!("cannot read Parquet batch: {error}"))?; - let batch_len = batch.num_rows().min(row_count - rows_read); - for (column_chunks, array) in chunks.iter_mut().zip(batch.columns()) { - column_chunks.push(array.slice(0, batch_len)); - } - rows_read += batch_len; - if rows_read == row_count { - break; - } - } - vortex_ensure!(rows_read > 0, "Parquet file contains no rows"); - - selected - .into_iter() - .zip(chunks) - .map(|(field_index, chunks)| { - let field = &schema.fields()[field_index]; - let chunk_refs = chunks - .iter() - .map(|chunk| chunk.as_ref()) - .collect::>(); - let combined = concat(&chunk_refs) - .map_err(|error| vortex_err!("cannot concatenate {}: {error}", field.name()))?; - let (combined, field, name) = match field.data_type() { - DataType::FixedSizeList(value_field, _) => { - let list = combined - .as_any() - .downcast_ref::() - .ok_or_else(|| vortex_err!("{} is not a fixed-size list", field.name()))?; - let values = list.values(); - ( - values.slice(0, values.len().min(row_count)), - Arc::clone(value_field), - format!("{}.values", field.name()), - ) - } - DataType::List(value_field) => { - let list = combined - .as_any() - .downcast_ref::() - .ok_or_else(|| vortex_err!("{} is not a list", field.name()))?; - let values = list.values(); - ( - values.slice(0, values.len().min(row_count)), - Arc::clone(value_field), - format!("{}.values", field.name()), - ) - } - DataType::LargeList(value_field) => { - let list = combined - .as_any() - .downcast_ref::() - .ok_or_else(|| vortex_err!("{} is not a large list", field.name()))?; - let values = list.values(); - ( - values.slice(0, values.len().min(row_count)), - Arc::clone(value_field), - format!("{}.values", field.name()), - ) - } - _ => (combined, Arc::clone(field), field.name().to_string()), - }; - let array = session.arrow().from_arrow_array(combined, field.as_ref())?; - let array = if matches!(field.data_type(), DataType::Timestamp(_, _)) { - array.cast(DType::Primitive(PType::I64, array.dtype().nullability()))? - } else { - array - }; - let primitive = array.execute::(&mut session.create_execution_ctx())?; - Ok(column(name, primitive)) - }) - .collect() -} - -fn encoding_tree(array: &ArrayRef) -> String { - let children = array.children(); - if children.is_empty() { - return array.encoding_id().to_string(); - } - let children = children - .iter() - .map(encoding_tree) - .collect::>() - .join(","); - format!("{}({children})", array.encoding_id()) -} - -fn profile_block_residual_array( - dataset: &str, - column: &str, - config: &str, - path: &str, - array: &ArrayRef, -) -> VortexResult<()> { - if let Some(residuals) = array.as_typed::() { - let residual_widths = residuals.residual_widths(); - let high_widths = residuals.high_widths(); - let blocks = residual_widths.len(); - let average_residual_width = residual_widths - .iter() - .map(|&width| f64::from(width)) - .sum::() - / blocks as f64; - let average_high_width = high_widths - .iter() - .map(|&width| f64::from(width)) - .sum::() - / blocks as f64; - let patch_starts = residuals.patch_starts(); - let mut maximum_patch_density = 0.0_f64; - let mut blocks_above_one_eighth = 0usize; - let mut blocks_at_one_quarter = 0usize; - for (block_index, starts) in patch_starts.windows(2).enumerate() { - let patch_count = usize::try_from(starts[1] - starts[0])?; - let block_start = block_index * 1_024; - let block_len = (array.len() - block_start).min(1_024); - maximum_patch_density = - maximum_patch_density.max(patch_count as f64 / block_len as f64); - blocks_above_one_eighth += usize::from(patch_count * 8 > block_len); - blocks_at_one_quarter += usize::from(patch_count * 4 >= block_len); - } - println!( - "block-residual-profile\t{dataset}\t{column}\t{config}\t{path}\t{}\t{}\t{}\t{blocks}\t{average_residual_width:.3}\t{average_high_width:.3}\t{maximum_patch_density:.3}\t{blocks_above_one_eighth}\t{blocks_at_one_quarter}\t{}", - array.dtype().as_ptype(), - array.len(), - residuals.patch_positions().len(), - array.nbytes(), - ); - } - for (child_index, child) in array.children().iter().enumerate() { - profile_block_residual_array( - dataset, - column, - config, - &format!("{path}/{child_index}"), - child, - )?; - } - Ok(()) -} - -fn encode_all( - compressor: &BtrBlocksCompressor, - columns: &[Column], - session: &VortexSession, -) -> VortexResult> { - let chunk_rows = std::env::var("VORTEX_BENCH_CHUNK_ROWS") - .ok() - .map(|value| { - value - .parse::() - .map_err(|error| vortex_err!("invalid VORTEX_BENCH_CHUNK_ROWS: {error}")) - }) - .transpose()?; - columns - .iter() - .map(|column| { - let Some(chunk_rows) = chunk_rows else { - return compressor.compress(&column.array, &mut session.create_execution_ctx()); - }; - vortex_ensure!(chunk_rows > 0, "VORTEX_BENCH_CHUNK_ROWS must be positive"); - let chunks = (0..column.array.len()) - .step_by(chunk_rows) - .map(|start| { - let stop = (start + chunk_rows).min(column.array.len()); - let chunk = column.array.slice(start..stop)?; - compressor.compress(&chunk, &mut session.create_execution_ctx()) - }) - .collect::>>()?; - Ok(ChunkedArray::try_new(chunks, column.array.dtype().clone())?.into_array()) - }) - .collect() -} - -fn decode_all(arrays: &[ArrayRef], session: &VortexSession) -> VortexResult<()> { - for array in arrays { - black_box( - array - .clone() - .execute::(&mut session.create_execution_ctx())?, - ); - } - Ok(()) -} - -fn percentile(durations: &mut [Duration], numerator: usize, denominator: usize) -> Duration { - durations.sort_unstable(); - durations[durations.len() * numerator / denominator] -} - -struct BinProfile { - count: usize, - ans_size_log: u32, - max_offset_bits: u32, - average_bits: f64, -} - -fn bin_profile(meta: &ChunkLatentVarMeta) -> BinProfile { - pco::match_latent_enum!(&meta.bins, DynBins(bins) => { - let total_weight = (1_u64 << meta.ans_size_log) as f64; - let average_bits = bins - .iter() - .map(|bin| { - let ans_bits = f64::from(meta.ans_size_log) - f64::from(bin.weight).log2(); - (ans_bits + f64::from(bin.offset_bits)) * f64::from(bin.weight) / total_weight - }) - .sum(); - BinProfile { - count: bins.len(), - ans_size_log: meta.ans_size_log, - max_offset_bits: bins - .iter() - .map(|bin| bin.offset_bits) - .max() - .unwrap_or_default(), - average_bits, - } - }) -} - -fn optional_bin_profile(meta: Option<&ChunkLatentVarMeta>) -> BinProfile { - meta.map(bin_profile).unwrap_or(BinProfile { - count: 0, - ans_size_log: 0, - max_offset_bits: 0, - average_bits: 0.0, - }) -} - -fn mode_name(mode: &Mode) -> String { - match mode { - Mode::Classic => "classic".to_string(), - Mode::IntMult(base) => format!("int-mult-{}", dyn_latent_value(*base)), - Mode::FloatMult(_) => "float-mult".to_string(), - Mode::FloatQuant(k) => format!("float-quant-{k}"), - Mode::Dict(_) => "dict".to_string(), - _ => "unknown".to_string(), - } -} - -fn dyn_latent_value(value: DynLatent) -> u64 { - match value { - DynLatent::U8(value) => u64::from(value), - DynLatent::U16(value) => u64::from(value), - DynLatent::U32(value) => u64::from(value), - DynLatent::U64(value) => value, - _ => unreachable!("unsupported Pco latent type"), - } -} - -fn delta_name(delta: &DeltaEncoding) -> String { - match delta { - DeltaEncoding::NoOp => "none".to_string(), - DeltaEncoding::Consecutive { - order, - secondary_uses_delta, - } => format!("consecutive-{order}-secondary-{secondary_uses_delta}"), - DeltaEncoding::Lookback { - config, - secondary_uses_delta, - } => format!( - "lookback-state-{}-window-{}-secondary-{secondary_uses_delta}", - config.state_n_log, config.window_n_log - ), - DeltaEncoding::Conv1(config) => format!("conv1-quantization-{}", config.quantization), - _ => "unknown".to_string(), - } -} - -fn profile_pco_values( - dataset: &str, - column: &str, - path: &str, - ptype: PType, - values: &[T], -) -> VortexResult<()> { - let file_compressor = FileCompressor::default(); - for (chunk_index, chunk) in values.chunks(pco::DEFAULT_MAX_PAGE_N).enumerate() { - let compressor = file_compressor - .chunk_compressor(chunk, &ChunkConfig::default()) - .map_err(|error| vortex_err!("cannot profile Pco chunk: {error}"))?; - let meta = compressor.meta(); - let delta = optional_bin_profile(meta.per_latent_var.delta.as_ref()); - let primary = bin_profile(&meta.per_latent_var.primary); - let secondary = optional_bin_profile(meta.per_latent_var.secondary.as_ref()); - println!( - "pco-profile\t{dataset}\t{column}\t{path}\t{ptype}\t{chunk_index}\t{}\t{}\t{}\t{}\t{}\t{:.3}\t{}\t{}\t{}\t{:.3}\t{}\t{}\t{}\t{:.3}", - chunk.len(), - mode_name(&meta.mode), - delta_name(&meta.delta_encoding), - delta.count, - delta.max_offset_bits, - delta.average_bits, - primary.count, - primary.ans_size_log, - primary.max_offset_bits, - primary.average_bits, - secondary.count, - secondary.ans_size_log, - secondary.max_offset_bits, - secondary.average_bits, - ); - } - Ok(()) -} - -fn reference_id_bits(reference_count: usize) -> usize { - match reference_count { - 0 | 1 => 0, - 2 => 1, - _ => 2, - } -} - -fn estimate_multi_reference_block( - values: &[u64], - requested_references: usize, - bits: usize, -) -> usize { - const BLOCK_LEN: usize = 1024; - const METADATA_BYTES: usize = 12; - const HIGH_PADDING_BYTES: usize = 15; - - let mut sorted = values.to_vec(); - sorted.sort_unstable(); - let mut references = (0..requested_references) - .map(|index| sorted[index * sorted.len() / requested_references]) - .collect::>(); - references.dedup(); - - let mut width_counts = [0_usize; 65]; - let mut maximum_width = 0_usize; - for &value in values { - let reference_index = references - .partition_point(|&reference| reference <= value) - .saturating_sub(1); - let residual = value - references[reference_index]; - let width = u64::BITS as usize - residual.leading_zeros() as usize; - width_counts[width] += 1; - maximum_width = maximum_width.max(width); - } - - let mut patch_count = values.len(); - let mut best_bits = usize::MAX; - for residual_width in 0..=maximum_width { - patch_count -= width_counts[residual_width]; - let high_width = if patch_count == 0 { - 0 - } else { - maximum_width - residual_width - }; - let cost_bits = residual_width * BLOCK_LEN - + reference_id_bits(references.len()) * BLOCK_LEN - + references.len() * bits - + patch_count * (u16::BITS as usize + high_width) - + METADATA_BYTES * 8 - + usize::from(patch_count > 0) * HIGH_PADDING_BYTES * 8; - best_bits = best_bits.min(cost_bits); - } - best_bits.div_ceil(8) -} - -fn estimate_multi_reference(values: &[u64], requested_references: usize, bits: usize) -> usize { - values - .chunks(1024) - .map(|block| estimate_multi_reference_block(block, requested_references, bits)) - .sum() -} - -fn estimate_bitmap_patches(values: &[u64], bits: usize) -> usize { - const BLOCK_LEN: usize = 1_024; - const METADATA_BYTES: usize = 12; - const HIGH_PADDING_BYTES: usize = 15; - - values - .chunks(BLOCK_LEN) - .map(|block| { - let base = block.iter().copied().min().unwrap_or_default(); - let mut width_counts = [0_usize; 65]; - let mut maximum_width = 0_usize; - for value in block { - let residual = value - base; - let width = u64::BITS as usize - residual.leading_zeros() as usize; - width_counts[width] += 1; - maximum_width = maximum_width.max(width); - } - - let mut patch_count = block.len(); - let mut best_bits = usize::MAX; - for residual_width in 0..=maximum_width { - patch_count -= width_counts[residual_width]; - let high_width = if patch_count == 0 { - 0 - } else { - maximum_width - residual_width - }; - let cost_bits = residual_width * BLOCK_LEN - + bits - + usize::from(patch_count > 0) * BLOCK_LEN - + patch_count * high_width - + METADATA_BYTES * 8 - + usize::from(patch_count > 0) * HIGH_PADDING_BYTES * 8; - best_bits = best_bits.min(cost_bits); - } - best_bits.div_ceil(8) - }) - .sum() -} - -fn estimate_mode_bitmap(values: &[u64], bits: usize) -> usize { - const BLOCK_LEN: usize = 1_024; - const METADATA_BYTES: usize = 8; - const VALUE_PADDING_BYTES: usize = 15; - - values - .chunks(BLOCK_LEN) - .map(|block| { - let mut counts = HashMap::::new(); - for value in block { - *counts.entry(*value).or_default() += 1; - } - let mode = counts - .into_iter() - .max_by_key(|(_, count)| *count) - .map(|(value, _)| value) - .unwrap_or_default(); - let exceptions = block.iter().filter(|&&value| value != mode).count(); - let exception_width = block - .iter() - .filter(|&&value| value != mode) - .map(|&value| u64::BITS as usize - value.leading_zeros() as usize) - .max() - .unwrap_or_default(); - let cost_bits = bits - + BLOCK_LEN - + exceptions * exception_width - + METADATA_BYTES * 8 - + usize::from(exceptions > 0) * VALUE_PADDING_BYTES * 8; - cost_bits.div_ceil(8) - }) - .sum() -} - -fn ordered_f32(value: f32) -> u64 { - let bits = value.to_bits(); - u64::from(if bits & (1_u32 << 31) == 0 { - bits ^ (1_u32 << 31) - } else { - !bits - }) -} - -fn ordered_f64(value: f64) -> u64 { - let bits = value.to_bits(); - if bits & (1_u64 << 63) == 0 { - bits ^ (1_u64 << 63) - } else { - !bits - } -} - -fn ordered_values(values: &PrimitiveArray) -> Vec { - match values.ptype() { - PType::F32 => values - .as_slice::() - .iter() - .copied() - .map(ordered_f32) - .collect(), - PType::F64 => values - .as_slice::() - .iter() - .copied() - .map(ordered_f64) - .collect(), - PType::I16 => values - .as_slice::() - .iter() - .map(|&value| u64::from((value as u16) ^ (1_u16 << 15))) - .collect(), - PType::I32 => values - .as_slice::() - .iter() - .map(|&value| u64::from((value as u32) ^ (1_u32 << 31))) - .collect(), - PType::I64 => values - .as_slice::() - .iter() - .map(|&value| (value as u64) ^ (1_u64 << 63)) - .collect(), - PType::U16 => values - .as_slice::() - .iter() - .map(|&value| u64::from(value)) - .collect(), - PType::U32 => values - .as_slice::() - .iter() - .map(|&value| u64::from(value)) - .collect(), - PType::U64 => values.as_slice::().to_vec(), - ptype => unreachable!("Pco does not support {ptype}"), - } -} - -fn profile_quotient_remainder( - dataset: &str, - column: &str, - path: &str, - ptype: PType, - pco_bytes: u64, - ordered: &[u64], - session: &VortexSession, -) -> VortexResult<()> { - const BASES: [u64; 9] = [2, 4, 5, 8, 10, 16, 32, 100, 1_000]; - - let compressor = BtrBlocksCompressor::default(); - for base in BASES { - let mut remainder_counts = HashMap::::new(); - for value in ordered { - *remainder_counts.entry(value % base).or_default() += 1; - } - let remainder_entropy = remainder_counts - .values() - .map(|&count| { - let probability = count as f64 / ordered.len() as f64; - -probability * probability.log2() - }) - .sum::(); - let most_common_remainder_share = - remainder_counts.values().copied().max().unwrap_or_default() as f64 - / ordered.len() as f64; - let quotients = ordered.iter().map(|value| value / base).collect::>(); - let remainders = ordered.iter().map(|value| value % base).collect::>(); - let quotient_bitmap_bytes = estimate_bitmap_patches("ients, ptype.bit_width()); - let remainder_mode_bitmap_bytes = estimate_mode_bitmap(&remainders, ptype.bit_width()); - let quotient = compressor.compress( - &latent_array("ients, ptype)?.into_array(), - &mut session.create_execution_ctx(), - )?; - let remainder = compressor.compress( - &latent_array(&remainders, ptype)?.into_array(), - &mut session.create_execution_ctx(), - )?; - println!( - "quotient-remainder-estimate\t{dataset}\t{column}\t{path}\t{ptype}\t{base}\t{pco_bytes}\t{remainder_entropy:.3}\t{most_common_remainder_share:.3}\t{}\t{}\t{}\t{quotient_bitmap_bytes}\t{remainder_mode_bitmap_bytes}\t{}\t{}\t{}", - quotient.nbytes(), - remainder.nbytes(), - quotient.nbytes() + remainder.nbytes(), - quotient_bitmap_bytes + remainder_mode_bitmap_bytes, - encoding_tree("ient), - encoding_tree(&remainder), - ); - } - if std::env::var_os("VORTEX_BENCH_INT_MULT").is_some() { - match ptype.bit_width() { - 32 => profile_int_mult_codec(dataset, column, path, ptype, pco_bytes, ordered)?, - 64 => profile_int_mult_dense_codec64(dataset, column, path, ptype, pco_bytes, ordered)?, - _ => {} - } - } - Ok(()) -} - -fn latent_array(values: &[u64], ptype: PType) -> VortexResult { - match ptype.bit_width() { - 16 => Ok(PrimitiveArray::from_iter( - values - .iter() - .map(|&value| u16::try_from(value)) - .collect::, _>>()?, - )), - 32 => Ok(PrimitiveArray::from_iter( - values - .iter() - .map(|&value| u32::try_from(value)) - .collect::, _>>()?, - )), - 64 => Ok(PrimitiveArray::from_iter(values.iter().copied())), - width => Err(vortex_err!( - "quotient and remainder profiling does not support {width}-bit values" - )), - } -} - -fn profile_int_mult_codec( - dataset: &str, - column: &str, - path: &str, - ptype: PType, - pco_bytes: u64, - ordered: &[u64], -) -> VortexResult<()> { - let values = ordered - .iter() - .map(|&value| u32::try_from(value)) - .collect::, _>>()?; - for base in [2, 5, 10, 16, 100, 1_000] { - let mut encode_durations = Vec::with_capacity(5); - let mut codec = IntMultCodec32::encode(&values, base)?; - for _ in 0..5 { - let start = Instant::now(); - codec = IntMultCodec32::encode(black_box(&values), base)?; - encode_durations.push(start.elapsed()); - } - let encode_median = percentile(&mut encode_durations, 1, 2); - - vortex_ensure!(codec.decode() == values, "IntMult codec decode differs"); - let decode_iterations = codec_decode_iterations()?; - let mut decode_durations = Vec::with_capacity(decode_iterations); - for _ in 0..decode_iterations { - let start = Instant::now(); - black_box(codec.decode()); - decode_durations.push(start.elapsed()); - } - let decode_median = percentile(&mut decode_durations, 1, 2); - - let scalar_iterations = 200_000; - let mut scalar_index = 0usize; - let mut scalar_checksum = 0u32; - let scalar_start = Instant::now(); - for _ in 0..scalar_iterations { - scalar_index = scalar_index.wrapping_add(2_654_435_761) % values.len(); - scalar_checksum ^= black_box(codec.scalar_at(scalar_index)?); - } - black_box(scalar_checksum); - let scalar_duration = scalar_start.elapsed(); - - let input_bytes = values.len() * ptype.byte_width(); - let encode_throughput = input_bytes as f64 / encode_median.as_secs_f64() / 1_000_000.0; - let decode_throughput = input_bytes as f64 / decode_median.as_secs_f64() / 1_000_000.0; - let scalar_nanoseconds = - scalar_duration.as_secs_f64() * 1_000_000_000.0 / scalar_iterations as f64; - println!( - "int-mult-checkpoint\t{dataset}\t{column}\t{path}\t{ptype}\t{base}\t{}\t{pco_bytes}\t{}\t{}\t{}\t{encode_throughput:.1}\t{decode_throughput:.1}\t{scalar_nanoseconds:.1}", - values.len(), - codec.encoded_size(), - codec.quotient_patch_count(), - codec.remainder_exception_count(), - ); - let mut gap_decode_durations = Vec::with_capacity(decode_iterations); - for _ in 0..decode_iterations { - let start = Instant::now(); - black_box(codec.decode_gaps()); - gap_decode_durations.push(start.elapsed()); - } - let gap_decode_median = percentile(&mut gap_decode_durations, 1, 2); - let gap_decode_throughput = - input_bytes as f64 / gap_decode_median.as_secs_f64() / 1_000_000.0; - let mut gap_scalar_index = 0usize; - let mut gap_scalar_checksum = 0u32; - let gap_scalar_start = Instant::now(); - for _ in 0..scalar_iterations { - gap_scalar_index = gap_scalar_index.wrapping_add(2_654_435_761) % values.len(); - gap_scalar_checksum ^= black_box(codec.scalar_at_gaps(gap_scalar_index)?); - } - black_box(gap_scalar_checksum); - let gap_scalar_nanoseconds = - gap_scalar_start.elapsed().as_secs_f64() * 1_000_000_000.0 / scalar_iterations as f64; - let (quotient_gap_bytes, remainder_gap_bytes) = codec.gap_bytes(); - let (quotient_bitmap_bytes, remainder_bitmap_bytes) = codec.bitmap_bytes(); - println!( - "int-mult-gaps-checkpoint\t{dataset}\t{column}\t{path}\t{ptype}\t{base}\t{}\t{pco_bytes}\t{}\t{quotient_gap_bytes}\t{remainder_gap_bytes}\t{quotient_bitmap_bytes}\t{remainder_bitmap_bytes}\t{encode_throughput:.1}\t{gap_decode_throughput:.1}\t{gap_scalar_nanoseconds:.1}", - values.len(), - codec.encoded_size_gaps(), - ); - if base <= 16 { - let mut pair_decode_durations = Vec::with_capacity(decode_iterations); - for _ in 0..decode_iterations { - let start = Instant::now(); - black_box(codec.decode_pairs()); - pair_decode_durations.push(start.elapsed()); - } - let pair_decode_median = percentile(&mut pair_decode_durations, 1, 2); - let pair_decode_throughput = - input_bytes as f64 / pair_decode_median.as_secs_f64() / 1_000_000.0; - println!( - "int-mult-pairs-checkpoint\t{dataset}\t{column}\t{path}\t{ptype}\t{base}\t{}\t{pco_bytes}\t{}\t{encode_throughput:.1}\t{pair_decode_throughput:.1}", - values.len(), - codec.encoded_size_pairs(), - ); - } - if base == 10 { - profile_int_mult_decode_breakdown(&codec, input_bytes, decode_iterations); - } - } - Ok(()) -} - -fn profile_int_mult_decode_breakdown( - codec: &IntMultCodec32, - input_bytes: usize, - iterations: usize, -) { - type DecodeVariant = fn(&IntMultCodec32) -> Vec; - let variants: [(&str, DecodeVariant); 9] = [ - ("full", IntMultCodec32::decode), - ( - "no-quotient-patches", - IntMultCodec32::decode_without_quotient_patches, - ), - ( - "no-remainder-exceptions", - IntMultCodec32::decode_without_remainder_exceptions, - ), - ("no-exceptions", IntMultCodec32::decode_without_exceptions), - ("gaps-full", IntMultCodec32::decode_gaps), - ( - "gaps-no-quotient-patches", - IntMultCodec32::decode_gaps_without_quotient_patches, - ), - ( - "gaps-no-remainder-exceptions", - IntMultCodec32::decode_gaps_without_remainder_exceptions, - ), - ( - "gaps-no-exceptions", - IntMultCodec32::decode_gaps_without_exceptions, - ), - ("pairs-full", IntMultCodec32::decode_pairs), - ]; - for (variant, decode) in variants { - let mut durations = Vec::with_capacity(iterations); - for _ in 0..iterations { - let start = Instant::now(); - black_box(decode(codec)); - durations.push(start.elapsed()); - } - let median = percentile(&mut durations, 1, 2); - let throughput = input_bytes as f64 / median.as_secs_f64() / 1_000_000.0; - println!("int-mult-decode-breakdown\t{variant}\t{throughput:.1}"); - } -} - -fn profile_int_mult_dense_codec64( - dataset: &str, - column: &str, - path: &str, - ptype: PType, - pco_bytes: u64, - values: &[u64], -) -> VortexResult<()> { - for base in [2, 5, 10, 16, 100, 1_000] { - let mut encode_durations = Vec::with_capacity(5); - let mut codec = IntMultDenseCodec64::encode(values, base)?; - for _ in 0..5 { - let start = Instant::now(); - codec = IntMultDenseCodec64::encode(black_box(values), base)?; - encode_durations.push(start.elapsed()); - } - let encode_median = percentile(&mut encode_durations, 1, 2); - - vortex_ensure!(codec.decode() == values, "dense IntMult decode differs"); - let decode_iterations = codec_decode_iterations()?; - let mut decode_durations = Vec::with_capacity(decode_iterations); - for _ in 0..decode_iterations { - let start = Instant::now(); - black_box(codec.decode()); - decode_durations.push(start.elapsed()); - } - let decode_median = percentile(&mut decode_durations, 1, 2); - - let scalar_iterations = 200_000; - let mut scalar_index = 0usize; - let mut scalar_checksum = 0u64; - let scalar_start = Instant::now(); - for _ in 0..scalar_iterations { - scalar_index = scalar_index.wrapping_add(2_654_435_761) % values.len(); - scalar_checksum ^= black_box(codec.scalar_at(scalar_index)?); - } - black_box(scalar_checksum); - let scalar_duration = scalar_start.elapsed(); - - let input_bytes = values.len() * ptype.byte_width(); - let encode_throughput = input_bytes as f64 / encode_median.as_secs_f64() / 1_000_000.0; - let decode_throughput = input_bytes as f64 / decode_median.as_secs_f64() / 1_000_000.0; - let scalar_nanoseconds = - scalar_duration.as_secs_f64() * 1_000_000_000.0 / scalar_iterations as f64; - println!( - "int-mult-dense64-checkpoint\t{dataset}\t{column}\t{path}\t{ptype}\t{base}\t{}\t{pco_bytes}\t{}\t{}\t{encode_throughput:.1}\t{decode_throughput:.1}\t{scalar_nanoseconds:.1}", - values.len(), - codec.encoded_size(), - codec.quotient_patch_count(), - ); - if base == 10 { - let mut no_patch_durations = Vec::with_capacity(decode_iterations); - for _ in 0..decode_iterations { - let start = Instant::now(); - black_box(codec.decode_without_quotient_patches()); - no_patch_durations.push(start.elapsed()); - } - let no_patch_median = percentile(&mut no_patch_durations, 1, 2); - let no_patch_throughput = - input_bytes as f64 / no_patch_median.as_secs_f64() / 1_000_000.0; - println!( - "int-mult-decode-breakdown\tdense64-no-quotient-patches\t{no_patch_throughput:.1}" - ); - } - } - Ok(()) -} - -fn profile_range_packed( - dataset: &str, - column: &str, - path: &str, - logical_bytes: usize, - pco_bytes: u64, - validity_bytes: u64, - ordered: &[u64], -) -> VortexResult<()> { - if ordered.is_empty() { - return Ok(()); - } - - let mut encode_durations = Vec::with_capacity(5); - let mut codec = RangePackedCodec::encode(ordered, 1_024)?; - for _ in 0..5 { - let start = Instant::now(); - codec = RangePackedCodec::encode(black_box(ordered), 1_024)?; - encode_durations.push(start.elapsed()); - } - let encode_median = percentile(&mut encode_durations, 1, 2); - - vortex_ensure!(codec.decode()? == ordered, "range packed decode differs"); - let decode_iterations = codec_decode_iterations()?; - let mut decode_durations = Vec::with_capacity(decode_iterations); - for _ in 0..decode_iterations { - let start = Instant::now(); - black_box(codec.decode()?); - decode_durations.push(start.elapsed()); - } - let decode_median = percentile(&mut decode_durations, 1, 2); - - let scalar_iterations = 200_000; - let mut scalar_index = 0usize; - let mut scalar_checksum = 0u64; - let scalar_start = Instant::now(); - for _ in 0..scalar_iterations { - scalar_index = scalar_index.wrapping_add(2_654_435_761) % ordered.len(); - scalar_checksum ^= black_box(codec.scalar_at(scalar_index)?); - } - black_box(scalar_checksum); - let scalar_duration = scalar_start.elapsed(); - let input_bytes = ordered.len() * logical_bytes; - let encode_throughput = input_bytes as f64 / encode_median.as_secs_f64() / 1_000_000.0; - let decode_throughput = input_bytes as f64 / decode_median.as_secs_f64() / 1_000_000.0; - let scalar_nanoseconds = - scalar_duration.as_secs_f64() * 1_000_000_000.0 / scalar_iterations as f64; - let encoded_bytes = u64::try_from(codec.encoded_size())? + validity_bytes; - println!( - "fixed-bin-checkpoint\t{dataset}\t{column}\t{path}\t{}\t{pco_bytes}\t{encoded_bytes}\t{}\t{}\t{}\t{encode_throughput:.1}\t{decode_throughput:.1}\t{scalar_nanoseconds:.1}", - ordered.len(), - codec.bin_count(), - codec.max_offset_bits(), - codec.offset_widths(), - ); - Ok(()) -} - -fn codec_decode_iterations() -> VortexResult { - std::env::var("VORTEX_BENCH_CODEC_DECODE_ITERATIONS") - .ok() - .map(|value| { - value.parse::().map_err(|error| { - vortex_err!("invalid VORTEX_BENCH_CODEC_DECODE_ITERATIONS: {error}") - }) - }) - .transpose() - .map(|iterations| iterations.unwrap_or(20)) -} - -fn tree_decode_iterations() -> VortexResult { - std::env::var("VORTEX_BENCH_TREE_DECODE_ITERATIONS") - .ok() - .map(|value| { - value.parse::().map_err(|error| { - vortex_err!("invalid VORTEX_BENCH_TREE_DECODE_ITERATIONS: {error}") - }) - }) - .transpose() - .map(|iterations| iterations.unwrap_or(20)) -} - -#[derive(Clone, Copy)] -enum FixedBinOffsetTree { - BlockResidual, - FoRBitPacked, -} - -impl FixedBinOffsetTree { - fn label(self) -> &'static str { - match self { - Self::BlockResidual => "block-residual", - Self::FoRBitPacked => "for-bitpacked", - } - } -} - -#[derive(Clone, Copy)] -enum FrequencyRankedCodeTree { - BitPacked, - Default, -} - -impl FrequencyRankedCodeTree { - fn label(self) -> &'static str { - match self { - Self::BitPacked => "bitpacked", - Self::Default => "default", - } - } -} - -fn decomposed_fixed_bin_integer_tree( - primitive: vortex_array::ArrayView<'_, Primitive>, - offset_tree: FixedBinOffsetTree, - session: &VortexSession, -) -> VortexResult> { - vortex_ensure!(primitive.ptype().is_int(), "fixed bins require integers"); - let primitive = fill_integer_nulls_with_first_valid( - primitive.into_owned(), - &mut session.create_execution_ctx(), - )?; - let decomposition = RangeDecomposition::encode(&ordered_integer_values(&primitive)?)?; - if decomposition.bin_starts().is_empty() - || !offsets_fit_ptype(decomposition.offsets(), primitive.ptype()) - { - return Ok(None); - } - - let codes = PrimitiveArray::new(decomposition.codes().to_vec(), primitive.validity()?); - // SAFETY: The decomposition computes the exact code width. - let codes = - unsafe { bitpack_encode_unchecked(codes, decomposition.code_width()) }?.into_array(); - let (starts, offsets) = range_components(&decomposition, primitive.ptype())?; - let references = DictArray::try_new(codes, starts)?.into_array(); - let offsets = match offset_tree { - FixedBinOffsetTree::BlockResidual => { - BlockResidual::from_primitive(offsets.as_view())?.into_array() - } - FixedBinOffsetTree::FoRBitPacked => { - let minimum = decomposition.offsets().iter().copied().min().unwrap_or(0); - let maximum = decomposition.offsets().iter().copied().max().unwrap_or(0); - let bit_width = u8::try_from(u64::BITS - (maximum - minimum).leading_zeros())?; - let mut ctx = session.create_execution_ctx(); - let encoded = FoR::encode(offsets, &mut ctx)?; - let packed = BitPacked::encode(encoded.encoded(), bit_width, &mut ctx)?; - FoR::try_new(packed.into_array(), encoded.reference_scalar().clone())?.into_array() - } - }; - Ok(Some(IntMult::try_new(references, offsets, 1)?.into_array())) -} - -fn int_mult_integer_tree( - primitive: vortex_array::ArrayView<'_, Primitive>, - base: u64, - session: &VortexSession, -) -> VortexResult { - let validity = primitive.validity()?; - let (primary, secondary) = match primitive.ptype() { - PType::I32 => { - let base = i32::try_from(base)?; - let values = primitive.as_slice::(); - ( - PrimitiveArray::new( - values - .iter() - .map(|value| value.div_euclid(base)) - .collect::>(), - validity, - ), - PrimitiveArray::from_iter( - values - .iter() - .map(|value| value.rem_euclid(base)) - .collect::>(), - ), - ) - } - PType::I64 => { - let base = i64::try_from(base)?; - let values = primitive.as_slice::(); - ( - PrimitiveArray::new( - values - .iter() - .map(|value| value.div_euclid(base)) - .collect::>(), - validity, - ), - PrimitiveArray::from_iter( - values - .iter() - .map(|value| value.rem_euclid(base)) - .collect::>(), - ), - ) - } - ptype => return Err(vortex_err!("ALP IntMult does not support {ptype}")), - }; - let compressor = BtrBlocksCompressor::default(); - let primary = - compressor.compress(&primary.into_array(), &mut session.create_execution_ctx())?; - let secondary = - compressor.compress(&secondary.into_array(), &mut session.create_execution_ctx())?; - Ok(IntMult::try_new(primary, secondary, base)?.into_array()) -} - -fn prefix_int_mult_integer_tree( - primitive: vortex_array::ArrayView<'_, Primitive>, - suffix_bits: u8, -) -> VortexResult> { - let validity = primitive.validity()?; - let base = 1_u64 << suffix_bits; - let (codes, dictionary, secondary) = match primitive.ptype() { - PType::I32 if suffix_bits < 31 => { - let base = i32::try_from(base)?; - let mut dictionary = Vec::::new(); - let mut code_by_value = HashMap::::new(); - let mut codes = Vec::with_capacity(primitive.len()); - let mut secondary = Vec::with_capacity(primitive.len()); - for &value in primitive.as_slice::() { - let quotient = value.div_euclid(base); - let code = match code_by_value.get("ient) { - Some(&code) => code, - None if dictionary.len() < 64 => { - let code = u8::try_from(dictionary.len())?; - dictionary.push(quotient); - code_by_value.insert(quotient, code); - code - } - None => return Ok(None), - }; - codes.push(code); - secondary.push(value.rem_euclid(base)); - } - ( - codes, - PrimitiveArray::from_iter(dictionary).into_array(), - PrimitiveArray::from_iter(secondary), - ) - } - PType::I64 if suffix_bits < 63 => { - let base = i64::try_from(base)?; - let mut dictionary = Vec::::new(); - let mut code_by_value = HashMap::::new(); - let mut codes = Vec::with_capacity(primitive.len()); - let mut secondary = Vec::with_capacity(primitive.len()); - for &value in primitive.as_slice::() { - let quotient = value.div_euclid(base); - let code = match code_by_value.get("ient) { - Some(&code) => code, - None if dictionary.len() < 64 => { - let code = u8::try_from(dictionary.len())?; - dictionary.push(quotient); - code_by_value.insert(quotient, code); - code - } - None => return Ok(None), - }; - codes.push(code); - secondary.push(value.rem_euclid(base)); - } - ( - codes, - PrimitiveArray::from_iter(dictionary).into_array(), - PrimitiveArray::from_iter(secondary), - ) - } - _ => return Ok(None), - }; - let code_width = - u8::try_from(u8::BITS - u8::try_from(dictionary.len().saturating_sub(1))?.leading_zeros())?; - let codes = PrimitiveArray::new(codes, validity); - // SAFETY: Every code fits the width computed from the dictionary length. - let codes = unsafe { bitpack_encode_unchecked(codes, code_width) }?.into_array(); - let primary = DictArray::try_new(codes, dictionary)?.into_array(); - // SAFETY: Euclidean remainders for a power-of-two base fit in `suffix_bits` bits. - let secondary = unsafe { bitpack_encode_unchecked(secondary, suffix_bits) }?.into_array(); - Ok(Some( - IntMult::try_new(primary, secondary, base)?.into_array(), - )) -} - -fn encode_int_mult_float_tree( - primitive: vortex_array::ArrayView<'_, Primitive>, - compact: &ArrayRef, - base: u64, - session: &VortexSession, -) -> VortexResult> { - if compact.encoding_id().as_ref() != "vortex.alp" { - return Ok(None); - } - let compact_alp = compact.as_::(); - if compact_alp.encoded().encoding_id().as_ref() != "vortex.pco" { - return Ok(None); - } - let alp = alp_encode(primitive, None, &mut session.create_execution_ctx())?; - let encoded = int_mult_integer_tree(alp.encoded().as_::(), base, session)?; - Ok(Some( - ALP::try_new(encoded, alp.exponents(), alp.patches())?.into_array(), - )) -} - -fn encode_prefix_int_mult_float_tree( - primitive: vortex_array::ArrayView<'_, Primitive>, - compact: &ArrayRef, - suffix_bits: u8, - session: &VortexSession, -) -> VortexResult> { - if compact.encoding_id().as_ref() != "vortex.alp" { - return Ok(None); - } - let compact_alp = compact.as_::(); - if compact_alp.encoded().encoding_id().as_ref() != "vortex.pco" { - return Ok(None); - } - let alp = alp_encode(primitive, None, &mut session.create_execution_ctx())?; - let Some(encoded) = - prefix_int_mult_integer_tree(alp.encoded().as_::(), suffix_bits)? - else { - return Ok(None); - }; - Ok(Some( - ALP::try_new(encoded, alp.exponents(), alp.patches())?.into_array(), - )) -} - -#[expect( - clippy::useless_conversion, - reason = "the generic unsigned-code path must reject values that exceed usize" -)] -fn frequency_ranked_dict_tree( - encoded: &ArrayRef, - code_tree: FrequencyRankedCodeTree, - session: &VortexSession, -) -> VortexResult> { - let Some(dict) = encoded.as_typed::() else { - return Ok(None); - }; - let mut ctx = session.create_execution_ctx(); - let codes = dict.codes().clone().execute::(&mut ctx)?; - let old_codes = match_each_unsigned_integer_ptype!(codes.ptype(), |T| { - codes - .as_slice::() - .iter() - .map(|&code| usize::try_from(u64::from(code))) - .collect::, _>>()? - }); - let validity = codes.validity()?; - let mask = validity.execute_mask(codes.len(), &mut ctx)?; - let mut counts = vec![0_usize; dict.values().len()]; - for (&code, valid) in old_codes.iter().zip(mask.iter()) { - if valid { - vortex_ensure!( - code < counts.len(), - "dictionary code {code} exceeds {} values", - counts.len() - ); - counts[code] += 1; - } - } - - let mut order = (0..counts.len()).collect::>(); - order.sort_unstable_by_key(|&code| (std::cmp::Reverse(counts[code]), code)); - let mut new_code_by_old = vec![0_usize; order.len()]; - for (new_code, &old_code) in order.iter().enumerate() { - new_code_by_old[old_code] = new_code; - } - let remapped = match_each_unsigned_integer_ptype!(codes.ptype(), |T| { - PrimitiveArray::new( - old_codes - .iter() - .zip(mask.iter()) - .map(|(&old_code, valid)| { - if valid { - T::try_from(new_code_by_old[old_code]) - } else { - T::try_from(0) - } - }) - .collect::, _>>()?, - validity.clone(), - ) - }); - let codes = match code_tree { - FrequencyRankedCodeTree::BitPacked => { - let bit_width_freq = bit_width_histogram(remapped.as_view(), &mut ctx)?; - let bit_width = find_best_bit_width(remapped.ptype(), &bit_width_freq)?; - bitpack_encode(&remapped, bit_width, Some(&bit_width_freq), &mut ctx)?.into_array() - } - FrequencyRankedCodeTree::Default => { - BtrBlocksCompressor::default().compress(&remapped.into_array(), &mut ctx)? - } - }; - - let order = order - .into_iter() - .map(u64::try_from) - .collect::, _>>()?; - let values = dict - .values() - .take(PrimitiveArray::from_iter(order).into_array())? - .execute::(&mut ctx)?; - let values = BtrBlocksCompressor::default().compress(&values.into_array(), &mut ctx)?; - Ok(Some(DictArray::try_new(codes, values)?.into_array())) -} - -fn fill_integer_nulls_with_first_valid( - primitive: PrimitiveArray, - ctx: &mut vortex_array::ExecutionCtx, -) -> VortexResult { - let validity = primitive.validity()?; - let mask = validity.execute_mask(primitive.len(), ctx)?; - if mask.all_true() { - return Ok(primitive); - } - let first_valid = mask.first(); - Ok(match_each_integer_ptype!(primitive.ptype(), |T| { - let values = primitive.as_slice::(); - let fill = first_valid.map_or_else(T::default, |index| values[index]); - PrimitiveArray::new::( - values - .iter() - .zip(mask.iter()) - .map(|(&value, valid)| if valid { value } else { fill }) - .collect::>(), - validity, - ) - })) -} - -fn ordered_integer_values(primitive: &PrimitiveArray) -> VortexResult> { - Ok(match primitive.ptype() { - PType::U8 => primitive - .as_slice::() - .iter() - .map(|&value| u64::from(value)) - .collect(), - PType::U16 => primitive - .as_slice::() - .iter() - .map(|&value| u64::from(value)) - .collect(), - PType::U32 => primitive - .as_slice::() - .iter() - .map(|&value| u64::from(value)) - .collect(), - PType::U64 => primitive.as_slice::().to_vec(), - PType::I8 => primitive - .as_slice::() - .iter() - .map(|&value| u64::from((value as u8) ^ (1_u8 << 7))) - .collect(), - PType::I16 => primitive - .as_slice::() - .iter() - .map(|&value| u64::from((value as u16) ^ (1_u16 << 15))) - .collect(), - PType::I32 => primitive - .as_slice::() - .iter() - .map(|&value| u64::from((value as u32) ^ (1_u32 << 31))) - .collect(), - PType::I64 => primitive - .as_slice::() - .iter() - .map(|&value| (value as u64) ^ (1_u64 << 63)) - .collect(), - ptype => return Err(vortex_err!("fixed bins do not support {ptype}")), - }) -} - -fn offsets_fit_ptype(offsets: &[u64], ptype: PType) -> bool { - let maximum = match ptype { - PType::U8 => u64::from(u8::MAX), - PType::U16 => u64::from(u16::MAX), - PType::U32 => u64::from(u32::MAX), - PType::U64 => u64::MAX, - PType::I8 => i8::MAX as u64, - PType::I16 => i16::MAX as u64, - PType::I32 => i32::MAX as u64, - PType::I64 => i64::MAX as u64, - _ => return false, - }; - offsets.iter().all(|&offset| offset <= maximum) -} - -fn range_components( - decomposition: &RangeDecomposition, - ptype: PType, -) -> VortexResult<(ArrayRef, PrimitiveArray)> { - let starts = decomposition.bin_starts(); - let offsets = decomposition.offsets(); - Ok(match ptype { - PType::U8 => ( - PrimitiveArray::from_iter( - starts - .iter() - .copied() - .map(u8::try_from) - .collect::, _>>()?, - ) - .into_array(), - PrimitiveArray::from_iter( - offsets - .iter() - .copied() - .map(u8::try_from) - .collect::, _>>()?, - ), - ), - PType::U16 => ( - PrimitiveArray::from_iter( - starts - .iter() - .copied() - .map(u16::try_from) - .collect::, _>>()?, - ) - .into_array(), - PrimitiveArray::from_iter( - offsets - .iter() - .copied() - .map(u16::try_from) - .collect::, _>>()?, - ), - ), - PType::U32 => ( - PrimitiveArray::from_iter( - starts - .iter() - .copied() - .map(u32::try_from) - .collect::, _>>()?, - ) - .into_array(), - PrimitiveArray::from_iter( - offsets - .iter() - .copied() - .map(u32::try_from) - .collect::, _>>()?, - ), - ), - PType::U64 => ( - PrimitiveArray::from_iter(starts.iter().copied()).into_array(), - PrimitiveArray::from_iter(offsets.iter().copied()), - ), - PType::I8 => ( - PrimitiveArray::from_iter( - starts - .iter() - .copied() - .map(|value| { - u8::try_from(value).map(|value| i8::from_le_bytes([value ^ (1_u8 << 7)])) - }) - .collect::, _>>()?, - ) - .into_array(), - PrimitiveArray::from_iter( - offsets - .iter() - .copied() - .map(i8::try_from) - .collect::, _>>()?, - ), - ), - PType::I16 => ( - PrimitiveArray::from_iter( - starts - .iter() - .copied() - .map(|value| { - u16::try_from(value) - .map(|value| i16::from_le_bytes((value ^ (1_u16 << 15)).to_le_bytes())) - }) - .collect::, _>>()?, - ) - .into_array(), - PrimitiveArray::from_iter( - offsets - .iter() - .copied() - .map(i16::try_from) - .collect::, _>>()?, - ), - ), - PType::I32 => ( - PrimitiveArray::from_iter( - starts - .iter() - .copied() - .map(|value| { - u32::try_from(value) - .map(|value| i32::from_le_bytes((value ^ (1_u32 << 31)).to_le_bytes())) - }) - .collect::, _>>()?, - ) - .into_array(), - PrimitiveArray::from_iter( - offsets - .iter() - .copied() - .map(i32::try_from) - .collect::, _>>()?, - ), - ), - PType::I64 => ( - PrimitiveArray::from_iter( - starts - .iter() - .copied() - .map(|value| i64::from_le_bytes((value ^ (1_u64 << 63)).to_le_bytes())), - ) - .into_array(), - PrimitiveArray::from_iter( - offsets - .iter() - .copied() - .map(i64::try_from) - .collect::, _>>()?, - ), - ), - ptype => return Err(vortex_err!("fixed bins do not support {ptype}")), - }) -} - -fn fixed_bin_float_tree( - compact: &ArrayRef, - session: &VortexSession, -) -> VortexResult> { - if compact.encoding_id().as_ref() == "vortex.alp" { - let alp = compact.as_::(); - if alp.encoded().encoding_id().as_ref() != "vortex.pco" { - return Ok(None); - } - let primitive = alp - .encoded() - .clone() - .execute::(&mut session.create_execution_ctx())?; - let primitive = primitive - .into_array() - .cast(alp.encoded().dtype().clone())? - .execute::(&mut session.create_execution_ctx())?; - let encoded = - range_packed_from_primitive(primitive.as_view(), &mut session.create_execution_ctx())?; - return Ok(Some( - ALP::try_new(encoded, alp.exponents(), alp.patches())?.into_array(), - )); - } - - if compact.encoding_id().as_ref() != "vortex.pco" || !compact.dtype().is_float() { - return Ok(None); - } - let primitive = compact - .clone() - .execute::(&mut session.create_execution_ctx())?; - let primitive = - fill_float_nulls_with_first_valid(primitive, &mut session.create_execution_ctx())?; - let ordered = OrderedFloat::from_primitive(primitive.as_view())?; - let encoded = range_packed_from_primitive( - ordered.encoded().as_::(), - &mut session.create_execution_ctx(), - )?; - Ok(Some( - OrderedFloat::try_new(encoded, primitive.ptype())?.into_array(), - )) -} - -fn range_packed_from_primitive( - primitive: vortex_array::ArrayView<'_, Primitive>, - ctx: &mut vortex_array::ExecutionCtx, -) -> VortexResult { - if std::env::var_os("VORTEX_BENCH_FIXED_BIN_FULL_POSITIONS").is_some() { - Ok(RangePacked::from_primitive_with_null_positions(primitive, ctx)?.into_array()) - } else { - Ok(RangePacked::from_primitive(primitive, ctx)?.into_array()) - } -} - -fn fill_float_nulls_with_first_valid( - primitive: PrimitiveArray, - ctx: &mut vortex_array::ExecutionCtx, -) -> VortexResult { - let validity = primitive.validity()?; - let mask = validity.execute_mask(primitive.len(), ctx)?; - if mask.all_true() { - return Ok(primitive); - } - let first_valid = mask.first(); - Ok(match primitive.ptype() { - PType::F32 => { - let values = primitive.as_slice::(); - let fill = first_valid.map_or(0.0, |index| values[index]); - PrimitiveArray::new::( - values - .iter() - .zip(mask.iter()) - .map(|(&value, valid)| if valid { value } else { fill }) - .collect::>(), - validity, - ) - } - PType::F64 => { - let values = primitive.as_slice::(); - let fill = first_valid.map_or(0.0, |index| values[index]); - PrimitiveArray::new::( - values - .iter() - .zip(mask.iter()) - .map(|(&value, valid)| if valid { value } else { fill }) - .collect::>(), - validity, - ) - } - ptype => return Err(vortex_err!("null fill does not support {ptype}")), - }) -} - -fn measure_fixed_bin_tree( - dataset: &str, - column: &str, - expected: &ArrayRef, - compact: &ArrayRef, - session: &VortexSession, -) -> VortexResult<()> { - let Some(encoded) = fixed_bin_float_tree(compact, session)? else { - return Ok(()); - }; - vortex_ensure!( - encoded.dtype() == expected.dtype(), - "fixed-bin tree dtype {} differs from expected {}", - encoded.dtype(), - expected.dtype() - ); - assert_arrays_eq!(encoded, expected, &mut session.create_execution_ctx()); - - let decode_iterations = tree_decode_iterations()?; - let mut decode_durations = Vec::with_capacity(decode_iterations); - for _ in 0..decode_iterations { - let start = Instant::now(); - black_box( - encoded - .clone() - .execute::(&mut session.create_execution_ctx())?, - ); - decode_durations.push(start.elapsed()); - } - let decode_median = percentile(&mut decode_durations, 1, 2); - let decode_throughput = expected.nbytes() as f64 / decode_median.as_secs_f64() / 1_000_000.0; - - let scalar_nanoseconds = measure_scalar_access(&encoded, session)?; - println!( - "fixed-bin-tree\t{dataset}\t{column}\t{}\t{}\t{decode_throughput:.1}\t{scalar_nanoseconds:.1}\t{}", - encoded.len(), - encoded.nbytes(), - encoding_tree(&encoded), - ); - - let mut fused_durations = Vec::with_capacity(decode_iterations); - for _ in 0..decode_iterations { - let start = Instant::now(); - black_box(decode_fixed_bin_float_tree(&encoded, session)?); - fused_durations.push(start.elapsed()); - } - let fused_median = percentile(&mut fused_durations, 1, 2); - let fused_throughput = expected.nbytes() as f64 / fused_median.as_secs_f64() / 1_000_000.0; - println!( - "fixed-bin-tree-fused\t{dataset}\t{column}\t{}\t{}\t{fused_throughput:.1}\t{}", - encoded.len(), - encoded.nbytes(), - encoding_tree(&encoded), - ); - - let mut encode_durations = Vec::with_capacity(5); - for _ in 0..5 { - let start = Instant::now(); - black_box(encode_fixed_bin_float_tree( - expected.as_::(), - compact, - session, - )?); - encode_durations.push(start.elapsed()); - } - let encode_median = percentile(&mut encode_durations, 1, 2); - let encode_throughput = expected.nbytes() as f64 / encode_median.as_secs_f64() / 1_000_000.0; - println!( - "fixed-bin-tree-encode\t{dataset}\t{column}\t{}\t{}\t{encode_throughput:.1}\t{}", - encoded.len(), - encoded.nbytes(), - encoding_tree(&encoded), - ); - Ok(()) -} - -fn measure_block_residual_float_tree( - dataset: &str, - column: &str, - expected: &ArrayRef, - compact: &ArrayRef, - session: &VortexSession, -) -> VortexResult<()> { - let Some(encoded) = block_residual_float_tree(compact, session)? else { - return Ok(()); - }; - measure_existing_tree( - dataset, - column, - "block-residual", - expected, - &encoded, - session, - )?; - - let mut encode_durations = Vec::with_capacity(5); - for _ in 0..5 { - let start = Instant::now(); - black_box(encode_block_residual_float_tree( - expected.as_::(), - compact, - session, - )?); - encode_durations.push(start.elapsed()); - } - let encode_median = percentile(&mut encode_durations, 1, 2); - let encode_throughput = expected.nbytes() as f64 / encode_median.as_secs_f64() / 1_000_000.0; - println!( - "candidate-tree-encode\t{dataset}\t{column}\tblock-residual\t{}\t{}\t{encode_throughput:.1}\t{}", - encoded.len(), - encoded.nbytes(), - encoding_tree(&encoded), - ); - Ok(()) -} - -fn measure_block_residual_patch_positions( - dataset: &str, - column: &str, - expected: &ArrayRef, - default: &ArrayRef, - compact: &ArrayRef, - session: &VortexSession, -) -> VortexResult<()> { - if let Some(encoded) = rewrite_alp_patch_positions(default, session)? { - measure_existing_tree( - dataset, - column, - "default-block-residual-patch-positions", - expected, - &encoded, - session, - )?; - } - - let Some(block_residual) = block_residual_float_tree(compact, session)? else { - return Ok(()); - }; - let Some(encoded) = rewrite_alp_patch_positions(&block_residual, session)? else { - return Ok(()); - }; - measure_existing_tree( - dataset, - column, - "block-residual-patch-positions", - expected, - &encoded, - session, - )?; - - let mut encode_durations = Vec::with_capacity(5); - for _ in 0..5 { - let start = Instant::now(); - black_box(encode_block_residual_float_tree_with_patch_positions( - expected.as_::(), - session, - )?); - encode_durations.push(start.elapsed()); - } - let encode_median = percentile(&mut encode_durations, 1, 2); - let encode_throughput = expected.nbytes() as f64 / encode_median.as_secs_f64() / 1_000_000.0; - println!( - "candidate-tree-encode\t{dataset}\t{column}\tblock-residual-patch-positions\t{}\t{}\t{encode_throughput:.1}\t{}", - encoded.len(), - encoded.nbytes(), - encoding_tree(&encoded), - ); - Ok(()) -} - -fn block_residual_float_tree( - compact: &ArrayRef, - session: &VortexSession, -) -> VortexResult> { - if compact.encoding_id().as_ref() == "vortex.alp" { - let alp = compact.as_::(); - if alp.encoded().encoding_id().as_ref() != "vortex.pco" { - return Ok(None); - } - let primitive = alp - .encoded() - .clone() - .execute::(&mut session.create_execution_ctx())?; - let primitive = primitive - .into_array() - .cast(alp.encoded().dtype().clone())? - .execute::(&mut session.create_execution_ctx())?; - let encoded = BlockResidual::from_primitive(primitive.as_view())?.into_array(); - return Ok(Some( - ALP::try_new(encoded, alp.exponents(), alp.patches())?.into_array(), - )); - } - - if compact.encoding_id().as_ref() != "vortex.pco" || !compact.dtype().is_float() { - return Ok(None); - } - let primitive = compact - .clone() - .execute::(&mut session.create_execution_ctx())?; - let primitive = - fill_float_nulls_with_first_valid(primitive, &mut session.create_execution_ctx())?; - let ordered = OrderedFloat::from_primitive(primitive.as_view())?; - let encoded = BlockResidual::from_primitive(ordered.encoded().as_::())?; - Ok(Some( - OrderedFloat::try_new(encoded.into_array(), primitive.ptype())?.into_array(), - )) -} - -fn block_residual_patch_positions( - patches: Patches, - session: &VortexSession, -) -> VortexResult { - let indices = patches - .indices() - .clone() - .execute::(&mut session.create_execution_ctx())?; - let encoded_indices = BlockResidual::from_primitive(indices.as_view())?.into_array(); - let indices = if encoded_indices.nbytes() < indices.nbytes() { - encoded_indices - } else { - indices.into_array() - }; - let chunk_offsets = patches - .chunk_offsets() - .as_ref() - .map(|offsets| { - let offsets = offsets - .clone() - .execute::(&mut session.create_execution_ctx())?; - let encoded = BlockResidual::from_primitive(offsets.as_view())?.into_array(); - Ok::(if encoded.nbytes() < offsets.nbytes() { - encoded - } else { - offsets.into_array() - }) - }) - .transpose()?; - Patches::new( - patches.array_len(), - patches.offset(), - indices, - patches.values().clone(), - chunk_offsets, - ) -} - -fn rewrite_alp_patch_positions( - encoded: &ArrayRef, - session: &VortexSession, -) -> VortexResult> { - if encoded.encoding_id().as_ref() != "vortex.alp" { - return Ok(None); - } - let alp = encoded.as_::(); - let Some(patches) = alp.patches() else { - return Ok(None); - }; - let patches = block_residual_patch_positions(patches, session)?; - Ok(Some( - ALP::try_new(alp.encoded().clone(), alp.exponents(), Some(patches))?.into_array(), - )) -} - -fn encode_block_residual_float_tree_with_patch_positions( - primitive: vortex_array::ArrayView<'_, Primitive>, - session: &VortexSession, -) -> VortexResult { - let alp = alp_encode(primitive, None, &mut session.create_execution_ctx())?; - let encoded = BlockResidual::from_primitive(alp.encoded().as_::())?; - let patches = alp - .patches() - .map(|patches| block_residual_patch_positions(patches, session)) - .transpose()?; - Ok(ALP::try_new(encoded.into_array(), alp.exponents(), patches)?.into_array()) -} - -fn encode_block_residual_float_tree( - primitive: vortex_array::ArrayView<'_, Primitive>, - compact: &ArrayRef, - session: &VortexSession, -) -> VortexResult { - if compact.encoding_id().as_ref() == "vortex.alp" { - let alp = alp_encode(primitive, None, &mut session.create_execution_ctx())?; - let encoded = BlockResidual::from_primitive(alp.encoded().as_::())?; - return Ok( - ALP::try_new(encoded.into_array(), alp.exponents(), alp.patches())?.into_array(), - ); - } - - let primitive = fill_float_nulls_with_first_valid( - primitive.into_owned(), - &mut session.create_execution_ctx(), - )?; - let ordered = OrderedFloat::from_primitive(primitive.as_view())?; - let encoded = BlockResidual::from_primitive(ordered.encoded().as_::())?; - Ok(OrderedFloat::try_new(encoded.into_array(), primitive.ptype())?.into_array()) -} - -fn encode_fixed_bin_float_tree( - primitive: vortex_array::ArrayView<'_, Primitive>, - compact: &ArrayRef, - session: &VortexSession, -) -> VortexResult { - if compact.encoding_id().as_ref() == "vortex.alp" { - let alp = alp_encode(primitive, None, &mut session.create_execution_ctx())?; - let packed = range_packed_from_primitive( - alp.encoded().as_::(), - &mut session.create_execution_ctx(), - )?; - return Ok(ALP::try_new(packed, alp.exponents(), alp.patches())?.into_array()); - } - - let primitive = fill_float_nulls_with_first_valid( - primitive.into_owned(), - &mut session.create_execution_ctx(), - )?; - let ordered = OrderedFloat::from_primitive(primitive.as_view())?; - let packed = range_packed_from_primitive( - ordered.encoded().as_::(), - &mut session.create_execution_ctx(), - )?; - Ok(OrderedFloat::try_new(packed, primitive.ptype())?.into_array()) -} - -fn encode_decomposed_fixed_bin_float_tree( - primitive: vortex_array::ArrayView<'_, Primitive>, - compact: &ArrayRef, - offset_tree: FixedBinOffsetTree, - session: &VortexSession, -) -> VortexResult> { - if compact.encoding_id().as_ref() == "vortex.alp" { - let compact_alp = compact.as_::(); - if compact_alp.encoded().encoding_id().as_ref() != "vortex.pco" { - return Ok(None); - } - let alp = alp_encode(primitive, None, &mut session.create_execution_ctx())?; - let Some(encoded) = decomposed_fixed_bin_integer_tree( - alp.encoded().as_::(), - offset_tree, - session, - )? - else { - return Ok(None); - }; - return Ok(Some( - ALP::try_new(encoded, alp.exponents(), alp.patches())?.into_array(), - )); - } - - if compact.encoding_id().as_ref() != "vortex.pco" || !compact.dtype().is_float() { - return Ok(None); - } - let primitive = fill_float_nulls_with_first_valid( - primitive.into_owned(), - &mut session.create_execution_ctx(), - )?; - let ordered = OrderedFloat::from_primitive(primitive.as_view())?; - let Some(encoded) = decomposed_fixed_bin_integer_tree( - ordered.encoded().as_::(), - offset_tree, - session, - )? - else { - return Ok(None); - }; - Ok(Some( - OrderedFloat::try_new(encoded, primitive.ptype())?.into_array(), - )) -} - -fn measure_decomposed_fixed_bin_tree( - dataset: &str, - column: &str, - expected: &ArrayRef, - compact: &ArrayRef, - offset_tree: FixedBinOffsetTree, - session: &VortexSession, -) -> VortexResult<()> { - let Some(encoded) = encode_decomposed_fixed_bin_float_tree( - expected.as_::(), - compact, - offset_tree, - session, - )? - else { - return Ok(()); - }; - let label = format!("decomposed-fixed-bin-{}", offset_tree.label()); - measure_existing_tree(dataset, column, &label, expected, &encoded, session)?; - - let mut encode_durations = Vec::with_capacity(5); - for _ in 0..5 { - let start = Instant::now(); - black_box(encode_decomposed_fixed_bin_float_tree( - expected.as_::(), - compact, - offset_tree, - session, - )?); - encode_durations.push(start.elapsed()); - } - let encode_median = percentile(&mut encode_durations, 1, 2); - let encode_throughput = expected.nbytes() as f64 / encode_median.as_secs_f64() / 1_000_000.0; - println!( - "candidate-tree-encode\t{dataset}\t{column}\t{label}\t{}\t{}\t{encode_throughput:.1}\t{}", - encoded.len(), - encoded.nbytes(), - encoding_tree(&encoded), - ); - Ok(()) -} - -fn measure_int_mult_float_tree( - dataset: &str, - column: &str, - expected: &ArrayRef, - compact: &ArrayRef, - base: u64, - session: &VortexSession, -) -> VortexResult<()> { - let Some(encoded) = - encode_int_mult_float_tree(expected.as_::(), compact, base, session)? - else { - return Ok(()); - }; - let label = format!("int-mult-{base}"); - measure_existing_tree(dataset, column, &label, expected, &encoded, session)?; - - let mut encode_durations = Vec::with_capacity(5); - for _ in 0..5 { - let start = Instant::now(); - black_box(encode_int_mult_float_tree( - expected.as_::(), - compact, - base, - session, - )?); - encode_durations.push(start.elapsed()); - } - let encode_median = percentile(&mut encode_durations, 1, 2); - let encode_throughput = expected.nbytes() as f64 / encode_median.as_secs_f64() / 1_000_000.0; - println!( - "candidate-tree-encode\t{dataset}\t{column}\t{label}\t{}\t{}\t{encode_throughput:.1}\t{}", - encoded.len(), - encoded.nbytes(), - encoding_tree(&encoded), - ); - Ok(()) -} - -fn measure_prefix_int_mult_float_tree( - dataset: &str, - column: &str, - expected: &ArrayRef, - compact: &ArrayRef, - suffix_bits: u8, - session: &VortexSession, -) -> VortexResult<()> { - let Some(encoded) = encode_prefix_int_mult_float_tree( - expected.as_::(), - compact, - suffix_bits, - session, - )? - else { - return Ok(()); - }; - let label = format!("prefix-int-mult-{suffix_bits}"); - measure_existing_tree(dataset, column, &label, expected, &encoded, session)?; - - let mut encode_durations = Vec::with_capacity(5); - for _ in 0..5 { - let start = Instant::now(); - black_box(encode_prefix_int_mult_float_tree( - expected.as_::(), - compact, - suffix_bits, - session, - )?); - encode_durations.push(start.elapsed()); - } - let encode_median = percentile(&mut encode_durations, 1, 2); - let encode_throughput = expected.nbytes() as f64 / encode_median.as_secs_f64() / 1_000_000.0; - println!( - "candidate-tree-encode\t{dataset}\t{column}\t{label}\t{}\t{}\t{encode_throughput:.1}\t{}", - encoded.len(), - encoded.nbytes(), - encoding_tree(&encoded), - ); - Ok(()) -} - -fn measure_frequency_ranked_dict_tree( - dataset: &str, - column: &str, - expected: &ArrayRef, - default: &ArrayRef, - code_tree: FrequencyRankedCodeTree, - session: &VortexSession, -) -> VortexResult<()> { - let Some(encoded) = frequency_ranked_dict_tree(default, code_tree, session)? else { - return Ok(()); - }; - let label = format!("frequency-ranked-dict-{}", code_tree.label()); - measure_existing_tree(dataset, column, &label, expected, &encoded, session)?; - - let mut encode_durations = Vec::with_capacity(5); - for _ in 0..5 { - let start = Instant::now(); - black_box(frequency_ranked_dict_tree(default, code_tree, session)?); - encode_durations.push(start.elapsed()); - } - let encode_median = percentile(&mut encode_durations, 1, 2); - let encode_throughput = expected.nbytes() as f64 / encode_median.as_secs_f64() / 1_000_000.0; - println!( - "candidate-tree-encode\t{dataset}\t{column}\t{label}\t{}\t{}\t{encode_throughput:.1}\t{}", - encoded.len(), - encoded.nbytes(), - encoding_tree(&encoded), - ); - Ok(()) -} - -fn measure_frequency_ranked_dict_trees( - dataset: &str, - column: &str, - expected: &ArrayRef, - default: &ArrayRef, - session: &VortexSession, -) -> VortexResult<()> { - if std::env::var_os("VORTEX_BENCH_FREQ_DICT").is_none() { - return Ok(()); - } - measure_existing_tree(dataset, column, "default", expected, default, session)?; - for code_tree in [ - FrequencyRankedCodeTree::BitPacked, - FrequencyRankedCodeTree::Default, - ] { - measure_frequency_ranked_dict_tree(dataset, column, expected, default, code_tree, session)?; - } - Ok(()) -} - -fn measure_existing_tree( - dataset: &str, - column: &str, - label: &str, - expected: &ArrayRef, - encoded: &ArrayRef, - session: &VortexSession, -) -> VortexResult<()> { - assert_arrays_eq!(encoded, expected, &mut session.create_execution_ctx()); - let mut decode_durations = Vec::with_capacity(20); - for _ in 0..20 { - let start = Instant::now(); - black_box( - encoded - .clone() - .execute::(&mut session.create_execution_ctx())?, - ); - decode_durations.push(start.elapsed()); - } - let decode_median = percentile(&mut decode_durations, 1, 2); - let decode_throughput = expected.nbytes() as f64 / decode_median.as_secs_f64() / 1_000_000.0; - let scalar_nanoseconds = measure_scalar_access(encoded, session)?; - println!( - "tree-throughput\t{dataset}\t{column}\t{label}\t{}\t{}\t{decode_throughput:.1}\t{scalar_nanoseconds:.1}\t{}", - encoded.len(), - encoded.nbytes(), - encoding_tree(encoded), - ); - Ok(()) -} - -fn measure_scalar_access(encoded: &ArrayRef, session: &VortexSession) -> VortexResult { - let scalar_iterations = 200_000; - let mut scalar_index = 0usize; - let mut scalar_checksum = 0u64; - let mut scalar_ctx = session.create_execution_ctx(); - let scalar_start = Instant::now(); - for _ in 0..scalar_iterations { - scalar_index = scalar_index.wrapping_add(2_654_435_761) % encoded.len(); - let scalar = encoded.execute_scalar(scalar_index, &mut scalar_ctx)?; - let bits = if scalar.is_null() { - u64::MAX - } else { - match encoded.dtype().as_ptype() { - PType::U8 => u64::from( - scalar - .as_primitive() - .typed_value::() - .ok_or_else(|| vortex_err!("tree produced an invalid u8 scalar"))?, - ), - PType::U16 => u64::from( - scalar - .as_primitive() - .typed_value::() - .ok_or_else(|| vortex_err!("tree produced an invalid u16 scalar"))?, - ), - PType::U32 => u64::from( - scalar - .as_primitive() - .typed_value::() - .ok_or_else(|| vortex_err!("tree produced an invalid u32 scalar"))?, - ), - PType::U64 => scalar - .as_primitive() - .typed_value::() - .ok_or_else(|| vortex_err!("tree produced an invalid u64 scalar"))?, - PType::I8 => u64::from(u8::from_le_bytes( - scalar - .as_primitive() - .typed_value::() - .ok_or_else(|| vortex_err!("tree produced an invalid i8 scalar"))? - .to_le_bytes(), - )), - PType::I16 => u64::from(u16::from_le_bytes( - scalar - .as_primitive() - .typed_value::() - .ok_or_else(|| vortex_err!("tree produced an invalid i16 scalar"))? - .to_le_bytes(), - )), - PType::I32 => u64::from(u32::from_le_bytes( - scalar - .as_primitive() - .typed_value::() - .ok_or_else(|| vortex_err!("tree produced an invalid i32 scalar"))? - .to_le_bytes(), - )), - PType::I64 => u64::from_le_bytes( - scalar - .as_primitive() - .typed_value::() - .ok_or_else(|| vortex_err!("tree produced an invalid i64 scalar"))? - .to_le_bytes(), - ), - PType::F16 => u64::from( - scalar - .as_primitive() - .typed_value::() - .ok_or_else(|| vortex_err!("tree produced an invalid f16 scalar"))? - .to_bits(), - ), - PType::F32 => u64::from( - scalar - .as_primitive() - .typed_value::() - .ok_or_else(|| vortex_err!("tree produced an invalid f32 scalar"))? - .to_bits(), - ), - PType::F64 => scalar - .as_primitive() - .typed_value::() - .ok_or_else(|| vortex_err!("tree produced an invalid f64 scalar"))? - .to_bits(), - } - }; - scalar_checksum ^= black_box(bits); - } - black_box(scalar_checksum); - Ok(scalar_start.elapsed().as_secs_f64() * 1_000_000_000.0 / scalar_iterations as f64) -} - -#[expect( - clippy::cast_possible_truncation, - reason = "the range-packed child retains the ALP integer word width" -)] -fn decode_fixed_bin_float_tree( - encoded: &ArrayRef, - session: &VortexSession, -) -> VortexResult { - if encoded.encoding_id().as_ref() == "vortex.ordered_float" { - let ordered = encoded.as_::(); - let packed = ordered.encoded().as_::(); - let validity = ordered.encoded().validity()?; - return Ok(match encoded.dtype().as_ptype() { - PType::F32 => { - let values = RangePacked::decode_mapped( - packed, - |ordered| f32::from_bits(unordered_u32(ordered as u32)), - 0.0, - )?; - PrimitiveArray::new::(values, validity) - } - PType::F64 => { - let values = RangePacked::decode_mapped( - packed, - |ordered| f64::from_bits(unordered_u64(ordered)), - 0.0, - )?; - PrimitiveArray::new::(values, validity) - } - ptype => return Err(vortex_err!("fixed-bin float tree does not support {ptype}")), - }); - } - - let alp = encoded.as_::(); - let packed = alp.encoded().as_::(); - let validity = alp.encoded().validity()?; - let exponents = alp.exponents(); - let decoded = match encoded.dtype().as_ptype() { - PType::F32 => { - let values = RangePacked::decode_mapped( - packed, - |ordered| { - let encoded = ((ordered as u32) ^ (1_u32 << 31)) as i32; - ::decode_single(encoded, exponents) - }, - 0.0, - )?; - PrimitiveArray::new::(values, validity) - } - PType::F64 => { - let values = RangePacked::decode_mapped( - packed, - |ordered| { - let encoded = (ordered ^ (1_u64 << 63)) as i64; - ::decode_single(encoded, exponents) - }, - 0.0, - )?; - PrimitiveArray::new::(values, validity) - } - ptype => return Err(vortex_err!("fixed-bin ALP tree does not support {ptype}")), - }; - if let Some(patches) = alp.patches() { - decoded.patch(&patches, &mut session.create_execution_ctx()) - } else { - Ok(decoded) - } -} - -fn unordered_u32(value: u32) -> u32 { - if value & (1_u32 << 31) == 0 { - !value - } else { - value ^ (1_u32 << 31) - } -} - -fn unordered_u64(value: u64) -> u64 { - if value & (1_u64 << 63) == 0 { - !value - } else { - value ^ (1_u64 << 63) - } -} - -fn profile_pco_array( - dataset: &str, - column: &str, - path: &str, - array: &ArrayRef, - session: &VortexSession, -) -> VortexResult<()> { - if array.encoding_id().as_ref() == "vortex.pco" { - let mut ctx = session.create_execution_ctx(); - let primitive = array.clone().execute::(&mut ctx)?; - let primitive_array = primitive.clone().into_array(); - let mask = primitive_array - .validity()? - .execute_mask(primitive.len(), &mut ctx)?; - let values = primitive_array - .filter(mask)? - .execute::(&mut ctx)?; - let ordered = ordered_values(&values); - let validity_bytes = array.children().iter().map(ArrayRef::nbytes).sum::(); - match values.ptype() { - PType::F32 => { - profile_pco_values(dataset, column, path, PType::F32, values.as_slice::())? - } - PType::F64 => { - profile_pco_values(dataset, column, path, PType::F64, values.as_slice::())? - } - PType::I16 => { - profile_pco_values(dataset, column, path, PType::I16, values.as_slice::())? - } - PType::I32 => { - profile_pco_values(dataset, column, path, PType::I32, values.as_slice::())? - } - PType::I64 => { - profile_pco_values(dataset, column, path, PType::I64, values.as_slice::())? - } - PType::U16 => { - profile_pco_values(dataset, column, path, PType::U16, values.as_slice::())? - } - PType::U32 => { - profile_pco_values(dataset, column, path, PType::U32, values.as_slice::())? - } - PType::U64 => { - profile_pco_values(dataset, column, path, PType::U64, values.as_slice::())? - } - ptype => return Err(vortex_err!("cannot profile Pco ptype {ptype}")), - } - if std::env::var_os("VORTEX_BENCH_PCO_MODES_ONLY").is_some() { - return Ok(()); - } - if std::env::var_os("VORTEX_BENCH_FIXED_BIN").is_some() { - profile_range_packed( - dataset, - column, - path, - values.ptype().byte_width(), - array.nbytes(), - validity_bytes, - &ordered, - )?; - } - profile_quotient_remainder( - dataset, - column, - path, - values.ptype(), - array.nbytes(), - &ordered, - session, - )?; - let bits = values.ptype().bit_width(); - let one_reference = - u64::try_from(estimate_multi_reference(&ordered, 1, bits))? + validity_bytes; - let two_references = - u64::try_from(estimate_multi_reference(&ordered, 2, bits))? + validity_bytes; - let four_references = - u64::try_from(estimate_multi_reference(&ordered, 4, bits))? + validity_bytes; - let bitmap_patches = - u64::try_from(estimate_bitmap_patches(&ordered, bits))? + validity_bytes; - println!( - "multi-ref-estimate\t{dataset}\t{column}\t{path}\t{}\t{}\t{}\t{}\t{}\t{}\t{}", - values.ptype(), - array.nbytes(), - one_reference, - two_references, - four_references, - one_reference.min(two_references).min(four_references), - bitmap_patches, - ); - } - for (child_index, child) in array.children().iter().enumerate() { - profile_pco_array( - dataset, - column, - &format!("{path}/{child_index}"), - child, - session, - )?; - } - Ok(()) -} - -#[expect( - clippy::cognitive_complexity, - reason = "benchmark options share the same encoded arrays" -)] -fn measure_dataset( - dataset: &str, - columns: &[Column], - configs: &[(&str, BtrBlocksCompressor)], - session: &VortexSession, -) -> VortexResult<()> { - let mut input_bytes = 0_u64; - for column in columns { - input_bytes += column.input_bytes; - } - let encoded = configs - .iter() - .map(|(name, compressor)| Ok((*name, encode_all(compressor, columns, session)?))) - .collect::>>()?; - - for (config, arrays) in &encoded { - for (column, array) in columns.iter().zip(arrays) { - assert_arrays_eq!(array, column.array, &mut session.create_execution_ctx()); - println!( - "structure\t{dataset}\t{}\t{}\t{config}\t{}\t{}", - column.name, - column.dtype_label, - encoding_tree(array), - array.nbytes() - ); - if std::env::var_os("VORTEX_BENCH_CHUNK_TREES").is_some() - && let Some(chunked) = array.as_typed::() - { - for (chunk_index, chunk) in chunked.iter_chunks().enumerate() { - println!( - "chunk-structure\t{dataset}\t{}\t{}\t{config}\t{chunk_index}\t{}\t{}\t{}", - column.name, - column.dtype_label, - chunk.len(), - encoding_tree(chunk), - chunk.nbytes(), - ); - } - } - if matches!( - *config, - "integer-block-residual-only" | "ordered-block-residual-only" - ) { - profile_block_residual_array(dataset, &column.name, config, "root", array)?; - } - } - } - - if std::env::var_os("VORTEX_BENCH_CONFIG_TREES").is_some() { - for (config, arrays) in &encoded { - for (column, array) in columns.iter().zip(arrays) { - if column.primitive.is_some() { - measure_existing_tree( - dataset, - &column.name, - config, - &column.array, - array, - session, - )?; - } - } - } - } - - if std::env::var_os("VORTEX_BENCH_SKIP_AUXILIARY_ANALYSIS").is_none() { - let prior_default = encoded - .iter() - .find(|(config, _)| *config == "prior-default") - .ok_or_else(|| vortex_err!("prior-default configuration is missing"))?; - let prior_compact = encoded - .iter() - .find(|(config, _)| *config == "prior-compact") - .ok_or_else(|| vortex_err!("prior-compact configuration is missing"))?; - for (column_index, column) in columns.iter().enumerate() { - let Some(primitive) = &column.primitive else { - continue; - }; - if !primitive.ptype().is_float() { - continue; - } - let default_array = &prior_default.1[column_index]; - let compact_array = &prior_compact.1[column_index]; - let input_bytes = primitive.len() * primitive.ptype().byte_width(); - let default_bytes = default_array.nbytes(); - let compact_bytes = compact_array.nbytes(); - let compact_savings = if default_bytes == 0 { - 0.0 - } else { - 100.0 * (1.0 - compact_bytes as f64 / default_bytes as f64) - }; - println!( - "float-column\t{dataset}\t{}\t{}\t{}\t{input_bytes}\t{default_bytes}\t{compact_bytes}\t{compact_savings:.3}\t{}\t{}", - column.name, - primitive.ptype(), - primitive.len(), - encoding_tree(default_array), - encoding_tree(compact_array), - ); - measure_frequency_ranked_dict_trees( - dataset, - &column.name, - &column.array, - default_array, - session, - )?; - if compact_bytes * 10 <= default_bytes * 9 - || std::env::var_os("VORTEX_BENCH_FIXED_BIN").is_some() - || std::env::var_os("VORTEX_BENCH_PCO_MODES_ONLY").is_some() - { - profile_pco_array(dataset, &column.name, "root", compact_array, session)?; - } - if std::env::var_os("VORTEX_BENCH_FIXED_BIN").is_some() { - measure_existing_tree( - dataset, - &column.name, - "default", - &column.array, - default_array, - session, - )?; - measure_existing_tree( - dataset, - &column.name, - "compact", - &column.array, - compact_array, - session, - )?; - measure_fixed_bin_tree( - dataset, - &column.name, - &column.array, - compact_array, - session, - )?; - measure_decomposed_fixed_bin_tree( - dataset, - &column.name, - &column.array, - compact_array, - FixedBinOffsetTree::BlockResidual, - session, - )?; - measure_decomposed_fixed_bin_tree( - dataset, - &column.name, - &column.array, - compact_array, - FixedBinOffsetTree::FoRBitPacked, - session, - )?; - measure_block_residual_float_tree( - dataset, - &column.name, - &column.array, - compact_array, - session, - )?; - } - if std::env::var_os("VORTEX_BENCH_INT_MULT_TREE").is_some() { - if std::env::var_os("VORTEX_BENCH_FIXED_BIN").is_none() { - measure_existing_tree( - dataset, - &column.name, - "default", - &column.array, - default_array, - session, - )?; - measure_existing_tree( - dataset, - &column.name, - "compact", - &column.array, - compact_array, - session, - )?; - measure_block_residual_float_tree( - dataset, - &column.name, - &column.array, - compact_array, - session, - )?; - } - for base in [5, 10, 100, 1_000] { - measure_int_mult_float_tree( - dataset, - &column.name, - &column.array, - compact_array, - base, - session, - )?; - } - let suffix_widths: &[u8] = match primitive.ptype() { - PType::F32 => &[12, 14, 16, 18, 20, 22, 24, 26, 28], - PType::F64 => &[24, 28, 32, 36, 40, 44, 48, 52, 56], - _ => &[], - }; - for &suffix_bits in suffix_widths { - measure_prefix_int_mult_float_tree( - dataset, - &column.name, - &column.array, - compact_array, - suffix_bits, - session, - )?; - } - } - if std::env::var_os("VORTEX_BENCH_PATCH_POSITIONS").is_some() { - if std::env::var_os("VORTEX_BENCH_FIXED_BIN").is_none() - && std::env::var_os("VORTEX_BENCH_INT_MULT_TREE").is_none() - { - measure_existing_tree( - dataset, - &column.name, - "default", - &column.array, - default_array, - session, - )?; - measure_existing_tree( - dataset, - &column.name, - "compact", - &column.array, - compact_array, - session, - )?; - } - measure_block_residual_patch_positions( - dataset, - &column.name, - &column.array, - default_array, - compact_array, - session, - )?; - } - } - } - - if std::env::var_os("VORTEX_BENCH_PROFILE_ONLY").is_some() { - return Ok(()); - } - - let encode_iterations = (128_000_000_u64 / input_bytes).clamp(3, 10) as usize; - let mut encode_durations = (0..configs.len()) - .map(|_| Vec::with_capacity(encode_iterations)) - .collect::>(); - for iteration in 0..encode_iterations { - for offset in 0..configs.len() { - let index = (iteration + offset) % configs.len(); - let start = Instant::now(); - black_box(encode_all(&configs[index].1, columns, session)?); - encode_durations[index].push(start.elapsed()); - } - } - - let decode_iterations = (512_000_000_u64 / input_bytes).clamp(5, 30) as usize; - let mut decode_durations = (0..configs.len()) - .map(|_| Vec::with_capacity(decode_iterations)) - .collect::>(); - for iteration in 0..decode_iterations { - for offset in 0..configs.len() { - let index = (iteration + offset) % configs.len(); - let start = Instant::now(); - black_box(decode_all(&encoded[index].1, session)?); - decode_durations[index].push(start.elapsed()); - } - } - - for (index, (config, arrays)) in encoded.iter().enumerate() { - let encoded_bytes = arrays.iter().map(ArrayRef::nbytes).sum::(); - let encode_median = percentile(&mut encode_durations[index], 1, 2); - let decode_median = percentile(&mut decode_durations[index], 1, 2); - let encode_throughput = input_bytes as f64 / encode_median.as_secs_f64() / 1_000_000.0; - let decode_throughput = input_bytes as f64 / decode_median.as_secs_f64() / 1_000_000.0; - println!( - "result\t{dataset}\t{config}\t{}\t{input_bytes}\t{encoded_bytes}\t{encode_throughput:.1}\t{decode_throughput:.1}", - columns[0].array.len() - ); - } - Ok(()) -} - -fn compressors() -> Vec<(&'static str, BtrBlocksCompressor)> { - let new_scheme_ids = [ - FloatQuantScheme.id(), - OrderedBlockResidualScheme.id(), - BlockResidualScheme.id(), - ]; - vec![ - ( - "prior-default", - BtrBlocksCompressorBuilder::default() - .exclude_schemes(new_scheme_ids) - .build(), - ), - ( - "float-quant-only", - BtrBlocksCompressorBuilder::default() - .exclude_schemes([OrderedBlockResidualScheme.id(), BlockResidualScheme.id()]) - .build(), - ), - ( - "ordered-block-residual-only", - BtrBlocksCompressorBuilder::default() - .exclude_schemes([FloatQuantScheme.id(), BlockResidualScheme.id()]) - .build(), - ), - ( - "integer-block-residual-only", - BtrBlocksCompressorBuilder::default() - .exclude_schemes([FloatQuantScheme.id(), OrderedBlockResidualScheme.id()]) - .build(), - ), - ( - "block-residual-bundle", - BtrBlocksCompressorBuilder::default() - .exclude_schemes([FloatQuantScheme.id()]) - .build(), - ), - ( - "proposed-default", - BtrBlocksCompressorBuilder::default().build(), - ), - ( - "range-packed-only", - BtrBlocksCompressorBuilder::default() - .exclude_schemes(new_scheme_ids) - .with_new_scheme(&RANGE_PACKED_SCHEME) - .build(), - ), - ( - "proposed-default-range-packed", - BtrBlocksCompressorBuilder::default() - .with_new_scheme(&RANGE_PACKED_SCHEME) - .build(), - ), - ( - "proposed-default-integer-fixed-bins", - BtrBlocksCompressorBuilder::default() - .with_new_scheme(&INTEGER_FIXED_BINS_SCHEME) - .build(), - ), - ( - "prior-compact", - BtrBlocksCompressorBuilder::default() - .with_compact() - .exclude_schemes(new_scheme_ids) - .build(), - ), - ( - "compact", - BtrBlocksCompressorBuilder::default().with_compact().build(), - ), - ] -} - -fn main() -> VortexResult<()> { - let row_count = std::env::var("VORTEX_BENCH_ROWS") - .ok() - .map(|value| { - value - .parse::() - .map_err(|error| vortex_err!("invalid VORTEX_BENCH_ROWS: {error}")) - }) - .transpose()? - .unwrap_or(DEFAULT_ROW_COUNT); - let session = array_session(); - vortex_range_packed::initialize(&session); - let mut configs = compressors(); - if let Ok(filter) = std::env::var("VORTEX_BENCH_CONFIGS") { - configs.retain(|(name, _)| filter.split(',').any(|requested| requested == *name)); - vortex_ensure!(!configs.is_empty(), "no benchmark configuration matched"); - } - - println!("structure\tdataset\tcolumn\tptype\tconfig\tencoding\tbytes"); - println!( - "float-column\tdataset\tcolumn\tptype\trows\tinput-bytes\tdefault-bytes\tcompact-bytes\tcompact-savings-pct\tdefault-encoding\tcompact-encoding" - ); - println!( - "pco-profile\tdataset\tcolumn\tpath\tptype\tchunk\trows\tmode\tdelta\tdelta-bins\tdelta-max-offset-bits\tdelta-average-bits\tprimary-bins\tprimary-ans-log\tprimary-max-offset-bits\tprimary-average-bits\tsecondary-bins\tsecondary-ans-log\tsecondary-max-offset-bits\tsecondary-average-bits" - ); - println!( - "multi-ref-estimate\tdataset\tcolumn\tpath\tptype\tpco-child-bytes\tone-reference-bytes\ttwo-reference-bytes\tfour-reference-bytes\tbest-reference-bytes\tbitmap-patch-bytes" - ); - println!( - "quotient-remainder-estimate\tdataset\tcolumn\tpath\tptype\tbase\tpco-child-bytes\tremainder-entropy\tmost-common-remainder-share\tquotient-bytes\tremainder-bytes\ttotal-bytes\tquotient-bitmap-bytes\tremainder-mode-bitmap-bytes\tbitmap-total-bytes\tquotient-encoding\tremainder-encoding" - ); - println!( - "int-mult-checkpoint\tdataset\tcolumn\tpath\tptype\tbase\trows\tpco-child-bytes\tint-mult-bytes\tquotient-patches\tremainder-exceptions\tencode-MB/s\tdecode-MB/s\tscalar-ns" - ); - println!( - "int-mult-gaps-checkpoint\tdataset\tcolumn\tpath\tptype\tbase\trows\tpco-child-bytes\tint-mult-bytes\tquotient-gap-bytes\tremainder-gap-bytes\tquotient-bitmap-bytes\tremainder-bitmap-bytes\tencode-MB/s\tdecode-MB/s\tscalar-ns" - ); - println!( - "int-mult-pairs-checkpoint\tdataset\tcolumn\tpath\tptype\tbase\trows\tpco-child-bytes\tint-mult-bytes\tencode-MB/s\tdecode-MB/s" - ); - println!("int-mult-decode-breakdown\tvariant\tdecode-MB/s"); - println!( - "int-mult-dense64-checkpoint\tdataset\tcolumn\tpath\tptype\tbase\trows\tpco-child-bytes\tint-mult-bytes\tquotient-patches\tencode-MB/s\tdecode-MB/s\tscalar-ns" - ); - println!( - "fixed-bin-checkpoint\tdataset\tcolumn\tpath\trows\tpco-bytes\tfixed-bin-bytes\tbins\tmax-offset-bits\toffset-widths\tencode-MB/s\tdecode-MB/s\tscalar-ns" - ); - println!("fixed-bin-tree\tdataset\tcolumn\trows\tbytes\tdecode-MB/s\tscalar-ns\tencoding"); - println!("fixed-bin-tree-fused\tdataset\tcolumn\trows\tbytes\tdecode-MB/s\tencoding"); - println!("fixed-bin-tree-encode\tdataset\tcolumn\trows\tbytes\tencode-MB/s\tencoding"); - println!("candidate-tree-encode\tdataset\tcolumn\tconfig\trows\tbytes\tencode-MB/s\tencoding"); - println!( - "tree-throughput\tdataset\tcolumn\tconfig\trows\tbytes\tdecode-MB/s\tscalar-ns\tencoding" - ); - println!("result\tdataset\tconfig\trows\tinput-bytes\tencoded-bytes\tencode-MB/s\tdecode-MB/s"); - println!( - "block-residual-profile\tdataset\tcolumn\tconfig\tpath\tptype\trows\tpatches\tblocks\taverage-residual-width\taverage-high-width\tmaximum-patch-density\tblocks-above-one-eighth\tblocks-at-one-quarter\tbytes" - ); - if std::env::var_os("VORTEX_BENCH_SKIP_SYNTHETIC").is_none() { - let synthetic_filter = std::env::var("VORTEX_BENCH_SYNTHETIC").ok(); - for (dataset, columns) in synthetic_datasets(row_count)? { - if synthetic_filter - .as_deref() - .is_some_and(|filter| filter != dataset) - { - continue; - } - measure_dataset(&dataset, &columns, &configs, &session)?; - } - } - for argument in std::env::args().skip(1) { - let path = Path::new(&argument); - let dataset = path - .file_stem() - .and_then(|name| name.to_str()) - .ok_or_else(|| vortex_err!("data path has no valid file name"))?; - let mut columns = if path - .extension() - .is_some_and(|extension| extension == "parquet") - { - read_parquet_numeric(path, row_count, &session)? - } else { - read_california(path, row_count)? - }; - if let Ok(column_filter) = std::env::var("VORTEX_BENCH_COLUMN") { - columns.retain(|column| column.name == column_filter); - vortex_ensure!( - !columns.is_empty(), - "dataset does not contain the requested column" - ); - } - measure_dataset(dataset, &columns, &configs, &session)?; - } - Ok(()) -} diff --git a/vortex-btrblocks/examples/float_compressor_bench/int_mult_codec.rs b/vortex-btrblocks/examples/float_compressor_bench/int_mult_codec.rs deleted file mode 100644 index a724044e956..00000000000 --- a/vortex-btrblocks/examples/float_compressor_bench/int_mult_codec.rs +++ /dev/null @@ -1,1057 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::mem::size_of; - -use fastlanes::BitPacking; -use vortex_error::VortexResult; - -const CHUNK_LEN: usize = 1_024; -const BITMAP_WORDS: usize = CHUNK_LEN / u64::BITS as usize; -const STREAM_PADDING: usize = 7; -const SERIALIZED_BLOCK_METADATA_BYTES: usize = 24; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct IntMultCodec32 { - len: usize, - base: u32, - blocks: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct IntMultDenseCodec64 { - len: usize, - base: u64, - blocks: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -struct IntMultDenseBlock64 { - len: u16, - quotient_base: u64, - quotient_width: u8, - quotient_high_width: u8, - quotient_lows: Vec, - quotient_patch_bitmap: Vec, - quotient_highs: Vec, - remainders: Vec, - quotient_patch_count: u16, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -struct IntMultBlock32 { - len: u16, - quotient_base: u32, - quotient_width: u8, - quotient_high_width: u8, - quotient_lows: Vec, - quotient_patch_bitmap: Vec, - quotient_patch_gaps: Vec, - quotient_highs: Vec, - remainder_mode: u32, - remainder_width: u8, - remainder_exception_bitmap: Vec, - remainder_exception_gaps: Vec, - remainder_pair_bitmap: Vec, - remainder_pair_values: Vec, - remainder_exceptions: Vec, - quotient_patch_count: u16, - remainder_exception_count: u16, -} - -impl IntMultCodec32 { - pub fn encode(values: &[u32], base: u32) -> VortexResult { - vortex_error::vortex_ensure!(base > 1, "IntMult base must exceed one"); - let blocks = values - .chunks(CHUNK_LEN) - .map(|block| encode_block(block, base)) - .collect::>>()?; - Ok(Self { - len: values.len(), - base, - blocks, - }) - } - - pub fn decode(&self) -> Vec { - self.decode_variant::() - } - - pub fn decode_gaps(&self) -> Vec { - self.decode_variant::() - } - - pub fn decode_pairs(&self) -> Vec { - self.decode_variant::() - } - - pub fn decode_gaps_without_quotient_patches(&self) -> Vec { - self.decode_variant::() - } - - pub fn decode_gaps_without_remainder_exceptions(&self) -> Vec { - self.decode_variant::() - } - - pub fn decode_gaps_without_exceptions(&self) -> Vec { - self.decode_variant::() - } - - pub fn decode_without_quotient_patches(&self) -> Vec { - self.decode_variant::() - } - - pub fn decode_without_remainder_exceptions(&self) -> Vec { - self.decode_variant::() - } - - pub fn decode_without_exceptions(&self) -> Vec { - self.decode_variant::() - } - - fn decode_variant< - const QUOTIENT_PATCHES: bool, - const REMAINDER_EXCEPTIONS: bool, - const GAPS: bool, - const PAIRS: bool, - >( - &self, - ) -> Vec { - let mut values = Vec::with_capacity(self.len); - let mut quotients = [0_u32; CHUNK_LEN]; - let mut remainder_exceptions = [0_u8; CHUNK_LEN]; - for block in &self.blocks { - if block.quotient_width == 0 { - quotients.fill(0); - } else { - // SAFETY: The encoder creates one complete FastLanes chunk. - unsafe { - u32::unchecked_unpack( - usize::from(block.quotient_width), - &block.quotient_lows, - &mut quotients, - ); - } - } - - if QUOTIENT_PATCHES { - let mut high_index = 0usize; - let mut apply = |position| { - let high = read_bits( - &block.quotient_highs, - high_index * usize::from(block.quotient_high_width), - block.quotient_high_width, - ); - quotients[position] |= high << block.quotient_width; - high_index += 1; - }; - if GAPS { - for_each_gap(&block.quotient_patch_gaps, &mut apply); - } else { - for_each_set_bit(&block.quotient_patch_bitmap, &mut apply); - } - } - let block_len = usize::from(block.len); - for quotient in &mut quotients[..block_len] { - *quotient = quotient - .wrapping_add(block.quotient_base) - .wrapping_mul(self.base) - .wrapping_add(block.remainder_mode); - } - if REMAINDER_EXCEPTIONS { - let mut exception_index = 0usize; - if PAIRS { - let mut pair_value_index = 0usize; - for_each_set_bit(&block.remainder_pair_bitmap, |pair_index| { - let position = pair_index * 2; - // SAFETY: The encoder stores one byte for every set pair bit. - let pair = - unsafe { *block.remainder_pair_values.get_unchecked(pair_value_index) }; - quotients[position] = quotients[position] - .wrapping_sub(block.remainder_mode) - .wrapping_add(u32::from(pair & 0x0f)); - if position + 1 < block_len { - quotients[position + 1] = quotients[position + 1] - .wrapping_sub(block.remainder_mode) - .wrapping_add(u32::from(pair >> 4)); - } - pair_value_index += 1; - }); - } else if GAPS { - for_each_gap(&block.remainder_exception_gaps, |position| { - let remainder = read_bits( - &block.remainder_exceptions, - exception_index * usize::from(block.remainder_width), - block.remainder_width, - ); - quotients[position] = quotients[position] - .wrapping_sub(block.remainder_mode) - .wrapping_add(remainder); - exception_index += 1; - }); - } else { - if block.remainder_width == 4 { - let exception_count = usize::from(block.remainder_exception_count); - for index in 0..exception_count.div_ceil(2) { - // SAFETY: The encoder stores two exceptions in each payload byte. - let byte = unsafe { *block.remainder_exceptions.get_unchecked(index) }; - remainder_exceptions[index * 2] = byte & 0x0f; - remainder_exceptions[index * 2 + 1] = byte >> 4; - } - for_each_set_bit(&block.remainder_exception_bitmap, |position| { - let remainder = u32::from(remainder_exceptions[exception_index]); - quotients[position] = quotients[position] - .wrapping_sub(block.remainder_mode) - .wrapping_add(remainder); - exception_index += 1; - }); - } else { - for_each_set_bit(&block.remainder_exception_bitmap, |position| { - let remainder = read_bits( - &block.remainder_exceptions, - exception_index * usize::from(block.remainder_width), - block.remainder_width, - ); - quotients[position] = quotients[position] - .wrapping_sub(block.remainder_mode) - .wrapping_add(remainder); - exception_index += 1; - }); - } - } - } - values.extend_from_slice("ients[..block_len]); - } - values - } - - pub fn scalar_at(&self, index: usize) -> VortexResult { - self.scalar_at_variant::(index) - } - - pub fn scalar_at_gaps(&self, index: usize) -> VortexResult { - self.scalar_at_variant::(index) - } - - fn scalar_at_variant(&self, index: usize) -> VortexResult { - vortex_error::vortex_ensure!( - index < self.len, - "index {index} is out of bounds for length {}", - self.len - ); - let block = &self.blocks[index / CHUNK_LEN]; - let index_in_block = index % CHUNK_LEN; - let mut quotient_residual = if block.quotient_width == 0 { - 0 - } else { - // SAFETY: The encoder creates one complete FastLanes chunk. - unsafe { - u32::unchecked_unpack_single( - usize::from(block.quotient_width), - &block.quotient_lows, - index_in_block, - ) - } - }; - let quotient_rank = if GAPS { - gap_rank(&block.quotient_patch_gaps, index_in_block) - } else if bitmap_value(&block.quotient_patch_bitmap, index_in_block) { - Some(bitmap_rank(&block.quotient_patch_bitmap, index_in_block)) - } else { - None - }; - if let Some(rank) = quotient_rank { - let high = read_bits( - &block.quotient_highs, - rank * usize::from(block.quotient_high_width), - block.quotient_high_width, - ); - quotient_residual |= high << block.quotient_width; - } - let quotient = block.quotient_base.wrapping_add(quotient_residual); - - let remainder_rank = if GAPS { - gap_rank(&block.remainder_exception_gaps, index_in_block) - } else if bitmap_value(&block.remainder_exception_bitmap, index_in_block) { - Some(bitmap_rank( - &block.remainder_exception_bitmap, - index_in_block, - )) - } else { - None - }; - let remainder = if let Some(rank) = remainder_rank { - read_bits( - &block.remainder_exceptions, - rank * usize::from(block.remainder_width), - block.remainder_width, - ) - } else { - block.remainder_mode - }; - Ok(quotient.wrapping_mul(self.base).wrapping_add(remainder)) - } - - pub fn encoded_size(&self) -> usize { - size_of::() - + self - .blocks - .iter() - .map(|block| { - SERIALIZED_BLOCK_METADATA_BYTES - + block.quotient_lows.len() * size_of::() - + block.quotient_patch_bitmap.len() * size_of::() - + block.quotient_highs.len() - + block.remainder_exception_bitmap.len() * size_of::() - + block.remainder_exceptions.len() - }) - .sum::() - } - - pub fn encoded_size_gaps(&self) -> usize { - size_of::() - + self - .blocks - .iter() - .map(|block| { - SERIALIZED_BLOCK_METADATA_BYTES - + block.quotient_lows.len() * size_of::() - + block.quotient_patch_gaps.len() - + block.quotient_highs.len() - + block.remainder_exception_gaps.len() - + block.remainder_exceptions.len() - }) - .sum::() - } - - pub fn encoded_size_pairs(&self) -> usize { - size_of::() - + self - .blocks - .iter() - .map(|block| { - SERIALIZED_BLOCK_METADATA_BYTES - + block.quotient_lows.len() * size_of::() - + block.quotient_patch_bitmap.len() * size_of::() - + block.quotient_highs.len() - + block.remainder_pair_bitmap.len() * size_of::() - + block.remainder_pair_values.len() - }) - .sum::() - } - - pub fn quotient_patch_count(&self) -> usize { - self.blocks - .iter() - .map(|block| usize::from(block.quotient_patch_count)) - .sum() - } - - pub fn remainder_exception_count(&self) -> usize { - self.blocks - .iter() - .map(|block| usize::from(block.remainder_exception_count)) - .sum() - } - - pub fn gap_bytes(&self) -> (usize, usize) { - self.blocks - .iter() - .fold((0, 0), |(quotient, remainder), block| { - ( - quotient + block.quotient_patch_gaps.len(), - remainder + block.remainder_exception_gaps.len(), - ) - }) - } - - pub fn bitmap_bytes(&self) -> (usize, usize) { - self.blocks - .iter() - .fold((0, 0), |(quotient, remainder), block| { - ( - quotient + block.quotient_patch_bitmap.len() * size_of::(), - remainder + block.remainder_exception_bitmap.len() * size_of::(), - ) - }) - } -} - -impl IntMultDenseCodec64 { - pub fn encode(values: &[u64], base: u64) -> VortexResult { - vortex_error::vortex_ensure!(base > 1, "IntMult base must exceed one"); - let blocks = values - .chunks(CHUNK_LEN) - .map(|block| encode_dense_block64(block, base)) - .collect::>>()?; - Ok(Self { - len: values.len(), - base, - blocks, - }) - } - - pub fn decode(&self) -> Vec { - self.decode_with_quotient_patches(true) - } - - pub fn decode_without_quotient_patches(&self) -> Vec { - self.decode_with_quotient_patches(false) - } - - fn decode_with_quotient_patches(&self, apply_patches: bool) -> Vec { - let mut values = Vec::with_capacity(self.len); - let mut quotients = [0_u64; CHUNK_LEN]; - let mut remainders = [0_u64; CHUNK_LEN]; - for block in &self.blocks { - if block.quotient_width == 0 { - quotients.fill(0); - } else { - // SAFETY: The encoder creates one complete FastLanes chunk. - unsafe { - u64::unchecked_unpack( - usize::from(block.quotient_width), - &block.quotient_lows, - &mut quotients, - ); - } - } - if apply_patches { - let mut high_index = 0usize; - for_each_set_bit(&block.quotient_patch_bitmap, |position| { - let high = read_bits64( - &block.quotient_highs, - high_index * usize::from(block.quotient_high_width), - block.quotient_high_width, - ); - quotients[position] |= high << block.quotient_width; - high_index += 1; - }); - } - - let remainder_width = bit_width64(self.base - 1); - if remainder_width == 0 { - remainders.fill(0); - } else { - // SAFETY: The encoder creates one complete FastLanes chunk. - unsafe { - u64::unchecked_unpack( - usize::from(remainder_width), - &block.remainders, - &mut remainders, - ); - } - } - - let block_len = usize::from(block.len); - for index in 0..block_len { - quotients[index] = quotients[index] - .wrapping_add(block.quotient_base) - .wrapping_mul(self.base) - .wrapping_add(remainders[index]); - } - values.extend_from_slice("ients[..block_len]); - } - values - } - - pub fn scalar_at(&self, index: usize) -> VortexResult { - vortex_error::vortex_ensure!( - index < self.len, - "index {index} is out of bounds for length {}", - self.len - ); - let block = &self.blocks[index / CHUNK_LEN]; - let index_in_block = index % CHUNK_LEN; - let mut quotient_residual = if block.quotient_width == 0 { - 0 - } else { - // SAFETY: The encoder creates one complete FastLanes chunk. - unsafe { - u64::unchecked_unpack_single( - usize::from(block.quotient_width), - &block.quotient_lows, - index_in_block, - ) - } - }; - if bitmap_value(&block.quotient_patch_bitmap, index_in_block) { - let rank = bitmap_rank(&block.quotient_patch_bitmap, index_in_block); - let high = read_bits64( - &block.quotient_highs, - rank * usize::from(block.quotient_high_width), - block.quotient_high_width, - ); - quotient_residual |= high << block.quotient_width; - } - let remainder_width = bit_width64(self.base - 1); - let remainder = if remainder_width == 0 { - 0 - } else { - // SAFETY: The encoder creates one complete FastLanes chunk. - unsafe { - u64::unchecked_unpack_single( - usize::from(remainder_width), - &block.remainders, - index_in_block, - ) - } - }; - Ok(quotient_residual - .wrapping_add(block.quotient_base) - .wrapping_mul(self.base) - .wrapping_add(remainder)) - } - - pub fn encoded_size(&self) -> usize { - size_of::() - + self - .blocks - .iter() - .map(|block| { - SERIALIZED_BLOCK_METADATA_BYTES - + block.quotient_lows.len() * size_of::() - + block.quotient_patch_bitmap.len() * size_of::() - + block.quotient_highs.len() - + block.remainders.len() * size_of::() - }) - .sum::() - } - - pub fn quotient_patch_count(&self) -> usize { - self.blocks - .iter() - .map(|block| usize::from(block.quotient_patch_count)) - .sum() - } -} - -fn encode_block(values: &[u32], base: u32) -> VortexResult { - let mut quotients = Vec::with_capacity(CHUNK_LEN); - let mut remainders = Vec::with_capacity(CHUNK_LEN); - for &value in values { - quotients.push(value / base); - remainders.push(value % base); - } - - let quotient_base = quotients.iter().copied().min().unwrap_or_default(); - let mut quotient_residuals = quotients - .iter() - .map(|"ient| quotient - quotient_base) - .collect::>(); - let (quotient_width, quotient_high_width, quotient_patch_count) = - choose_quotient_width("ient_residuals, values.len()); - quotient_residuals.resize(CHUNK_LEN, 0); - let quotient_mask = low_mask(quotient_width); - let quotient_lows = quotient_residuals - .iter() - .map(|&residual| residual & quotient_mask) - .collect::>(); - - let mut quotient_patch_bitmap = vec![0_u64; BITMAP_WORDS]; - let mut quotient_patch_positions = Vec::with_capacity(quotient_patch_count); - let mut quotient_highs = - BitWriter::with_capacity(quotient_patch_count * usize::from(quotient_high_width)); - if quotient_high_width > 0 { - for (position, &residual) in quotient_residuals[..values.len()].iter().enumerate() { - let high = residual >> quotient_width; - if high != 0 { - set_bitmap(&mut quotient_patch_bitmap, position); - quotient_patch_positions.push(u16::try_from(position)?); - quotient_highs.write(high, quotient_high_width); - } - } - } - if quotient_patch_count == 0 { - quotient_patch_bitmap.clear(); - } - - let remainder_mode = mode(&remainders, base)?; - let remainder_width = bit_width(base - 1); - let mut remainder_exception_bitmap = vec![0_u64; BITMAP_WORDS]; - let remainder_exception_count = remainders - .iter() - .filter(|&&remainder| remainder != remainder_mode) - .count(); - let mut remainder_exceptions = - BitWriter::with_capacity(remainder_exception_count * usize::from(remainder_width)); - let mut remainder_exception_positions = Vec::with_capacity(remainder_exception_count); - for (position, &remainder) in remainders.iter().enumerate() { - if remainder != remainder_mode { - set_bitmap(&mut remainder_exception_bitmap, position); - remainder_exception_positions.push(u16::try_from(position)?); - remainder_exceptions.write(remainder, remainder_width); - } - } - if remainder_exception_count == 0 { - remainder_exception_bitmap.clear(); - } - - let mut remainder_pair_bitmap = vec![0_u64; BITMAP_WORDS / 2]; - let mut remainder_pair_values = Vec::with_capacity(values.len().div_ceil(2)); - let mut remainder_pair_count = 0usize; - if base <= 16 { - for (pair_index, pair) in remainders.chunks(2).enumerate() { - let left_exception = pair[0] != remainder_mode; - let right_exception = pair - .get(1) - .is_some_and(|&remainder| remainder != remainder_mode); - if !left_exception && !right_exception { - continue; - } - set_bitmap(&mut remainder_pair_bitmap, pair_index); - let right = pair.get(1).copied().unwrap_or(remainder_mode); - remainder_pair_values.push(u8::try_from(pair[0] | (right << 4))?); - remainder_pair_count += 1; - } - } - if remainder_pair_count == 0 { - remainder_pair_bitmap.clear(); - } - - Ok(IntMultBlock32 { - len: u16::try_from(values.len())?, - quotient_base, - quotient_width, - quotient_high_width, - quotient_lows: fast_pack("ient_lows, quotient_width), - quotient_patch_bitmap, - quotient_patch_gaps: encode_gaps("ient_patch_positions), - quotient_highs: quotient_highs.finish(), - remainder_mode, - remainder_width, - remainder_exception_bitmap, - remainder_exception_gaps: encode_gaps(&remainder_exception_positions), - remainder_pair_bitmap, - remainder_pair_values, - remainder_exceptions: remainder_exceptions.finish(), - quotient_patch_count: u16::try_from(quotient_patch_count)?, - remainder_exception_count: u16::try_from(remainder_exception_count)?, - }) -} - -fn encode_dense_block64(values: &[u64], base: u64) -> VortexResult { - let mut quotient_residuals = Vec::with_capacity(CHUNK_LEN); - let mut remainders = Vec::with_capacity(CHUNK_LEN); - let quotient_base = values - .iter() - .map(|value| value / base) - .min() - .unwrap_or_default(); - for &value in values { - quotient_residuals.push(value / base - quotient_base); - remainders.push(value % base); - } - let (quotient_width, quotient_high_width, quotient_patch_count) = - choose_quotient_width64("ient_residuals, values.len()); - quotient_residuals.resize(CHUNK_LEN, 0); - remainders.resize(CHUNK_LEN, 0); - let quotient_mask = low_mask64(quotient_width); - let quotient_lows = quotient_residuals - .iter() - .map(|&residual| residual & quotient_mask) - .collect::>(); - - let mut quotient_patch_bitmap = vec![0_u64; BITMAP_WORDS]; - let mut quotient_highs = - BitWriter64::with_capacity(quotient_patch_count * usize::from(quotient_high_width)); - if quotient_high_width > 0 { - for (position, &residual) in quotient_residuals[..values.len()].iter().enumerate() { - let high = residual >> quotient_width; - if high != 0 { - set_bitmap(&mut quotient_patch_bitmap, position); - quotient_highs.write(high, quotient_high_width); - } - } - } - if quotient_patch_count == 0 { - quotient_patch_bitmap.clear(); - } - - Ok(IntMultDenseBlock64 { - len: u16::try_from(values.len())?, - quotient_base, - quotient_width, - quotient_high_width, - quotient_lows: fast_pack64("ient_lows, quotient_width), - quotient_patch_bitmap, - quotient_highs: quotient_highs.finish(), - remainders: fast_pack64(&remainders, bit_width64(base - 1)), - quotient_patch_count: u16::try_from(quotient_patch_count)?, - }) -} - -fn choose_quotient_width(residuals: &[u32], value_count: usize) -> (u8, u8, usize) { - let mut width_counts = [0_usize; 33]; - let mut maximum_width = 0u8; - for &residual in residuals { - let width = bit_width(residual); - width_counts[usize::from(width)] += 1; - maximum_width = maximum_width.max(width); - } - - let mut patch_count = value_count; - let mut best = (maximum_width, 0_u8, 0_usize); - let mut best_bits = CHUNK_LEN * usize::from(maximum_width); - for residual_width in 0..=maximum_width { - patch_count -= width_counts[usize::from(residual_width)]; - let high_width = if patch_count == 0 { - 0 - } else { - maximum_width - residual_width - }; - let bits = CHUNK_LEN * usize::from(residual_width) - + usize::from(patch_count > 0) * CHUNK_LEN - + patch_count * usize::from(high_width); - if bits < best_bits { - best = (residual_width, high_width, patch_count); - best_bits = bits; - } - } - best -} - -fn choose_quotient_width64(residuals: &[u64], value_count: usize) -> (u8, u8, usize) { - let mut width_counts = [0_usize; 65]; - let mut maximum_width = 0u8; - for &residual in residuals { - let width = bit_width64(residual); - width_counts[usize::from(width)] += 1; - maximum_width = maximum_width.max(width); - } - - let mut patch_count = value_count; - let mut best = (maximum_width, 0_u8, 0_usize); - let mut best_bits = CHUNK_LEN * usize::from(maximum_width); - for residual_width in 0..=maximum_width { - patch_count -= width_counts[usize::from(residual_width)]; - let high_width = if patch_count == 0 { - 0 - } else { - maximum_width - residual_width - }; - let bits = CHUNK_LEN * usize::from(residual_width) - + usize::from(patch_count > 0) * CHUNK_LEN - + patch_count * usize::from(high_width); - if bits < best_bits { - best = (residual_width, high_width, patch_count); - best_bits = bits; - } - } - best -} - -fn mode(values: &[u32], base: u32) -> VortexResult { - let mut counts = vec![0_u16; usize::try_from(base)?]; - for &value in values { - counts[usize::try_from(value)?] += 1; - } - Ok(counts - .iter() - .enumerate() - .max_by_key(|(_, count)| **count) - .map(|(value, _)| u32::try_from(value)) - .transpose()? - .unwrap_or_default()) -} - -fn fast_pack(values: &[u32], width: u8) -> Vec { - if width == 0 { - return Vec::new(); - } - let mut packed = vec![0_u32; CHUNK_LEN * usize::from(width) / u32::BITS as usize]; - // SAFETY: Both slices have the exact lengths required for one FastLanes chunk. - unsafe { u32::unchecked_pack(usize::from(width), values, &mut packed) }; - packed -} - -fn fast_pack64(values: &[u64], width: u8) -> Vec { - if width == 0 { - return Vec::new(); - } - let mut packed = vec![0_u64; CHUNK_LEN * usize::from(width) / u64::BITS as usize]; - // SAFETY: Both slices have the exact lengths required for one FastLanes chunk. - unsafe { u64::unchecked_pack(usize::from(width), values, &mut packed) }; - packed -} - -fn bit_width(value: u32) -> u8 { - u8::try_from(u32::BITS - value.leading_zeros()).unwrap_or(u8::MAX) -} - -fn low_mask(width: u8) -> u32 { - if width == 32 { - u32::MAX - } else if width == 0 { - 0 - } else { - (1_u32 << width) - 1 - } -} - -fn bit_width64(value: u64) -> u8 { - u8::try_from(u64::BITS - value.leading_zeros()).unwrap_or(u8::MAX) -} - -fn low_mask64(width: u8) -> u64 { - if width == 64 { - u64::MAX - } else if width == 0 { - 0 - } else { - (1_u64 << width) - 1 - } -} - -fn set_bitmap(bitmap: &mut [u64], index: usize) { - bitmap[index / 64] |= 1_u64 << (index % 64); -} - -fn bitmap_value(bitmap: &[u64], index: usize) -> bool { - bitmap - .get(index / 64) - .is_some_and(|word| word & (1_u64 << (index % 64)) != 0) -} - -fn bitmap_rank(bitmap: &[u64], index: usize) -> usize { - let full_words = index / 64; - let prior = bitmap[..full_words] - .iter() - .map(|word| word.count_ones() as usize) - .sum::(); - let bit_index = index % 64; - let current = bitmap - .get(full_words) - .map(|word| (word & ((1_u64 << bit_index).wrapping_sub(1))).count_ones() as usize) - .unwrap_or_default(); - prior + current -} - -fn encode_gaps(positions: &[u16]) -> Vec { - let mut gaps = Vec::with_capacity(positions.len()); - let mut next_position = 0usize; - for &position in positions { - let mut gap = usize::from(position) - next_position; - while gap >= 255 { - gaps.push(255); - gap -= 255; - } - gaps.push(u8::try_from(gap).unwrap_or_else(|_| unreachable!("gap is below 255"))); - next_position = usize::from(position) + 1; - } - gaps -} - -#[inline(always)] -fn for_each_gap(gaps: &[u8], mut function: impl FnMut(usize)) { - let mut next_position = 0usize; - let mut gap = 0usize; - for &part in gaps { - gap += usize::from(part); - if part < 255 { - let position = next_position + gap; - function(position); - next_position = position + 1; - gap = 0; - } - } -} - -fn gap_rank(gaps: &[u8], target: usize) -> Option { - let mut next_position = 0usize; - let mut gap = 0usize; - let mut rank = 0usize; - for &part in gaps { - gap += usize::from(part); - if part < 255 { - let position = next_position + gap; - if position >= target { - return (position == target).then_some(rank); - } - next_position = position + 1; - gap = 0; - rank += 1; - } - } - None -} - -#[inline(always)] -fn for_each_set_bit(bitmap: &[u64], mut function: impl FnMut(usize)) { - for (word_index, &word) in bitmap.iter().enumerate() { - let mut remaining = word; - while remaining != 0 { - let bit = remaining.trailing_zeros() as usize; - function(word_index * 64 + bit); - remaining &= remaining - 1; - } - } -} - -#[inline(always)] -fn read_bits(bytes: &[u8], bit_offset: usize, width: u8) -> u32 { - if width == 0 { - return 0; - } - let byte_offset = bit_offset / 8; - let shift = bit_offset % 8; - // SAFETY: Each nonempty stream contains seven readable padding bytes. - let word = unsafe { - bytes - .as_ptr() - .add(byte_offset) - .cast::() - .read_unaligned() - }; - u32::try_from((u64::from_le(word) >> shift) & u64::from(low_mask(width))) - .unwrap_or_else(|_| unreachable!("masked bit stream value fits u32")) -} - -#[inline(always)] -fn read_bits64(bytes: &[u8], bit_offset: usize, width: u8) -> u64 { - if width == 0 { - return 0; - } - let byte_offset = bit_offset / 8; - let shift = bit_offset % 8; - // SAFETY: Each nonempty stream contains fifteen readable padding bytes. - let low = unsafe { - bytes - .as_ptr() - .add(byte_offset) - .cast::() - .read_unaligned() - }; - let mut value = u64::from_le(low) >> shift; - if shift + usize::from(width) > 64 { - // SAFETY: Each nonempty stream contains fifteen readable padding bytes. - let high = unsafe { - bytes - .as_ptr() - .add(byte_offset + size_of::()) - .cast::() - .read_unaligned() - }; - value |= u64::from_le(high) << (64 - shift); - } - value & low_mask64(width) -} - -struct BitWriter { - bytes: Vec, - bit_len: usize, -} - -struct BitWriter64 { - bytes: Vec, - bit_len: usize, -} - -impl BitWriter64 { - fn with_capacity(bits: usize) -> Self { - Self { - bytes: Vec::with_capacity(bits.div_ceil(8) + 15), - bit_len: 0, - } - } - - fn write(&mut self, value: u64, width: u8) { - for bit in 0..width { - let position = self.bit_len + usize::from(bit); - let byte_index = position / 8; - if byte_index == self.bytes.len() { - self.bytes.push(0); - } - self.bytes[byte_index] |= (((value >> bit) & 1) as u8) << (position % 8); - } - self.bit_len += usize::from(width); - } - - fn finish(mut self) -> Vec { - if self.bit_len == 0 { - return Vec::new(); - } - self.bytes.extend_from_slice(&[0; 15]); - self.bytes - } -} - -impl BitWriter { - fn with_capacity(bits: usize) -> Self { - Self { - bytes: Vec::with_capacity(bits.div_ceil(8) + STREAM_PADDING), - bit_len: 0, - } - } - - fn write(&mut self, value: u32, width: u8) { - for bit in 0..width { - let position = self.bit_len + usize::from(bit); - let byte_index = position / 8; - if byte_index == self.bytes.len() { - self.bytes.push(0); - } - self.bytes[byte_index] |= (((value >> bit) & 1) as u8) << (position % 8); - } - self.bit_len += usize::from(width); - } - - fn finish(mut self) -> Vec { - if self.bit_len == 0 { - return Vec::new(); - } - self.bytes.extend_from_slice(&[0; STREAM_PADDING]); - self.bytes - } -} - -#[cfg(test)] -mod tests { - use vortex_error::VortexResult; - - use super::IntMultCodec32; - use super::IntMultDenseCodec64; - - #[test] - fn roundtrip_and_scalar_access() -> VortexResult<()> { - let values = (0_u32..20_000) - .map(|index| match index % 5 { - 0 => u32::MAX - index, - 1 => 2_000_000_000 + index % 10, - _ => 1_000_000 + (index % 31) * 10, - }) - .collect::>(); - for base in [2, 10, 100, 1_000] { - let codec = IntMultCodec32::encode(&values, base)?; - assert_eq!(codec.decode(), values); - assert_eq!(codec.decode_gaps(), values); - if base <= 16 { - assert_eq!(codec.decode_pairs(), values); - } - for index in [0, 1, 1_023, 1_024, values.len() - 1] { - assert_eq!(codec.scalar_at(index)?, values[index]); - assert_eq!(codec.scalar_at_gaps(index)?, values[index]); - } - } - Ok(()) - } - - #[test] - fn dense_u64_roundtrip_and_scalar_access() -> VortexResult<()> { - let values = (0_u64..20_000) - .map(|index| match index % 4 { - 0 => u64::MAX - index, - 1 => (1_u64 << 63) + index % 10, - _ => 1_000_000 + (index % 31) * 10, - }) - .collect::>(); - for base in [2, 10, 100, 1_000] { - let codec = IntMultDenseCodec64::encode(&values, base)?; - assert_eq!(codec.decode(), values); - for index in [0, 1, 1_023, 1_024, values.len() - 1] { - assert_eq!(codec.scalar_at(index)?, values[index]); - } - } - Ok(()) - } -} diff --git a/vortex-btrblocks/src/schemes/fixed_bins.rs b/vortex-btrblocks/src/schemes/fixed_bins.rs deleted file mode 100644 index 01e730e4cb8..00000000000 --- a/vortex-btrblocks/src/schemes/fixed_bins.rs +++ /dev/null @@ -1,74 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! Shared coarse models for fixed-bin scheme rejection. - -#[derive(Clone, Copy, Debug)] -pub(crate) struct CoarseModel { - pub(crate) range_ratio: f64, - pub(crate) block_ratio: f64, -} - -pub(crate) fn coarse_model(values: &[u64], source_width: usize) -> CoarseModel { - CoarseModel { - range_ratio: coarse_prefix_ratio(values, source_width), - block_ratio: coarse_block_ratio(values, source_width), - } -} - -fn coarse_prefix_ratio(values: &[u64], source_width: usize) -> f64 { - let Some((&minimum, &maximum)) = values.iter().min().zip(values.iter().max()) else { - return 0.0; - }; - let span_width = bit_width(maximum - minimum); - let mut best_bits = values.len() * usize::from(span_width); - for code_width in 1_u8..=6 { - if code_width > span_width { - break; - } - let bucket_count = 1usize << code_width; - let shift = span_width - code_width; - let mut counts = [0usize; 64]; - let mut minima = [u64::MAX; 64]; - let mut maxima = [0_u64; 64]; - for &value in values { - let relative = value - minimum; - let bucket = usize::try_from(relative >> shift).unwrap_or(bucket_count - 1); - let bucket = bucket.min(bucket_count - 1); - counts[bucket] += 1; - minima[bucket] = minima[bucket].min(value); - maxima[bucket] = maxima[bucket].max(value); - } - let offset_bits = (0..bucket_count) - .filter(|&bucket| counts[bucket] != 0) - .map(|bucket| counts[bucket] * usize::from(bit_width(maxima[bucket] - minima[bucket]))) - .sum::(); - best_bits = best_bits.min(values.len() * usize::from(code_width) + offset_bits); - } - - if best_bits == 0 { - f64::INFINITY - } else { - values.len() as f64 * source_width as f64 / best_bits as f64 - } -} - -fn coarse_block_ratio(values: &[u64], source_width: usize) -> f64 { - let block_bits = values - .chunks(64) - .map(|block| { - let minimum = block.iter().copied().min().unwrap_or_default(); - let maximum = block.iter().copied().max().unwrap_or_default(); - block.len() * usize::from(bit_width(maximum - minimum)) - }) - .sum::(); - if block_bits == 0 { - f64::INFINITY - } else { - values.len() as f64 * source_width as f64 / block_bits as f64 - } -} - -fn bit_width(value: u64) -> u8 { - u8::try_from(u64::BITS - value.leading_zeros()).unwrap_or(u8::MAX) -} diff --git a/vortex-btrblocks/src/schemes/float/mod.rs b/vortex-btrblocks/src/schemes/float/mod.rs index dc4cf90edf6..e2f8aecb398 100644 --- a/vortex-btrblocks/src/schemes/float/mod.rs +++ b/vortex-btrblocks/src/schemes/float/mod.rs @@ -7,7 +7,6 @@ mod alp; mod alprd; mod float_quant; mod ordered_block_residual; -mod range_packed; mod rle; mod sparse; @@ -20,7 +19,6 @@ pub use float_quant::FloatQuantScheme; pub use ordered_block_residual::OrderedBlockResidualScheme; #[cfg(feature = "pco")] pub use pco::PcoScheme; -pub use range_packed::OrderedFloatRangePackedScheme; pub use rle::FloatRLEScheme; pub use sparse::NullDominatedSparseScheme; // Re-export builtin schemes from vortex-compressor. diff --git a/vortex-btrblocks/src/schemes/float/range_packed.rs b/vortex-btrblocks/src/schemes/float/range_packed.rs deleted file mode 100644 index 7f9db4147bf..00000000000 --- a/vortex-btrblocks/src/schemes/float/range_packed.rs +++ /dev/null @@ -1,526 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! Fixed-bin range packing for ordered floating-point values. - -use vortex_array::ArrayId; -use vortex_array::ArrayRef; -use vortex_array::Canonical; -use vortex_array::ExecutionCtx; -use vortex_array::IntoArray; -use vortex_array::VTable; -use vortex_array::arrays::Dict; -use vortex_array::arrays::DictArray; -use vortex_array::arrays::Primitive; -use vortex_array::arrays::PrimitiveArray; -use vortex_array::dtype::PType; -use vortex_array::dtype::half::f16; -use vortex_array::match_each_float_ptype; -use vortex_block_residual::BlockResidual; -use vortex_block_residual::OrderedFloat; -use vortex_block_residual::OrderedFloatArraySlotsExt; -use vortex_compressor::scheme::CompressionEstimate; -use vortex_compressor::scheme::DeferredEstimate; -use vortex_compressor::scheme::EstimateScore; -use vortex_compressor::scheme::EstimateVerdict; -use vortex_error::VortexResult; -use vortex_fastlanes::BitPacked; -use vortex_fastlanes::bitpack_compress::bitpack_encode_unchecked; -use vortex_int_mult::IntMult; -use vortex_range_packed::RangeDecomposition; - -use crate::ArrayAndStats; -use crate::CascadingCompressor; -use crate::CompressorContext; -use crate::Scheme; -use crate::SchemeExt; -use crate::schemes::fixed_bins::CoarseModel; -use crate::schemes::fixed_bins::coarse_model; -use crate::schemes::sample_primitive_one_percent; - -/// The default factor accounts for RangePacked decode cost during scheme selection. -const DEFAULT_DECODE_COST_FACTOR: f64 = 1.20; -const PREFILTER_MODEL_MARGIN: f64 = 1.50; -const MIN_PREFILTER_MODEL_RATIO: f64 = 1.15; -const MAX_BLOCK_TO_RANGE_RATIO: f64 = 1.10; - -/// Compress floats through ordered IEEE bits, then use fixed-bin range packing. -#[derive(Debug, Copy, Clone, PartialEq)] -pub struct OrderedFloatRangePackedScheme { - decode_cost_factor: f64, -} - -impl OrderedFloatRangePackedScheme { - /// Creates a scheme with the specified decode cost factor. - pub const fn new(decode_cost_factor: f64) -> Self { - Self { decode_cost_factor } - } -} - -impl Default for OrderedFloatRangePackedScheme { - fn default() -> Self { - Self::new(DEFAULT_DECODE_COST_FACTOR) - } -} - -impl Scheme for OrderedFloatRangePackedScheme { - fn scheme_name(&self) -> &'static str { - "vortex.ordered_float.range_packed" - } - - fn matches(&self, canonical: &Canonical) -> bool { - canonical.dtype().is_float() - } - - fn produced_encodings(&self) -> Vec { - vec![ - OrderedFloat.id(), - IntMult.id(), - Dict.id(), - BitPacked.id(), - BlockResidual.id(), - ] - } - - fn num_children(&self) -> usize { - 1 - } - - fn expected_compression_ratio( - &self, - _data: &ArrayAndStats, - compress_ctx: CompressorContext, - _exec_ctx: &mut ExecutionCtx, - ) -> CompressionEstimate { - if compress_ctx.is_sample() { - return CompressionEstimate::Verdict(EstimateVerdict::Skip); - } - - let decode_cost_factor = self.decode_cost_factor; - let scheme_id = self.id(); - CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - move |compressor, data, best_so_far, compress_ctx, exec_ctx| { - let bit_width = data.array_as_primitive().ptype().bit_width() as f64; - let maximum_adjusted_ratio = bit_width / decode_cost_factor; - let best_ratio = best_so_far - .and_then(EstimateScore::finite_ratio) - .unwrap_or(1.0); - if maximum_adjusted_ratio <= best_ratio { - return Ok(EstimateVerdict::Skip); - } - - let sample = sample_primitive_one_percent(data.array_as_primitive(), exec_ctx)?; - let sample = normalize_float_null_values(sample.as_view(), exec_ctx)?; - let before_nbytes = sample.nbytes(); - let Some(after_nbytes) = estimate_ordered_float_if_promising( - sample.as_view(), - best_ratio, - decode_cost_factor, - exec_ctx, - )? - else { - return Ok(EstimateVerdict::Skip); - }; - if after_nbytes == 0 { - return Ok(EstimateVerdict::Skip); - } - - let adjusted_ratio = - before_nbytes as f64 / after_nbytes as f64 / decode_cost_factor; - if adjusted_ratio <= best_ratio { - return Ok(EstimateVerdict::Skip); - } - - let sample = sample.into_array(); - let incumbent = compressor.compress_sample_without_scheme( - &sample, - scheme_id, - compress_ctx, - exec_ctx, - )?; - if after_nbytes as f64 * decode_cost_factor >= incumbent.nbytes() as f64 { - return Ok(EstimateVerdict::Skip); - } - Ok(EstimateVerdict::Ratio(adjusted_ratio)) - }, - ))) - } - - fn compress( - &self, - _compressor: &CascadingCompressor, - data: &ArrayAndStats, - _compress_ctx: CompressorContext, - exec_ctx: &mut ExecutionCtx, - ) -> VortexResult { - let primitive = normalize_float_null_values(data.array_as_primitive(), exec_ctx)?; - encode_ordered_float(primitive.as_view(), exec_ctx) - } -} - -fn estimate_ordered_float_if_promising( - primitive: vortex_array::ArrayView<'_, Primitive>, - best_ratio: f64, - decode_cost_factor: f64, - exec_ctx: &mut ExecutionCtx, -) -> VortexResult> { - let ordered_model = coarse_ordered_float_model(primitive); - if model_prefers_blocks(ordered_model) { - return Ok(None); - } - if !prefilter_passes(ordered_model, best_ratio, decode_cost_factor) { - return Ok(None); - } - Ok(Some(estimate_ordered_float(primitive, exec_ctx)?)) -} - -fn prefilter_passes(model: CoarseModel, best_ratio: f64, decode_cost_factor: f64) -> bool { - model.range_ratio.is_finite() - && model.range_ratio >= MIN_PREFILTER_MODEL_RATIO - && model.range_ratio / decode_cost_factor * PREFILTER_MODEL_MARGIN > best_ratio - && !model_prefers_blocks(model) -} - -fn model_prefers_blocks(model: CoarseModel) -> bool { - model.block_ratio > model.range_ratio * MAX_BLOCK_TO_RANGE_RATIO -} - -fn coarse_ordered_float_model(primitive: vortex_array::ArrayView<'_, Primitive>) -> CoarseModel { - let values: Vec = match primitive.ptype() { - PType::F16 => primitive - .as_slice::() - .iter() - .map(|value| u64::from(ordered_u16(value.to_bits()))) - .collect(), - PType::F32 => primitive - .as_slice::() - .iter() - .map(|value| u64::from(ordered_u32(value.to_bits()))) - .collect(), - PType::F64 => primitive - .as_slice::() - .iter() - .map(|value| ordered_u64(value.to_bits())) - .collect(), - _ => { - return CoarseModel { - range_ratio: 0.0, - block_ratio: 0.0, - }; - } - }; - coarse_model(&values, primitive.ptype().bit_width()) -} - -fn ordered_u16(bits: u16) -> u16 { - if bits & (1_u16 << 15) == 0 { - bits ^ (1_u16 << 15) - } else { - !bits - } -} - -fn ordered_u32(bits: u32) -> u32 { - if bits & (1_u32 << 31) == 0 { - bits ^ (1_u32 << 31) - } else { - !bits - } -} - -fn ordered_u64(bits: u64) -> u64 { - if bits & (1_u64 << 63) == 0 { - bits ^ (1_u64 << 63) - } else { - !bits - } -} - -fn estimate_ordered_float( - primitive: vortex_array::ArrayView<'_, Primitive>, - exec_ctx: &mut ExecutionCtx, -) -> VortexResult { - Ok(encode_ordered_float(primitive, exec_ctx)?.nbytes()) -} - -fn encode_ordered_float( - primitive: vortex_array::ArrayView<'_, Primitive>, - _exec_ctx: &mut ExecutionCtx, -) -> VortexResult { - let ordered = OrderedFloat::from_primitive(primitive)?; - let ordered_values = ordered.encoded().as_::(); - if ordered_values.is_empty() { - return Ok(ordered.into_array()); - } - - let decomposition = match ordered_values.ptype() { - PType::U16 => RangeDecomposition::encode( - &ordered_values - .as_slice::() - .iter() - .map(|&value| u64::from(value)) - .collect::>(), - )?, - PType::U32 => RangeDecomposition::encode( - &ordered_values - .as_slice::() - .iter() - .map(|&value| u64::from(value)) - .collect::>(), - )?, - PType::U64 => RangeDecomposition::encode(ordered_values.as_slice::())?, - ptype => vortex_error::vortex_bail!( - "range decomposition requires unsigned integers, got {ptype}" - ), - }; - let code_width = decomposition.code_width(); - let codes = PrimitiveArray::new(decomposition.codes().to_vec(), ordered_values.validity()?); - // SAFETY: The decomposition computes the exact code width. - let codes = unsafe { bitpack_encode_unchecked(codes, code_width) }?.into_array(); - let starts = narrow_unsigned(decomposition.bin_starts(), ordered_values.ptype())?.into_array(); - let references = DictArray::try_new(codes, starts)?.into_array(); - let offsets = narrow_unsigned(decomposition.offsets(), ordered_values.ptype())?; - let offsets = BlockResidual::from_primitive(offsets.as_view())?.into_array(); - let reconstructed = IntMult::try_new(references, offsets, 1)?; - Ok(OrderedFloat::try_new(reconstructed.into_array(), primitive.ptype())?.into_array()) -} - -fn narrow_unsigned(values: &[u64], ptype: PType) -> VortexResult { - Ok(match ptype { - PType::U16 => PrimitiveArray::from_iter( - values - .iter() - .map(|&value| u16::try_from(value)) - .collect::, _>>()?, - ), - PType::U32 => PrimitiveArray::from_iter( - values - .iter() - .map(|&value| u32::try_from(value)) - .collect::, _>>()?, - ), - PType::U64 => PrimitiveArray::from_iter(values.iter().copied()), - _ => vortex_error::vortex_bail!( - "range decomposition requires unsigned integers, got {ptype}" - ), - }) -} - -fn normalize_float_null_values( - array: vortex_array::ArrayView<'_, Primitive>, - exec_ctx: &mut ExecutionCtx, -) -> VortexResult { - let validity = array.validity()?; - if validity.definitely_no_nulls() { - return Ok(array.into_owned()); - } - - let mask = validity.execute_mask(array.len(), exec_ctx)?; - Ok(match_each_float_ptype!(array.ptype(), |T| { - let values = array.as_slice::(); - let replacement = mask - .iter() - .position(|valid| valid) - .map(|index| values[index]) - .unwrap_or_default(); - array - .into_owned() - .map_each_with_validity::( - exec_ctx, - |(value, valid)| if valid { value } else { replacement }, - )? - })) -} - -#[cfg(test)] -mod tests { - use rstest::rstest; - use vortex_array::IntoArray; - use vortex_array::VortexSessionExecute; - use vortex_array::array_session; - use vortex_array::arrays::PrimitiveArray; - use vortex_array::assert_arrays_eq; - use vortex_array::dtype::half::f16; - use vortex_error::VortexResult; - - use super::OrderedFloatRangePackedScheme; - use super::encode_ordered_float; - use super::estimate_ordered_float; - use super::estimate_ordered_float_if_promising; - use crate::BtrBlocksCompressorBuilder; - use crate::CascadingCompressor; - - const TEST_SCHEME: OrderedFloatRangePackedScheme = OrderedFloatRangePackedScheme::new(1.0); - - #[rstest] - #[case::f16(PrimitiveArray::from_iter((0_u16..4_096).map(|index| { - f16::from_bits(0x3c00 + (index.wrapping_mul(101) % 1_024)) - })))] - #[case::f32(PrimitiveArray::from_iter((0_u32..4_096).map(|index| { - f32::from_bits(0x3f80_0000 + (index.wrapping_mul(7_919) % 65_536)) - })))] - #[case::f64(PrimitiveArray::from_iter((0_u64..4_096).map(|index| { - f64::from_bits(0x3ff0_0000_0000_0000 + (index.wrapping_mul(7_919) % 65_536)) - })))] - fn ordered_float_tree_roundtrips(#[case] expected: PrimitiveArray) -> VortexResult<()> { - let session = array_session(); - let mut ctx = session.create_execution_ctx(); - let compressed = CascadingCompressor::new(vec![&TEST_SCHEME]) - .compress(&expected.clone().into_array(), &mut ctx)?; - - assert_arrays_eq!(compressed, expected, &mut ctx); - Ok(()) - } - - #[test] - fn transform_estimate_matches_encoded_size() -> VortexResult<()> { - let expected = PrimitiveArray::from_option_iter((0_u64..65_536).map(|index| { - (index % 19 != 0).then(|| { - let value = (index.wrapping_mul(7_919) % 100_000) as f64 / 100.0; - if index % 101 == 0 { - f64::from_bits(value.to_bits() | 1) - } else { - value - } - }) - })); - let session = array_session(); - let mut ctx = session.create_execution_ctx(); - - let estimate = estimate_ordered_float(expected.as_view(), &mut ctx)?; - let encoded = encode_ordered_float(expected.as_view(), &mut ctx)?; - - assert_eq!(estimate, encoded.nbytes()); - Ok(()) - } - - #[test] - fn prefilter_rejects_uniform_float_bits() -> VortexResult<()> { - let mut state = 0x4d59_5df4_d0f3_3173_u64; - let expected = PrimitiveArray::from_iter((0..65_536).map(|_| { - state ^= state << 13; - state ^= state >> 7; - state ^= state << 17; - let unit = ((state >> 11) as f64 + 0.5) / (1_u64 << 53) as f64; - unit * 2_000_000.0 - 1_000_000.0 - })); - let session = array_session(); - let mut ctx = session.create_execution_ctx(); - - let candidate = - estimate_ordered_float_if_promising(expected.as_view(), 1.13, 1.20, &mut ctx)?; - - assert!(candidate.is_none()); - Ok(()) - } - - #[test] - fn prefilter_rejects_constant_sample() -> VortexResult<()> { - let expected = PrimitiveArray::from_iter(vec![1.0_f64; 1_024]); - let session = array_session(); - let mut ctx = session.create_execution_ctx(); - - let candidate = - estimate_ordered_float_if_promising(expected.as_view(), 1.0, 1.20, &mut ctx)?; - - assert!(candidate.is_none()); - Ok(()) - } - - #[test] - fn prefilter_retains_clustered_float_bits() -> VortexResult<()> { - let expected = PrimitiveArray::from_iter((0_u64..65_536).map(|index| { - let cluster = index % 8; - let residual = index.wrapping_mul(7_919) % 1_024; - f64::from_bits(0x3ff0_0000_0000_0000 + cluster * 0x1_0000_0000 + residual) - })); - let session = array_session(); - let mut ctx = session.create_execution_ctx(); - - let candidate = - estimate_ordered_float_if_promising(expected.as_view(), 1.36, 1.20, &mut ctx)?; - - assert!(candidate.is_some()); - Ok(()) - } - - #[test] - fn prefilter_rejects_block_local_float_bits() -> VortexResult<()> { - let expected = PrimitiveArray::from_iter((0_u64..65_536).map(|index| { - let block = index / 64; - let residual = index.wrapping_mul(7_919) % 1_024; - f64::from_bits(0x3ff0_0000_0000_0000 + block * 0x10_0000 + residual) - })); - let session = array_session(); - let mut ctx = session.create_execution_ctx(); - - let candidate = - estimate_ordered_float_if_promising(expected.as_view(), 1.0, 1.20, &mut ctx)?; - - assert!(candidate.is_none()); - Ok(()) - } - - #[test] - fn completed_incumbent_tree_beats_range_tree() -> VortexResult<()> { - let expected = PrimitiveArray::from_iter((0_u64..65_536).map(|index| { - let cluster = index % 8; - f64::from_bits(0x3ff0_0000_0000_0000 + cluster * 0x1_0000_0000) - })); - let expected_array = expected.clone().into_array(); - let session = array_session(); - let mut ctx = session.create_execution_ctx(); - - let retained = estimate_ordered_float_if_promising( - expected.as_view(), - 1.0, - TEST_SCHEME.decode_cost_factor, - &mut ctx, - )?; - assert!(retained.is_some()); - - let range_only = - CascadingCompressor::new(vec![&TEST_SCHEME]).compress(&expected_array, &mut ctx)?; - assert!(range_only.is::()); - assert!(range_only.children()[0].is::()); - - let selected = BtrBlocksCompressorBuilder::default() - .with_new_scheme(&TEST_SCHEME) - .build() - .compress(&expected_array, &mut ctx)?; - let selected_is_range = selected.is::() - && selected - .children() - .first() - .is_some_and(|child| child.is::()); - assert!(!selected_is_range); - assert!(selected.nbytes() < range_only.nbytes()); - assert_arrays_eq!(selected, expected, &mut ctx); - Ok(()) - } - - #[test] - fn nullable_tree_stores_full_positions() -> VortexResult<()> { - let expected = PrimitiveArray::from_option_iter((0_u64..65_536).map(|index| { - (index % 17 != 0) - .then(|| f64::from_bits(0x3ff0_0000_0000_0000 + index.wrapping_mul(7_919) % 1_024)) - })); - let expected_array = expected.clone().into_array(); - let session = array_session(); - let mut ctx = session.create_execution_ctx(); - let compressed = - CascadingCompressor::new(vec![&TEST_SCHEME]).compress(&expected_array, &mut ctx)?; - - let int_mult = &compressed.children()[0]; - assert!(int_mult.is::()); - let references = &int_mult.children()[0]; - let offsets = &int_mult.children()[1]; - assert!(references.is::()); - assert!(references.children()[0].is::()); - assert!(offsets.is::()); - assert_eq!(references.len(), expected.len()); - assert_eq!(offsets.len(), expected.len()); - assert_arrays_eq!(compressed, expected, &mut ctx); - Ok(()) - } -} diff --git a/vortex-btrblocks/src/schemes/integer/block_residual.rs b/vortex-btrblocks/src/schemes/integer/block_residual.rs index 1c05ff19e33..c3780dd75f9 100644 --- a/vortex-btrblocks/src/schemes/integer/block_residual.rs +++ b/vortex-btrblocks/src/schemes/integer/block_residual.rs @@ -78,11 +78,15 @@ impl Scheme for BlockResidualScheme { fn expected_compression_ratio( &self, - _data: &ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { - if compress_ctx.finished_cascading() || compress_ctx.is_sample() { + // A single block cannot amortize a block-local reference against FoR. + if data.array().len() <= BLOCK_LEN + || compress_ctx.finished_cascading() + || compress_ctx.is_sample() + { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( diff --git a/vortex-btrblocks/src/schemes/integer/fixed_bins.rs b/vortex-btrblocks/src/schemes/integer/fixed_bins.rs deleted file mode 100644 index 68908e8381c..00000000000 --- a/vortex-btrblocks/src/schemes/integer/fixed_bins.rs +++ /dev/null @@ -1,446 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! Integer compression with fixed ranges and block-local offsets. - -use vortex_array::ArrayId; -use vortex_array::ArrayRef; -use vortex_array::Canonical; -use vortex_array::ExecutionCtx; -use vortex_array::IntoArray; -use vortex_array::VTable; -use vortex_array::arrays::Dict; -use vortex_array::arrays::DictArray; -use vortex_array::arrays::Primitive; -use vortex_array::arrays::PrimitiveArray; -use vortex_array::dtype::PType; -use vortex_block_residual::BlockResidual; -use vortex_compressor::scheme::CompressionEstimate; -use vortex_compressor::scheme::DeferredEstimate; -use vortex_compressor::scheme::EstimateScore; -use vortex_compressor::scheme::EstimateVerdict; -use vortex_error::VortexResult; -use vortex_fastlanes::BitPacked; -use vortex_fastlanes::bitpack_compress::bitpack_encode_unchecked; -use vortex_int_mult::IntMult; -use vortex_range_packed::RangeDecomposition; - -use crate::ArrayAndStats; -use crate::CascadingCompressor; -use crate::CompressorContext; -use crate::Scheme; -use crate::SchemeExt; -use crate::normalize_null_values; -use crate::schemes::fixed_bins::CoarseModel; -use crate::schemes::fixed_bins::coarse_model; -use crate::schemes::float::ALPScheme; -use crate::schemes::sample_primitive_one_percent; - -const DEFAULT_DECODE_COST_FACTOR: f64 = 1.20; -const PREFILTER_MODEL_MARGIN: f64 = 1.50; -const MIN_PREFILTER_MODEL_RATIO: f64 = 1.15; -const MAX_BLOCK_TO_RANGE_RATIO: f64 = 1.10; - -/// Compress integers with at most 64 range starts and block-local offsets. -#[derive(Debug, Copy, Clone, PartialEq)] -pub struct IntegerFixedBinsScheme { - decode_cost_factor: f64, -} - -impl IntegerFixedBinsScheme { - /// Creates a scheme with the specified decode cost factor. - pub const fn new(decode_cost_factor: f64) -> Self { - Self { decode_cost_factor } - } -} - -impl Default for IntegerFixedBinsScheme { - fn default() -> Self { - Self::new(DEFAULT_DECODE_COST_FACTOR) - } -} - -impl Scheme for IntegerFixedBinsScheme { - fn scheme_name(&self) -> &'static str { - "vortex.int.fixed_bins" - } - - fn matches(&self, canonical: &Canonical) -> bool { - canonical.dtype().is_int() - } - - fn produced_encodings(&self) -> Vec { - vec![IntMult.id(), Dict.id(), BitPacked.id(), BlockResidual.id()] - } - - fn num_children(&self) -> usize { - 2 - } - - fn expected_compression_ratio( - &self, - _data: &ArrayAndStats, - compress_ctx: CompressorContext, - _exec_ctx: &mut ExecutionCtx, - ) -> CompressionEstimate { - let is_alp_child = compress_ctx - .cascade_history() - .last() - .is_some_and(|(scheme, child)| *scheme == ALPScheme.id() && *child == 0); - if compress_ctx.is_sample() || !is_alp_child { - return CompressionEstimate::Verdict(EstimateVerdict::Skip); - } - - let decode_cost_factor = self.decode_cost_factor; - let scheme_id = self.id(); - CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - move |compressor, data, best_so_far, compress_ctx, exec_ctx| { - let source_width = data.array_as_primitive().ptype().bit_width() as f64; - let best_ratio = best_so_far - .and_then(EstimateScore::finite_ratio) - .unwrap_or(1.0); - if source_width / decode_cost_factor <= best_ratio { - return Ok(EstimateVerdict::Skip); - } - - let sample = sample_primitive_one_percent(data.array_as_primitive(), exec_ctx)?; - let sample = normalize_null_values(sample.as_view(), exec_ctx)?; - let ordered = ordered_integer_values(sample.as_view()); - let model = coarse_model(&ordered, sample.ptype().bit_width()); - if !prefilter_passes(model, best_ratio, decode_cost_factor) { - return Ok(EstimateVerdict::Skip); - } - - let encoded = encode_fixed_bins(sample.as_view())?; - let after_nbytes = encoded.nbytes(); - if after_nbytes == 0 { - return Ok(EstimateVerdict::Skip); - } - let adjusted_ratio = - sample.nbytes() as f64 / after_nbytes as f64 / decode_cost_factor; - if adjusted_ratio <= best_ratio { - return Ok(EstimateVerdict::Skip); - } - - let incumbent = compressor.compress_sample_without_scheme( - &sample.into_array(), - scheme_id, - compress_ctx, - exec_ctx, - )?; - if after_nbytes as f64 * decode_cost_factor >= incumbent.nbytes() as f64 { - return Ok(EstimateVerdict::Skip); - } - Ok(EstimateVerdict::Ratio(adjusted_ratio)) - }, - ))) - } - - fn compress( - &self, - _compressor: &CascadingCompressor, - data: &ArrayAndStats, - _compress_ctx: CompressorContext, - exec_ctx: &mut ExecutionCtx, - ) -> VortexResult { - let primitive = normalize_null_values(data.array_as_primitive(), exec_ctx)?; - encode_fixed_bins(primitive.as_view()) - } -} - -fn prefilter_passes(model: CoarseModel, best_ratio: f64, decode_cost_factor: f64) -> bool { - model.range_ratio.is_finite() - && model.range_ratio >= MIN_PREFILTER_MODEL_RATIO - && model.range_ratio / decode_cost_factor * PREFILTER_MODEL_MARGIN > best_ratio - && model.block_ratio <= model.range_ratio * MAX_BLOCK_TO_RANGE_RATIO -} - -fn encode_fixed_bins(primitive: vortex_array::ArrayView<'_, Primitive>) -> VortexResult { - let ordered = ordered_integer_values(primitive); - let decomposition = RangeDecomposition::encode(&ordered)?; - let code_width = decomposition.code_width(); - let codes = PrimitiveArray::new(decomposition.codes().to_vec(), primitive.validity()?); - // SAFETY: The decomposition computes the exact code width. - let codes = unsafe { bitpack_encode_unchecked(codes, code_width) }?.into_array(); - let starts = restore_starts(decomposition.bin_starts(), primitive.ptype())?.into_array(); - let references = DictArray::try_new(codes, starts)?.into_array(); - let offsets = restore_offsets(decomposition.offsets(), primitive.ptype())?; - let offsets = BlockResidual::from_primitive(offsets.as_view())?.into_array(); - Ok(IntMult::try_new(references, offsets, 1)?.into_array()) -} - -fn ordered_integer_values(primitive: vortex_array::ArrayView<'_, Primitive>) -> Vec { - match primitive.ptype() { - PType::U8 => primitive - .as_slice::() - .iter() - .map(|&value| u64::from(value)) - .collect(), - PType::U16 => primitive - .as_slice::() - .iter() - .map(|&value| u64::from(value)) - .collect(), - PType::U32 => primitive - .as_slice::() - .iter() - .map(|&value| u64::from(value)) - .collect(), - PType::U64 => primitive.as_slice::().to_vec(), - PType::I8 => primitive - .as_slice::() - .iter() - .map(|&value| u64::from((value as u8) ^ (1 << 7))) - .collect(), - PType::I16 => primitive - .as_slice::() - .iter() - .map(|&value| u64::from((value as u16) ^ (1 << 15))) - .collect(), - PType::I32 => primitive - .as_slice::() - .iter() - .map(|&value| u64::from((value as u32) ^ (1 << 31))) - .collect(), - PType::I64 => primitive - .as_slice::() - .iter() - .map(|&value| (value as u64) ^ (1 << 63)) - .collect(), - ptype => unreachable!("fixed bins require integers, got {ptype}"), - } -} - -fn restore_starts(values: &[u64], ptype: PType) -> VortexResult { - Ok(match ptype { - PType::U8 => PrimitiveArray::from_iter( - values - .iter() - .copied() - .map(u8::try_from) - .collect::, _>>()?, - ), - PType::U16 => PrimitiveArray::from_iter( - values - .iter() - .copied() - .map(u16::try_from) - .collect::, _>>()?, - ), - PType::U32 => PrimitiveArray::from_iter( - values - .iter() - .copied() - .map(u32::try_from) - .collect::, _>>()?, - ), - PType::U64 => PrimitiveArray::from_iter(values.iter().copied()), - PType::I8 => PrimitiveArray::from_iter( - values - .iter() - .copied() - .map(u8::try_from) - .collect::, _>>()? - .into_iter() - .map(|value| i8::from_le_bytes([(value ^ (1 << 7))])), - ), - PType::I16 => PrimitiveArray::from_iter( - values - .iter() - .copied() - .map(u16::try_from) - .collect::, _>>()? - .into_iter() - .map(|value| i16::from_le_bytes((value ^ (1 << 15)).to_le_bytes())), - ), - PType::I32 => PrimitiveArray::from_iter( - values - .iter() - .copied() - .map(u32::try_from) - .collect::, _>>()? - .into_iter() - .map(|value| i32::from_le_bytes((value ^ (1 << 31)).to_le_bytes())), - ), - PType::I64 => PrimitiveArray::from_iter( - values - .iter() - .map(|&value| i64::from_le_bytes((value ^ (1 << 63)).to_le_bytes())), - ), - _ => vortex_error::vortex_bail!("fixed bins require integers, got {ptype}"), - }) -} - -fn restore_offsets(values: &[u64], ptype: PType) -> VortexResult { - Ok(match ptype { - PType::U8 => PrimitiveArray::from_iter( - values - .iter() - .copied() - .map(u8::try_from) - .collect::, _>>()?, - ), - PType::U16 => PrimitiveArray::from_iter( - values - .iter() - .copied() - .map(u16::try_from) - .collect::, _>>()?, - ), - PType::U32 => PrimitiveArray::from_iter( - values - .iter() - .copied() - .map(u32::try_from) - .collect::, _>>()?, - ), - PType::U64 => PrimitiveArray::from_iter(values.iter().copied()), - PType::I8 => PrimitiveArray::from_iter( - values - .iter() - .copied() - .map(u8::try_from) - .collect::, _>>()? - .into_iter() - .map(|value| i8::from_le_bytes([value])), - ), - PType::I16 => PrimitiveArray::from_iter( - values - .iter() - .copied() - .map(u16::try_from) - .collect::, _>>()? - .into_iter() - .map(|value| i16::from_le_bytes(value.to_le_bytes())), - ), - PType::I32 => PrimitiveArray::from_iter( - values - .iter() - .copied() - .map(u32::try_from) - .collect::, _>>()? - .into_iter() - .map(|value| i32::from_le_bytes(value.to_le_bytes())), - ), - PType::I64 => PrimitiveArray::from_iter( - values - .iter() - .map(|&value| i64::from_le_bytes(value.to_le_bytes())), - ), - _ => vortex_error::vortex_bail!("fixed bins require integers, got {ptype}"), - }) -} - -#[cfg(test)] -mod tests { - #![allow(clippy::cast_possible_truncation)] - - use rstest::rstest; - use vortex_array::IntoArray; - use vortex_array::VortexSessionExecute; - use vortex_array::arrays::PrimitiveArray; - use vortex_array::assert_arrays_eq; - use vortex_error::VortexResult; - - use super::IntegerFixedBinsScheme; - use super::encode_fixed_bins; - use crate::CascadingCompressor; - use crate::schemes::float::ALPScheme; - use crate::schemes::integer::BitPackingScheme; - use crate::schemes::integer::FoRScheme; - - const TEST_SCHEME: IntegerFixedBinsScheme = IntegerFixedBinsScheme::new(1.0); - - #[rstest] - #[case::u8(PrimitiveArray::from_iter((0_u16..4_096).map(|index| { - ((index % 4) * 64 + (index.wrapping_mul(37) % 8)) as u8 - })))] - #[case::u16(PrimitiveArray::from_iter((0_u32..4_096).map(|index| { - ((index % 4) * 16_000 + (index.wrapping_mul(37) % 32)) as u16 - })))] - #[case::u32(PrimitiveArray::from_iter((0_u64..4_096).map(|index| { - ((index % 4) * 1_000_000_000 + index.wrapping_mul(37) % 1_024) as u32 - })))] - #[case::u64(PrimitiveArray::from_iter((0_u64..4_096).map(|index| { - (index % 4) * 4_000_000_000_000_000_000 + index.wrapping_mul(37) % 1_024 - })))] - #[case::i8(PrimitiveArray::from_iter((0_i16..4_096).map(|index| { - (-120 + (index % 4) * 64 + (index.wrapping_mul(37) % 8)) as i8 - })))] - #[case::i16(PrimitiveArray::from_iter((0_i32..4_096).map(|index| { - (-30_000 + (index % 4) * 16_000 + (index.wrapping_mul(37) % 32)) as i16 - })))] - #[case::i32(PrimitiveArray::from_iter((0_i64..4_096).map(|index| { - (-2_000_000_000 + (index % 4) * 1_000_000_000 + index.wrapping_mul(37) % 1_024) as i32 - })))] - #[case::i64(PrimitiveArray::from_iter((0_i64..4_096).map(|index| { - [-8_000_000_000_000_000_000, -4_000_000_000_000_000_000, 0, 4_000_000_000_000_000_000] - [usize::try_from(index % 4).unwrap_or_default()] - + index.wrapping_mul(37) % 1_024 - })))] - fn fixed_bin_tree_roundtrips(#[case] expected: PrimitiveArray) -> VortexResult<()> { - let session = vortex_array::array_session(); - let mut ctx = session.create_execution_ctx(); - let compressed = encode_fixed_bins(expected.as_view())?; - - assert_arrays_eq!(compressed, expected, &mut ctx); - Ok(()) - } - - #[test] - fn nullable_signed_values_roundtrip() -> VortexResult<()> { - let expected = PrimitiveArray::from_option_iter((0_i64..65_536).map(|index| { - (index % 17 != 0).then(|| { - [ - -8_000_000_000_000_000_000, - -4_000_000_000_000_000_000, - 0, - 4_000_000_000_000_000_000, - ][usize::try_from(index % 4).unwrap_or_default()] - + index.wrapping_mul(37) % 1_024 - }) - })); - let session = vortex_array::array_session(); - let mut ctx = session.create_execution_ctx(); - let compressed = encode_fixed_bins(expected.as_view())?; - - assert_arrays_eq!(compressed, expected, &mut ctx); - Ok(()) - } - - #[test] - fn selector_composes_below_alp() -> VortexResult<()> { - let expected = PrimitiveArray::from_iter((0_i64..65_536).map(|index| { - [0_i64, 1_000_000_000, 2_000_000_000, 3_000_000_000] - [usize::try_from(index % 4).unwrap_or_default()] as f64 - + (index.wrapping_mul(37) % 1_024) as f64 - })); - let expected_array = expected.clone().into_array(); - let session = vortex_array::array_session(); - let mut ctx = session.create_execution_ctx(); - let compressed = CascadingCompressor::new(vec![ - &ALPScheme, - &TEST_SCHEME, - &FoRScheme, - &BitPackingScheme, - ]) - .compress(&expected_array, &mut ctx)?; - - assert!( - compressed.is::(), - "expected ALP, got tree:\n{}", - compressed.display_tree() - ); - let encoded = &compressed.children()[0]; - assert!( - encoded.is::(), - "expected IntMult child, got tree:\n{}", - compressed.display_tree() - ); - assert!(encoded.children()[0].is::()); - assert!(encoded.children()[0].children()[0].is::()); - assert!(encoded.children()[1].is::()); - assert_arrays_eq!(compressed, expected, &mut ctx); - Ok(()) - } -} diff --git a/vortex-btrblocks/src/schemes/integer/mod.rs b/vortex-btrblocks/src/schemes/integer/mod.rs index 101f2a1ab0f..43ff8bbb6cd 100644 --- a/vortex-btrblocks/src/schemes/integer/mod.rs +++ b/vortex-btrblocks/src/schemes/integer/mod.rs @@ -7,7 +7,6 @@ mod bitpacking; mod block_residual; #[cfg(feature = "unstable_encodings")] mod delta; -mod fixed_bins; mod for_; mod rle; mod runend; @@ -23,7 +22,6 @@ pub use block_residual::BlockResidualScheme; pub(crate) use block_residual::patch_adjusted_estimate_nbytes; #[cfg(feature = "unstable_encodings")] pub use delta::DeltaScheme; -pub use fixed_bins::IntegerFixedBinsScheme; pub use for_::FoRScheme; #[cfg(feature = "pco")] pub use pco::PcoScheme; diff --git a/vortex-btrblocks/src/schemes/mod.rs b/vortex-btrblocks/src/schemes/mod.rs index 60938b5b15b..0b9baab0b16 100644 --- a/vortex-btrblocks/src/schemes/mod.rs +++ b/vortex-btrblocks/src/schemes/mod.rs @@ -11,7 +11,6 @@ pub mod string; pub mod decimal; pub mod temporal; -pub(crate) mod fixed_bins; pub(crate) mod patches; use std::ops::Range; diff --git a/vortex-btrblocks/tests/snapshots/golden__default__list_of_int_runs.snap b/vortex-btrblocks/tests/snapshots/golden__default__list_of_int_runs.snap index c9554add05a..e5f7bbb6eee 100644 --- a/vortex-btrblocks/tests/snapshots/golden__default__list_of_int_runs.snap +++ b/vortex-btrblocks/tests/snapshots/golden__default__list_of_int_runs.snap @@ -3,7 +3,7 @@ source: vortex-btrblocks/tests/golden.rs expression: rendered --- input: list(i32), len=4066, nbytes=81804 -root: vortex.list(list(i32), len=4066) nbytes=11146 +root: vortex.list(list(i32), len=4066) nbytes=10304 metadata: elements: vortex.runend(i32, len=16384) nbytes=3968 metadata: offset: 0 @@ -15,11 +15,5 @@ root: vortex.list(list(i32), len=4066) nbytes=11146 metadata: reference: -49931i32 encoded: fastlanes.bitpacked(i32, len=1020) nbytes=2176 metadata: bit_width: 17, offset: 0 - offsets: fastlanes.bitpacked(u16, len=4067) nbytes=7178 - metadata: bit_width: 14, offset: 0 - patch_indices: vortex.primitive(u16, len=1) nbytes=2 - metadata: ptype: u16 - patch_values: vortex.constant(u16, len=1) nbytes=4 - metadata: scalar: 16384u16 - patch_chunk_offsets: vortex.primitive(u8, len=4) nbytes=4 - metadata: ptype: u8 + offsets: vortex.block_residual(u16, len=4067) nbytes=6336 + metadata: blocks: 4, slice: 0..4067 diff --git a/vortex-file/Cargo.toml b/vortex-file/Cargo.toml index 7eeeca59ec0..920f012f0f4 100644 --- a/vortex-file/Cargo.toml +++ b/vortex-file/Cargo.toml @@ -54,7 +54,6 @@ vortex-mask = { workspace = true } vortex-metrics = { workspace = true } vortex-onpair = { workspace = true } vortex-pco = { workspace = true } -vortex-range-packed = { workspace = true } vortex-runend = { workspace = true } vortex-scan = { workspace = true } vortex-sequence = { workspace = true } diff --git a/vortex-file/src/lib.rs b/vortex-file/src/lib.rs index ed27018089c..64a0f000514 100644 --- a/vortex-file/src/lib.rs +++ b/vortex-file/src/lib.rs @@ -193,7 +193,6 @@ pub fn register_default_encodings(session: &VortexSession) { vortex_float_quant::initialize(session); vortex_int_mult::initialize(session); vortex_block_residual::initialize(session); - vortex_range_packed::initialize(session); vortex_runend::initialize(session); vortex_sequence::initialize(session); vortex_sparse::initialize(session); diff --git a/vortex/Cargo.toml b/vortex/Cargo.toml index 30106c914d0..3fd56b80578 100644 --- a/vortex/Cargo.toml +++ b/vortex/Cargo.toml @@ -47,7 +47,6 @@ vortex-layout = { workspace = true } vortex-mask = { workspace = true } vortex-metrics = { workspace = true } vortex-pco = { workspace = true } -vortex-range-packed = { workspace = true } vortex-proto = { workspace = true, default-features = true } vortex-runend = { workspace = true } vortex-scan = { workspace = true } diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index 159c4ec9a59..be31c5bcf47 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -20,7 +20,6 @@ use vortex::VortexSessionDefault; use vortex::array::Canonical; use vortex::array::ExecutionCtx; use vortex::array::IntoArray; -use vortex::array::arrays::DictArray; use vortex::array::arrays::Primitive; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::VarBinViewArray; @@ -53,7 +52,6 @@ use vortex::encodings::fsst::fsst_train_compressor; use vortex::encodings::int_mult::IntMult; use vortex::encodings::int_mult::IntMultArraySlotsExt; use vortex::encodings::pco::Pco; -use vortex::encodings::range_packed::RangeDecomposition; use vortex::encodings::runend::RunEnd; use vortex::encodings::sequence::sequence_encode; use vortex::encodings::zigzag::zigzag_encode; @@ -397,13 +395,6 @@ fn setup_int_mult_integer_array() -> PrimitiveArray { } } -fn setup_ranged_u64_array() -> PrimitiveArray { - PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { - let cluster = index % 10; - cluster * 1_000_000_000 + index.wrapping_mul(7_919) % 1_024 - })) -} - fn encode_int_mult_packed_tree(array: &PrimitiveArray) -> vortex::array::ArrayRef { let split = IntMult::from_primitive(array.as_view(), 10).unwrap(); let primary = split @@ -436,25 +427,6 @@ fn encode_int_mult_packed_tree(array: &PrimitiveArray) -> vortex .into_array() } -fn encode_decomposed_range_tree(array: &PrimitiveArray) -> vortex::array::ArrayRef { - let decomposition = RangeDecomposition::encode(array.as_slice::()).unwrap(); - let code_width = decomposition.code_width(); - let codes = PrimitiveArray::from_iter(decomposition.codes().iter().copied()); - // SAFETY: The decomposition computes the exact code width. - let codes = unsafe { bitpack_encode_unchecked(codes, code_width) } - .unwrap() - .into_array(); - let starts = PrimitiveArray::from_iter(decomposition.bin_starts().iter().copied()).into_array(); - let references = DictArray::try_new(codes, starts).unwrap().into_array(); - let offsets = PrimitiveArray::from_iter(decomposition.offsets().iter().copied()); - let offsets = BlockResidual::from_primitive(offsets.as_view()) - .unwrap() - .into_array(); - IntMult::try_new(references, offsets, 1) - .unwrap() - .into_array() -} - fn encode_for_bitpacked_tree(array: &PrimitiveArray, bit_width: u8) -> vortex::array::ArrayRef { let mut ctx = SESSION.create_execution_ctx(); let encoded = FoR::encode(array.clone(), &mut ctx).unwrap(); @@ -999,40 +971,6 @@ fn int_mult_packed_tree_scalar_at(bencher: Bencher) { .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); } -#[divan::bench(name = "decomposed_range_compress_u64")] -fn bench_decomposed_range_compress_u64(bencher: Bencher) { - let array = setup_ranged_u64_array(); - - with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) - .with_inputs(|| &array) - .bench_refs(|array| encode_decomposed_range_tree(array)); -} - -#[divan::bench(name = "decomposed_range_decompress_u64")] -fn bench_decomposed_range_decompress_u64(bencher: Bencher) { - let encoded = encode_decomposed_range_tree(&setup_ranged_u64_array()); - - with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * 8) - .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) - .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); -} - -#[divan::bench(name = "decomposed_range_scalar_at_u64")] -fn bench_decomposed_range_scalar_at_u64(bencher: Bencher) { - let encoded = encode_decomposed_range_tree(&setup_ranged_u64_array()); - let next_index = AtomicUsize::new(0); - - bencher - .with_inputs(|| { - ( - &encoded, - SESSION.create_execution_ctx(), - next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), - ) - }) - .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); -} - #[divan::bench(name = "block_residual_compress_u64")] fn bench_block_residual_compress_u64(bencher: Bencher) { let float_array = setup_random_walk_array(); diff --git a/vortex/src/editions/core/v2026_08.rs b/vortex/src/editions/core/v2026_08.rs index 6cc1a3c1c8d..d60ac754e22 100644 --- a/vortex/src/editions/core/v2026_08.rs +++ b/vortex/src/editions/core/v2026_08.rs @@ -31,7 +31,6 @@ pub static DECLARATION: EditionDeclaration = EditionDeclaration { EditionMember::array(&"vortex.int_mult"), EditionMember::array(&"vortex.map"), EditionMember::array(&"vortex.ordered_float"), - EditionMember::array(&"vortex.range_packed"), EditionMember::aggregate(&"vortex.bounded_max"), EditionMember::aggregate(&"vortex.bounded_min"), EditionMember::aggregate(&"vortex.max"), diff --git a/vortex/src/lib.rs b/vortex/src/lib.rs index 92d52efddfd..4c6479ebab7 100644 --- a/vortex/src/lib.rs +++ b/vortex/src/lib.rs @@ -285,11 +285,6 @@ pub mod encodings { pub use vortex_block_residual::*; } - /// Fixed-bin range-packed integer encoding. - pub mod range_packed { - pub use vortex_range_packed::*; - } - /// Arrow-compatible run-end encoding. pub mod runend { pub use vortex_runend::*; From 15e7e6b900f7ec31567bb43c3105730cf7ca5ea0 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Fri, 21 Aug 2026 14:59:46 -0400 Subject: [PATCH 75/78] refactor: remove unselected IntMult prototype Signed-off-by: Will Manning --- Cargo.lock | 17 - Cargo.toml | 2 - docs/plans/native-pcodec.md | 48 +- encodings/block-residual/Cargo.toml | 2 - .../block-residual/src/ordered_float_array.rs | 178 +--- encodings/int-mult/Cargo.toml | 29 - encodings/int-mult/src/array.rs | 863 ------------------ encodings/int-mult/src/lib.rs | 17 - encodings/int-mult/src/rules.rs | 10 - encodings/int-mult/src/slice.rs | 27 - vortex-file/Cargo.toml | 1 - vortex-file/src/lib.rs | 1 - vortex/Cargo.toml | 1 - vortex/benches/single_encoding_throughput.rs | 161 ---- vortex/src/editions/core/v2026_08.rs | 1 - vortex/src/lib.rs | 5 - 16 files changed, 19 insertions(+), 1344 deletions(-) delete mode 100644 encodings/int-mult/Cargo.toml delete mode 100644 encodings/int-mult/src/array.rs delete mode 100644 encodings/int-mult/src/lib.rs delete mode 100644 encodings/int-mult/src/rules.rs delete mode 100644 encodings/int-mult/src/slice.rs diff --git a/Cargo.lock b/Cargo.lock index 9efe0098e44..0b03204bf80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9552,7 +9552,6 @@ dependencies = [ "vortex-flatbuffers", "vortex-float-quant", "vortex-fsst", - "vortex-int-mult", "vortex-io", "vortex-ipc", "vortex-layout", @@ -9759,12 +9758,10 @@ name = "vortex-block-residual" version = "0.1.0" dependencies = [ "fastlanes", - "num-traits", "rstest", "vortex-array", "vortex-buffer", "vortex-error", - "vortex-int-mult", "vortex-session", ] @@ -10190,7 +10187,6 @@ dependencies = [ "vortex-flatbuffers", "vortex-float-quant", "vortex-fsst", - "vortex-int-mult", "vortex-io", "vortex-layout", "vortex-mask", @@ -10274,19 +10270,6 @@ dependencies = [ "vortex-utils", ] -[[package]] -name = "vortex-int-mult" -version = "0.1.0" -dependencies = [ - "num-traits", - "rstest", - "vortex-array", - "vortex-buffer", - "vortex-error", - "vortex-fastlanes", - "vortex-session", -] - [[package]] name = "vortex-io" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 18d2f2b9718..4e4791a9a48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,6 @@ members = [ "encodings/onpair", "encodings/block-residual", "encodings/float-quant", - "encodings/int-mult", # Benchmarks "benchmarks/bench-support", "benchmarks/lance-bench", @@ -326,7 +325,6 @@ vortex-parquet-variant = { version = "0.1.0", path = "./encodings/parquet-varian vortex-pco = { version = "0.1.0", path = "./encodings/pco", default-features = false } vortex-block-residual = { version = "0.1.0", path = "./encodings/block-residual", default-features = false } vortex-float-quant = { version = "0.1.0", path = "./encodings/float-quant", default-features = false } -vortex-int-mult = { version = "0.1.0", path = "./encodings/int-mult", default-features = false } vortex-proto = { version = "0.1.0", path = "./vortex-proto", default-features = false } vortex-row = { version = "0.1.0", path = "./vortex-row", default-features = false } vortex-runend = { version = "0.1.0", path = "./encodings/runend", default-features = false } diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index b8aed1e1736..7f017afd60a 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -36,7 +36,6 @@ Focus the production work on these encodings: - `OrderedFloatArray`. - `BlockResidualArray` for all integer types, with one reference per 1,024-value block. - `FloatQuantArray`. -- `IntMultArray` as a composable integer transform. Keep `FloatQuantScheme`, `OrderedBlockResidualScheme`, and `BlockResidualScheme` as BtrBlocks candidates. @@ -68,9 +67,9 @@ Do not add adjacent Delta, Delta-of-delta, Delta with lookback, or convolution D ## Current state -The branch implements `OrderedFloatArray`, `BlockResidualArray`, `FloatQuantArray`, and `IntMultArray`. +The branch implements `OrderedFloatArray`, `BlockResidualArray`, and `FloatQuantArray`. -The evaluation set includes BtrBlocks schemes for the first three arrays. +The evaluation set includes BtrBlocks schemes for all three arrays. `OrderedFloat(BlockResidual)` now supports `f16`, `f32`, and `f64` inputs. @@ -100,15 +99,11 @@ The final calibration retains the 1.05 BlockResidual ratio floor. It retains the 1.02 ordered-float factor, 1.10 FloatQuant factor, 1.12 8-bit factor, and nonlinear patch cost. -`IntMultArray` owns only the quotient and remainder transform. - -Its two children remain generic arrays. Child compression owns patches and other integer models. - The range decomposition prototype used `IntMult(base=1)` as generic addition. Its fused decode path skipped multiplication and avoided dictionary reference materialization. -IntMult has no Default scheme. +No tested IntMult tree passed the Default evidence bar. The focused branch no longer contains `IntMultArray`. Decomposed fixed bins do not participate in the Default selector. @@ -131,14 +126,11 @@ The Default selector can exclude a supported type when measured costs do not jus | OrderedFloat | `f16`, `f32`, `f64` | All float types remain eligible. | | BlockResidual | `u8`, `u16`, `u32`, `u64`, `i8`, `i16`, `i32`, `i64` | All widths are eligible. | | FloatQuant | `f16`, `f32`, `f64` | All float types remain eligible. | -| IntMult | `u8`, `u16`, `u32`, `u64`, `i8`, `i16`, `i32`, `i64` | No Default scheme exists yet. | OrderedFloat validates the logical float type, unsigned child width, child nullability, child length, and empty metadata. FloatQuant validates the metadata version, split width, latent child types, child nullability, and child lengths. -IntMult validates the metadata version, positive base, base range, matching child types, child nullability, and child lengths. - BlockResidual validates every payload offset table before decode or scalar access. It also validates bases, bit widths, packed word counts, patch counts, patch order, patch bounds, and validity length. @@ -210,35 +202,35 @@ The automatic scheme accepts `f16`, `f32`, and `f64` inputs. Native `f32` and two-child selection meet the selected-column and rejected-column throughput limits. -Final selector factors still require broad corpus validation. +The final corpus pass validates the retained selector factors. -## IntMultArray +## IntMult prototype -`IntMultArray` reconstructs each integer as `base * primary + secondary`. +The prototype reconstructed each integer as `base * primary + secondary`. -The array supports every signed and unsigned integer type. +It supported every signed and unsigned integer type. -The primary and secondary children can use any compatible Vortex array encoding. +The primary and secondary children used compatible Vortex array encodings. -The primary child owns validity. The secondary child is nonnullable. +The primary child owned validity. The secondary child was nonnullable. -The array supports canonical decode, scalar access, slices, serialization, and validation. +The prototype supported canonical decode, scalar access, slices, serialization, and validation. -`IntMult::from_primitive` creates quotient and remainder children with the source integer width. +`IntMult::from_primitive` created quotient and remainder children with the source integer width. -A future selector can compress both children through specialized or recursive integer schemes. +The tested selectors compressed both children through specialized or recursive integer schemes. -IntMult does not own exception positions or exception payloads. +IntMult did not own exception positions or exception payloads. -Child encodings can use `PatchArray`, `BlockResidual`, or other integer arrays when those trees win. +Child encodings used `PatchArray`, `BlockResidual`, or other integer arrays when those trees won. -When the base equals one, bulk decode and scalar access skip multiplication. +When the base equaled one, bulk decode and scalar access skipped multiplication. -For a dictionary primary, the fused path decodes the secondary into the output buffer. +For a dictionary primary, the fused path decoded the secondary into the output buffer. -It then unpacks dictionary codes and adds the selected reference directly. +It then unpacked dictionary codes and added the selected reference directly. -This path avoids a full materialized array of references. +This path avoided a full materialized array of references. ## Selection policy @@ -2900,8 +2892,6 @@ The bundle contains these schemes: - `OrderedBlockResidualScheme` for floating-point values. - `FloatQuantScheme` for floating-point values. -Keep `IntMultArray` as a composable primitive without a Default scheme. - Retain these selector controls: - A 1.05 minimum ratio for integer BlockResidual. @@ -2930,8 +2920,6 @@ Prepare one focused stack for `OrderedFloatArray`, `BlockResidualArray`, and `Or Prepare a separate stack for `FloatQuantArray` and `FloatQuantScheme`. -Prepare a separate primitive-array change for `IntMultArray`. - Keep entropy, bit-split, and alternate residual models on `wm/pcodec-entropy-experiments`. ## References diff --git a/encodings/block-residual/Cargo.toml b/encodings/block-residual/Cargo.toml index bd3b06ba582..e0bb52093d0 100644 --- a/encodings/block-residual/Cargo.toml +++ b/encodings/block-residual/Cargo.toml @@ -18,11 +18,9 @@ workspace = true [dependencies] fastlanes = { workspace = true } -num-traits = { workspace = true } vortex-array = { workspace = true } vortex-buffer = { workspace = true } vortex-error = { workspace = true } -vortex-int-mult = { workspace = true } vortex-session = { workspace = true } [dev-dependencies] diff --git a/encodings/block-residual/src/ordered_float_array.rs b/encodings/block-residual/src/ordered_float_array.rs index efcee20949f..d489b72b40f 100644 --- a/encodings/block-residual/src/ordered_float_array.rs +++ b/encodings/block-residual/src/ordered_float_array.rs @@ -6,7 +6,6 @@ use std::fmt::Formatter; use std::hash::Hasher; use std::ops::Range; -use num_traits::AsPrimitive; use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; @@ -20,18 +19,14 @@ use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::TypedArrayRef; use vortex_array::array_slots; -use vortex_array::arrays::Dict; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; -use vortex_array::arrays::dict::DictArraySlotsExt; use vortex_array::arrays::slice::SliceReduce; use vortex_array::arrays::slice::SliceReduceAdaptor; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; -use vortex_array::dtype::NativePType; use vortex_array::dtype::PType; use vortex_array::dtype::half::f16; -use vortex_array::match_each_integer_ptype; use vortex_array::optimizer::rules::ParentRuleSet; use vortex_array::scalar::Scalar; use vortex_array::serde::ArrayChildren; @@ -46,9 +41,6 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; -use vortex_int_mult::IntMult; -use vortex_int_mult::IntMultArrayExt; -use vortex_int_mult::IntMultArraySlotsExt; use vortex_session::VortexSession; use vortex_session::registry::CachedId; @@ -183,13 +175,7 @@ impl VTable for OrderedFloat { } fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { - let decoded = if let Some(int_mult) = array.encoded().as_typed::() { - if let Some(decoded) = decompress_ordered_int_mult(array.as_view(), int_mult, ctx)? { - decoded - } else { - decode_primitive(array.as_view(), ctx)? - } - } else if let Some(block_residual) = array.encoded().as_typed::() { + let decoded = if let Some(block_residual) = array.encoded().as_typed::() { match array.dtype().as_ptype() { PType::F16 => decompress_ordered_f16(block_residual, ctx)?, PType::F32 => decompress_ordered_f32(block_residual, ctx)?, @@ -211,137 +197,6 @@ impl VTable for OrderedFloat { } } -fn decompress_ordered_int_mult( - array: ArrayView<'_, OrderedFloat>, - int_mult: ArrayView<'_, IntMult>, - ctx: &mut ExecutionCtx, -) -> VortexResult> { - if int_mult.base() != 1 { - return Ok(None); - } - let Some(dict) = int_mult.primary().as_typed::() else { - return Ok(None); - }; - let Some(offsets) = int_mult.secondary().as_typed::() else { - return Ok(None); - }; - let starts = dict.values().clone().execute::(ctx)?; - if !starts.validity()?.definitely_no_nulls() || starts.ptype() != offsets.dtype().as_ptype() { - return Ok(None); - } - let codes = dict.codes().clone().execute::(ctx)?; - let validity = codes.validity()?; - let offsets = int_mult - .secondary() - .clone() - .execute::(ctx)?; - - decode_ordered_int_mult_parts( - array.dtype().as_ptype(), - starts, - offsets, - codes, - validity, - ctx, - ) -} - -fn decode_ordered_int_mult_parts( - float_ptype: PType, - starts: PrimitiveArray, - offsets: PrimitiveArray, - codes: PrimitiveArray, - validity: vortex_array::validity::Validity, - ctx: &mut ExecutionCtx, -) -> VortexResult> { - match_each_integer_ptype!(codes.ptype(), |C| { - decode_ordered_int_mult_codes::( - float_ptype, - starts, - offsets, - codes.as_slice::(), - validity, - ctx, - ) - }) -} - -fn decode_ordered_int_mult_codes( - float_ptype: PType, - starts: PrimitiveArray, - offsets: PrimitiveArray, - codes: &[C], - validity: vortex_array::validity::Validity, - ctx: &mut ExecutionCtx, -) -> VortexResult> -where - C: NativePType + AsPrimitive, -{ - match float_ptype { - PType::F16 if starts.ptype() == PType::U16 => { - decode_ordered_u16(starts.as_slice::(), offsets, codes, validity, ctx).map(Some) - } - PType::F32 if starts.ptype() == PType::U32 => { - decode_ordered_u32(starts.as_slice::(), offsets, codes, validity, ctx).map(Some) - } - PType::F64 if starts.ptype() == PType::U64 => { - decode_ordered_u64(starts.as_slice::(), offsets, codes, validity, ctx).map(Some) - } - _ => Ok(None), - } -} - -macro_rules! decode_ordered_words { - ($name:ident, $word:ty, $float:ty, $unordered:ident) => { - fn $name( - starts: &[$word], - offsets: PrimitiveArray, - codes: &[C], - validity: vortex_array::validity::Validity, - ctx: &mut ExecutionCtx, - ) -> VortexResult - where - C: NativePType + AsPrimitive, - { - let mut offsets = offsets.into_buffer_mut::<$word>(); - let mut invalid_codes = Vec::new(); - for (index, (offset, code)) in offsets.iter_mut().zip(codes).enumerate() { - let code = >::as_(*code); - let Some(start) = starts.get(code) else { - invalid_codes.push(index); - continue; - }; - *offset = $unordered(start.wrapping_add(*offset)); - } - validate_invalid_code_payloads(&validity, offsets.len(), &invalid_codes, ctx)?; - // SAFETY: Every integer bit pattern represents a valid float value of equal width. - let values = unsafe { offsets.transmute::<$float>() }.freeze(); - Ok(PrimitiveArray::new(values, validity)) - } - }; -} - -decode_ordered_words!(decode_ordered_u16, u16, f16, unordered_u16); -decode_ordered_words!(decode_ordered_u32, u32, f32, unordered_u32); -decode_ordered_words!(decode_ordered_u64, u64, f64, unordered_u64); - -fn validate_invalid_code_payloads( - validity: &vortex_array::validity::Validity, - len: usize, - invalid_codes: &[usize], - ctx: &mut ExecutionCtx, -) -> VortexResult<()> { - if !invalid_codes.is_empty() { - let mask = validity.execute_mask(len, ctx)?; - for &index in invalid_codes { - if mask.value(index) { - vortex_bail!("OrderedFloat IntMult dictionary code is invalid"); - } - } - } - Ok(()) -} - impl OperationsVTable for OrderedFloat { fn scalar_at( array: ArrayView<'_, OrderedFloat>, @@ -599,7 +454,6 @@ mod tests { use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::array_session; - use vortex_array::arrays::DictArray; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -611,7 +465,6 @@ mod tests { use vortex_buffer::Buffer; use vortex_buffer::ByteBufferMut; use vortex_error::VortexResult; - use vortex_int_mult::IntMult; use vortex_session::registry::ReadContext; use super::OrderedFloat; @@ -752,35 +605,6 @@ mod tests { Ok(()) } - #[test] - fn roundtrip_nullable_int_mult_dictionary_block_residual() -> VortexResult<()> { - let primitive = PrimitiveArray::new( - Buffer::from(vec![1.0_f64, 0.0, 2.5, -3.0]), - Validity::from_iter([true, false, true, true]), - ); - let ordered = OrderedFloat::from_primitive(primitive.as_view())?; - let ordered_primitive = ordered.encoded().as_::(); - let ordered_values = ordered_primitive.as_slice::(); - let starts = - PrimitiveArray::from_iter([ordered_values[0], ordered_values[2], ordered_values[3]]); - let codes = PrimitiveArray::new( - Buffer::from(vec![0_u8, 9, 1, 2]), - Validity::from_iter([true, false, true, true]), - ); - let references = DictArray::try_new(codes.into_array(), starts.into_array())?; - let offsets = - BlockResidual::from_primitive(PrimitiveArray::from_iter([0_u64; 4]).as_view())?; - let int_mult = IntMult::try_new(references.into_array(), offsets.into_array(), 1)?; - let encoded = OrderedFloat::try_new(int_mult.into_array(), PType::F64)?; - let session = array_session(); - crate::initialize(&session); - vortex_int_mult::initialize(&session); - let mut ctx = session.create_execution_ctx(); - - assert_arrays_eq!(encoded, primitive, &mut ctx); - Ok(()) - } - #[test] fn nullable_serialized_slice_roundtrip() -> VortexResult<()> { let primitive = PrimitiveArray::new( diff --git a/encodings/int-mult/Cargo.toml b/encodings/int-mult/Cargo.toml deleted file mode 100644 index 273e7f22da5..00000000000 --- a/encodings/int-mult/Cargo.toml +++ /dev/null @@ -1,29 +0,0 @@ -[package] -name = "vortex-int-mult" -authors = { workspace = true } -categories = { workspace = true } -description = "Vortex integer multiplier transform array encoding" -edition = { workspace = true } -homepage = { workspace = true } -include = { workspace = true } -keywords = { workspace = true } -license = { workspace = true } -readme = { workspace = true } -repository = { workspace = true } -rust-version = { workspace = true } -version = { workspace = true } - -[dependencies] -num-traits = { workspace = true } -vortex-array = { workspace = true } -vortex-buffer = { workspace = true } -vortex-error = { workspace = true } -vortex-fastlanes = { workspace = true } -vortex-session = { workspace = true } - -[dev-dependencies] -rstest = { workspace = true } -vortex-array = { workspace = true, features = ["_test-harness"] } - -[lints] -workspace = true diff --git a/encodings/int-mult/src/array.rs b/encodings/int-mult/src/array.rs deleted file mode 100644 index 606ab51cb4b..00000000000 --- a/encodings/int-mult/src/array.rs +++ /dev/null @@ -1,863 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::fmt::Display; -use std::fmt::Formatter; -use std::hash::Hash; -use std::hash::Hasher; - -use num_traits::AsPrimitive; -use vortex_array::Array; -use vortex_array::ArrayEq; -use vortex_array::ArrayHash; -use vortex_array::ArrayId; -use vortex_array::ArrayParts; -use vortex_array::ArrayRef; -use vortex_array::ArrayView; -use vortex_array::EqMode; -use vortex_array::ExecutionCtx; -use vortex_array::ExecutionResult; -use vortex_array::IntoArray; -use vortex_array::TypedArrayRef; -use vortex_array::array_slots; -use vortex_array::arrays::Dict; -use vortex_array::arrays::Primitive; -use vortex_array::arrays::PrimitiveArray; -use vortex_array::arrays::dict::DictArraySlotsExt; -use vortex_array::buffer::BufferHandle; -use vortex_array::dtype::DType; -use vortex_array::dtype::NativePType; -use vortex_array::dtype::Nullability::NonNullable; -use vortex_array::dtype::PType; -use vortex_array::match_each_integer_ptype; -use vortex_array::scalar::Scalar; -use vortex_array::serde::ArrayChildren; -use vortex_array::vtable::OperationsVTable; -use vortex_array::vtable::VTable; -use vortex_array::vtable::ValidityChild; -use vortex_array::vtable::ValidityVTableFromChild; -use vortex_buffer::Buffer; -use vortex_buffer::BufferMut; -use vortex_error::VortexExpect; -use vortex_error::VortexResult; -use vortex_error::vortex_bail; -use vortex_error::vortex_ensure; -use vortex_error::vortex_panic; -use vortex_fastlanes::BitPacked; -use vortex_fastlanes::BitPackedArrayExt; -use vortex_fastlanes::bitpack_decompress::unpack_pair_map; -use vortex_session::VortexSession; -use vortex_session::registry::CachedId; - -use crate::rules::RULES; - -const METADATA_VERSION: u8 = 1; -const METADATA_LEN: usize = 9; - -/// An integer transform represented as multiplied and additive child arrays. -pub type IntMultArray = Array; - -#[array_slots(IntMult)] -pub struct IntMultSlots { - /// Values multiplied by the array base. - #[slot(0)] - pub primary: ArrayRef, - /// Values added after multiplication. - #[slot(1)] - pub secondary: ArrayRef, -} - -#[derive(Clone, Debug)] -pub struct IntMultData { - base: u64, -} - -impl Display for IntMultData { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "base: {}", self.base) - } -} - -impl ArrayHash for IntMultData { - fn array_hash(&self, state: &mut H, _accuracy: EqMode) { - self.base.hash(state); - } -} - -impl ArrayEq for IntMultData { - fn array_eq(&self, other: &Self, _accuracy: EqMode) -> bool { - self.base == other.base - } -} - -#[derive(Clone, Debug)] -pub struct IntMult; - -impl VTable for IntMult { - type TypedArrayData = IntMultData; - type OperationsVTable = Self; - type ValidityVTable = ValidityVTableFromChild; - - fn id(&self) -> ArrayId { - static ID: CachedId = CachedId::new("vortex.int_mult"); - *ID - } - - fn validate( - &self, - data: &Self::TypedArrayData, - dtype: &DType, - len: usize, - slots: &[Option], - ) -> VortexResult<()> { - validate_children(data.base, dtype, len, IntMultSlotsView::from_slots(slots)) - } - - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { - 0 - } - - fn buffer(_array: ArrayView<'_, Self>, index: usize) -> BufferHandle { - vortex_panic!("IntMultArray buffer index {index} is invalid") - } - - fn buffer_name(_array: ArrayView<'_, Self>, index: usize) -> Option { - vortex_panic!("IntMultArray buffer index {index} is invalid") - } - - fn with_buffers( - &self, - array: ArrayView<'_, Self>, - buffers: &[BufferHandle], - ) -> VortexResult> { - vortex_array::vtable::with_empty_buffers(self, array, buffers) - } - - fn serialize( - array: ArrayView<'_, Self>, - _session: &VortexSession, - ) -> VortexResult>> { - let mut metadata = Vec::with_capacity(METADATA_LEN); - metadata.push(METADATA_VERSION); - metadata.extend_from_slice(&array.data().base.to_le_bytes()); - Ok(Some(metadata)) - } - - fn deserialize( - &self, - dtype: &DType, - len: usize, - metadata: &[u8], - _buffers: &[BufferHandle], - children: &dyn ArrayChildren, - _session: &VortexSession, - ) -> VortexResult> { - vortex_ensure!( - metadata.len() == METADATA_LEN, - "IntMult metadata requires {METADATA_LEN} bytes" - ); - vortex_ensure!( - metadata[0] == METADATA_VERSION, - "unsupported IntMult metadata version {}", - metadata[0] - ); - vortex_ensure!(children.len() == 2, "IntMult requires two children"); - - let mut base_bytes = [0_u8; size_of::()]; - base_bytes.copy_from_slice(&metadata[1..]); - let base = u64::from_le_bytes(base_bytes); - let ptype = PType::try_from(dtype)?; - ensure_base_fits(base, ptype)?; - let primary = children.get(0, dtype, len)?; - let secondary_dtype = DType::Primitive(ptype, NonNullable); - let secondary = children.get(1, &secondary_dtype, len)?; - let slots = IntMultSlots { primary, secondary }.into_slots(); - Ok( - ArrayParts::new(self.clone(), dtype.clone(), len, IntMultData { base }) - .with_slots(slots), - ) - } - - fn slot_name(_array: ArrayView<'_, Self>, index: usize) -> String { - IntMultSlots::NAMES[index].to_string() - } - - fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { - if array.base() == 1 - && let Some(decoded) = decode_dict_add(array.as_view(), ctx)? - { - return Ok(ExecutionResult::done(decoded.into_array())); - } - if let Some(decoded) = decode_fastlanes_pair(array.as_view())? { - return Ok(ExecutionResult::done(decoded.into_array())); - } - let primary = array.primary().clone().execute::(ctx)?; - let secondary = array.secondary().clone().execute::(ctx)?; - Ok(ExecutionResult::done( - decode_primitive(primary, secondary, array.base())?.into_array(), - )) - } - - fn reduce_parent( - array: ArrayView<'_, Self>, - parent: &ArrayRef, - child_idx: usize, - ) -> VortexResult> { - RULES.evaluate(array, parent, child_idx) - } -} - -fn decode_fastlanes_pair(array: ArrayView<'_, IntMult>) -> VortexResult> { - let Some(primary) = array.primary().as_opt::() else { - return Ok(None); - }; - let Some(secondary) = array.secondary().as_opt::() else { - return Ok(None); - }; - if primary.patches().is_some() - || secondary.patches().is_some() - || primary.offset() != secondary.offset() - || !array.dtype().as_ptype().is_unsigned_int() - { - return Ok(None); - } - - let validity = primary.validity()?; - let base = array.base(); - Ok(Some(vortex_array::match_each_unsigned_integer_ptype!( - array.dtype().as_ptype(), - |T| { - let base = T::try_from(base).vortex_expect("validated IntMult base"); - PrimitiveArray::new( - unpack_pair_map::(primary, secondary, |primary, secondary| { - T::wrapping_mul_add(base, primary, secondary) - })?, - validity, - ) - } - ))) -} - -fn decode_dict_add( - array: ArrayView<'_, IntMult>, - ctx: &mut ExecutionCtx, -) -> VortexResult> { - let Some(dict) = array.primary().as_typed::() else { - return Ok(None); - }; - let starts = dict.values().clone().execute::(ctx)?; - if !starts.validity()?.definitely_no_nulls() { - return Ok(None); - } - let secondary = array.secondary().clone().execute::(ctx)?; - - if let Some(codes) = dict.codes().as_typed::() { - let validity = BitPackedArrayExt::validity(&codes); - if codes.patches().is_none() { - return decode_bitpacked_dict(starts, secondary, codes, validity, ctx).map(Some); - } - } - - let codes = dict.codes().clone().execute::(ctx)?; - let validity = codes.validity()?; - decode_primitive_dict(starts, secondary, codes, validity, ctx).map(Some) -} - -fn decode_bitpacked_dict( - starts: PrimitiveArray, - secondary: PrimitiveArray, - codes: ArrayView<'_, BitPacked>, - validity: vortex_array::validity::Validity, - ctx: &mut ExecutionCtx, -) -> VortexResult { - match_each_integer_ptype!(secondary.ptype(), |T| { - let starts = starts.as_slice::(); - let (output, invalid_codes) = - add_bitpacked_codes(starts, secondary.into_buffer_mut::(), codes)?; - validate_invalid_codes(&validity, output.len(), &invalid_codes, ctx)?; - VortexResult::Ok(PrimitiveArray::new(output, validity)) - }) -} - -fn decode_primitive_dict( - starts: PrimitiveArray, - secondary: PrimitiveArray, - codes: PrimitiveArray, - validity: vortex_array::validity::Validity, - ctx: &mut ExecutionCtx, -) -> VortexResult { - match_each_integer_ptype!(secondary.ptype(), |T| { - let starts = starts.as_slice::(); - match_each_integer_ptype!(codes.ptype(), |C| { - let (output, invalid_codes) = add_primitive_codes( - starts, - secondary.into_buffer_mut::(), - codes.as_slice::(), - )?; - validate_invalid_codes(&validity, output.len(), &invalid_codes, ctx)?; - VortexResult::Ok(PrimitiveArray::new(output, validity)) - }) - }) -} - -fn add_primitive_codes( - starts: &[T], - mut output: BufferMut, - codes: &[C], -) -> VortexResult<(BufferMut, Vec)> -where - T: NativePType + WrappingMulAdd, - C: NativePType + AsPrimitive, -{ - let mut invalid_codes = Vec::new(); - for (index, (value, code)) in output.as_mut_slice().iter_mut().zip(codes).enumerate() { - let code: usize = (*code).as_(); - let Some(start) = starts.get(code) else { - invalid_codes.push(index); - continue; - }; - *value = ::wrapping_add(*start, *value); - } - Ok((output, invalid_codes)) -} - -fn validate_invalid_codes( - validity: &vortex_array::validity::Validity, - len: usize, - invalid_codes: &[usize], - ctx: &mut ExecutionCtx, -) -> VortexResult<()> { - if !invalid_codes.is_empty() { - let mask = validity.execute_mask(len, ctx)?; - for &index in invalid_codes { - if mask.value(index) { - vortex_bail!("IntMult dictionary code is invalid"); - } - } - } - Ok(()) -} - -fn add_bitpacked_codes( - starts: &[T], - mut output: BufferMut, - codes: ArrayView<'_, BitPacked>, -) -> VortexResult<(BufferMut, Vec)> -where - T: NativePType + WrappingMulAdd, -{ - let mut invalid_codes = Vec::new(); - if codes.bit_width() == 0 { - if let Some(start) = starts.first() { - for value in output.as_mut_slice() { - *value = ::wrapping_add(*start, *value); - } - } else { - invalid_codes.extend(0..output.len()); - } - return Ok((output, invalid_codes)); - } - - match_each_integer_ptype!(codes.dtype().as_ptype(), |C| { - let mut chunks = codes.unpacked_chunks::()?; - chunks.for_each_unpacked_chunk(|codes, range| { - add_dictionary_starts( - starts, - &mut output.as_mut_slice()[range.clone()], - codes, - range.start, - &mut invalid_codes, - ); - }); - VortexResult::Ok((output, invalid_codes)) - }) -} - -fn add_dictionary_starts( - starts: &[T], - output: &mut [T], - codes: &[C], - offset: usize, - invalid_codes: &mut Vec, -) where - T: NativePType + WrappingMulAdd, - C: NativePType + AsPrimitive, -{ - for (index, (value, code)) in output.iter_mut().zip(codes).enumerate() { - let code: usize = (*code).as_(); - let Some(start) = starts.get(code) else { - invalid_codes.push(offset + index); - continue; - }; - *value = ::wrapping_add(*start, *value); - } -} - -impl OperationsVTable for IntMult { - fn scalar_at( - array: ArrayView<'_, IntMult>, - index: usize, - ctx: &mut ExecutionCtx, - ) -> VortexResult { - let primary = array.primary().execute_scalar(index, ctx)?; - if primary.is_null() { - return Ok(Scalar::null(array.dtype().clone())); - } - let secondary = array.secondary().execute_scalar(index, ctx)?; - let primary = primary.as_primitive(); - let secondary = secondary.as_primitive(); - let base = array.base(); - let nullability = array.dtype().nullability(); - Ok(match_each_integer_ptype!(primary.ptype(), |T| { - let primary = primary - .typed_value::() - .vortex_expect("validated IntMult primary scalar"); - let secondary = secondary - .typed_value::() - .vortex_expect("validated IntMult secondary scalar"); - let base = T::try_from(base).vortex_expect("validated IntMult base"); - let value = if array.base() == 1 { - ::wrapping_add(primary, secondary) - } else { - T::wrapping_mul_add(base, primary, secondary) - }; - Scalar::primitive(value, nullability) - })) - } -} - -impl ValidityChild for IntMult { - fn validity_child(array: ArrayView<'_, IntMult>) -> ArrayRef { - array.primary().clone() - } -} - -pub trait IntMultArrayExt: TypedArrayRef + IntMultArraySlotsExt { - /// Return the multiplier that reconstructs each value. - fn base(&self) -> u64 { - self.deref().base - } -} - -impl> IntMultArrayExt for T {} - -impl IntMult { - /// Construct an integer multiplication array from generic child arrays. - pub fn try_new( - primary: ArrayRef, - secondary: ArrayRef, - base: u64, - ) -> VortexResult { - let dtype = primary.dtype().clone(); - let len = primary.len(); - let slots = IntMultSlots { primary, secondary }.into_slots(); - Array::try_from_parts( - ArrayParts::new(IntMult, dtype, len, IntMultData { base }).with_slots(slots), - ) - } - - /// Split integers into quotient and remainder child arrays. - pub fn from_primitive( - array: ArrayView<'_, Primitive>, - base: u64, - ) -> VortexResult { - let ptype = array.ptype(); - ensure_base_fits(base, ptype)?; - let validity = array.validity()?; - let (primary, secondary) = match_each_integer_ptype!(ptype, |T| { - let base = T::try_from(base).vortex_expect("validated IntMult base"); - let (primary, secondary) = split_buffer(array.as_slice::(), base); - ( - PrimitiveArray::new(primary, validity).into_array(), - PrimitiveArray::new(secondary, NonNullable.into()).into_array(), - ) - }); - Self::try_new(primary, secondary, base) - } -} - -fn validate_children( - base: u64, - dtype: &DType, - len: usize, - slots: IntMultSlotsView<'_>, -) -> VortexResult<()> { - let ptype = PType::try_from(dtype)?; - ensure_base_fits(base, ptype)?; - vortex_ensure!( - slots.primary.dtype() == dtype, - "IntMult primary dtype {} differs from {dtype}", - slots.primary.dtype() - ); - vortex_ensure!(slots.primary.len() == len, "IntMult primary length differs"); - let secondary_dtype = DType::Primitive(ptype, NonNullable); - vortex_ensure!( - slots.secondary.dtype() == &secondary_dtype, - "IntMult secondary dtype {} differs from {secondary_dtype}", - slots.secondary.dtype() - ); - vortex_ensure!( - slots.secondary.len() == len, - "IntMult secondary length differs" - ); - Ok(()) -} - -fn ensure_base_fits(base: u64, ptype: PType) -> VortexResult<()> { - vortex_ensure!(ptype.is_int(), "IntMult requires integers"); - vortex_ensure!(base >= 1, "IntMult base must be positive"); - let maximum = match ptype { - PType::U8 => u64::from(u8::MAX), - PType::U16 => u64::from(u16::MAX), - PType::U32 => u64::from(u32::MAX), - PType::U64 => u64::MAX, - PType::I8 => i8::MAX as u64, - PType::I16 => i16::MAX as u64, - PType::I32 => i32::MAX as u64, - PType::I64 => i64::MAX as u64, - _ => vortex_bail!("IntMult requires integers"), - }; - vortex_ensure!(base <= maximum, "IntMult base does not fit {ptype}"); - Ok(()) -} - -fn split_buffer(values: &[T], base: T) -> (Buffer, Buffer) -where - T: NativePType + Copy + std::ops::Div + std::ops::Rem, -{ - let mut primary = BufferMut::with_capacity(values.len()); - let mut secondary = BufferMut::with_capacity(values.len()); - for value in values { - primary.push(*value / base); - secondary.push(*value % base); - } - (primary.freeze(), secondary.freeze()) -} - -fn decode_primitive( - primary: PrimitiveArray, - secondary: PrimitiveArray, - base: u64, -) -> VortexResult { - let ptype = primary.ptype(); - ensure_base_fits(base, ptype)?; - let validity = primary.validity()?; - Ok(match_each_integer_ptype!(ptype, |T| { - let base = T::try_from(base).vortex_expect("validated IntMult base"); - let values = compose_buffer( - primary.into_buffer_mut::(), - secondary.into_buffer::(), - base, - ); - PrimitiveArray::new(values, validity) - })) -} - -fn compose_buffer(mut primary: BufferMut, secondary: Buffer, base: T) -> Buffer -where - T: NativePType + WrappingMulAdd, -{ - if ::is_one(base) { - for (primary, secondary) in primary.as_mut_slice().iter_mut().zip(secondary.as_slice()) { - *primary = ::wrapping_add(*primary, *secondary); - } - return primary.freeze(); - } - - for (primary, secondary) in primary.as_mut_slice().iter_mut().zip(secondary.as_slice()) { - *primary = T::wrapping_mul_add(base, *primary, *secondary); - } - primary.freeze() -} - -trait WrappingMulAdd: Copy { - fn is_one(value: Self) -> bool; - - fn wrapping_add(lhs: Self, rhs: Self) -> Self; - - fn wrapping_mul_add(base: Self, primary: Self, secondary: Self) -> Self; -} - -macro_rules! impl_wrapping_mul_add { - ($type:ty) => { - impl WrappingMulAdd for $type { - #[inline] - fn is_one(value: Self) -> bool { - value == 1 - } - - #[inline] - fn wrapping_add(lhs: Self, rhs: Self) -> Self { - lhs.wrapping_add(rhs) - } - - #[inline] - fn wrapping_mul_add(base: Self, primary: Self, secondary: Self) -> Self { - base.wrapping_mul(primary).wrapping_add(secondary) - } - } - }; -} - -impl_wrapping_mul_add!(u8); -impl_wrapping_mul_add!(u16); -impl_wrapping_mul_add!(u32); -impl_wrapping_mul_add!(u64); -impl_wrapping_mul_add!(i8); -impl_wrapping_mul_add!(i16); -impl_wrapping_mul_add!(i32); -impl_wrapping_mul_add!(i64); - -#[cfg(test)] -mod tests { - use rstest::rstest; - use vortex_array::ArrayContext; - use vortex_array::ArrayRef; - use vortex_array::IntoArray; - use vortex_array::VortexSessionExecute; - use vortex_array::array_session; - use vortex_array::arrays::DictArray; - use vortex_array::arrays::PrimitiveArray; - use vortex_array::assert_arrays_eq; - use vortex_array::dtype::NativePType; - use vortex_array::serde::SerializeOptions; - use vortex_array::serde::SerializedArray; - use vortex_array::validity::Validity; - use vortex_buffer::ByteBufferMut; - use vortex_buffer::buffer; - use vortex_fastlanes::bitpack_compress::bitpack_encode_unchecked; - use vortex_session::registry::ReadContext; - - use super::*; - - fn round_trip(values: Vec, base: u64) -> VortexResult<()> - where - T: NativePType + Copy, - { - let input = PrimitiveArray::from_iter(values); - let encoded = IntMult::from_primitive(input.as_view(), base)?; - let decoded = encoded - .into_array() - .execute::(&mut array_session().create_execution_ctx())?; - assert_arrays_eq!(decoded, input, &mut array_session().create_execution_ctx()); - Ok(()) - } - - #[rstest] - #[case(vec![0_u8, 1, 6, 7, 8, 254, 255], 7)] - #[case(vec![0_u16, 1, 6, 7, 8, 65_534, 65_535], 7)] - #[case(vec![0_u32, 1, 6, 7, 8, u32::MAX - 1, u32::MAX], 7)] - #[case(vec![0_u64, 1, 6, 7, 8, u64::MAX - 1, u64::MAX], 7)] - #[case(vec![i8::MIN, -8, -7, -1, 0, 1, 7, 8, i8::MAX], 7)] - #[case(vec![i16::MIN, -8, -7, -1, 0, 1, 7, 8, i16::MAX], 7)] - #[case(vec![i32::MIN, -8, -7, -1, 0, 1, 7, 8, i32::MAX], 7)] - #[case(vec![i64::MIN, -8, -7, -1, 0, 1, 7, 8, i64::MAX], 7)] - fn round_trip_integer_ptype(#[case] values: Vec, #[case] base: u64) -> VortexResult<()> - where - T: NativePType + Copy, - { - round_trip(values, base) - } - - #[test] - fn exposes_exact_children() -> VortexResult<()> { - let input = PrimitiveArray::from_iter([0_u32, 11, 22, 33, 44]); - let encoded = IntMult::from_primitive(input.as_view(), 10)?; - assert_arrays_eq!( - encoded.primary(), - PrimitiveArray::from_iter([0_u32, 1, 2, 3, 4]), - &mut array_session().create_execution_ctx() - ); - assert_arrays_eq!( - encoded.secondary(), - PrimitiveArray::from_iter([0_u32, 1, 2, 3, 4]), - &mut array_session().create_execution_ctx() - ); - Ok(()) - } - - #[test] - fn rejects_invalid_child_shapes() { - let primary = PrimitiveArray::from_iter([1_u32, 2, 3]).into_array(); - let wrong_ptype = PrimitiveArray::from_iter([1_u16, 2, 3]).into_array(); - assert!(IntMult::try_new(primary.clone(), wrong_ptype, 10).is_err()); - - let wrong_length = PrimitiveArray::from_iter([1_u32, 2]).into_array(); - assert!(IntMult::try_new(primary.clone(), wrong_length, 10).is_err()); - - let nullable = PrimitiveArray::from_option_iter([Some(1_u32), None, Some(3)]).into_array(); - assert!(IntMult::try_new(primary, nullable, 10).is_err()); - } - - #[test] - fn base_one_adds_generic_children() -> VortexResult<()> { - let primary = PrimitiveArray::from_iter([100_u32, 200, 300]).into_array(); - let secondary = PrimitiveArray::from_iter([1_u32, 2, 3]).into_array(); - let encoded = IntMult::try_new(primary, secondary, 1)?; - assert_arrays_eq!( - encoded, - PrimitiveArray::from_iter([101_u32, 202, 303]), - &mut array_session().create_execution_ctx() - ); - Ok(()) - } - - #[test] - fn base_one_adds_bitpacked_dictionary_references() -> VortexResult<()> { - let codes = PrimitiveArray::from_iter([0_u8, 1, 2, 1, 0]); - // SAFETY: Two bits represent every code. - let codes = unsafe { bitpack_encode_unchecked(codes, 2) }?.into_array(); - let starts = PrimitiveArray::from_iter([100_u32, 1_000, 10_000]).into_array(); - let references = DictArray::try_new(codes, starts)?.into_array(); - let offsets = PrimitiveArray::from_iter([1_u32, 2, 3, 4, 5]).into_array(); - let encoded = IntMult::try_new(references, offsets, 1)?; - - assert_arrays_eq!( - encoded, - PrimitiveArray::from_iter([101_u32, 1_002, 10_003, 1_004, 105]), - &mut array_session().create_execution_ctx() - ); - Ok(()) - } - - #[test] - fn base_one_adds_nullable_bitpacked_dictionary_references() -> VortexResult<()> { - let codes = PrimitiveArray::new( - buffer![0_u8, 3, 1, 0, 3], - Validity::from_iter([true, false, true, true, false]), - ); - // SAFETY: Two bits represent every code payload, including invalid null payloads. - let codes = unsafe { bitpack_encode_unchecked(codes, 2) }?.into_array(); - let starts = PrimitiveArray::from_iter([100_u32, 1_000]).into_array(); - let references = DictArray::try_new(codes, starts)?.into_array(); - let offsets = PrimitiveArray::from_iter([1_u32, 2, 3, 4, 5]).into_array(); - let encoded = IntMult::try_new(references, offsets, 1)?; - let expected = - PrimitiveArray::from_option_iter([Some(101_u32), None, Some(1_003), Some(104), None]); - let mut ctx = array_session().create_execution_ctx(); - - assert!(decode_dict_add(encoded.as_view(), &mut ctx)?.is_some()); - assert_arrays_eq!(encoded.clone(), expected, &mut ctx); - assert_arrays_eq!( - encoded.into_array().slice(1..4)?, - expected.into_array().slice(1..4)?, - &mut ctx - ); - Ok(()) - } - - #[test] - fn base_one_adds_zero_width_dictionary_references() -> VortexResult<()> { - let codes = PrimitiveArray::from_iter([0_u8; 3]); - // SAFETY: Zero bits represent the only code. - let codes = unsafe { bitpack_encode_unchecked(codes, 0) }?.into_array(); - let starts = PrimitiveArray::from_iter([100_u32]).into_array(); - let references = DictArray::try_new(codes, starts)?.into_array(); - let offsets = PrimitiveArray::from_iter([1_u32, 2, 3]).into_array(); - let encoded = IntMult::try_new(references, offsets, 1)?; - - assert_arrays_eq!( - encoded, - PrimitiveArray::from_iter([101_u32, 102, 103]), - &mut array_session().create_execution_ctx() - ); - Ok(()) - } - - #[test] - fn multiplies_bitpacked_pair_with_validity_and_slice() -> VortexResult<()> { - let input = PrimitiveArray::from_option_iter( - (0..2_050_u32) - .map(|index| (index != 1_024).then_some((index % 1_000) * 10 + index % 10)), - ); - let split = IntMult::from_primitive(input.as_view(), 10)?; - let mut ctx = array_session().create_execution_ctx(); - let primary = split - .primary() - .clone() - .execute::(&mut ctx)?; - let secondary = split - .secondary() - .clone() - .execute::(&mut ctx)?; - // SAFETY: Ten bits represent every quotient in the test input. - let primary = unsafe { bitpack_encode_unchecked(primary, 10) }?.into_array(); - // SAFETY: Four bits represent every remainder in the test input. - let secondary = unsafe { bitpack_encode_unchecked(secondary, 4) }?.into_array(); - let encoded = IntMult::try_new(primary, secondary, 10)?; - - assert!(decode_fastlanes_pair(encoded.as_view())?.is_some()); - assert_arrays_eq!(encoded, input, &mut ctx); - assert_arrays_eq!( - encoded.into_array().slice(1_000..1_050)?, - input.into_array().slice(1_000..1_050)?, - &mut ctx - ); - Ok(()) - } - - #[test] - fn preserves_validity() -> VortexResult<()> { - let input = PrimitiveArray::new( - buffer![10_u32, 20, 30, 40], - Validity::from_iter([true, false, true, false]), - ); - let encoded = IntMult::from_primitive(input.as_view(), 10)?; - let mut ctx = array_session().create_execution_ctx(); - assert!( - encoded - .clone() - .into_array() - .execute_scalar(1, &mut ctx)? - .is_null() - ); - let decoded = encoded.into_array().execute::(&mut ctx)?; - assert_arrays_eq!(decoded, input, &mut ctx); - Ok(()) - } - - #[test] - fn slices_children() -> VortexResult<()> { - let input = PrimitiveArray::from_iter([10_u32, 21, 32, 43, 54]); - let encoded = IntMult::from_primitive(input.as_view(), 10)?; - let sliced = encoded.into_array().slice(1..4)?; - assert_arrays_eq!( - sliced, - PrimitiveArray::from_iter([21_u32, 32, 43]), - &mut array_session().create_execution_ctx() - ); - Ok(()) - } - - #[test] - fn serialized_generic_children_round_trip() -> VortexResult<()> { - let session = array_session(); - crate::initialize(&session); - let primary = PrimitiveArray::from_option_iter([Some(100_u32), None, Some(300)]); - let secondary = PrimitiveArray::from_iter([1_u32, 2, 3]); - let encoded = - IntMult::try_new(primary.into_array(), secondary.into_array(), 1)?.into_array(); - let dtype = encoded.dtype().clone(); - let len = encoded.len(); - let array_ctx = ArrayContext::empty(); - let serialized = encoded.serialize(&array_ctx, &session, &SerializeOptions::default())?; - let mut bytes = ByteBufferMut::empty(); - for buffer in serialized { - bytes.extend_from_slice(buffer.as_ref()); - } - let parts = SerializedArray::try_from(bytes.freeze())?; - let decoded = parts.decode(&dtype, len, &ReadContext::new(array_ctx.to_ids()), &session)?; - let expected: ArrayRef = - PrimitiveArray::from_option_iter([Some(101_u32), None, Some(303)]).into_array(); - assert_arrays_eq!(decoded, expected, &mut session.create_execution_ctx()); - Ok(()) - } - - #[rstest] - #[case(0)] - #[case(256)] - fn rejects_invalid_u8_base(#[case] base: u64) { - let input = PrimitiveArray::from_iter([1_u8, 2, 3]); - assert!(IntMult::from_primitive(input.as_view(), base).is_err()); - } -} diff --git a/encodings/int-mult/src/lib.rs b/encodings/int-mult/src/lib.rs deleted file mode 100644 index 1050c146f8e..00000000000 --- a/encodings/int-mult/src/lib.rs +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! Integer multiplication transform with independent multiplied and additive children. - -mod array; -mod rules; -mod slice; - -pub use array::*; -use vortex_array::session::ArraySessionExt; -use vortex_session::VortexSession; - -/// Register the integer multiplication encoding in one session. -pub fn initialize(session: &VortexSession) { - session.arrays().register(IntMult); -} diff --git a/encodings/int-mult/src/rules.rs b/encodings/int-mult/src/rules.rs deleted file mode 100644 index 9d2be58f5ba..00000000000 --- a/encodings/int-mult/src/rules.rs +++ /dev/null @@ -1,10 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use vortex_array::arrays::slice::SliceReduceAdaptor; -use vortex_array::optimizer::rules::ParentRuleSet; - -use crate::IntMult; - -pub(crate) static RULES: ParentRuleSet = - ParentRuleSet::new(&[ParentRuleSet::lift(&SliceReduceAdaptor(IntMult))]); diff --git a/encodings/int-mult/src/slice.rs b/encodings/int-mult/src/slice.rs deleted file mode 100644 index 0ff8bffa94b..00000000000 --- a/encodings/int-mult/src/slice.rs +++ /dev/null @@ -1,27 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::ops::Range; - -use vortex_array::ArrayRef; -use vortex_array::ArrayView; -use vortex_array::IntoArray; -use vortex_array::arrays::slice::SliceReduce; -use vortex_error::VortexResult; - -use crate::IntMult; -use crate::IntMultArrayExt; -use crate::IntMultArraySlotsExt; - -impl SliceReduce for IntMult { - fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { - Ok(Some( - IntMult::try_new( - array.primary().slice(range.clone())?, - array.secondary().slice(range)?, - array.base(), - )? - .into_array(), - )) - } -} diff --git a/vortex-file/Cargo.toml b/vortex-file/Cargo.toml index 920f012f0f4..5b86c11467c 100644 --- a/vortex-file/Cargo.toml +++ b/vortex-file/Cargo.toml @@ -47,7 +47,6 @@ vortex-fastlanes = { workspace = true } vortex-flatbuffers = { workspace = true, features = ["file"] } vortex-float-quant = { workspace = true } vortex-fsst = { workspace = true } -vortex-int-mult = { workspace = true } vortex-io = { workspace = true } vortex-layout = { workspace = true } vortex-mask = { workspace = true } diff --git a/vortex-file/src/lib.rs b/vortex-file/src/lib.rs index 64a0f000514..465c7fd6e3c 100644 --- a/vortex-file/src/lib.rs +++ b/vortex-file/src/lib.rs @@ -191,7 +191,6 @@ pub fn register_default_encodings(session: &VortexSession) { vortex_decimal_byte_parts::initialize(session); vortex_fastlanes::initialize(session); vortex_float_quant::initialize(session); - vortex_int_mult::initialize(session); vortex_block_residual::initialize(session); vortex_runend::initialize(session); vortex_sequence::initialize(session); diff --git a/vortex/Cargo.toml b/vortex/Cargo.toml index 3fd56b80578..3d98384ecaa 100644 --- a/vortex/Cargo.toml +++ b/vortex/Cargo.toml @@ -40,7 +40,6 @@ vortex-file = { workspace = true, optional = true, default-features = true } vortex-flatbuffers = { workspace = true } vortex-float-quant = { workspace = true } vortex-fsst = { workspace = true } -vortex-int-mult = { workspace = true } vortex-io = { workspace = true } vortex-ipc = { workspace = true } vortex-layout = { workspace = true } diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index be31c5bcf47..bf04576ca80 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -49,8 +49,6 @@ use vortex::encodings::float_quant::FloatQuantArraySlotsExt; use vortex::encodings::float_quant::analyze_float_quant; use vortex::encodings::fsst::fsst_compress; use vortex::encodings::fsst::fsst_train_compressor; -use vortex::encodings::int_mult::IntMult; -use vortex::encodings::int_mult::IntMultArraySlotsExt; use vortex::encodings::pco::Pco; use vortex::encodings::runend::RunEnd; use vortex::encodings::sequence::sequence_encode; @@ -344,89 +342,6 @@ fn setup_block_local_integer_array() -> PrimitiveArray { } } -fn setup_int_mult_i32_array() -> PrimitiveArray { - PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { - let primary = (index as i32).wrapping_mul(7_919) % 1_000_000; - let secondary = index as i32 % 10 - 5; - primary.wrapping_mul(10).wrapping_add(secondary) - })) -} - -fn setup_int_mult_u64_array() -> PrimitiveArray { - PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { - let primary = index.wrapping_mul(7_919) % 1_000_000_000; - let secondary = index % 10; - primary.wrapping_mul(10).wrapping_add(secondary) - })) -} - -fn setup_int_mult_integer_array() -> PrimitiveArray { - match T::PTYPE { - PType::U8 => PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { - let primary = index.wrapping_mul(79) % 20; - (primary * 10 + index % 10) as u8 - })), - PType::U16 => PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { - let primary = index.wrapping_mul(7_919) % 6_000; - (primary * 10 + index % 10) as u16 - })), - PType::U32 => PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { - let primary = index.wrapping_mul(7_919) % 100_000_000; - (primary * 10 + index % 10) as u32 - })), - PType::U64 => setup_int_mult_u64_array(), - PType::I8 => PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { - let primary = (index.wrapping_mul(79) % 20) as i16 - 10; - let secondary = (index % 10) as i16 - 5; - (primary * 10 + secondary) as i8 - })), - PType::I16 => PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { - let primary = (index.wrapping_mul(7_919) % 3_000) as i32 - 1_500; - let secondary = (index % 10) as i32 - 5; - (primary * 10 + secondary) as i16 - })), - PType::I32 => setup_int_mult_i32_array(), - PType::I64 => PrimitiveArray::from_iter((0..FLOAT_CODEC_NUM_VALUES).map(|index| { - let primary = (index.wrapping_mul(7_919) % 1_000_000_000) as i64; - let secondary = (index % 10) as i64 - 5; - primary * 10 + secondary - })), - ptype => unreachable!("unsupported IntMult benchmark type {ptype}"), - } -} - -fn encode_int_mult_packed_tree(array: &PrimitiveArray) -> vortex::array::ArrayRef { - let split = IntMult::from_primitive(array.as_view(), 10).unwrap(); - let primary = split - .primary() - .clone() - .execute::(&mut SESSION.create_execution_ctx()) - .unwrap(); - let secondary = split - .secondary() - .clone() - .execute::(&mut SESSION.create_execution_ctx()) - .unwrap(); - let primary_width = match T::PTYPE { - PType::U8 => 5, - PType::U16 => 13, - PType::U32 => 27, - PType::U64 => 30, - ptype => unreachable!("unsupported packed IntMult benchmark type {ptype}"), - }; - // SAFETY: The benchmark generator bounds the quotient to the selected width. - let primary = unsafe { bitpack_encode_unchecked(primary, primary_width) } - .unwrap() - .into_array(); - // SAFETY: The benchmark generator bounds the remainder below ten. - let secondary = unsafe { bitpack_encode_unchecked(secondary, 4) } - .unwrap() - .into_array(); - IntMult::try_new(primary, secondary, 10) - .unwrap() - .into_array() -} - fn encode_for_bitpacked_tree(array: &PrimitiveArray, bit_width: u8) -> vortex::array::ArrayRef { let mut ctx = SESSION.create_execution_ctx(); let encoded = FoR::encode(array.clone(), &mut ctx).unwrap(); @@ -895,82 +810,6 @@ fn bench_ordered_float_scalar_at_f32(bencher: Bencher) { .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); } -#[divan::bench(types = [u8, u16, u32, u64, i8, i16, i32, i64])] -fn int_mult_split_compress(bencher: Bencher) { - let array = setup_int_mult_integer_array::(); - let byte_width = u64::try_from(T::PTYPE.byte_width()).unwrap(); - - with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * byte_width) - .with_inputs(|| &array) - .bench_refs(|array| IntMult::from_primitive(array.as_view(), 10).unwrap()); -} - -#[divan::bench(types = [u8, u16, u32, u64, i8, i16, i32, i64])] -fn int_mult_split_decompress(bencher: Bencher) { - let encoded = IntMult::from_primitive(setup_int_mult_integer_array::().as_view(), 10) - .unwrap() - .into_array(); - let byte_width = u64::try_from(T::PTYPE.byte_width()).unwrap(); - - with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * byte_width) - .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) - .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); -} - -#[divan::bench(types = [u8, u16, u32, u64, i8, i16, i32, i64])] -fn int_mult_split_scalar_at(bencher: Bencher) { - let encoded = IntMult::from_primitive(setup_int_mult_integer_array::().as_view(), 10) - .unwrap() - .into_array(); - let next_index = AtomicUsize::new(0); - - bencher - .with_inputs(|| { - ( - &encoded, - SESSION.create_execution_ctx(), - next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), - ) - }) - .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); -} - -#[divan::bench(types = [u8, u16, u32, u64])] -fn int_mult_packed_tree_compress(bencher: Bencher) { - let array = setup_int_mult_integer_array::(); - let byte_width = u64::try_from(T::PTYPE.byte_width()).unwrap(); - - with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * byte_width) - .with_inputs(|| &array) - .bench_refs(|array| encode_int_mult_packed_tree::(array)); -} - -#[divan::bench(types = [u8, u16, u32, u64])] -fn int_mult_packed_tree_decompress(bencher: Bencher) { - let encoded = encode_int_mult_packed_tree::(&setup_int_mult_integer_array::()); - let byte_width = u64::try_from(T::PTYPE.byte_width()).unwrap(); - - with_byte_counter(bencher, FLOAT_CODEC_NUM_VALUES * byte_width) - .with_inputs(|| (&encoded, SESSION.create_execution_ctx())) - .bench_refs(|(array, ctx)| canonicalize((**array).clone(), ctx)); -} - -#[divan::bench(types = [u8, u16, u32, u64])] -fn int_mult_packed_tree_scalar_at(bencher: Bencher) { - let encoded = encode_int_mult_packed_tree::(&setup_int_mult_integer_array::()); - let next_index = AtomicUsize::new(0); - - bencher - .with_inputs(|| { - ( - &encoded, - SESSION.create_execution_ctx(), - next_index.fetch_add(2_654_435_761, Ordering::Relaxed) % encoded.len(), - ) - }) - .bench_values(|(array, mut ctx, index)| array.execute_scalar(index, &mut ctx).unwrap()); -} - #[divan::bench(name = "block_residual_compress_u64")] fn bench_block_residual_compress_u64(bencher: Bencher) { let float_array = setup_random_walk_array(); diff --git a/vortex/src/editions/core/v2026_08.rs b/vortex/src/editions/core/v2026_08.rs index d60ac754e22..b64faf56025 100644 --- a/vortex/src/editions/core/v2026_08.rs +++ b/vortex/src/editions/core/v2026_08.rs @@ -28,7 +28,6 @@ pub static DECLARATION: EditionDeclaration = EditionDeclaration { added: &[ EditionMember::array(&"vortex.block_residual"), EditionMember::array(&"vortex.float_quant"), - EditionMember::array(&"vortex.int_mult"), EditionMember::array(&"vortex.map"), EditionMember::array(&"vortex.ordered_float"), EditionMember::aggregate(&"vortex.bounded_max"), diff --git a/vortex/src/lib.rs b/vortex/src/lib.rs index 4c6479ebab7..cb6e8aff091 100644 --- a/vortex/src/lib.rs +++ b/vortex/src/lib.rs @@ -265,11 +265,6 @@ pub mod encodings { pub use vortex_float_quant::*; } - /// Integer multiplication transform with generic multiplied and additive children. - pub mod int_mult { - pub use vortex_int_mult::*; - } - /// Fast Static Symbol Table string encoding. pub mod fsst { pub use vortex_fsst::*; From c3691782eb76c4f9a9bdb2fbff94b0acaa0be7b4 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Fri, 21 Aug 2026 15:31:59 -0400 Subject: [PATCH 76/78] refactor: harden native numeric encodings Signed-off-by: Will Manning --- Cargo.lock | 3 - Cargo.toml | 4 +- docs/plans/native-pcodec.md | 42 +- .../src/block_residual_array.rs | 29 +- encodings/block-residual/src/codec.rs | 376 +----------------- encodings/block-residual/src/lib.rs | 4 +- .../block-residual/src/ordered_float_array.rs | 23 +- .../src/bitpacking/array/unpack_iter.rs | 4 +- encodings/float-quant/src/array.rs | 74 +++- encodings/pco/src/array.rs | 36 +- encodings/pco/src/tests.rs | 19 - vortex-bench/src/lib.rs | 7 + vortex-btrblocks/Cargo.toml | 7 - vortex-btrblocks/examples/pco_probe.rs | 272 ------------- vortex-btrblocks/src/builder.rs | 43 +- .../src/schemes/float/float_quant.rs | 26 ++ .../schemes/float/scheme_selection_tests.rs | 4 +- .../src/schemes/integer/bitpacking.rs | 120 +++--- vortex-btrblocks/src/schemes/integer/for_.rs | 49 ++- .../schemes/integer/scheme_selection_tests.rs | 4 +- vortex-btrblocks/src/schemes/mod.rs | 1 + vortex-btrblocks/src/trace_tests.rs | 5 + vortex-compressor/src/builtins/dict/float.rs | 24 +- .../src/builtins/dict/integer.rs | 20 +- vortex-compressor/src/compressor/cascade.rs | 29 -- vortex-compressor/src/stats/float.rs | 22 +- vortex-compressor/src/trace.rs | 1 - vortex-file/Cargo.toml | 2 +- vortex/benches/single_encoding_throughput.rs | 1 + vortex/src/editions/core/mod.rs | 2 + vortex/src/editions/core/v2026_08_4.rs | 25 ++ vortex/src/editions/mod.rs | 2 + vortex/src/editions/tests.rs | 34 +- 33 files changed, 382 insertions(+), 932 deletions(-) delete mode 100644 vortex-btrblocks/examples/pco_probe.rs create mode 100644 vortex/src/editions/core/v2026_08_4.rs diff --git a/Cargo.lock b/Cargo.lock index cb278295857..67b69a74d43 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9766,12 +9766,9 @@ name = "vortex-btrblocks" version = "0.1.0" dependencies = [ "arrow-array 58.4.0", - "arrow-schema 58.4.0", - "arrow-select 58.4.0", "codspeed-divan-compat", "insta", "itertools 0.14.0", - "parquet 58.4.0", "pco", "rand 0.10.2", "rstest", diff --git a/Cargo.toml b/Cargo.toml index fe973ff0f6f..c62cce728a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -299,6 +299,7 @@ vortex = { version = "0.1.0", path = "./vortex" } vortex-alp = { version = "0.1.0", path = "./encodings/alp", default-features = false } vortex-array = { version = "0.1.0", path = "./vortex-array", default-features = false } vortex-arrow = { version = "0.1.0", path = "./vortex-arrow", default-features = false } +vortex-block-residual = { version = "0.1.0", path = "./encodings/block-residual", default-features = false } vortex-btrblocks = { version = "0.1.0", path = "./vortex-btrblocks", default-features = false } vortex-buffer = { version = "0.1.0", path = "./vortex-buffer", default-features = false } vortex-bytebool = { version = "0.1.0", path = "./encodings/bytebool", default-features = false } @@ -313,6 +314,7 @@ vortex-error = { version = "0.1.0", path = "./vortex-error", default-features = vortex-fastlanes = { version = "0.1.0", path = "./encodings/fastlanes", default-features = false } vortex-file = { version = "0.1.0", path = "./vortex-file", default-features = false } vortex-flatbuffers = { version = "0.1.0", path = "./vortex-flatbuffers", default-features = false } +vortex-float-quant = { version = "0.1.0", path = "./encodings/float-quant", default-features = false } vortex-fsst = { version = "0.1.0", path = "./encodings/fsst", default-features = false } vortex-io = { version = "0.1.0", path = "./vortex-io", default-features = false } vortex-ipc = { version = "0.1.0", path = "./vortex-ipc", default-features = false } @@ -323,8 +325,6 @@ vortex-metrics = { version = "0.1.0", path = "./vortex-metrics", default-feature vortex-onpair = { version = "0.1.0", path = "./encodings/onpair", default-features = false } vortex-parquet-variant = { version = "0.1.0", path = "./encodings/parquet-variant" } vortex-pco = { version = "0.1.0", path = "./encodings/pco", default-features = false } -vortex-block-residual = { version = "0.1.0", path = "./encodings/block-residual", default-features = false } -vortex-float-quant = { version = "0.1.0", path = "./encodings/float-quant", default-features = false } vortex-proto = { version = "0.1.0", path = "./vortex-proto", default-features = false } vortex-row = { version = "0.1.0", path = "./vortex-row", default-features = false } vortex-runend = { version = "0.1.0", path = "./encodings/runend", default-features = false } diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md index 7f017afd60a..eeaa5894497 100644 --- a/docs/plans/native-pcodec.md +++ b/docs/plans/native-pcodec.md @@ -111,6 +111,12 @@ The final calibration includes the complete corpus and selected-tree evidence. The Opportunistic bundle cuts aggregate size by 11.7 percent against Prior Default. +The three array encodings belong to the draft `core:2026.08.4` edition. + +The default session still writes the frozen `core:2026.08.1` edition. + +Focused file tests and benchmarks enable the draft numeric edition explicitly. + Future work targets repeated Compact mechanisms with native, bounded-access trees. ## Array support and validation @@ -2084,7 +2090,9 @@ A later chunk-level trace identified completed incumbent cascades as the cause. The current broad size gain does not justify Default inclusion by itself. -IntMult now belongs to the August 2026 core edition, so files can serialize the decomposed tree. +The prototype temporarily added IntMult to a draft edition for file tests. + +The focused branch removed IntMult and that edition membership. Fast rejection remains a cost advantage, not an admission rule. @@ -2778,11 +2786,11 @@ This round completed these steps: - Rejected the dense-remainder IntMult layout on CMS Payments. - Rejected pure IntMult on a no-Delta CMS standard-deviation column. - Rejected centered block residuals after a complete Euro subjectivity comparison. -- Assigned dictionary codes by descending value frequency. -- Added float frequency counts to the existing distinct-value statistics. +- Tested dictionary codes in descending value frequency, then removed the policy with IntMult. +- Tested float frequency counts, then restored the existing distinct-value statistics. - Reopened BlockResidual descendants for integer and float dictionary codes. - Validated the ranked Dict policy across eight numeric datasets. -- Audited constructor and deserialization validation for all four production arrays. +- Audited constructor and deserialization validation for all three production arrays. - Added round-trip tests for every supported integer and float type. - Added single-encoding benchmarks for every supported integer and float type. - Added packed-child IntMult benchmarks for every unsigned integer type. @@ -2794,7 +2802,7 @@ This round completed these steps: - Added a locality rejection test that retains Euro and rejects clear BlockResidual fits. - Updated stale golden trees after the BlockResidual payload and selector changes. - Added four numeric bundle controls to the complete compression benchmark. -- Added explicit winner result events to the compressor trace. +- Tested explicit winner result events, then removed the unused trace event. - Added optional chunk-level encoding trees to the focused compressor benchmark. - Rejected constant FloatQuant samples before exact sample encoding. - Added and calibrated a 1.10 FloatQuant selection factor. @@ -2916,12 +2924,32 @@ Test these items in future research: ## Pull request structure -Prepare one focused stack for `OrderedFloatArray`, `BlockResidualArray`, and `OrderedBlockResidualScheme`. +Prepare one pull request for the three arrays, their schemes, fused FastLanes paths, and benchmark support. -Prepare a separate stack for `FloatQuantArray` and `FloatQuantScheme`. +Split the pull request only when review reveals a clear independent boundary. Keep entropy, bit-split, and alternate residual models on `wm/pcodec-entropy-experiments`. +## Adversarial review checkpoint + +The review followed the merge from `origin/develop`. + +It preserved the frozen August editions and added `core:2026.08.4` as a draft. + +It added the array conformance suite to BlockResidual, OrderedFloat, and FloatQuant. + +It added empty-buffer validation to OrderedFloat and FloatQuant deserialization. + +It added signed coverage for the fused nonzero-secondary FloatQuant encoder. + +It changed invalid FloatQuant references from a debug panic to an error. + +It removed the Pco probe, the custom Pco API, abandoned compressor hooks, and public BlockResidual codec internals. + +It removed unrelated dictionary frequency ranking and float frequency statistics. + +It retained the three production candidates, their selectors, and their focused benchmarks. + ## References - [PCodec paper](https://arxiv.org/html/2502.06112v2) diff --git a/encodings/block-residual/src/block_residual_array.rs b/encodings/block-residual/src/block_residual_array.rs index 02d7f77d322..5e5babd5cde 100644 --- a/encodings/block-residual/src/block_residual_array.rs +++ b/encodings/block-residual/src/block_residual_array.rs @@ -336,7 +336,7 @@ impl SliceReduce for BlockResidual { static RULES: ParentRuleSet = ParentRuleSet::new(&[ParentRuleSet::lift(&SliceReduceAdaptor(BlockResidual))]); -pub trait BlockResidualArrayExt: TypedArrayRef { +pub(crate) trait BlockResidualArrayExt: TypedArrayRef { fn unsliced_validity(&self) -> Validity { child_to_validity( self.as_ref().slots()[BlockResidualSlots::VALIDITY].as_ref(), @@ -388,11 +388,6 @@ pub trait BlockResidualArrayExt: TypedArrayRef { fn patch_highs(&self) -> &[u8] { &self.patch_highs } - - /// Decode the logical slice. - fn decompress(&self, ctx: &mut ExecutionCtx) -> VortexResult { - decompress_array(self.to_owned().as_view(), ctx) - } } impl> BlockResidualArrayExt for T {} @@ -1419,6 +1414,7 @@ mod tests { use vortex_array::array_session; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; + use vortex_array::compute::conformance::consistency::test_array_consistency; use vortex_array::dtype::NativePType; use vortex_array::dtype::PType; use vortex_array::serde::SerializeOptions; @@ -1507,7 +1503,8 @@ mod tests { #[test] fn rejects_components_outside_logical_width() -> VortexResult<()> { - let mut parts = BlockResidualCodec::encode(&[0_u64, 1, 2])?.into_parts()?; + let mut parts = + BlockResidualCodec::encode_with_word_width(&[0_u64, 1, 2], 64)?.into_parts()?; parts.bases[0] = u64::from(u8::MAX) + 1; assert!(BlockResidual::try_new(parts, Validity::NonNullable, PType::U8).is_err()); Ok(()) @@ -1515,7 +1512,8 @@ mod tests { #[test] fn rejects_invalid_component_offsets() -> VortexResult<()> { - let mut parts = BlockResidualCodec::encode(&[0_u64, 1, 2])?.into_parts()?; + let mut parts = + BlockResidualCodec::encode_with_word_width(&[0_u64, 1, 2], 64)?.into_parts()?; parts.residual_starts[0] = 1; assert!(BlockResidual::try_new(parts, Validity::NonNullable, PType::U64).is_err()); Ok(()) @@ -1640,4 +1638,19 @@ mod tests { assert_arrays_eq!(decoded, expected, &mut session.create_execution_ctx()); Ok(()) } + + #[test] + fn conformance() -> VortexResult<()> { + let mut values = vec![42_i16; 2_050]; + values[1_023] = i16::MAX; + let primitive = PrimitiveArray::new( + Buffer::from(values), + Validity::from_iter((0..2_050).map(|index| index != 1_024)), + ); + let array = BlockResidual::from_primitive(primitive.as_view())?.into_array(); + let session = array_session(); + crate::initialize(&session); + test_array_consistency(&array, &mut session.create_execution_ctx()); + Ok(()) + } } diff --git a/encodings/block-residual/src/codec.rs b/encodings/block-residual/src/codec.rs index 8c91b14ccff..b99846b0776 100644 --- a/encodings/block-residual/src/codec.rs +++ b/encodings/block-residual/src/codec.rs @@ -12,24 +12,24 @@ const SERIALIZED_BLOCK_METADATA_BYTES: usize = 12; /// Block-local residual codec for ordered unsigned latents. #[derive(Clone, Debug, PartialEq, Eq)] -pub struct BlockResidualCodec { +pub(crate) struct BlockResidualCodec { len: usize, blocks: Vec, } /// Serialized children for the one-reference block residual codec. #[derive(Clone, Debug, PartialEq, Eq)] -pub struct BlockResidualParts { - pub len: usize, - pub bases: Vec, - pub residual_widths: Vec, - pub high_widths: Vec, - pub residual_starts: Vec, - pub patch_starts: Vec, - pub high_starts: Vec, - pub residual_words: Vec, - pub patch_positions: Vec, - pub patch_highs: Vec, +pub(crate) struct BlockResidualParts { + pub(crate) len: usize, + pub(crate) bases: Vec, + pub(crate) residual_widths: Vec, + pub(crate) high_widths: Vec, + pub(crate) residual_starts: Vec, + pub(crate) patch_starts: Vec, + pub(crate) high_starts: Vec, + pub(crate) residual_words: Vec, + pub(crate) patch_positions: Vec, + pub(crate) patch_highs: Vec, } pub(crate) struct BlockResidualCodecEstimate { @@ -39,7 +39,6 @@ pub(crate) struct BlockResidualCodecEstimate { #[derive(Clone, Debug, PartialEq, Eq)] struct BlockResidualBlock { - len: u16, base: u64, residual_width: u8, high_width: u8, @@ -49,11 +48,6 @@ struct BlockResidualBlock { } impl BlockResidualCodec { - /// Encode ordered unsigned latents in independent 1024-value blocks. - pub fn encode(values: &[u64]) -> VortexResult { - Self::encode_with_word_width(values, 64) - } - pub(crate) fn encode_with_word_width(values: &[u64], word_width: u8) -> VortexResult { vortex_error::vortex_ensure!( matches!(word_width, 8 | 16 | 32 | 64), @@ -97,8 +91,7 @@ impl BlockResidualCodec { } } - /// Convert the codec into serialized array children. - pub fn into_parts(self) -> VortexResult { + pub(crate) fn into_parts(self) -> VortexResult { let mut parts = BlockResidualParts { len: self.len, bases: Vec::with_capacity(self.blocks.len()), @@ -133,258 +126,7 @@ impl BlockResidualCodec { } Ok(parts) } - - /// Reconstruct the codec from serialized array children. - pub fn try_from_parts(parts: BlockResidualParts) -> VortexResult { - let block_count = parts.len.div_ceil(CHUNK_LEN); - vortex_error::vortex_ensure!( - parts.bases.len() == block_count - && parts.residual_widths.len() == block_count - && parts.high_widths.len() == block_count, - "block residual metadata child lengths are invalid" - ); - validate_starts( - &parts.residual_starts, - block_count, - parts.residual_words.len(), - "residual", - )?; - validate_starts( - &parts.patch_starts, - block_count, - parts.patch_positions.len(), - "patch", - )?; - validate_starts( - &parts.high_starts, - block_count, - parts.patch_highs.len(), - "patch high", - )?; - - let mut blocks = Vec::with_capacity(block_count); - for block_index in 0..block_count { - let residual_start = usize::try_from(parts.residual_starts[block_index])?; - let residual_stop = usize::try_from(parts.residual_starts[block_index + 1])?; - let patch_start = usize::try_from(parts.patch_starts[block_index])?; - let patch_stop = usize::try_from(parts.patch_starts[block_index + 1])?; - let high_start = usize::try_from(parts.high_starts[block_index])?; - let high_stop = usize::try_from(parts.high_starts[block_index + 1])?; - let block_start = block_index * CHUNK_LEN; - let block_len = (parts.len - block_start).min(CHUNK_LEN); - let residual_width = parts.residual_widths[block_index]; - let high_width = parts.high_widths[block_index]; - vortex_error::vortex_ensure!( - residual_width <= 64 - && high_width <= 64 - && u16::from(residual_width) + u16::from(high_width) <= 64, - "block residual bit widths are invalid" - ); - vortex_error::vortex_ensure!( - residual_stop - residual_start - == CHUNK_LEN * usize::from(residual_width) / u64::BITS as usize, - "block residual packed word count is invalid" - ); - let patch_positions = &parts.patch_positions[patch_start..patch_stop]; - vortex_error::vortex_ensure!( - patch_positions.is_empty() || (high_width > 0 && residual_width < 64), - "block residual patches require nonzero high bits" - ); - vortex_error::vortex_ensure!( - patch_positions - .iter() - .all(|&position| usize::from(position) < block_len) - && patch_positions - .windows(2) - .all(|window| window[0] < window[1]), - "block residual patch positions are invalid" - ); - let expected_high_len = if patch_positions.is_empty() { - 0 - } else { - (patch_positions.len() * usize::from(high_width)).div_ceil(8) + HIGH_PADDING - }; - vortex_error::vortex_ensure!( - high_stop - high_start == expected_high_len, - "block residual patch high payload length is invalid" - ); - blocks.push(BlockResidualBlock { - len: u16::try_from(block_len)?, - base: parts.bases[block_index], - residual_width, - high_width, - residuals: parts.residual_words[residual_start..residual_stop].to_vec(), - patch_positions: parts.patch_positions[patch_start..patch_stop].to_vec(), - patch_highs: parts.patch_highs[high_start..high_stop].to_vec(), - }); - } - Ok(Self { - len: parts.len, - blocks, - }) - } - - /// Decode all values. - pub fn decode(&self) -> VortexResult> { - let mut values = Vec::with_capacity(self.len); - let mut residuals = [0u64; CHUNK_LEN]; - for block in &self.blocks { - residuals.fill(0); - if block.residual_width > 0 { - // SAFETY: The encoder creates one complete FastLanes chunk. - unsafe { - u64::unchecked_unpack( - usize::from(block.residual_width), - &block.residuals, - &mut residuals, - ); - } - } - - let mut high_bit_position = 0usize; - for &position in &block.patch_positions { - // SAFETY: The encoder appends fifteen readable padding bytes. - let high = unsafe { - read_wide_bits(&block.patch_highs, high_bit_position, block.high_width) - }; - residuals[usize::from(position)] |= high << block.residual_width; - high_bit_position += usize::from(block.high_width); - } - - let block_len = usize::from(block.len); - for residual in &mut residuals[..block_len] { - *residual = residual.wrapping_add(block.base); - } - values.extend_from_slice(&residuals[..block_len]); - } - Ok(values) - } - - /// Decode ordered `f64` latents with a fused inverse transform. - pub fn decode_ordered_f64(&self) -> VortexResult> { - let mut values = Vec::with_capacity(self.len); - let mut residuals = [0_u64; CHUNK_LEN]; - for block in &self.blocks { - residuals.fill(0); - if block.residual_width > 0 { - // SAFETY: The encoder creates one complete FastLanes chunk. - unsafe { - u64::unchecked_unpack( - usize::from(block.residual_width), - &block.residuals, - &mut residuals, - ); - } - } - let mut high_bit_position = 0_usize; - for &position in &block.patch_positions { - // SAFETY: The encoder appends fifteen readable padding bytes. - let high = unsafe { - read_wide_bits(&block.patch_highs, high_bit_position, block.high_width) - }; - residuals[usize::from(position)] |= high << block.residual_width; - high_bit_position += usize::from(block.high_width); - } - let block_len = usize::from(block.len); - for residual in &mut residuals[..block_len] { - *residual = residual.wrapping_add(block.base); - } - values.extend(residuals[..block_len].iter().map(|&ordered| { - let bits = if ordered & (1_u64 << 63) == 0 { - !ordered - } else { - ordered ^ (1_u64 << 63) - }; - f64::from_bits(bits) - })); - } - Ok(values) - } - - /// Decode one value with direct packed access and a binary patch search. - pub fn scalar_at(&self, index: usize) -> VortexResult { - vortex_error::vortex_ensure!( - index < self.len, - "index {index} is out of bounds for length {}", - self.len - ); - let block = &self.blocks[index / CHUNK_LEN]; - let index_in_block = index % CHUNK_LEN; - let mut residual = if block.residual_width == 0 { - 0 - } else { - // SAFETY: The encoder creates one complete FastLanes chunk. - unsafe { - u64::unchecked_unpack_single( - usize::from(block.residual_width), - &block.residuals, - index_in_block, - ) - } - }; - if let Ok(patch_index) = block - .patch_positions - .binary_search(&u16::try_from(index_in_block)?) - { - // SAFETY: The encoder appends fifteen readable padding bytes. - let high = unsafe { - read_wide_bits( - &block.patch_highs, - patch_index * usize::from(block.high_width), - block.high_width, - ) - }; - residual |= high << block.residual_width; - } - Ok(block.base.wrapping_add(residual)) - } - - /// Return the logical value count. - pub fn len(&self) -> usize { - self.len - } - - /// Return true when the codec contains no values. - pub fn is_empty(&self) -> bool { - self.len == 0 - } - - /// Return the encoded bytes, including estimated serialized block metadata. - pub fn encoded_size(&self) -> usize { - self.blocks - .iter() - .map(|block| { - SERIALIZED_BLOCK_METADATA_BYTES - + size_of::() - + block.residuals.len() * size_of::() - + block.patch_positions.len() * size_of::() - + block.patch_highs.len() - }) - .sum() - } - - /// Return the logical block count. - pub fn block_count(&self) -> usize { - self.blocks.len() - } - - /// Return the total patch count. - pub fn patch_count(&self) -> usize { - self.blocks - .iter() - .map(|block| block.patch_positions.len()) - .sum() - } - - /// Return the sum of main residual widths across all blocks. - pub fn total_residual_width(&self) -> usize { - self.blocks - .iter() - .map(|block| usize::from(block.residual_width)) - .sum() - } } - fn encode_block(values: &[u64], word_width: u8) -> VortexResult { let base = values.iter().copied().min().unwrap_or(0); let mut residuals = Vec::with_capacity(CHUNK_LEN); @@ -463,27 +205,6 @@ fn choose_width( } } -fn validate_starts( - starts: &[u32], - block_count: usize, - payload_len: usize, - name: &str, -) -> VortexResult<()> { - vortex_error::vortex_ensure!( - starts.len() == block_count + 1, - "block residual {name} offsets have an invalid length" - ); - vortex_error::vortex_ensure!( - starts.first() == Some(&0) && usize::try_from(*starts.last().unwrap_or(&0))? == payload_len, - "block residual {name} offsets do not cover the payload" - ); - vortex_error::vortex_ensure!( - starts.windows(2).all(|window| window[0] <= window[1]), - "block residual {name} offsets are not ordered" - ); - Ok(()) -} - struct BlockPlan { base: u64, residual_width: u8, @@ -524,7 +245,6 @@ fn materialize_block( }; Ok(BlockResidualBlock { - len: u16::try_from(values.len())?, base: plan.base, residual_width: plan.residual_width, high_width: plan.high_width, @@ -710,79 +430,11 @@ mod tests { use super::BlockResidualCodec; - #[test] - fn roundtrip_lengths_and_domains() -> VortexResult<()> { - for len in [0, 1, 1_023, 1_024, 1_025, 20_000] { - let values = (0..len) - .map(|index| match index % 4 { - 0 => index as u64, - 1 => 1_000_000 + index as u64 % 31, - 2 => u64::MAX - index as u64 % 13, - _ => 42, - }) - .collect::>(); - let codec = BlockResidualCodec::encode(&values)?; - assert_eq!(codec.decode()?, values); - for (index, &value) in values.iter().enumerate() { - assert_eq!(codec.scalar_at(index)?, value); - } - } - Ok(()) - } - - #[test] - fn constant_roundtrip() -> VortexResult<()> { - let values = vec![42; 10_000]; - let codec = BlockResidualCodec::encode(&values)?; - assert_eq!(codec.decode()?, values); - assert_eq!(codec.scalar_at(4_321)?, 42); - Ok(()) - } - - #[test] - fn parts_roundtrip() -> VortexResult<()> { - let values = (0..4_099) - .map(|index| { - let value = u64::try_from(index)?; - Ok(1_000_000_u64.wrapping_add(value * value)) - }) - .collect::>>()?; - let parts = BlockResidualCodec::encode(&values)?.into_parts()?; - let codec = BlockResidualCodec::try_from_parts(parts)?; - assert_eq!(codec.decode()?, values); - Ok(()) - } - - #[test] - fn rejects_incompatible_patch_widths() -> VortexResult<()> { - let mut values = vec![0_u64; 1_024]; - values[1_023] = u64::MAX; - let mut parts = BlockResidualCodec::encode(&values)?.into_parts()?; - assert!(!parts.patch_positions.is_empty()); - - parts.residual_widths[0] = 64; - parts.high_widths[0] = 1; - assert!(BlockResidualCodec::try_from_parts(parts).is_err()); - Ok(()) - } - - #[test] - fn rejects_patches_without_high_bits() -> VortexResult<()> { - let mut values = vec![0_u64; 1_024]; - values[1_023] = u64::MAX; - let mut parts = BlockResidualCodec::encode(&values)?.into_parts()?; - assert!(!parts.patch_positions.is_empty()); - - parts.high_widths[0] = 0; - assert!(BlockResidualCodec::try_from_parts(parts).is_err()); - Ok(()) - } - #[test] fn patch_penalty_prefers_dense_residuals() -> VortexResult<()> { let mut values = vec![0_u64; 1_024]; values[..307].fill(4_095); - let parts = BlockResidualCodec::encode(&values)?.into_parts()?; + let parts = BlockResidualCodec::encode_with_word_width(&values, 64)?.into_parts()?; assert_eq!(parts.residual_widths, [12]); assert!(parts.patch_positions.is_empty()); diff --git a/encodings/block-residual/src/lib.rs b/encodings/block-residual/src/lib.rs index f1d0b7b3070..b4298389790 100644 --- a/encodings/block-residual/src/lib.rs +++ b/encodings/block-residual/src/lib.rs @@ -8,8 +8,8 @@ mod codec; mod ordered_float_array; pub use block_residual_array::*; -pub use codec::BlockResidualCodec; -pub use codec::BlockResidualParts; +pub(crate) use codec::BlockResidualCodec; +pub(crate) use codec::BlockResidualParts; pub use ordered_float_array::*; use vortex_array::session::ArraySessionExt; use vortex_session::VortexSession; diff --git a/encodings/block-residual/src/ordered_float_array.rs b/encodings/block-residual/src/ordered_float_array.rs index d489b72b40f..3d293205f6c 100644 --- a/encodings/block-residual/src/ordered_float_array.rs +++ b/encodings/block-residual/src/ordered_float_array.rs @@ -152,10 +152,11 @@ impl VTable for OrderedFloat { dtype: &DType, len: usize, metadata: &[u8], - _buffers: &[BufferHandle], + buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult> { + vortex_ensure!(buffers.is_empty(), "OrderedFloatArray expects no buffers"); vortex_ensure!( metadata.is_empty(), "OrderedFloatArray metadata must be empty" @@ -457,6 +458,7 @@ mod tests { use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; + use vortex_array::compute::conformance::consistency::test_array_consistency; use vortex_array::dtype::PType; use vortex_array::dtype::half::f16; use vortex_array::serde::SerializeOptions; @@ -639,4 +641,23 @@ mod tests { assert_arrays_eq!(decoded, expected, &mut ctx); Ok(()) } + + #[test] + fn conformance() -> VortexResult<()> { + let primitive = PrimitiveArray::new( + Buffer::from(vec![f32::NEG_INFINITY, -0.0, 0.0, 42.25, f32::INFINITY]), + Validity::from_iter([true, false, true, true, true]), + ); + let direct = OrderedFloat::from_primitive(primitive.as_view())?; + let residuals = BlockResidual::from_primitive(direct.encoded().as_::())?; + let fused = OrderedFloat::try_new(residuals.into_array(), PType::F32)?; + let session = array_session(); + crate::initialize(&session); + let mut ctx = session.create_execution_ctx(); + + for array in [direct.into_array(), fused.into_array()] { + test_array_consistency(&array, &mut ctx); + } + Ok(()) + } } diff --git a/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs b/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs index 2c79c6173f2..3c77e146ad0 100644 --- a/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs +++ b/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs @@ -221,8 +221,8 @@ impl> UnpackedChunks { }); } - /// Walk every unpacked chunk in logical order and reuse one scratch buffer. - pub fn for_each_unpacked_chunk(&mut self, mut f: F) + /// Walk every unpacked chunk in array order, reusing the internal scratch buffer. + pub(crate) fn for_each_unpacked_chunk(&mut self, mut f: F) where F: FnMut(&mut [T], Range), { diff --git a/encodings/float-quant/src/array.rs b/encodings/float-quant/src/array.rs index 5c7cb989885..c25c49c28f4 100644 --- a/encodings/float-quant/src/array.rs +++ b/encodings/float-quant/src/array.rs @@ -176,10 +176,11 @@ impl VTable for FloatQuant { dtype: &DType, len: usize, metadata: &[u8], - _buffers: &[BufferHandle], + buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult> { + vortex_ensure!(buffers.is_empty(), "FloatQuant expects no buffers"); vortex_ensure!( metadata.len() == METADATA_LEN, "FloatQuant metadata requires {METADATA_LEN} bytes" @@ -472,11 +473,6 @@ pub struct FloatQuantAnalysis { pub secondary_bit_width: u8, } -/// Estimate a useful low-bit split width for a canonical float array. -pub fn estimate_k(array: ArrayView<'_, Primitive>) -> Option { - analyze_float_quant(array).map(|analysis| analysis.k) -} - /// Analyze a canonical float array for a FloatQuant split. pub fn analyze_float_quant(array: ArrayView<'_, Primitive>) -> Option { match array.ptype() { @@ -733,10 +729,14 @@ fn split_primary_for_f32(values: &[f32], k: u8, primary_min: u32) -> VortexResul values.iter().all(|value| value.to_bits() & low_mask == 0), "FloatQuant constant secondary requires zero low bits" ); - Ok(values + values .iter() - .map(|value| (ordered_u32(value.to_bits()) >> k) - primary_min) - .collect()) + .map(|value| { + (ordered_u32(value.to_bits()) >> k) + .checked_sub(primary_min) + .ok_or_else(|| vortex_error::vortex_err!("FloatQuant primary minimum is invalid")) + }) + .collect::>>() } fn split_primary_for_f16(values: &[f16], k: u8, primary_min: u16) -> VortexResult> { @@ -746,10 +746,14 @@ fn split_primary_for_f16(values: &[f16], k: u8, primary_min: u16) -> VortexResul values.iter().all(|value| value.to_bits() & low_mask == 0), "FloatQuant constant secondary requires zero low bits" ); - Ok(values + values .iter() - .map(|value| (ordered_u16(value.to_bits()) >> k) - primary_min) - .collect()) + .map(|value| { + (ordered_u16(value.to_bits()) >> k) + .checked_sub(primary_min) + .ok_or_else(|| vortex_error::vortex_err!("FloatQuant primary minimum is invalid")) + }) + .collect::>>() } fn split_primary_for_f64(values: &[f64], k: u8, primary_min: u64) -> VortexResult> { @@ -759,10 +763,14 @@ fn split_primary_for_f64(values: &[f64], k: u8, primary_min: u64) -> VortexResul values.iter().all(|value| value.to_bits() & low_mask == 0), "FloatQuant constant secondary requires zero low bits" ); - Ok(values + values .iter() - .map(|value| (ordered_u64(value.to_bits()) >> k) - primary_min) - .collect()) + .map(|value| { + (ordered_u64(value.to_bits()) >> k) + .checked_sub(primary_min) + .ok_or_else(|| vortex_error::vortex_err!("FloatQuant primary minimum is invalid")) + }) + .collect::>>() } fn join_f16(primary: u16, secondary: u16, k: u8) -> f16 { @@ -1043,6 +1051,7 @@ mod tests { use vortex_array::array_session; use vortex_array::assert_arrays_eq; use vortex_array::assert_nth_scalar; + use vortex_array::compute::conformance::consistency::test_array_consistency; use vortex_array::serde::SerializeOptions; use vortex_array::serde::SerializedArray; use vortex_array::validity::Validity; @@ -1249,6 +1258,14 @@ mod tests { assert!(FloatQuant::primary_for_primitive(f64_values.as_view(), 29, 0).is_err()); } + #[test] + fn primary_for_primitive_rejects_invalid_reference() { + let values = PrimitiveArray::from_iter([1.0_f32, 2.0]); + assert!( + FloatQuant::primary_for_primitive(values.as_view(), 1, u64::from(u32::MAX)).is_err() + ); + } + #[test] fn fastlanes_zero_decode_roundtrip_and_slice() -> VortexResult<()> { let original = PrimitiveArray::from_option_iter((0_u32..4097).map(|index| { @@ -1382,4 +1399,31 @@ mod tests { ); Ok(()) } + + #[test] + fn conformance() -> VortexResult<()> { + let explicit = PrimitiveArray::from_option_iter([ + Some(f32::NEG_INFINITY), + None, + Some(-0.0), + Some(0.0), + Some(42.25), + Some(f32::INFINITY), + ]); + let explicit = FloatQuant::from_primitive(explicit.as_view(), 8)?.into_array(); + let implicit = PrimitiveArray::from_option_iter([ + Some(f64::from(-10.5_f32)), + None, + Some(f64::from(-0.0_f32)), + Some(f64::from(42.25_f32)), + ]); + let implicit = + FloatQuant::from_primitive_constant_secondary(implicit.as_view(), 29)?.into_array(); + let mut ctx = SESSION.create_execution_ctx(); + + for array in [explicit, implicit] { + test_array_consistency(&array, &mut ctx); + } + Ok(()) + } } diff --git a/encodings/pco/src/array.rs b/encodings/pco/src/array.rs index e0783ac36ea..7dc3a371bda 100644 --- a/encodings/pco/src/array.rs +++ b/encodings/pco/src/array.rs @@ -352,24 +352,6 @@ impl Pco { let data = PcoData::from_primitive(parray, level, values_per_page, ctx)?; Self::try_new(dtype, data, validity) } - - /// Compress a primitive array with a custom Pco configuration. - /// - /// # Errors - /// - /// Returns an error if the configuration is invalid or compression fails. - pub fn from_primitive_with_config( - parray: ArrayView<'_, Primitive>, - chunk_config: &ChunkConfig, - values_per_chunk: usize, - ctx: &mut ExecutionCtx, - ) -> VortexResult { - let dtype = parray.dtype().clone(); - let validity = parray.validity()?; - let data = - PcoData::from_primitive_with_config(parray, chunk_config, values_per_chunk, ctx)?; - Self::try_new(dtype, data, validity) - } } #[array_slots(Pco)] @@ -551,6 +533,7 @@ impl PcoData { values_per_page: usize, ctx: &mut ExecutionCtx, ) -> VortexResult { + let number_type = number_type_from_dtype(parray.dtype()); let values_per_page = if values_per_page == 0 { values_per_chunk } else { @@ -562,21 +545,6 @@ impl PcoData { .with_compression_level(level) .with_paging_spec(PagingSpec::EqualPagesUpTo(values_per_page)); - Self::from_primitive_with_config(parray, &chunk_config, values_per_chunk, ctx) - } - - fn from_primitive_with_config( - parray: ArrayView<'_, Primitive>, - chunk_config: &ChunkConfig, - values_per_chunk: usize, - ctx: &mut ExecutionCtx, - ) -> VortexResult { - vortex_ensure!( - values_per_chunk > 0, - "values per chunk must be greater than zero" - ); - let number_type = number_type_from_dtype(parray.dtype()); - let values = collect_valid(parray, ctx)?; let n_values = values.len(); @@ -595,7 +563,7 @@ impl PcoData { let values = values.to_buffer::(); let chunk = &values.as_slice()[chunk_start..chunk_end]; fc - .chunk_compressor(chunk, chunk_config) + .chunk_compressor(chunk, &chunk_config) .map_err(vortex_err_from_pco)? } ); diff --git a/encodings/pco/src/tests.rs b/encodings/pco/src/tests.rs index 7d9613b4ea5..8a0e1083b16 100644 --- a/encodings/pco/src/tests.rs +++ b/encodings/pco/src/tests.rs @@ -4,10 +4,6 @@ use std::sync::LazyLock; -use pco::ChunkConfig; -use pco::DeltaSpec; -use pco::ModeSpec; -use pco::PagingSpec; use vortex_array::ArrayContext; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; @@ -74,21 +70,6 @@ fn test_compress_decompress() { ); } -#[test] -fn test_custom_config_roundtrip() -> VortexResult<()> { - let mut ctx = SESSION.create_execution_ctx(); - let array = PrimitiveArray::from_iter((0..1_000).map(|value| f64::from(value).sin())); - let config = ChunkConfig::default() - .with_mode_spec(ModeSpec::Classic) - .with_delta_spec(DeltaSpec::NoOp) - .with_paging_spec(PagingSpec::EqualPagesUpTo(128)); - - let compressed = Pco::from_primitive_with_config(array.as_view(), &config, 512, &mut ctx)?; - - assert_arrays_eq!(compressed, array, &mut ctx); - Ok(()) -} - #[test] fn test_compress_decompress_small() { let mut ctx = SESSION.create_execution_ctx(); diff --git a/vortex-bench/src/lib.rs b/vortex-bench/src/lib.rs index 196991ede46..8f3b2ec1fba 100644 --- a/vortex-bench/src/lib.rs +++ b/vortex-bench/src/lib.rs @@ -74,6 +74,8 @@ pub use datasets::BenchmarkDataset; pub use output::BenchmarkOutput; pub use output::create_output_writer; use vortex::VortexSessionDefault; +use vortex::editions::CORE_2026_08_4; +use vortex::editions::EditionSessionExt; pub use vortex::error::vortex_panic; use vortex::io::session::RuntimeSessionExt; use vortex::session::VortexSession; @@ -84,6 +86,11 @@ static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; pub static SESSION: LazyLock = LazyLock::new(|| { let session = VortexSession::default().with_tokio(); + session + .enable_edition(CORE_2026_08_4) + .unwrap_or_else(|error| { + vortex_panic!("numeric benchmark edition is not registered: {error}") + }); vortex_spatial::initialize(&session); session }); diff --git a/vortex-btrblocks/Cargo.toml b/vortex-btrblocks/Cargo.toml index f517a92d942..38da11310c0 100644 --- a/vortex-btrblocks/Cargo.toml +++ b/vortex-btrblocks/Cargo.toml @@ -42,11 +42,8 @@ vortex-zstd = { workspace = true, optional = true } [dev-dependencies] arrow-array = { workspace = true } -arrow-schema = { workspace = true } -arrow-select = { workspace = true } divan = { workspace = true } insta = { workspace = true } -parquet = { workspace = true } rstest = { workspace = true } test-with = { workspace = true } tpchgen = { workspace = true } @@ -74,7 +71,3 @@ test = false name = "compress_listview" harness = false test = false - -[[example]] -name = "pco_probe" -required-features = ["pco"] diff --git a/vortex-btrblocks/examples/pco_probe.rs b/vortex-btrblocks/examples/pco_probe.rs deleted file mode 100644 index 02f1feea8d8..00000000000 --- a/vortex-btrblocks/examples/pco_probe.rs +++ /dev/null @@ -1,272 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::f64::consts::TAU; -use std::time::Duration; -use std::time::Instant; - -use pco::ChunkConfig; -use pco::DeltaSpec; -use pco::ModeSpec; -use pco::PagingSpec; -use pco::metadata::DeltaEncoding; -use pco::metadata::Mode; -use pco::wrapped::FileCompressor; -use vortex_array::ArrayRef; -use vortex_array::IntoArray; -use vortex_array::VortexSessionExecute; -use vortex_array::array_session; -use vortex_array::arrays::PrimitiveArray; -use vortex_btrblocks::BtrBlocksCompressor; -use vortex_btrblocks::BtrBlocksCompressorBuilder; -use vortex_btrblocks::schemes::float::PcoScheme; -use vortex_error::VortexResult; -use vortex_error::vortex_err; -use vortex_pco::Pco; - -const N: usize = 1 << 18; -const VALUES_PER_PAGE: usize = 8192; -const STAGE_ITERATIONS: usize = 15; - -struct Rng(u64); - -impl Rng { - fn next_u64(&mut self) -> u64 { - self.0 ^= self.0 << 13; - self.0 ^= self.0 >> 7; - self.0 ^= self.0 << 17; - self.0 - } - - fn uniform(&mut self) -> f64 { - ((self.next_u64() >> 11) as f64 + 0.5) * (1.0 / ((1_u64 << 53) as f64)) - } - - fn normal(&mut self) -> f64 { - let radius = (-2.0 * self.uniform().ln()).sqrt(); - radius * (TAU * self.uniform()).cos() - } -} - -#[expect( - clippy::cast_possible_truncation, - reason = "the probe models f32 values stored as f64" -)] -fn widen_f32(value: f64) -> f64 { - value as f32 as f64 -} - -fn datasets() -> Vec<(&'static str, Vec)> { - let mut rng = Rng(0x4d59_5df4_d0f3_3173); - let gaussian = (0..N).map(|_| 1_000.0 + 17.0 * rng.normal()).collect(); - let lognormal = (0..N).map(|_| rng.normal().exp()).collect(); - let decimal = (0..N) - .map(|_| (rng.next_u64() % 1_000_000) as f64 / 100.0) - .collect(); - let widened_f32 = (0..N) - .map(|_| widen_f32(1_000.0 + 17.0 * rng.normal())) - .collect(); - let mut value = 0.0; - let random_walk = (0..N) - .map(|_| { - value += rng.normal() * 0.01; - value - }) - .collect(); - - vec![ - ("gaussian", gaussian), - ("lognormal", lognormal), - ("decimal", decimal), - ("widened-f32", widened_f32), - ("random-walk", random_walk), - ] -} - -fn pco_config(mode: ModeSpec, delta: DeltaSpec) -> ChunkConfig { - ChunkConfig::default() - .with_mode_spec(mode) - .with_delta_spec(delta) - .with_paging_spec(PagingSpec::EqualPagesUpTo(VALUES_PER_PAGE)) -} - -fn percentile(durations: &mut [Duration], numerator: usize, denominator: usize) -> Duration { - durations.sort_unstable(); - durations[durations.len() * numerator / denominator] -} - -fn selected_config(dataset: &str) -> ChunkConfig { - match dataset { - "gaussian" | "lognormal" => pco_config(ModeSpec::Classic, DeltaSpec::NoOp), - "decimal" => pco_config(ModeSpec::TryFloatMult(0.01), DeltaSpec::NoOp), - "widened-f32" => pco_config(ModeSpec::TryFloatQuant(29), DeltaSpec::NoOp), - "random-walk" => pco_config(ModeSpec::Classic, DeltaSpec::TryConsecutive(1)), - _ => unreachable!("unknown synthetic dataset"), - } -} - -fn pco_stage_once( - values: &[f64], - config: &ChunkConfig, -) -> VortexResult<(Duration, Duration, usize)> { - let compressor = FileCompressor::default(); - let prepare_start = Instant::now(); - let mut chunk = compressor - .chunk_compressor(values, config) - .map_err(|error| vortex_err!("{}", error))?; - let prepare = prepare_start.elapsed(); - - let emit_start = Instant::now(); - let mut encoded_bytes = 0usize; - let mut metadata = Vec::with_capacity(chunk.meta_size_hint()); - chunk - .write_meta(&mut metadata) - .map_err(|error| vortex_err!("{}", error))?; - encoded_bytes += metadata.len(); - for page_idx in 0..chunk.n_per_page().len() { - let mut page = Vec::with_capacity(chunk.page_size_hint(page_idx)); - chunk - .write_page(page_idx, &mut page) - .map_err(|error| vortex_err!("{}", error))?; - encoded_bytes += page.len(); - } - let emit = emit_start.elapsed(); - Ok((prepare, emit, encoded_bytes)) -} - -fn report_pco_stages( - dataset: &str, - config_name: &str, - values: &[f64], - config: &ChunkConfig, -) -> VortexResult<()> { - for _ in 0..3 { - std::hint::black_box(pco_stage_once(std::hint::black_box(values), config)?); - } - - let mut prepare_durations = Vec::with_capacity(STAGE_ITERATIONS); - let mut emit_durations = Vec::with_capacity(STAGE_ITERATIONS); - let mut total_durations = Vec::with_capacity(STAGE_ITERATIONS); - let mut encoded_bytes = 0usize; - for _ in 0..STAGE_ITERATIONS { - let (prepare, emit, bytes) = pco_stage_once(std::hint::black_box(values), config)?; - prepare_durations.push(prepare); - emit_durations.push(emit); - total_durations.push(prepare + emit); - encoded_bytes = std::hint::black_box(bytes); - } - - let prepare = percentile(&mut prepare_durations, 1, 2); - let emit = percentile(&mut emit_durations, 1, 2); - let total = percentile(&mut total_durations, 1, 2); - let input_bytes = size_of_val(values); - let throughput = input_bytes as f64 / total.as_secs_f64() / 1_000_000.0; - println!( - "pco-stage\t{dataset}\t{config_name}\t{encoded_bytes}\t{:.3}\t{:.3}\t{:.3}\t{throughput:.1}", - prepare.as_secs_f64() * 1_000.0, - emit.as_secs_f64() * 1_000.0, - total.as_secs_f64() * 1_000.0, - ); - Ok(()) -} - -fn report_auto_selection(dataset: &str, array: &PrimitiveArray) -> VortexResult<()> { - let config = pco_config(ModeSpec::Auto, DeltaSpec::Auto); - let compressor = FileCompressor::default(); - let chunk = compressor - .chunk_compressor(array.as_slice::(), &config) - .map_err(|error| vortex_err!("{}", error))?; - let mode = match &chunk.meta().mode { - Mode::Classic => "classic".to_owned(), - Mode::IntMult(_) => "int-mult".to_owned(), - Mode::FloatMult(_) => "float-mult".to_owned(), - Mode::FloatQuant(bits) => format!("float-quant-{bits}"), - Mode::Dict(_) => "dict".to_owned(), - _ => "unknown".to_owned(), - }; - let delta = match &chunk.meta().delta_encoding { - DeltaEncoding::NoOp => "none".to_owned(), - DeltaEncoding::Consecutive { order, .. } => format!("consecutive-{order}"), - DeltaEncoding::Lookback { .. } => "lookback".to_owned(), - DeltaEncoding::Conv1(_) => "conv1".to_owned(), - _ => "unknown".to_owned(), - }; - eprintln!("{dataset}: pco-auto selected {mode} with {delta}"); - Ok(()) -} - -fn report( - dataset: &str, - encoder: &str, - input_bytes: u64, - encode: impl FnOnce() -> VortexResult, -) -> VortexResult<()> { - let start = Instant::now(); - let compressed = encode()?; - let elapsed = start.elapsed(); - let encoded_bytes = compressed.nbytes(); - let bits_per_value = 8.0 * encoded_bytes as f64 / N as f64; - let throughput = input_bytes as f64 / elapsed.as_secs_f64() / 1_000_000.0; - println!("{dataset}\t{encoder}\t{encoded_bytes}\t{bits_per_value:.3}\t{throughput:.1}"); - Ok(()) -} - -fn main() -> VortexResult<()> { - let session = array_session(); - let default = BtrBlocksCompressor::default(); - let with_auto = BtrBlocksCompressorBuilder::default() - .with_new_scheme(&PcoScheme) - .build(); - - println!("dataset\tencoder\tbytes\tbits/value\tMB/s"); - println!("pco-stage columns: dataset, config, bytes, prepare-ms, emit-ms, total-ms, MB/s"); - for (name, values) in datasets() { - let array = PrimitiveArray::from_iter(values.clone()); - let array_ref = array.clone().into_array(); - let input_bytes = array_ref.nbytes(); - report_auto_selection(name, &array)?; - report_pco_stages( - name, - "auto", - &values, - &pco_config(ModeSpec::Auto, DeltaSpec::Auto), - )?; - for level in [0, 4, 8, 12] { - let config = selected_config(name).with_compression_level(level); - report_pco_stages(name, &format!("selected-level-{level}"), &values, &config)?; - } - report(name, "btrblocks", input_bytes, || { - default.compress(&array_ref, &mut session.create_execution_ctx()) - })?; - report(name, "btrblocks+pco-auto", input_bytes, || { - with_auto.compress(&array_ref, &mut session.create_execution_ctx()) - })?; - for (encoder, config) in [ - ( - "pco-classic", - pco_config(ModeSpec::Classic, DeltaSpec::NoOp), - ), - ( - "pco-classic-delta1", - pco_config(ModeSpec::Classic, DeltaSpec::TryConsecutive(1)), - ), - ( - "pco-classic-delta2", - pco_config(ModeSpec::Classic, DeltaSpec::TryConsecutive(2)), - ), - ("pco-auto", pco_config(ModeSpec::Auto, DeltaSpec::Auto)), - ] { - report(name, encoder, input_bytes, || { - Ok(Pco::from_primitive_with_config( - array.as_view(), - &config, - pco::DEFAULT_MAX_PAGE_N, - &mut session.create_execution_ctx(), - )? - .into_array()) - })?; - } - } - - Ok(()) -} diff --git a/vortex-btrblocks/src/builder.rs b/vortex-btrblocks/src/builder.rs index fce29c74b3d..cb34d38a4e6 100644 --- a/vortex-btrblocks/src/builder.rs +++ b/vortex-btrblocks/src/builder.rs @@ -225,6 +225,7 @@ impl BtrBlocksCompressorBuilder { #[cfg(test)] mod tests { + use rstest::rstest; use vortex_array::VTable; use vortex_fastlanes::FoR; @@ -239,12 +240,16 @@ mod tests { #[test] fn default_includes_all_schemes() { let builder = BtrBlocksCompressorBuilder::default(); - assert_eq!(builder.schemes.len(), ALL_SCHEMES.len()); - assert!( + assert_eq!( builder .schemes .iter() - .any(|scheme| scheme.id() == float::FloatQuantScheme.id()) + .map(|scheme| scheme.id()) + .collect::>(), + ALL_SCHEMES + .iter() + .map(|scheme| scheme.id()) + .collect::>() ); } @@ -281,36 +286,18 @@ mod tests { assert_eq!(builder.schemes.len(), ALL_SCHEMES.len()); } - #[test] - fn cuda_compatible_excludes_alprd() { - let builder = BtrBlocksCompressorBuilder::default().only_cuda_compatible(); - assert!( - !builder - .schemes - .iter() - .any(|s| s.id() == float::ALPRDScheme.id()) - ); - } - - #[test] - fn cuda_compatible_excludes_float_quant() { - let builder = BtrBlocksCompressorBuilder::default().only_cuda_compatible(); - assert!( - !builder - .schemes - .iter() - .any(|scheme| scheme.id() == float::FloatQuantScheme.id()) - ); - } - - #[test] - fn cuda_compatible_excludes_block_residual() { + #[rstest] + #[case(float::ALPRDScheme.id())] + #[case(float::FloatQuantScheme.id())] + #[case(float::OrderedBlockResidualScheme.id())] + #[case(integer::BlockResidualScheme.id())] + fn cuda_compatible_excludes_non_cuda_schemes(#[case] scheme_id: SchemeId) { let builder = BtrBlocksCompressorBuilder::default().only_cuda_compatible(); assert!( !builder .schemes .iter() - .any(|scheme| scheme.id() == integer::BlockResidualScheme.id()) + .any(|scheme| scheme.id() == scheme_id) ); } diff --git a/vortex-btrblocks/src/schemes/float/float_quant.rs b/vortex-btrblocks/src/schemes/float/float_quant.rs index 9c3befb72e3..2edc224e7ab 100644 --- a/vortex-btrblocks/src/schemes/float/float_quant.rs +++ b/vortex-btrblocks/src/schemes/float/float_quant.rs @@ -132,6 +132,8 @@ fn encode_float_quant( primitive: vortex_array::ArrayView<'_, Primitive>, analysis: FloatQuantAnalysis, ) -> VortexResult { + // The ordered transform complements the low bits of negative values. Decode complements the + // secondary bits again, so both signs store the original low bits. let (primary_packed, secondary_packed, latent_ptype, reference) = match primitive.ptype() { PType::F16 => { let primary_min = u16::try_from(analysis.primary_min)?; @@ -296,7 +298,11 @@ fn ordered_u64(bits: u64) -> u64 { #[cfg(test)] mod tests { + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::array_session; use vortex_array::arrays::PrimitiveArray; + use vortex_array::assert_arrays_eq; use vortex_array::dtype::half::f16; use vortex_compressor::scheme::EstimateVerdict; use vortex_error::VortexResult; @@ -371,4 +377,24 @@ mod tests { } Ok(()) } + + #[test] + fn nonzero_secondary_round_trips_negative_values() -> VortexResult<()> { + let values = PrimitiveArray::from_iter((0_u32..2_050).flat_map(|index| { + let high_bits = index.wrapping_mul(7_919) & 0x007f_ff00; + let low_bits = index % 7; + let positive = f32::from_bits(0x3f80_0000 | high_bits | low_bits); + [positive, -positive] + })); + let analysis = analyze_float_quant(values.as_view()) + .ok_or_else(|| vortex_err!("FloatQuant test input did not produce an analysis"))?; + assert!(analysis.secondary_bit_width > 0); + let encoded = encode_float_quant(values.as_view(), analysis)?; + assert_arrays_eq!( + encoded, + values.into_array(), + &mut array_session().create_execution_ctx() + ); + Ok(()) + } } diff --git a/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs index 6150763d8cf..7237bb31ae5 100644 --- a/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs @@ -7,6 +7,8 @@ use std::f64::consts::TAU; use std::sync::LazyLock; use vortex_alp::ALP; +use vortex_array::ArrayEq; +use vortex_array::EqMode; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::Constant; @@ -187,7 +189,7 @@ fn test_float_quant_ignores_null_payloads() -> VortexResult<()> { assert!(first.is::()); assert!(second.is::()); - assert_eq!(first.nbytes(), second.nbytes()); + assert!(first.array_eq(&second, EqMode::Value)); Ok(()) } diff --git a/vortex-btrblocks/src/schemes/integer/bitpacking.rs b/vortex-btrblocks/src/schemes/integer/bitpacking.rs index b11e54299b9..5ac7d0e4078 100644 --- a/vortex-btrblocks/src/schemes/integer/bitpacking.rs +++ b/vortex-btrblocks/src/schemes/integer/bitpacking.rs @@ -10,7 +10,6 @@ use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::VTable; use vortex_array::arrays::Patched; -use vortex_array::arrays::Primitive; use vortex_array::arrays::patched::use_experimental_patches; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_compressor::scheme::CompressionEstimate; @@ -32,65 +31,6 @@ use crate::compress_patches; #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct BitPackingScheme; -pub(crate) fn compress_bitpacked( - primitive_array: vortex_array::ArrayView<'_, Primitive>, - exec_ctx: &mut ExecutionCtx, -) -> VortexResult { - let histogram = bit_width_histogram(primitive_array, exec_ctx)?; - let bw = find_best_bit_width(primitive_array.ptype(), &histogram)?; - - if bw as usize == primitive_array.ptype().bit_width() { - return Ok(primitive_array.array().clone()); - } - - let primitive_array = primitive_array.into_owned(); - let packed = bitpack_encode(&primitive_array, bw, Some(&histogram), exec_ctx)?; - let packed_stats = packed.statistics().to_owned(); - let ptype = packed.dtype().as_ptype(); - let mut parts = BitPacked::into_parts(packed); - - let array = if use_experimental_patches() { - let patches = parts.patches.take(); - let array = BitPacked::try_new( - parts.packed, - ptype, - parts.validity, - None, - parts.bit_width, - parts.len, - parts.offset, - )? - .into_array(); - - match patches { - None => array, - Some(patches) => Patched::from_array_and_patches(array, &patches, exec_ctx)? - .with_stats_set(packed_stats) - .into_array(), - } - } else { - let patches = parts - .patches - .take() - .map(|patches| compress_patches(patches, exec_ctx)) - .transpose()?; - parts.patches = patches; - BitPacked::try_new( - parts.packed, - ptype, - parts.validity, - parts.patches, - parts.bit_width, - parts.len, - parts.offset, - )? - .with_stats_set(packed_stats) - .into_array() - }; - - Ok(array) -} - impl Scheme for BitPackingScheme { fn scheme_name(&self) -> &'static str { "vortex.int.bitpacking" @@ -132,6 +72,64 @@ impl Scheme for BitPackingScheme { exec_ctx: &mut ExecutionCtx, ) -> VortexResult { let primitive_array = data.array_as_primitive(); - compress_bitpacked(primitive_array, exec_ctx) + + let histogram = bit_width_histogram(primitive_array, exec_ctx)?; + let bw = find_best_bit_width(primitive_array.ptype(), &histogram)?; + + // If best bw is determined to be the current bit-width, return the original array. + if bw as usize == primitive_array.ptype().bit_width() { + return Ok(primitive_array.array().clone()); + } + + // Otherwise we can bitpack the array. + let primitive_array = primitive_array.into_owned(); + let packed = bitpack_encode(&primitive_array, bw, Some(&histogram), exec_ctx)?; + + let packed_stats = packed.statistics().to_owned(); + let ptype = packed.dtype().as_ptype(); + let mut parts = BitPacked::into_parts(packed); + + let array = if use_experimental_patches() { + let patches = parts.patches.take(); + // Transpose patches into G-ALP style PatchedArray, wrapping an inner BitPackedArray. + let array = BitPacked::try_new( + parts.packed, + ptype, + parts.validity, + None, + parts.bit_width, + parts.len, + parts.offset, + )? + .into_array(); + + match patches { + None => array, + Some(p) => Patched::from_array_and_patches(array, &p, exec_ctx)? + .with_stats_set(packed_stats) + .into_array(), + } + } else { + // Compress patches and place back into BitPackedArray. + let patches = parts + .patches + .take() + .map(|p| compress_patches(p, exec_ctx)) + .transpose()?; + parts.patches = patches; + BitPacked::try_new( + parts.packed, + ptype, + parts.validity, + parts.patches, + parts.bit_width, + parts.len, + parts.offset, + )? + .with_stats_set(packed_stats) + .into_array() + }; + + Ok(array) } } diff --git a/vortex-btrblocks/src/schemes/integer/for_.rs b/vortex-btrblocks/src/schemes/integer/for_.rs index 3c43a4a5c0b..476a0dec282 100644 --- a/vortex-btrblocks/src/schemes/integer/for_.rs +++ b/vortex-btrblocks/src/schemes/integer/for_.rs @@ -31,31 +31,6 @@ use crate::CompressorContext; use crate::Scheme; use crate::SchemeExt; -pub(crate) fn compress_for_primitive( - compressor: &CascadingCompressor, - primitive: PrimitiveArray, - compress_ctx: CompressorContext, - exec_ctx: &mut ExecutionCtx, -) -> VortexResult { - let for_array = FoR::encode(primitive, exec_ctx)?; - let biased = for_array - .encoded() - .clone() - .execute::(exec_ctx)?; - - let leaf_ctx = compress_ctx.clone().as_leaf(); - let biased_data = ArrayAndStats::new(biased.into_array(), compress_ctx.merged_stats_options()); - let compressed = BitPackingScheme.compress(compressor, &biased_data, leaf_ctx, exec_ctx)?; - - let for_compressed = FoR::try_new(compressed, for_array.reference_scalar().clone())?; - for_compressed - .as_ref() - .statistics() - .inherit_from(for_array.as_ref().statistics()); - - Ok(for_compressed.into_array()) -} - /// Frame of Reference encoding. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct FoRScheme; @@ -155,6 +130,28 @@ impl Scheme for FoRScheme { exec_ctx: &mut ExecutionCtx, ) -> VortexResult { let primitive = data.array().clone().execute::(exec_ctx)?; - compress_for_primitive(compressor, primitive, compress_ctx, exec_ctx) + let for_array = FoR::encode(primitive, exec_ctx)?; + let biased = for_array + .encoded() + .clone() + .execute::(exec_ctx)?; + + // Immediately bitpack. If any other scheme was preferable, it would be chosen instead + // of bitpacking. + // NOTE: we could delegate in the future if we had another downstream codec that performs + // as well. + let leaf_ctx = compress_ctx.clone().as_leaf(); + let biased_data = + ArrayAndStats::new(biased.into_array(), compress_ctx.merged_stats_options()); + let compressed = BitPackingScheme.compress(compressor, &biased_data, leaf_ctx, exec_ctx)?; + + // TODO(connor): This should really be `new_unchecked`. + let for_compressed = FoR::try_new(compressed, for_array.reference_scalar().clone())?; + for_compressed + .as_ref() + .statistics() + .inherit_from(for_array.as_ref().statistics()); + + Ok(for_compressed.into_array()) } } diff --git a/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs index 477cafb91fd..f94ed226a4a 100644 --- a/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs @@ -9,6 +9,8 @@ use std::sync::LazyLock; use rand::Rng; use rand::SeedableRng; use rand::rngs::StdRng; +use vortex_array::ArrayEq; +use vortex_array::EqMode; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::Constant; @@ -106,7 +108,7 @@ fn test_block_residual_ignores_null_payloads() -> VortexResult<()> { assert!(first.is::()); assert!(second.is::()); - assert_eq!(first.nbytes(), second.nbytes()); + assert!(first.array_eq(&second, EqMode::Value)); Ok(()) } diff --git a/vortex-btrblocks/src/schemes/mod.rs b/vortex-btrblocks/src/schemes/mod.rs index 0b9baab0b16..2aa2b5ad03f 100644 --- a/vortex-btrblocks/src/schemes/mod.rs +++ b/vortex-btrblocks/src/schemes/mod.rs @@ -37,6 +37,7 @@ use crate::schemes::integer::SparseScheme; const SAMPLE_BLOCK_LEN: usize = 64; const MIN_SAMPLE_BLOCKS: usize = 16; const SAMPLE_BLOCK_MULTIPLE: usize = 16; + fn sample_primitive_one_percent( primitive: ArrayView<'_, Primitive>, exec_ctx: &mut ExecutionCtx, diff --git a/vortex-btrblocks/src/trace_tests.rs b/vortex-btrblocks/src/trace_tests.rs index 668e5dbcf35..60aa85a429b 100644 --- a/vortex-btrblocks/src/trace_tests.rs +++ b/vortex-btrblocks/src/trace_tests.rs @@ -263,6 +263,11 @@ fn trace_scan_compare_on_compressed_quantity() -> VortexResult<()> { iter 4 current=vortex.dict(bool, len=4096) builder_active=false ExecuteSlot slot=1 parent=vortex.dict(bool, len=4096) child=vortex.binary(bool, len=50) iter 5 current=vortex.binary(bool, len=50) stack_parent=vortex.dict(bool, len=4096) slot=1 builder_active=false + execute_until target=AnyCanonical root=vortex.sequence(i16, len=50) + iter 0 current=vortex.sequence(i16, len=50) builder_active=false + Done array=vortex.primitive(i16, len=50) + iter 1 current=vortex.primitive(i16, len=50) builder_active=false + return output=vortex.primitive(i16, len=50) Done array=vortex.bool(bool, len=50) iter 6 current=vortex.bool(bool, len=50) stack_parent=vortex.dict(bool, len=4096) slot=1 builder_active=false pop_frame slot=1 output=vortex.dict(bool, len=4096) diff --git a/vortex-compressor/src/builtins/dict/float.rs b/vortex-compressor/src/builtins/dict/float.rs index 1de5b7ce986..f962c3ff967 100644 --- a/vortex-compressor/src/builtins/dict/float.rs +++ b/vortex-compressor/src/builtins/dict/float.rs @@ -165,13 +165,7 @@ macro_rules! typed_encode { }; let codes_validity = $source_array.validity()?; - let mut values = distinct - .distinct_values() - .iter() - .map(|(value, &count)| (value.0, count)) - .collect::>(); - values.sort_unstable_by_key(|&(_, count)| std::cmp::Reverse(count)); - let values: Buffer<$typ> = values.into_iter().map(|(value, _)| value).collect(); + let values: Buffer<$typ> = distinct.distinct_values().iter().map(|x| x.0).collect(); let max_code = values.len(); let codes = if max_code <= u8::MAX as usize { @@ -277,10 +271,10 @@ mod tests { #[test] fn test_float_dict_encode() -> VortexResult<()> { let mut ctx = vortex_array::array_session().create_execution_ctx(); - let values = buffer![1f32, 2f32, 2f32, 0f32, 2f32]; + let values = buffer![1f32, 2f32, 2f32, 0f32, 1f32]; let validity = Validity::Array(BoolArray::from_iter([true, true, true, false, true]).into_array()); - let array = PrimitiveArray::new(values, validity.clone()); + let array = PrimitiveArray::new(values, validity); let stats = FloatStats::generate_opts( &array, @@ -292,19 +286,9 @@ mod tests { let dict_array = dictionary_encode(array.as_view(), &stats)?; assert_eq!(dict_array.values().len(), 2); assert_eq!(dict_array.codes().len(), 5); - assert_arrays_eq!( - dict_array.values(), - PrimitiveArray::new(buffer![2f32, 1f32], Validity::AllValid).into_array(), - &mut ctx - ); - assert_arrays_eq!( - dict_array.codes(), - PrimitiveArray::new(buffer![1u8, 0, 0, 0, 0], validity).into_array(), - &mut ctx - ); let expected = PrimitiveArray::new( - buffer![1f32, 2f32, 2f32, 2f32, 2f32], + buffer![1f32, 2f32, 2f32, 1f32, 1f32], Validity::Array(BoolArray::from_iter([true, true, true, false, true]).into_array()), ) .into_array(); diff --git a/vortex-compressor/src/builtins/dict/integer.rs b/vortex-compressor/src/builtins/dict/integer.rs index 27955ededde..27a17ef94ad 100644 --- a/vortex-compressor/src/builtins/dict/integer.rs +++ b/vortex-compressor/src/builtins/dict/integer.rs @@ -157,13 +157,7 @@ macro_rules! typed_encode { }; let codes_validity = $source_array.validity()?; - let mut values = distinct - .distinct_values() - .iter() - .map(|(value, &count)| (value.0, count)) - .collect::>(); - values.sort_unstable_by_key(|&(_, count)| std::cmp::Reverse(count)); - let values: Buffer<$typ> = values.into_iter().map(|(value, _)| value).collect(); + let values: Buffer<$typ> = distinct.distinct_values().keys().map(|x| x.0).collect(); let max_code = values.len(); let codes = if max_code <= u8::MAX as usize { @@ -286,7 +280,7 @@ mod tests { let data = buffer![100i32, 200, 100, 0, 100]; let validity = Validity::Array(BoolArray::from_iter([true, true, true, false, true]).into_array()); - let array = PrimitiveArray::new(data, validity.clone()); + let array = PrimitiveArray::new(data, validity); let stats = IntegerStats::generate_opts( &array, @@ -298,16 +292,6 @@ mod tests { let dict_array = dictionary_encode(array.as_view(), &stats)?; assert_eq!(dict_array.values().len(), 2); assert_eq!(dict_array.codes().len(), 5); - assert_arrays_eq!( - dict_array.values(), - PrimitiveArray::new(buffer![100i32, 200], Validity::AllValid).into_array(), - &mut ctx - ); - assert_arrays_eq!( - dict_array.codes(), - PrimitiveArray::new(buffer![0u8, 1, 0, 0, 0], validity).into_array(), - &mut ctx - ); let expected = PrimitiveArray::new( buffer![100i32, 200, 100, 100, 100], diff --git a/vortex-compressor/src/compressor/cascade.rs b/vortex-compressor/src/compressor/cascade.rs index ce4e0f2f674..86d45d2c0d9 100644 --- a/vortex-compressor/src/compressor/cascade.rs +++ b/vortex-compressor/src/compressor/cascade.rs @@ -66,35 +66,6 @@ impl CascadingCompressor { Ok(compressed) } - /// Compresses an estimator sample after it removes one scheme from the current cascade. - /// - /// Fused schemes use this method to compare their output with the completed incumbent tree. - /// The method preserves the current cascade depth and marks the input as a sample. - /// - /// # Errors - /// - /// Returns an error if canonicalization or compression fails. - pub fn compress_sample_without_scheme( - &self, - array: &ArrayRef, - excluded: SchemeId, - compress_ctx: CompressorContext, - exec_ctx: &mut ExecutionCtx, - ) -> VortexResult { - let compressor = Self { - schemes: self - .schemes - .iter() - .copied() - .filter(|scheme| scheme.id() != excluded) - .collect(), - root_exclusions: self.root_exclusions.clone(), - }; - let canonical = array.clone().execute::(exec_ctx)?.0; - let compact = canonical.compact(exec_ctx)?; - compressor.compress_canonical(compact, compress_ctx.with_sampling(), exec_ctx) - } - /// Compresses a child array produced by a cascading scheme. /// /// If the cascade budget is exhausted, the canonical array is returned as-is. Otherwise, the diff --git a/vortex-compressor/src/stats/float.rs b/vortex-compressor/src/stats/float.rs index fa61f0c6506..c505993e9e5 100644 --- a/vortex-compressor/src/stats/float.rs +++ b/vortex-compressor/src/stats/float.rs @@ -19,22 +19,22 @@ use vortex_error::VortexResult; use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_mask::AllOr; -use vortex_utils::aliases::hash_map::HashMap; +use vortex_utils::aliases::hash_set::HashSet; use super::GenerateStatsOptions; /// Information about the distinct values in a float array. #[derive(Debug, Clone)] pub struct DistinctInfo { - /// The distinct float values and their occurrence counts. - distinct_values: HashMap, u32, FxBuildHasher>, + /// The set of distinct float values. + distinct_values: HashSet, FxBuildHasher>, /// The count of unique values. This _must_ be non-zero. distinct_count: u32, } impl DistinctInfo { - /// Returns a reference to the distinct values map. - pub fn distinct_values(&self) -> &HashMap, u32, FxBuildHasher> { + /// Returns a reference to the distinct values set. + pub fn distinct_values(&self) -> &HashSet, FxBuildHasher> { &self.distinct_values } } @@ -188,7 +188,7 @@ where average_run_length: 0, erased: TypedStats { distinct: Some(DistinctInfo { - distinct_values: HashMap::with_capacity_and_hasher(0, FxBuildHasher), + distinct_values: HashSet::with_capacity_and_hasher(0, FxBuildHasher), distinct_count: 0, }), } @@ -202,10 +202,12 @@ where .ok_or_else(|| vortex_err!("Failed to compute null_count"))?; let value_count = array.len() - null_count; + // Keep a HashMap of T, then convert the keys into PValue afterward since value is + // so much more efficient to hash and search for. let mut distinct_values = if count_distinct_values { - HashMap::with_capacity_and_hasher(array.len() / 2, FxBuildHasher) + HashSet::with_capacity_and_hasher(array.len() / 2, FxBuildHasher) } else { - HashMap::with_hasher(FxBuildHasher) + HashSet::with_hasher(FxBuildHasher) }; let validity = array @@ -225,7 +227,7 @@ where AllOr::All => { for value in first_valid_buff { if count_distinct_values { - *distinct_values.entry(NativeValue(value)).or_insert(0) += 1; + distinct_values.insert(NativeValue(value)); } if value != prev { @@ -242,7 +244,7 @@ where { if valid { if count_distinct_values { - *distinct_values.entry(NativeValue(value)).or_insert(0) += 1; + distinct_values.insert(NativeValue(value)); } if value != prev { diff --git a/vortex-compressor/src/trace.rs b/vortex-compressor/src/trace.rs index 0a7a4126cfc..ee594621199 100644 --- a/vortex-compressor/src/trace.rs +++ b/vortex-compressor/src/trace.rs @@ -120,7 +120,6 @@ pub(super) fn record_winner_compress_result( span.record("achieved_ratio", r); } span.record("accepted", accepted); - tracing::debug!(target: TARGET_TRACE, "winner.result"); } /// Records the final output size and, when finite, the top-level compression ratio. diff --git a/vortex-file/Cargo.toml b/vortex-file/Cargo.toml index 57f8ca7015b..44c7331b3d4 100644 --- a/vortex-file/Cargo.toml +++ b/vortex-file/Cargo.toml @@ -33,8 +33,8 @@ tracing = { workspace = true } url = { workspace = true } vortex-alp = { workspace = true } vortex-array = { workspace = true } -vortex-btrblocks = { workspace = true } vortex-block-residual = { workspace = true } +vortex-btrblocks = { workspace = true } vortex-buffer = { workspace = true } vortex-bytebool = { workspace = true } diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index 31ab75b9996..54aa0805ac3 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors #![expect(clippy::unwrap_used)] +#![expect(clippy::cast_possible_truncation)] use std::sync::LazyLock; use std::sync::atomic::AtomicUsize; diff --git a/vortex/src/editions/core/mod.rs b/vortex/src/editions/core/mod.rs index 84a0dcb8a05..f344baf6136 100644 --- a/vortex/src/editions/core/mod.rs +++ b/vortex/src/editions/core/mod.rs @@ -12,6 +12,7 @@ pub mod v2025_10; pub mod v2026_08; pub mod v2026_08_2; pub mod v2026_08_3; +pub mod v2026_08_4; pub use v2025_05::CORE_2025_05_0; pub use v2025_06::CORE_2025_06_0; @@ -20,3 +21,4 @@ pub use v2026_08::CORE_2026_08_0; pub use v2026_08::CORE_2026_08_1; pub use v2026_08_2::CORE_2026_08_2; pub use v2026_08_3::CORE_2026_08_3; +pub use v2026_08_4::CORE_2026_08_4; diff --git a/vortex/src/editions/core/v2026_08_4.rs b/vortex/src/editions/core/v2026_08_4.rs new file mode 100644 index 00000000000..c91d31d691f --- /dev/null +++ b/vortex/src/editions/core/v2026_08_4.rs @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The August 2026 draft core edition with numeric array encodings. + +use vortex_edition::Edition; +use vortex_edition::EditionDeclaration; +use vortex_edition::EditionId; +use vortex_edition::EditionMember; + +/// The fifth August 2026 edition of the `core` family. +pub const CORE_2026_08_4: EditionId = EditionId::new("core", 2026, 8, 4); + +/// The declaration of [`CORE_2026_08_4`] and its new components. +pub static DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: CORE_2026_08_4, + min_vortex_version: None, + }, + added: &[ + EditionMember::array(&"vortex.block_residual"), + EditionMember::array(&"vortex.float_quant"), + EditionMember::array(&"vortex.ordered_float"), + ], +}; diff --git a/vortex/src/editions/mod.rs b/vortex/src/editions/mod.rs index a6dd8ee7fe9..d770f14c165 100644 --- a/vortex/src/editions/mod.rs +++ b/vortex/src/editions/mod.rs @@ -42,6 +42,7 @@ pub use self::core::CORE_2026_08_0; pub use self::core::CORE_2026_08_1; pub use self::core::CORE_2026_08_2; pub use self::core::CORE_2026_08_3; +pub use self::core::CORE_2026_08_4; pub use self::preview::PREVIEW_2025_05_0; pub use self::preview::PREVIEW_2026_02_0; pub use self::preview::PREVIEW_2026_04_0; @@ -63,6 +64,7 @@ pub static EDITION_DECLARATIONS: &[&EditionDeclaration] = &[ &core::v2026_08::DECLARATION_1, &core::v2026_08_2::DECLARATION, &core::v2026_08_3::DECLARATION, + &core::v2026_08_4::DECLARATION, &preview::v2025_05::DECLARATION, &preview::v2026_02::DECLARATION, &preview::v2026_04::DECLARATION, diff --git a/vortex/src/editions/tests.rs b/vortex/src/editions/tests.rs index 9ec890bcaf0..c8ef1622e20 100644 --- a/vortex/src/editions/tests.rs +++ b/vortex/src/editions/tests.rs @@ -53,6 +53,7 @@ use super::CORE_2026_08_0; use super::CORE_2026_08_1; use super::CORE_2026_08_2; use super::CORE_2026_08_3; +use super::CORE_2026_08_4; use super::DEFAULT_CORE_EDITION; use super::DEFAULT_PREVIEW_EDITION; use super::EDITION_DECLARATIONS; @@ -193,6 +194,29 @@ fn core_2026_08_3_adds_variants() { ); } +#[test] +fn core_2026_08_4_adds_numeric_arrays() { + let session = session().unwrap_or_else(|e| panic!("registering editions: {e}")); + assert!( + session + .find(&CORE_2026_08_4) + .unwrap_or_else(|| panic!("{CORE_2026_08_4} is not registered")) + .is_draft() + ); + let arrays = session.components_in(&CORE_2026_08_4, ComponentKind::Array); + for id in [ + "vortex.block_residual", + "vortex.float_quant", + "vortex.ordered_float", + ] { + assert!( + arrays + .iter() + .any(|inclusion| inclusion.component_id.as_str() == id) + ); + } +} + #[test] fn encodings_in_editions_unions_families() { let session = session().unwrap_or_else(|e| panic!("registering editions: {e}")); @@ -696,10 +720,13 @@ async fn default_strategy_round_trip_uses_only_enabled_encodings() -> VortexResu } #[tokio::test] -async fn default_writer_round_trips_float_quant() -> VortexResult<()> { +async fn numeric_draft_writer_round_trips_float_quant() -> VortexResult<()> { use crate::VortexSessionDefault; let session = VortexSession::default(); + session + .enable_edition(CORE_2026_08_4) + .map_err(|error| vortex_err!("{error}"))?; let values = (0u32..65_536) .map(|index| { let mantissa = index.wrapping_mul(7_919) & 0x007f_ffff; @@ -726,7 +753,7 @@ async fn default_writer_round_trips_float_quant() -> VortexResult<()> { } #[tokio::test] -async fn default_writer_round_trips_ordered_block_residual() -> VortexResult<()> { +async fn numeric_draft_writer_round_trips_ordered_block_residual() -> VortexResult<()> { use crate::VortexSessionDefault; fn uniform(state: &mut u64) -> f64 { @@ -737,6 +764,9 @@ async fn default_writer_round_trips_ordered_block_residual() -> VortexResult<()> } let session = VortexSession::default(); + session + .enable_edition(CORE_2026_08_4) + .map_err(|error| vortex_err!("{error}"))?; let mut state = 0x4d59_5df4_d0f3_3173_u64; let mut value = 0.0_f64; let values = (0..65_536) From 6a234562231295c435de990428097d924b38dea9 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Fri, 21 Aug 2026 15:55:24 -0400 Subject: [PATCH 77/78] docs: remove native compression work log Signed-off-by: Will Manning --- docs/plans/native-pcodec.md | 2956 ----------------------------------- 1 file changed, 2956 deletions(-) delete mode 100644 docs/plans/native-pcodec.md diff --git a/docs/plans/native-pcodec.md b/docs/plans/native-pcodec.md deleted file mode 100644 index eeaa5894497..00000000000 --- a/docs/plans/native-pcodec.md +++ /dev/null @@ -1,2956 +0,0 @@ -# Native PCodec-inspired numeric compression - -## Goal - -Improve default numeric compression where ALP and ALP-RD lose to Pco. - -Keep native Vortex arrays, bounded metadata, fast canonical decode, and bounded random access. - -Pco remains a compression oracle. The new arrays do not preserve Pco byte compatibility. - -The production target is the default compressor, not Compact. - -Use Compact to identify numeric columns where the default compressor leaves a material size gap. - -For each repeated gap, identify the Pco model that creates the gain. - -Transfer that model into a native array only when it preserves these Default properties: - -- Full decode remains close to the displaced encoding. -- Compression throughput remains close to the displaced encoding. -- Scalar access remains O(1) or bounded O(log N). -- Selector cost remains proportionate to measured gains. -- Cheap rejection lowers the evidence bar, but it is not required. -- The size gain remains material after native array overhead. - -Compact can use experimental schemes during evaluation. Default selection remains the release decision. - -Measure each candidate against both the prior Default tree and the Compact tree. - -Use Compact gap recovery as the size metric. Use the displaced Default tree as the speed baseline. - -## Current decision - -Focus the production work on these encodings: - -- `OrderedFloatArray`. -- `BlockResidualArray` for all integer types, with one reference per 1,024-value block. -- `FloatQuantArray`. - -Keep `FloatQuantScheme`, `OrderedBlockResidualScheme`, and `BlockResidualScheme` as BtrBlocks candidates. - -Remove `FloatMultArray` and `FloatMultScheme` from the focused branch. - -Remove `RangeEntropyArray`, `RangeEntropyScheme`, and `BitSplitCodec` from the focused branch. - -The `wm/pcodec-entropy-experiments` branch preserves the complete entropy and bit-split prototypes. - -Remove `RangePackedArray` and its manual benchmark from the focused branch. - -The Git history and this plan preserve the fixed-bin experiment. - -The fixed-bin experiment used this composed tree: - -- `Dict(BitPacked(bin_codes), bin_starts)` reconstructs one reference per value. -- `BlockResidual(offsets)` stores each distance from the selected reference. -- `IntMult(base=1, references, offsets)` adds both components. - -The prototype accepted any bin count from one through 64. - -The experiment also tested an `IntegerFixedBinsScheme` below ALP. - -The complete corpus did not justify either scheme. The focused branch no longer contains either scheme. - -Do not add the fused RangePacked array to the Default or Compact selector. - -Do not add adjacent Delta, Delta-of-delta, Delta with lookback, or convolution Delta. - -## Current state - -The branch implements `OrderedFloatArray`, `BlockResidualArray`, and `FloatQuantArray`. - -The evaluation set includes BtrBlocks schemes for all three arrays. - -`OrderedFloat(BlockResidual)` now supports `f16`, `f32`, and `f64` inputs. - -The float scheme applies `OrderedFloat` first, then `BlockResidual` to the unsigned child. - -The serialized tree uses `OrderedFloat(BlockResidual(...))` because the outer array restores the float dtype. - -The FloatQuant candidate accepts native `f16`, `f32`, and `f64` inputs. - -Direct integer BlockResidual supports every integer type. The Default selector accepts every integer width. - -The retained schemes win on specific structures. They do not replace ALP or ALP-RD across general float data. - -FloatQuant now passes the speed gates for zero-secondary and one-bit-secondary inputs. - -The complete file corpus justifies FloatQuant in Default. - -OpenAI-on-C4 f64 embeddings select zero-secondary FloatQuant and shrink by 40.5 percent. - -BlockResidual now passes the direct speed gates for every integer width. - -The GloVe result identifies a separate entropy gap inside the ALP integer child. - -The previously tested quotient and remainder trees do not close that gap. - -The final calibration retains the 1.05 BlockResidual ratio floor. - -It retains the 1.02 ordered-float factor, 1.10 FloatQuant factor, 1.12 8-bit factor, and nonlinear patch cost. - -The range decomposition prototype used `IntMult(base=1)` as generic addition. - -Its fused decode path skipped multiplication and avoided dictionary reference materialization. - -No tested IntMult tree passed the Default evidence bar. The focused branch no longer contains `IntMultArray`. - -Decomposed fixed bins do not participate in the Default selector. - -The final calibration includes the complete corpus and selected-tree evidence. - -The Opportunistic bundle cuts aggregate size by 11.7 percent against Prior Default. - -The three array encodings belong to the draft `core:2026.08.4` edition. - -The default session still writes the frozen `core:2026.08.1` edition. - -Focused file tests and benchmarks enable the draft numeric edition explicitly. - -Future work targets repeated Compact mechanisms with native, bounded-access trees. - -## Array support and validation - -The array API and the Default selector use separate type policies. - -An array supports each natural logical type unless the transform has a structural restriction. - -The Default selector can exclude a supported type when measured costs do not justify selection. - -| Array | Supported logical types | Default policy | -| --- | --- | --- | -| OrderedFloat | `f16`, `f32`, `f64` | All float types remain eligible. | -| BlockResidual | `u8`, `u16`, `u32`, `u64`, `i8`, `i16`, `i32`, `i64` | All widths are eligible. | -| FloatQuant | `f16`, `f32`, `f64` | All float types remain eligible. | - -OrderedFloat validates the logical float type, unsigned child width, child nullability, child length, and empty metadata. - -FloatQuant validates the metadata version, split width, latent child types, child nullability, and child lengths. - -BlockResidual validates every payload offset table before decode or scalar access. - -It also validates bases, bit widths, packed word counts, patch counts, patch order, patch bounds, and validity length. - -Bit-exact tests cover signed zero values, infinities, and NaN payloads for every float width. - -Round-trip tests cover every signed and unsigned integer width. - -## OrderedFloatArray - -`OrderedFloatArray` maps IEEE float bits to unsigned integers with the same order. - -The transform preserves every bit pattern. It also preserves nulls and signed zero values. - -The array stores one unsigned child. Empty metadata identifies the transform. - -The array supports `f16`, `f32`, and `f64` values. - -It supports canonical decode, scalar access, slice reduction, serialization, and validation. - -## BlockResidualArray - -`BlockResidualArray` divides integers into independent blocks of 1,024 values. - -Unsigned values retain their bit pattern. Signed values first flip the sign bit to preserve numeric order. - -Each block stores one minimum value. Packed residuals store the difference from that minimum. - -Rare wide residuals use sorted positions and packed high bits. - -Scalar access uses one packed read and one binary search over the patch positions. - -One aligned parent buffer stores references, widths, offsets, packed words, patch positions, and patch high bits. - -Fixed metadata stores only lengths and slice bounds. - -Typed zero-copy views point into the parent buffer after a file read. - -The array supports canonical decode, scalar access, slice reduction, serialization, and validation. - -The `OrderedFloat(BlockResidual)` execute kernel combines residual decode with the inverse float transform. - -The residual payload uses the logical integer width. A 16-bit array uses 16-bit FastLanes pack and unpack operations. - -The serialized residual payload remains a `u64` section. The logical type defines the packed word interpretation. - -The production codec uses one reference per block. The multi-reference prototype did not justify its encode and decode costs. - -## FloatQuantArray - -`FloatQuantArray` splits each float into a primary quantum and a secondary adjustment. - -Metadata stores only the split width. The array dtype stores the source type. - -One or two child arrays store the integer latents. - -The BtrBlocks scheme uses a fixed `FoR(BitPacked)` primary tree. - -A common path uses `FloatQuant(FoR(BitPacked))` for `f32` values stored in `f64` columns. -An absent secondary child represents zero low bits. - -Nonzero low bits use `FloatQuant(FoR(BitPacked), BitPacked)`. - -The kernel unpacks both aligned children and reconstructs each float in one pass. - -The array supports exact IEEE bit-pattern round trips, nulls, slices, scalar access, and serialization. - -The automatic scheme accepts `f16`, `f32`, and `f64` inputs. - -Native `f32` and two-child selection meet the selected-column and rejected-column throughput limits. - -The final corpus pass validates the retained selector factors. - -## IntMult prototype - -The prototype reconstructed each integer as `base * primary + secondary`. - -It supported every signed and unsigned integer type. - -The primary and secondary children used compatible Vortex array encodings. - -The primary child owned validity. The secondary child was nonnullable. - -The prototype supported canonical decode, scalar access, slices, serialization, and validation. - -`IntMult::from_primitive` created quotient and remainder children with the source integer width. - -The tested selectors compressed both children through specialized or recursive integer schemes. - -IntMult did not own exception positions or exception payloads. - -Child encodings used `PatchArray`, `BlockResidual`, or other integer arrays when those trees won. - -When the base equaled one, bulk decode and scalar access skipped multiplication. - -For a dictionary primary, the fused path decoded the secondary into the output buffer. - -It then unpacked dictionary codes and added the selected reference directly. - -This path avoided a full materialized array of references. - -## Selection policy - -Fast rejection is a selection advantage for schemes in normal Default recursion. - -A cheap rejection path reads a bounded sample and avoids child compression when the model does not fit. - -This path lets Default test more encodings with little throughput loss on rejected columns. - -Fast rejection is not a hard gate. - -Do not use fast rejection as an admission criterion. - -A scheme with costly analysis can enter Default when stronger corpus evidence justifies its analysis cost. - -Specialized schemes remain useful when they bypass depth limits, recursive search, or an unfused common tree. - -Measure rejected-column cost separately from selected-column encode cost. - -`FloatQuantScheme` uses the normal sample comparison against ALP, ALP-RD, dictionary, sparse, and RLE schemes. - -The sample builds the exact one-child or two-child fixed tree. - -The prefilter tracks the bitwise union of all low float bits. - -For each split, it compares the removed bit count with the required secondary width. - -A candidate must save at least two fixed bits through the split. - -The prefilter uses a stratified one-percent sample. - -If the sample qualifies, the estimator builds the exact fixed tree on that sample. - -The final encoder maps float blocks directly into one or two FastLanes packers. - -This fused path does not allocate full primary or secondary integer buffers. - -The scheme does not call the recursive integer selector. - -`OrderedBlockResidualScheme` uses eight locality-preserving sample blocks. - -The ordinary BtrBlocks sample does not preserve the block-local float structure. - -The estimator runs the exact production width planner on native integer or float slices. - -It does not create packed payloads, patch payloads, temporary ordered integers, or child arrays. - -All-valid samples use direct block copies. Nullable samples retain the validity mask. - -`BlockResidualScheme` requires at least a 1.05 compression ratio before outer comparison. - -The nonlinear patch cost represents the measured decode cliff for dense patches. - -The selector does not apply another width-specific size factor. - -Direct 32-bit and 64-bit BlockResidual decode now matches or exceeds the main FoR paths. - -BlockResidual does not occur below ZigZag. That composition lost decode throughput on GloVe for a small size reduction. - -The ordered-float residual scheme requires a 1.05 compression ratio. - -Its adjusted score includes a 1.02 decode-cost factor. - -`BlockResidualScheme` uses the same locality probe for integer arrays. - -The Default scheme accepts every integer width. - -The scheme does not run inside trial compression for an outer scheme. Generic 64-row samples do not preserve 1,024-row locality. - -The selected outer scheme can still choose BlockResidual for its full child. - -The integer BlockResidual scheme does not use a width-specific decode-cost factor. - -The block planner adds 16 synthetic cost bits per patch. - -This cost selects the physical residual width. It preserves useful sparse-patch trees. - -A global 96-bit patch cost prevented the synthetic dense-patch regressions. It also removed useful sparse-patch compression on HashTags. - -The BtrBlocks estimator adds a separate nonlinear decode cost. - -For `p` patches and `n` values, it adds `320 * p * p / n` synthetic bits. - -This formula adds 80 bits per patch at 25 percent density. It adds 32 bits per patch at 10 percent density. - -The physical encoding remains size-optimal. The adjusted size only affects selection. - -The selector excludes BlockResidual from dictionary-code children. A complete BlockResidual tree can still displace a complete dictionary tree. - -Both schemes remain eligible to displace ALP or ALP-RD when their sample size scores win. - -## Single-encoding performance ledger - -This snapshot dates from 2026-08-20. - -Each benchmark uses two million logical values and 100 Divan samples. - -Encode and decode results report median logical input throughput. - -Scalar results report median random-access latency. - -### Primitive transforms - -These cases use canonical primitive children. They isolate each outer array transform. - -| Array and input | Encode | Decode | Scalar access | -| --- | ---: | ---: | ---: | -| OrderedFloat `f16` | 59.11 GB/s | 59.09 GB/s | 77 ns | -| OrderedFloat `f32` | 52.38 GB/s | 60.47 GB/s | 82 ns | -| OrderedFloat `f64` | 58.78 GB/s | 42.78 GB/s | 90 ns | -| IntMult base-ten `i8` | 0.72 GB/s | 29.98 GB/s | 125 ns | -| IntMult base-ten `i16` | 1.41 GB/s | 29.68 GB/s | 105 ns | -| IntMult base-ten `i32` | 2.82 GB/s | 29.84 GB/s | 100 ns | -| IntMult base-ten `i64` | 5.57 GB/s | 24.80 GB/s | 169 ns | -| IntMult base-ten `u8` | 0.70 GB/s | 29.63 GB/s | 125 ns | -| IntMult base-ten `u16` | 1.42 GB/s | 30.59 GB/s | 160 ns | -| IntMult base-ten `u32` | 2.84 GB/s | 30.51 GB/s | 136 ns | -| IntMult base-ten `u64` | 5.66 GB/s | 24.57 GB/s | 150 ns | -| FloatQuant zero-secondary `f16` | 7.62 GB/s | 33.02 GB/s | 83 ns | -| FloatQuant zero-secondary `f32` | 13.63 GB/s | 33.58 GB/s | 75 ns | -| FloatQuant zero-secondary `f64` | 20.90 GB/s | 26.88 GB/s | 82 ns | -| FloatQuant one-bit-secondary `f16` | 2.41 GB/s | 2.98 GB/s | 118 ns | -| FloatQuant one-bit-secondary `f32` | 4.87 GB/s | 4.57 GB/s | 136 ns | -| FloatQuant one-bit-secondary `f64` | 9.18 GB/s | 8.23 GB/s | 148 ns | - -IntMult encode includes quotient and remainder creation. It excludes compression of both children. - -FloatQuant zero-secondary encode uses separate validation and transform passes. - -The prior fallible per-value loop reached 1.99 GB/s for `f32` and 3.72 GB/s for `f64`. - -The two-pass implementation improved those results by 5.9 times and 5.1 times. - -### Production child trees - -These cases include the current compressed children and fused decode paths. - -| Array tree and input | Encode | Decode | Scalar access | -| --- | ---: | ---: | ---: | -| BlockResidual `i8` | 0.56 GB/s | 30.53 GB/s | 44 ns | -| BlockResidual `i16` | 1.40 GB/s | 31.31 GB/s | 39 ns | -| BlockResidual `i32` | 2.72 GB/s | 33.89 GB/s | 40 ns | -| BlockResidual `i64` | 4.93 GB/s | 35.88 GB/s | 42 ns | -| BlockResidual `u8` | 0.55 GB/s | 30.14 GB/s | 35 ns | -| BlockResidual `u16` | 1.31 GB/s | 32.55 GB/s | 61 ns | -| BlockResidual `u32` | 2.67 GB/s | 46.61 GB/s | 48 ns | -| BlockResidual `u64` | 4.97 GB/s | 35.56 GB/s | 40 ns | -| IntMult with packed children `u8` | 0.70 GB/s | 24.81 GB/s | 125 ns | -| IntMult with packed children `u16` | 1.38 GB/s | 22.90 GB/s | 125 ns | -| IntMult with packed children `u32` | 2.64 GB/s | 22.62 GB/s | 140 ns | -| IntMult with packed children `u64` | 4.34 GB/s | 17.96 GB/s | 125 ns | -| OrderedFloat with BlockResidual `f16` | 1.26 GB/s | 26.36 GB/s | 79 ns | -| OrderedFloat with BlockResidual `f32` | 2.43 GB/s | 29.64 GB/s | 78 ns | -| OrderedFloat with BlockResidual `f64` | 2.70 GB/s | 20.44 GB/s | 125 ns | -| FloatQuant with packed primary `f16` | 3.21 GB/s | 20.05 GB/s | 125 ns | -| FloatQuant with packed primary `f32` | 5.72 GB/s | 19.79 GB/s | 129 ns | -| FloatQuant with packed primary `f64` | 7.76 GB/s | 16.19 GB/s | 125 ns | -| FloatQuant with two packed children `f16` | 3.11 GB/s | 16.75 GB/s | 174 ns | -| FloatQuant with two packed children `f32` | 5.17 GB/s | 16.30 GB/s | 183 ns | -| FloatQuant with two packed children `f64` | 6.72 GB/s | 13.13 GB/s | 208 ns | -| Decomposed fixed bins `u64` | 1.20 GB/s | 14.27 GB/s | 332 ns | - -The FloatQuant production encode rows use the single-scheme compressor. - -They include sample analysis and direct packing of the final child tree. - -The implicit-zero `f64` decoder maps FastLanes output directly to reconstructed floats. - -A controlled five-second comparison measured 14.08 GB/s before fusion and 16.19 GB/s after it. - -The fused path improves median decode throughput by 15.0 percent. - -The same trial reduced `f16` and `f32` throughput by 2 to 3 percent. Those types retain the generic path. - -The paired decoder remains within 18 percent of the zero-secondary decoder for every float width. - -The packed IntMult tree uses a base of ten and patch-free BitPacked children. - -The fused pair decoder unpacks both children directly into the reconstructed output. - -It removes the full secondary buffer and the separate multiplication pass. - -The paired `u64` benchmark improved decode throughput from 13.10 GB/s to 17.73 GB/s. - -This change increased decode throughput by 35.3 percent. - -The decomposed fixed-bin row uses ten separated clusters and a BlockResidual offset child. - -The fixed-bin result still lacks complete Default and Compact comparisons on real columns. - -## Performance requirements - -A default candidate must meet these limits: - -- Compression throughput can regress by at most 20 percent on a selected column. -- Full-decode throughput can regress by at most 20 percent. -- Scalar access must remain bounded and competitive with the displaced encoding. -- The selected candidate must reduce size materially. -- Rejected candidates must add little analysis cost. - -Use at least two million logical rows for throughput and scalar-access decisions. - -Exclude source-array construction and storage input from codec throughput measurements. - -## Evidence for the retained candidates - -### FloatQuant on widened f32 values - -The input contains two million arbitrary `f32` values stored in an `f64` column. - -| Configuration | Bytes | Compression MB/s | Decode MB/s | Scalar access ns | -| --- | ---: | ---: | ---: | ---: | -| Prior default | 14,057,966 | 534.4 | 10,072.7 | 208.7 | -| Default with FloatQuant | 8,753,920 | 565.4 | 14,032.5 | 145.5 | -| Compact | 6,051,737 | 272.2 | 3,912.7 | Not measured | - -FloatQuant reduced size by 37.7 percent. It remained 44.7 percent larger than Compact. - -Full compressor throughput increased by 5.8 percent. Decode throughput increased by 39.3 percent. - -Scalar access latency decreased by 30.3 percent. - -The selected tree was `FloatQuant(FoR(BitPacked))` with an implicit-zero secondary. - -The fused FloatQuant scheme compressed at 4,799 MB/s. - -It decoded between 14,480 and 14,810 MB/s. - -Compact Pco compressed the same input at 272 MB/s and decoded it at 3,913 MB/s. - -The scheme exceeded Compact throughput by 17.6 times for compression and 3.6 times for decode. - -FloatQuant recovered 66.2 percent of the size gap between the prior default and Compact Pco. - -The direct sample tree removed the recursive integer selector from the estimate and final compression paths. - -FloatQuant meets the selected-column throughput limit on this input. - -### FloatQuant with a nonzero secondary - -The input changed the lowest bit for ten percent of the widened-`f32` values. - -The fixed tree was `FloatQuant(FoR(BitPacked), BitPacked)`. The secondary used one bit per value. - -| Configuration | Bytes | Encode MB/s | Decode MB/s | Scalar access ns | -| --- | ---: | ---: | ---: | ---: | -| Prior ALP-RD default | 14,057,966 | 558.0 | 11,420 | 250 | -| Default with FloatQuant | 9,004,032 | 603.2 | 13,060 | 209 | -| Compact Pco | 6,171,139 | 255.5 | 2,914 | Not measured | - -FloatQuant reduced size by 36.0 percent. It was 2.9 percent larger than the zero-secondary tree. - -FloatQuant recovered 64.1 percent of the size gap between ALP-RD and Compact Pco. - -The first generic decode path reached 8,375 MB/s. - -The fused pair kernel increased decode throughput by 59.4 percent. - -The complete default encoded 8.1 percent faster and decoded 14.4 percent faster than the prior default. - -Scalar access latency decreased by 16.4 percent. - -The direct two-child scheme compressed at 6,298 MB/s. - -The fused kernel supports aligned, patch-free BitPacked secondary children of any width. - -Decode throughput was 13,010 MB/s with a one-bit secondary. - -It was 12,410 MB/s with a 16-bit secondary. - -The real profile selected this tree only for `HashTags_1.twitter#id`. - -It reduced that column by 6.1 percent against ALP-RD. - -OrderedFloat with BlockResidual reduced the column by 14.5 percent and won the final comparison. - -No other tested Pcodec or Public BI column selected the two-child tree. - -Rejected analysis changed encode throughput by less than one percent on most measured datasets. - -Retain the two-child form in the default candidate set for final corpus calibration. - -### All-width two-child FloatQuant revalidation - -The August 20 pass added two-child benchmarks for `f16` and `f32`. - -Each case changes the lowest bit for ten percent of two million quantized values. - -| Input | Incumbent bytes | Candidate bytes | Size change | Encode change | Decode change | -| --- | ---: | ---: | ---: | ---: | ---: | -| `f16` | 1,751,040 | 1,751,040 | 0.0 percent | -0.3 percent | +0.2 percent | -| `f32` | 5,752,576 | 4,001,792 | -30.4 percent | +29.6 percent | +43.1 percent | -| `f64` | 14,057,966 | 9,004,032 | -36.0 percent | +10.0 percent | +14.2 percent | - -The `f16` selector retains `Dict(BitPacked, Primitive)`. - -The `f32` and `f64` selectors choose `FloatQuant(FoR(BitPacked), BitPacked)`. - -The paired decoder stays within 18 percent of the zero-secondary decoder for all three widths. - -Paired scalar access uses 174 ns for `f16`, 183 ns for `f32`, and 208 ns for `f64`. - -The two selected complete trees improve size, encode throughput, and decode throughput. - -This evidence supports the opportunistic FloatQuant bundle despite the zero-selection real corpus pass. - -### FloatQuant rejection cost - -The August 20 pass added isolated Default controls that differ only by FloatQuant. - -Each benchmark compresses two million values with 100 samples. - -| Input | Default without FloatQuant | Default with FloatQuant | Time change | -| --- | ---: | ---: | ---: | -| General `f16` | 4.383 ms | 4.348 ms | Noise | -| General `f32` | 17.84 ms | 18.03 ms | +1.1 percent | -| General `f64` | 23.62 ms | 23.43 ms | Noise | -| Near-miss `f32` | 23.77 ms | 23.78 ms | Noise | -| Near-miss `f64` | 20.67 ms | 20.58 ms | Noise | - -The near-miss inputs pass FloatQuant transform analysis but fail the adjusted ratio test. - -The estimator now computes the exact padded FastLanes buffer size and validity size from the analysis. - -It no longer packs the sample payload only to count its bytes. - -A focused test verifies the estimate against complete `f16`, `f32`, and `f64` trees. - -The test includes a nullable two-child case. - -The optimization did not produce a measurable complete-compressor change on the near-miss controls. - -Sample construction, statistics, and the other schemes dominate those complete times. - -The isolated controls place the marginal rejected cost between noise and 1.1 percent. - -The 16-file pass remains the broader result. It measured a 0.9 percent write-throughput cost. - -### OrderedFloat with BlockResidual on random walks - -| Configuration | Bytes | Encode MB/s | Decode MB/s | Scalar access ns | -| --- | ---: | ---: | ---: | ---: | -| Prior default | 12,255,488 | 589.8 | 12,438.1 | 207.7 | -| Default with the scheme | 10,425,690 | 529.6 | 21,728.1 | 166.7 | -| Compact Pco | 9,342,749 | 318.1 | 4,559.8 | Not measured | - -The residual scheme saved 14.9 percent against the prior default. It remained 11.6 percent larger than Compact. - -Decode throughput increased by 74.7 percent. Random access latency decreased by 19.7 percent. - -Compression throughput decreased by 10.2 percent on the selected column. - -The isolated tree compressed at 2,969 MB/s and decoded at 20,870 MB/s. - -Compact Pco compressed the same input at 318 MB/s and decoded it at 4,560 MB/s. - -Ordered BlockResidual recovered 62.8 percent of the size gap between ALP-RD and Compact Pco. - -The selector rejected the scheme on Gaussian, lognormal, decimal, widened-f32, and four-cluster inputs. - -The rejected locality probe reduced compression throughput by 0.5 to 2.2 percent. - -The HashTags dataset selected this scheme for two columns. - -It reduced the numeric subset by 6.0 percent with no isolated compressor regression. - -### Integer BlockResidual throughput - -The direct benchmark uses two million block-local values. - -| Logical type and tree | Encode GB/s | Decode GB/s | Scalar access ns | -| --- | ---: | ---: | ---: | -| `u64` BlockResidual | 5.11 | 37.39 | 84 | -| `u64` FoR plus BitPacked | 4.51 | 36.22 | 125 | -| `i16` BlockResidual | 1.39 | 31.86 | 125 | -| `i16` FoR plus BitPacked | 1.15 | 42.14 | 125 | - -The first `i16` implementation unpacked through `u64` residuals. It decoded at 11.43 GB/s. - -Native-width unpack increased `i16` decode throughput by 2.78 times. - -The synthetic `i16` BlockResidual tree uses 1,793,784 bytes. The FoR plus BitPacked tree uses 3,500,000 bytes. - -BlockResidual is 48.7 percent smaller on that input. - -Its `i16` decode throughput is 24.4 percent lower, but it still exceeds 31 GB/s. - -The absolute decode rate makes this a useful Default trade. The selector now includes 16-bit integers. - -### 16-bit selector revalidation - -The complete synthetic control uses the production compressor without experimental Delta. - -| Configuration | Bytes | Encode MB/s | Decode MB/s | -| --- | ---: | ---: | ---: | -| Prior Default | 3,501,568 | 599.4 | 38,415 | -| Default with 16-bit BlockResidual | 1,793,269 | 653.8 | 31,600 | - -The selected tree reduces size by 48.8 percent and increases encode throughput by 9.1 percent. - -Decode throughput decreases by 17.7 percent, but it remains 31.6 GB/s. - -Three real files selected direct or nested 16-bit BlockResidual trees. - -| Dataset | 32-bit and 64-bit bundle | Bundle with 16-bit | Size change | Encode change | Decode change | -| --- | ---: | ---: | ---: | ---: | ---: | -| Bimbo | 10,573,121 | 9,671,397 | -8.53 percent | -1.01 percent | -2.20 percent | -| Food | 12,297,725 | 11,443,009 | -6.95 percent | -1.28 percent | -5.66 percent | -| HashTags | 21,283,222 | 21,254,341 | -0.14 percent | +1.78 percent | +8.81 percent | -| Geometric mean | — | — | -5.27 percent | -0.18 percent | +0.13 percent | - -The throughput changes come from adjacent runs and include ordinary benchmark noise. - -The exact size gains and high absolute decode rates support 16-bit Default eligibility. - -### 8-bit selector validation - -The direct benchmark already showed 30.53 GB/s decode and 44 ns scalar access for `i8`. - -The complete synthetic control uses two million block-local `i8` values. - -| Configuration | Bytes | Encode MB/s | -| --- | ---: | ---: | -| Prior Default with Primitive | 2,000,000 | 820.6 | -| Default with BlockResidual | 1,043,448 | 316.3 | - -BlockResidual reduces size by 47.8 percent and decodes at 29.76 GB/s. - -The selected tree has a material encode cost because the prior Default stores the input without compression. - -A uniform `i8` control rejects BlockResidual and retains Primitive at 2,000,000 bytes. - -The rejected analysis changes encode throughput from 720.5 MB/s to 706.3 MB/s, a 2.0 percent decrease. - -The original eight numeric files contain no 8-bit columns. - -A second pass quantizes 20 million real GloVe values into six `i8` and `u8` variants. - -It contains these quantizers: - -- Symmetric quantization uses the maximum absolute corpus value. The unsigned form adds 128. -- Global affine quantization maps the corpus minimum and maximum to 0 and 255. The signed form subtracts 128. -- Per-vector affine quantization uses each 200-value vector minimum and maximum. The signed form subtracts 128. - -The global affine `i8` and `u8` variants select BlockResidual. The unsigned symmetric variant also selects BlockResidual. - -The signed symmetric variant retains ZigZag with BitPacked. Both per-vector affine variants retain Primitive. - -| Selected variant | Prior bytes | BlockResidual bytes | Size change | Encode MB/s | Decode MB/s | Scalar access ns | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| Global affine `i8` | 20,000,000 | 16,775,243 | -16.12 percent | 277.8 | 21,791 | 80 | -| Global affine `u8` | 20,000,000 | 16,775,243 | -16.12 percent | 278.3 | 20,745 | 82 | -| Symmetric `u8` | 20,000,000 | 15,558,250 | -22.21 percent | 294.4 | 32,870 | 78 | - -The isolated Primitive encoders reach 754 to 777 MB/s on these selected columns. - -The complete writer reduces that isolated encode gap: - -| File | Size change | Write time change | Read throughput | -| --- | ---: | ---: | ---: | -| Global affine `i8` only | -16.12 percent | +10.06 percent | 17.3 GB/s | -| Two `i8` variants | -8.06 percent | +0.21 percent | 30.7 GB/s | -| Four `u8` variants | -10.19 percent | +1.62 to +2.25 percent | 35.4 GB/s | - -The single-column result isolates the maximum measured writer cost. The multi-column results represent mixed accepted and rejected candidates. - -A patch-free boundary control uses one seven-bit local range per block. - -It reduces complete file size by 10.35 percent and increases write time by 1.43 percent. - -Its complete read throughput is 31.7 GB/s. Scalar access takes 81 ns for `u8` and 110 ns for `i8`. - -Residual widths change in whole bits. Therefore, the first patch-free 8-bit win already saves about ten percent after file overhead. - -The patch-density adjustment protects cases between those discrete widths. - -The bulk and writer results support Default eligibility for `i8` and `u8`. - -The file random-access pass adds a material selection trade: - -| Input | Size change | Prior correlated | BlockResidual correlated | Prior uniform | BlockResidual uniform | -| --- | ---: | ---: | ---: | ---: | ---: | -| Global affine `i8` | -16.12 percent | 78.6 us | 156.1 us | 497.7 us | 835.6 us | -| Symmetric `u8` | -22.16 percent | 79.5 us | 136.4 us | 490.6 us | 708.1 us | -| Seven-bit boundary | -10.35 percent | 112.7 us | 174.3 us | 785.3 us | 1,014.3 us | - -Each pattern requests about 100 rows through a cached file handle. - -The candidate increases correlated latency by 55 to 99 percent. It increases uniform latency by 29 to 68 percent. - -Every measured lookup remains below 1.1 ms. Sparse patches make the global affine case the slowest relative result. - -The selector keeps the common 1.05 raw-ratio floor. It applies a 1.12 access cost factor to 8-bit estimates. - -The factor rejects the seven-bit boundary case. That case saved 10.35 percent and increased random-access latency by 29 to 55 percent. - -The factor retains the global affine `i8` and `u8` variants. It also retains the unsigned symmetric variant. - -These retained variants save 16.12 to 22.21 percent. The measured file-access latency remains below one millisecond for about 100 requested rows. - -### Narrow BlockResidual in Compact - -The Compact comparison uses the same two million block-local `i16` values. - -| Tree | Bytes | Encode MB/s | Decode MB/s | -| --- | ---: | ---: | ---: | -| FoR plus BitPacked | 3,501,568 | 1,129 | 40,970 | -| BlockResidual | 1,793,784 | 1,361 | 31,330 | -| Compact Pco | 241,832 | 391 | 1,680 | - -BlockResidual uses 7.4 times as many bytes as Pco. - -It encodes 3.5 times faster and decodes 18.6 times faster than Pco. - -Compact gives priority to the Pco size point. Narrow BlockResidual therefore has no Compact-only role. - -The 48.8 percent saving against FoR plus BitPacked supports one more narrow decode optimization pass. - -The BlockResidual estimator measures its sampled tree exactly, with all payload sections. - -The incumbent and outer tree estimates remain approximate. Trial compression previously mis-ranked Dict and ALP on Taxi tips. - -The outer-sample exclusion removed that error from the measured tree. - -### Broad numeric revalidation before patch-density calibration - -This table predates the nonlinear patch cost. The later patch-density section contains the current HashTags result. - -The focused run uses two million rows when the source contains that many rows. - -The integer-only configuration adds `BlockResidualScheme` to the prior default. - -| Dataset | Prior bytes | Integer BlockResidual bytes | Complete default bytes | Integer size change | Complete size change | -| --- | ---: | ---: | ---: | ---: | ---: | -| California Housing | 307,427 | 301,125 | 301,125 | -2.0 percent | -2.0 percent | -| NYC Taxi | 52,407,972 | 52,407,972 | 52,407,972 | 0.0 percent | 0.0 percent | -| CMS Payments | 30,061,670 | 28,472,040 | 28,472,040 | -5.3 percent | -5.3 percent | -| Arade | 26,937,026 | 26,937,026 | 26,937,026 | 0.0 percent | 0.0 percent | -| Euro2016 | 44,698,131 | 39,566,892 | 39,566,892 | -11.5 percent | -11.5 percent | -| Food | 13,579,790 | 13,579,790 | 13,579,790 | 0.0 percent | 0.0 percent | -| HashTags | 22,602,522 | 22,193,089 | 21,018,949 | -1.8 percent | -7.0 percent | - -Across these numeric inputs, integer BlockResidual reduced size by 3.7 percent. - -Its aggregate encode throughput decreased by 0.4 percent. Its aggregate decode throughput increased by 1.7 percent. - -California Housing contains only 20,433 rows. Fixed analysis cost dominates its encode result. - -The complete default reduced aggregate size by 4.4 percent. Encode throughput decreased by 1.2 percent. - -Aggregate decode throughput increased by 2.3 percent. - -Euro2016 reduced size by 11.5 percent and increased decode throughput by 10.2 percent. - -CMS reduced size by 5.3 percent and increased decode throughput by 5.5 percent. - -HashTags reduced size by 7.0 percent and increased decode throughput by 4.9 percent. - -The direct narrow-type exclusion changed no selected tree in this corpus. - -Before that exclusion, an earlier 1.40 trial factor rejected each direct `i16` candidate. - -A FastLanes fused FoR decode trial did not improve `u64` throughput. It reduced `i16` throughput, so the implementation retains native unpack. - -The zero-width residual path now writes base values and patches directly. It skips the scratch residual block. - -### Native f32 OrderedFloat with BlockResidual - -The input contains two million f32 values with narrow ordered-bit ranges inside each 1,024-value block. - -| Operation | Result | -| --- | ---: | -| Encode | 2.45 GB/s | -| Decode | 30.30 GB/s | -| Scalar access | 167 ns | - -The fused decode now handles `OrderedFloat(BlockResidual)` trees with u32 latents. - -The BtrBlocks candidate now accepts f32 and f64 inputs. - -These results remove the prior f32 type exclusion. Corpus comparisons against ALP and ALP-RD remain necessary. - -### Native f32 FloatQuant - -The input contains two million native `f32` values with eight zero low mantissa bits. - -| Configuration | Selected tree | Bytes | Encode MB/s | Decode MB/s | -| --- | --- | ---: | ---: | ---: | -| Prior default | `ALP-RD(BitPacked, BitPacked)` | 5,752,576 | 664.7 | 10,284.4 | -| Default with FloatQuant | `FloatQuant(FoR(BitPacked))` | 3,751,680 | 745.4 | 18,461.6 | -| FloatQuant only | `FloatQuant(FoR(BitPacked))` | 3,751,680 | 746.6 | 18,169.8 | -| Compact Pco | `Pco` | 201,374 | 294.2 | 2,885.0 | - -FloatQuant reduced size by 34.8 percent against ALP-RD. - -Decode throughput increased by 79.5 percent. - -Full compression throughput increased by 12.1 percent in the interleaved compressor benchmark. - -The fused FloatQuant scheme compressed at 2.33 GB/s. - -The prior materialized tree compressed at 1.25 GB/s in the same direct benchmark. - -The proposed default compressed at 753 MB/s in the Divan benchmark. - -The prior default compressed at 776 MB/s in that benchmark. The difference is 2.9 percent. - -On rejected general `f32` data, the proposed default compressed at 390 MB/s. - -The prior default compressed at 394 MB/s. The difference is 1.2 percent. - -The Compact size is specific to this synthetic pattern. It does not establish a general real-float result. - -### Native f16 coverage - -The audit added `f16` support to OrderedFloat, OrderedFloat with BlockResidual, FloatQuant, and FloatQuantScheme. - -The test corpus covers zero low bits, nonzero secondary bits, signed zero values, infinities, and NaN payloads. - -The quantized input contains two million `f16` values with four zero low mantissa bits. - -| Configuration | Bytes | Encode MB/s | Decode MB/s | -| --- | ---: | ---: | ---: | -| Prior default | 1,500,800 | 368.1 | 7,268 | -| Default with FloatQuant | 1,500,672 | 1,018 | 19,450 | - -The selected Default tree is `FloatQuant(FoR(BitPacked))`. - -It uses nearly the same space as the prior dictionary-like tree. - -Compression throughput increased by 2.77 times. Decode throughput increased by 2.68 times. - -On general rejected `f16` values, median compression throughput decreased by 1.8 percent. - -The first `f16` BlockResidual tree decoded at 20.50 GB/s. - -A later same-benchmark comparison measured 24.31 GB/s before a fused narrow transform and 26.36 GB/s after it. - -The fused path improves median decode throughput by 8.4 percent. It reconstructs residuals and float bits in one output pass. - -Retain `f16` eligibility. Revisit its selector factors during final corpus calibration. - -### Direct integer comparison after decode specialization - -The direct benchmark compares two million block-local values against a whole-column FoR plus BitPacked tree. - -The `u32` decode path now unpacks residuals and adds the block base directly into the output buffer. - -Signed integers and ordered floats still require a transform after residual decode. - -| Type and tree | Decode GB/s | Difference | -| --- | ---: | ---: | -| `u32` BlockResidual | 46.27 | +16.1 percent | -| `u32` FoR plus BitPacked | 39.85 | Baseline | -| `i32` BlockResidual | 33.86 | +31.8 percent | -| `i32` FoR plus BitPacked | 25.69 | Baseline | -| `u64` BlockResidual | 36.88 | +0.6 percent | -| `u64` FoR plus BitPacked | 36.65 | Baseline | -| `i16` BlockResidual | 32.34 | -22.2 percent | -| `i16` FoR plus BitPacked | 41.59 | Baseline | - -The specialization removes the prior `u32` decode disadvantage. - -Native-width unpack leaves a percentage gap for `i16`, but its absolute decode rate remains high. - -One width factor does not predict both signed and unsigned results. The integer selector uses no width-specific factor. - -### Serialized BlockResidual topology - -The first serialized layout stored nine primitive child arrays per BlockResidual array. - -The codec kernels remained fast, but the file reader executed nine extra array nodes for each chunk. - -An intermediate layout replaced those children with nine direct parent buffers. - -That layout improved file decode, but each buffer still produced a separate file segment. - -The final layout stores every table and payload in one aligned parent buffer. - -The section order preserves native alignment without padding: - -1. Block references and residual words use 64-bit sections. -2. Residual and patch offsets use 32-bit sections. -3. Patch positions use one 16-bit section. -4. Widths and packed patch high bits use byte sections. - -Typed zero-copy views share the same allocation. Scalar access and bulk decode read those views directly. - -The single-encoding benchmark uses two million block-local values. - -| Logical type | Encode GB/s | Decode GB/s | Scalar access ns | -| --- | ---: | ---: | ---: | -| `u64` | 5.23 | 36.87 | 83 | -| `u32` | 2.79 | 46.69 | 38 | -| `i32` | 2.78 | 32.23 | 41 | -| `i16` | 1.42 | 32.18 | 83 | -| `i8` | 0.56 | 30.53 | 44 | - -The 16-bit path now meets the revised size and absolute-throughput preference. - -The 8-bit path also clears the decode and scalar-access gates. - -The broad file benchmark uses three iterations across eight datasets. - -Positive throughput changes indicate faster execution. - -| Dataset | Size change | Encode throughput change | Decode throughput change | -| --- | ---: | ---: | ---: | -| Taxi | -3.24 percent | -1.06 percent | +2.51 percent | -| GloVe | 0.00 percent | +2.45 percent | +0.53 percent | -| Arade | -0.19 percent | +0.47 percent | +1.72 percent | -| Bimbo | -1.38 percent | +0.57 percent | +0.05 percent | -| CMSprovider | -1.41 percent | +0.20 percent | -0.06 percent | -| Euro2016 | -2.68 percent | +1.68 percent | -5.74 percent | -| Food | -1.76 percent | +0.50 percent | -3.50 percent | -| HashTags | -0.63 percent | +1.77 percent | -9.71 percent | -| Geometric mean | -1.42 percent | +0.82 percent | -1.86 percent | - -Five datasets decoded at parity or faster. - -The focused five-iteration runs showed smaller HashTags decode losses of 3.7 to 5.6 percent. - -The focused Euro2016 decode loss ranged from 3.8 to 5.1 percent. - -Focused encode throughput remained within one percent of the prior default. - -This layout recovers most of the Compact-like size gain without a material aggregate throughput loss. - -The remaining threshold calibration must use selected-column evidence and the complete corpus. - -### BlockResidual selector calibration - -The final numeric trial removes the prior 1.10 and 1.20 width factors. - -The 1.05 minimum ratio and nonlinear patch cost remain active. - -An initial trial selected `ALP(ZigZag(BlockResidual))` on GloVe. - -That tree reduced size by 1.2 percent and reduced decode throughput by 19.9 percent. - -No other selected corpus tree placed BlockResidual below ZigZag. An ancestor exclusion removes that composition. - -The final run reads two million rows from seven Public BI files and GloVe. - -| Dataset | Size change | Encode throughput change | Decode throughput change | -| --- | ---: | ---: | ---: | -| Arade | -3.13 percent | -4.44 percent | +8.80 percent | -| Bimbo | -2.97 percent | -0.36 percent | +1.78 percent | -| CMSprovider 1 | -1.63 percent | -3.42 percent | -1.29 percent | -| CMSprovider 2 | -1.56 percent | -3.32 percent | -0.65 percent | -| Euro2016 | -11.47 percent | -3.61 percent | +6.23 percent | -| Food | -6.89 percent | -0.85 percent | -4.00 percent | -| HashTags | -5.83 percent | -0.47 percent | +4.36 percent | -| GloVe | 0.00 percent | +0.43 percent | +3.63 percent | -| Geometric mean | -4.25 percent | -2.02 percent | +2.28 percent | - -The prior width factors reduced size by 2.28 percent in the same eight-dataset scope. - -They changed encode throughput by -0.49 percent and decode throughput by +3.30 percent. - -The new policy recovers another 1.97 percent of geometric-mean size. - -It keeps every dataset throughput change far inside the 20-percent gate. - -The complete file corpus remains necessary before the policy becomes final. - -### Compact transfer baseline - -The local corpus comparison uses three iterations on all 16 available datasets. - -The proposed Default includes FloatQuant and BlockResidual. The prior Default excludes all new numeric schemes. - -The numeric scope contains eight numeric datasets. The real scope adds two TPC-H comment variants. - -Positive throughput changes indicate faster execution. - -| Scope | Size change | Encode throughput change | Decode throughput change | Proposed gap above Compact | Proposed gap above Parquet with Zstd | -| --- | ---: | ---: | ---: | ---: | ---: | -| Numeric, 8 datasets | -0.84 percent | -0.35 percent | -2.10 percent | +40.84 percent | +1.46 percent | -| Real, 10 datasets | -1.12 percent | -0.47 percent | -1.85 percent | +37.92 percent | +2.28 percent | -| All, 16 datasets | -0.70 percent | -1.45 percent | -1.85 percent | +22.26 percent | -0.84 percent | - -The proposed Default preserves the aggregate speed and Parquet size constraints. - -It recovers little of Compact's numeric advantage. Compact remains 40.84 percent smaller across the numeric scope. - -Taxi gains 3.23 percent, CMS gains 1.34 percent, and Euro2016 gains 2.42 percent against the prior Default. - -GloVe size does not change. The remaining datasets change by less than one percent. - -This result defines the next objective. Recover repeated Compact gains without a large loss in Default throughput or scalar access. - -### Column attribution for Compact wins - -The profile reads up to two million numeric rows from seven Public BI files and GloVe. - -Twenty-nine float columns give Compact a size advantage of at least ten percent. - -The prior Default uses 190,686,999 bytes across those columns. Compact uses 143,939,147 bytes. - -Compact saves 24.52 percent, or 46,747,852 bytes. - -The Pco trees use these main modes: - -- Classic bins without Delta cover 20,200,768 profiled values. -- Classic bins with consecutive Delta cover 9,311,364 values. -- `IntMult` with consecutive Delta covers 5,048,574 values. -- `IntMult` without Delta covers 4,469,863 values. -- `FloatMult` with consecutive Delta covers 3,999,998 values. -- FloatQuant variants cover about four million values. -- Classic bins with lookback Delta cover 1,345,593 values. - -Classic bins without Delta form the largest transferable target. - -`IntMult` forms a separate target for ALP integer children. GloVe demonstrates this target with `IntMult(10)`. - -Consecutive Delta explains several Arade wins. It conflicts with the Default random-access and decode goals. - -The largest single gap is Euro2016 `subjectivity_confidence`. - -The prior Default uses 14,234,326 bytes. Compact uses 6,860,457 bytes with classic bins and no Delta. - -CMS standard-deviation columns also use classic bins without Delta. - -CMS payment fields use `IntMult`. CMS submitted-charge fields use FloatQuant. - -Food `volume_total_bytes` uses classic bins inside an ALP child. - -The column attribution separates numeric Compact gains from file-level string compression gains. - -### Resolved selector calibration miss on a Compact gap - -The CMS ALP integer child exposed a historical BlockResidual threshold miss. - -The current Default tree uses 11,064,308 bytes and decodes at 24.98 GB/s. - -`ALP(BlockResidual)` uses 10,716,519 bytes and decodes at 22.27 GB/s. - -The candidate is 3.1 percent smaller and 10.8 percent slower to decode. - -The historical 1.20 factor rejected it. - -Later calibration removed the width factors. - -The current integer scheme uses a 1.05 raw-ratio floor and no decode-cost factor. - -The current compressor selects `ALP(BlockResidual)` at 10,716,519 bytes. - -This result resolves the threshold miss. The nonlinear patch cost still protects dense-patch decode. - -### Patch-density sweep - -The u32 input uses one large outlier at each configured stride. - -| Outlier stride | Approximate patch share | Decode GB/s | Scalar access ns | -| ---: | ---: | ---: | ---: | -| 256 | 0.4 percent | 57.88 | 84 | -| 64 | 1.6 percent | 49.31 | 86 | -| 16 | 6.3 percent | 26.32 | 104 | -| 4 | 25.0 percent | 9.57 | 118 | -| 1 | Packed residuals | 33.02 | 99 | - -Scalar access remains bounded across the sweep. - -Bulk decode has a severe patch-density cliff before the planner changes to packed residuals. - -Before the decode specialization, the complete selector exposed two failures with the 16-bit patch cost. - -| Outlier stride | Prior bytes | BlockResidual bytes | Prior decode GB/s | BlockResidual decode GB/s | -| ---: | ---: | ---: | ---: | ---: | -| 4 | 5,508,488 | 3,072,310 | 18.11 | 9.01 | -| 1 | 5,252,352 | 2,543,221 | 44.73 | 33.39 | - -The stride-4 tree remains a bad selection because one quarter of the values use patches. - -A 96-bit cost selected a near-full residual width for both cases. - -The complete selector then retained BitPacked at stride 4. It retained patch-free BlockResidual at stride 1. - -With the direct-output path, decode reached 18.06 GB/s at stride 4 and 45.94 GB/s at stride 1. - -These results combine the 96-bit planner with the decode specialization. The next sweep must isolate both effects. - -The global 96-bit cost also increased proposed-default size on real data: - -| Dataset | Size change from 16-bit cost | -| --- | ---: | -| Euro2016 | +0.1 percent | -| HashTags | +3.2 percent | -| Air Quality | +0.04 percent | - -On HashTags, size increased from 21,306,418 bytes to 21,989,476 bytes. - -Decode throughput changed from 22.54 GB/s to 22.77 GB/s. Encode throughput did not change materially. - -The global 96-bit cost is rejected. - -The first per-block density gate also rejected the two useful HashTags trees. - -The selector-level cost uses total sample density instead. It preserves the physical encoding and compares the complete tree against its incumbent. - -The dense synthetic tree now retains BitPacked. - -| Synthetic input | Prior tree | Proposed tree | Proposed bytes | Result | -| --- | --- | --- | ---: | --- | -| 25 percent patches | BitPacked | BitPacked | 5,508,488 | Reject BlockResidual | -| Packed residuals | FoR with BitPacked | BlockResidual | 2,543,221 | Select BlockResidual | - -The packed-residual prior tree uses 5,252,352 bytes. - -BlockResidual reduced its size by 51.6 percent. Encode throughput decreased by 9.4 percent, and decode throughput increased by 8.0 percent. - -Two HashTags trees explain the real-data tradeoff: - -| Column and candidate | Prior bytes | Candidate bytes | Encode change | Decode change | Decision | -| --- | ---: | ---: | ---: | ---: | --- | -| `twitter#in_reply_to_user_id` BlockResidual | 701,203 | 399,111 | -23.4 percent | -3.0 percent | Reject | -| `interaction#received_at` Ordered BlockResidual | 3,263,939 | 3,018,443 | -38.3 percent | -28.5 percent | Reject | - -The first tree fails the selected-column encode gate. The second tree fails both throughput gates. - -With the nonlinear cost, complete HashTags size is 21,874,126 bytes. - -The prior default uses 22,602,522 bytes. The unpenalized new default used 21,306,418 bytes but selected both failing trees. - -The gated default reduces size by 3.2 percent against the prior default. - -Its measured aggregate encode throughput increased by 0.9 percent. Decode throughput increased by 5.6 percent. - -BlockResidual also composes with outer encodings. Sparse and RunEnd children selected BlockResidual in HashTags and the synthetic low-density sweep. - -### BlockResidual analysis cost - -California Housing contains 20,433 rows and nine nonnullable `f32` columns. - -Every new scheme rejects every column. The output remains 290,737 bytes. - -The first eight-block estimator reduced complete encode throughput by 14 to 15 percent. - -The exact width planner and direct all-valid copies reduced that cost to 6.3 percent. - -| Configuration | Encode MB/s | -| --- | ---: | -| Prior default | 214.4 | -| FloatQuant only | 216.4 | -| Ordered BlockResidual only | 208.1 | -| Integer BlockResidual only | 213.2 | -| Complete proposed default | 200.8 | - -A four-block trial reduced analysis cost further. It mis-ranked the California longitude column. - -The selected `ALP(BlockResidual)` tree saved only 5.7 percent against the prior tree. - -That result did not meet the intended 32-bit speed-adjusted margin. The production candidate retains eight sample blocks. - -The two-million-value random walk retained `OrderedFloat(BlockResidual)`. - -It used 10,428,451 bytes, encoded at 526.7 MB/s, and decoded at 22.29 GB/s. - -The prior default used 12,255,488 bytes, encoded at 595.8 MB/s, and decoded at 12.55 GB/s. - -The selected candidate remained inside both throughput gates. - -### Complete parent compositions - -The finalized benchmark decodes every nested child to recursive canonical form. - -Each numeric case contains two million source values. The FSST case contains 500,000 strings. - -| Complete tree | Prior bytes | Proposed bytes | Encode change | Decode change | -| --- | ---: | ---: | ---: | ---: | -| Timestamp storage with direct BlockResidual | 9,254,235 | 7,293,370 | -5.1 percent | +348.9 percent | -| `List(BlockResidual)` | 12,755,712 | 2,543,268 | +1.0 percent | +24.8 percent | -| FSST with two BlockResidual children | 4,675,014 | 4,118,592 | -0.4 percent | +8.1 percent | -| `ALP(BlockResidual)` | 7,753,472 | 2,543,268 | -0.2 percent | +1.0 percent | -| `Sparse(BlockResidual, BlockResidual)` | 1,070,594 | 383,297 | +1.9 percent | +6.3 percent | -| `RunEnd(Sequence, BlockResidual)` | 739,968 | 159,124 | +0.5 percent | +1.6 percent | - -Each composition reduced size and passed both throughput gates. - -The timestamp candidate displaced `DateTimeParts`. It reduced size by 21.2 percent and decoded 4.5 times faster. - -The list, ALP, Sparse, and RunEnd cases reduced size by 64.2 to 80.1 percent. - -The FSST case reduced size by 11.9 percent with no material encode cost. - -A separate float case selected direct `OrderedFloat(BlockResidual)`. - -It reduced size by 69.9 percent, increased encode throughput by 128.7 percent, and increased decode throughput by 56.4 percent. - -The attempted ALP-RD case did not select ALP-RD. Evidence for BlockResidual under an ALP-RD child remains missing. - -Focused selection tests now cover ALP, Sparse, and RunEnd child composition. - -Golden snapshots cover temporal and FSST parent trees. - -### GloVe embeddings - -The corpus now includes the real GloVe dataset with 100,000 rows and 200 f32 values per row. - -The complete proposed and previous Vortex defaults both use 68,375,768 bytes. - -The Vortex file uses 0.92 times the bytes of Parquet with Zstd. - -The first two million embedding values use `ALP(ZigZag(BitPacked))` under both default configurations. - -That tree uses 6,274,834 bytes for 8,000,000 raw bytes. - -Compact uses `ALP(Pco)` and 5,287,510 bytes. Compact is 15.7 percent smaller than the default tree. - -FloatQuant and OrderedFloat with BlockResidual both lose to ALP on this dataset. - -Pco receives the `i32` primary child from ALP. Each 262,144-value Pco chunk selects `IntMult(10)` with no Delta encoding. - -`IntMult(10)` splits each integer into a quotient and a remainder modulo ten. - -The quotient uses 23 to 31 entropy bins. Its average encoded cost is 16.95 to 16.99 bits per value. - -The remainder uses ten entropy bins and no offset bits. Its average encoded cost is about 1.08 bits per value. - -The complete Pco child uses 4,518,870 bytes for two million values. This result includes model metadata and page overhead. - -The size gap comes from a decimal quotient and remainder split plus entropy coding. Pco does not use a Delta or float mode here. - -GloVe therefore identifies an ALP-child compression gap. It does not identify a failure in the ALP float transform. - -The current Vortex tree does not leave the embedding values uncompressed. - -It uses ALP for the float transform, then ZigZag and BitPacked for the integer child. - -The Pco advantage comes from a different encoding of the same ALP integer child. - -### GloVe quotient and remainder prototypes - -The first prototype splits the exact ALP integer child with a constant base. - -The estimator stores each child at the source latent width. GloVe `i32` latents produce `u32` quotient and remainder children. - -The native-width correction did not change the GloVe byte totals. Each selected child still uses the same packed bit width. - -| Candidate | Bytes | -| --- | ---: | -| Pco `IntMult(10)` child | 4,518,870 | -| Base ten with ordinary integer children | 6,002,688 | -| Best tested ordinary base, base 100 | 5,572,096 | -| Base ten with bitmap-patched quotient and mode bitmap remainder | 5,221,476 | -| Direct bitmap patches on the unsplit child | 5,541,496 | -| Direct position patches on the unsplit child | 5,507,862 | - -Ordinary integer trees lose materially to Pco on the quotient and remainder. - -Bitmap patches improve size, but they do not match the Pco child. - -The results reject a general quotient and remainder array with ordinary children. - -They support a small-alphabet entropy experiment and a fixed integer split experiment. - -### Bounded IntMult prototypes - -The direct prototypes use 1,024-value blocks and exact quotient and remainder splits. - -Each quotient block stores one reference, FastLanes-packed low bits, and bounded bitmap patches. - -The GloVe variants tested three remainder layouts: - -- A mode value with bitmap exceptions. -- Gap positions with a maximum scan of 1,024 values. -- One exception bit per value pair with one payload byte per exceptional pair. - -| GloVe base-ten child | Bytes | Decode MB/s | Scalar ns | -| --- | ---: | ---: | ---: | -| Bitmap remainder | 5,197,126 | 10,400 to 11,200 | 14 | -| Gap positions | 5,348,130 | 10,800 to 12,400 | 78 to 88 | -| Pair bytes | 5,184,309 | 10,600 to 10,700 | Not measured | - -The current Default GloVe tree decodes at 16,700 to 17,500 MB/s. - -The bitmap variant encodes at 650 to 666 MB/s. It applies 142,861 quotient patches and 298,831 remainder exceptions. - -The gap layout loses size and scalar speed. The pair layout does not improve bulk decode. - -The CMS prototype stores every remainder with one dense FastLanes stream. This layout avoids sparse remainder expansion. - -| CMS payment child, base ten | Bytes | Encode MB/s | Decode MB/s | Scalar ns | -| --- | ---: | ---: | ---: | ---: | -| Pco child | 9,141,872 | Not isolated | Not isolated | Not isolated | -| Dense remainder prototype | 9,811,885 | 879 | 8,094 | 19 | - -The complete Default CMS tree uses 10,494,404 bytes and decodes at 25,226 MB/s. - -The dense prototype applies 853,533 quotient patches across two million values. - -A diagnostic decode without quotient patches reaches 12,628 MB/s. The split and reconstruction costs still miss the Default decode target. - -These results reject the tested bespoke IntMult layouts for Default. - -The layouts retain bounded scalar access, but their bulk decode cost is too high. - -They do not reject a pure IntMult transform with independently compressed children. - -The result does not reject an IntMult backend for Compact. Compact accepts a different throughput and random-access tradeoff. - -### Exact ALP child comparison - -The benchmark replaces a Compact `ALP(Pco)` child with the same latent integers in BlockResidual. - -This comparison separates float transform quality from integer child compression. - -| Input | Current default bytes | `ALP(BlockResidual)` bytes | Current decode MB/s | Candidate decode MB/s | Candidate encode MB/s | -| --- | ---: | ---: | ---: | ---: | ---: | -| CMS Payments | 4,146,124 | 3,918,239 | 14,202 | 20,280 | 2,605 | -| GloVe | 6,274,834 | 6,298,751 | 16,738 | 19,310 | 1,201 | - -The CMS candidate is 5.5 percent smaller and 42.8 percent faster to decode. - -The selector at the time rejected it because the 64-bit BlockResidual score included a 1.20 factor. - -The current selector removed that factor. The resolved calibration section records the selected 10,716,519-byte tree. - -The GloVe candidate is 0.4 percent larger than the current default. - -It does not recover the 15.7 percent Pco size advantage. - -GloVe still requires a better integer split or small-alphabet backend for the ALP child. - -### Fixed-bin range backend experiments - -The experimental branch compares ANS symbols with fixed-width bin identifiers. - -The packed variants retain variable-width offsets inside each bin. - -| Input and backend | Bytes | Encode MB/s | Decode MB/s | -| --- | ---: | ---: | ---: | -| GloVe ALP child, ANS | 5,264,329 | 414.2 | 3,476.5 | -| GloVe ALP child, packed bins | 5,573,968 | 765.8 | 7,700.0 | -| GloVe `IntMult(10)`, ANS | 4,716,002 | 457.8 | 1,944.6 | -| GloVe `IntMult(10)`, packed bins | 5,562,181 | 821.1 | 3,661.7 | -| CMS ALP child, ANS | 3,293,477 | 406.6 | 3,823.0 | -| CMS ALP child, packed bins | 3,647,593 | 642.7 | 8,358.2 | -| Food ALP child, ANS | 5,405,851 | 392.5 | 3,936.6 | -| Food ALP child, packed bins | 5,525,544 | 607.4 | 8,525.6 | - -ANS approaches Pco size, but it retains an expensive decode path. - -Packed bins decode faster, but they lose part of the size benefit. - -Packed bins remain plausible for Compact on selected columns. - -Fast scalar access requires checkpoints for the variable offset stream. - -A checkpoint interval of 32 values bounds scalar work to 31 offset widths. - -This design adds about 0.5 bits per value with 16-bit checkpoints. - -### Fixed-bin checkpoint prototype - -The checkpoint prototype uses 1,024-value blocks and one 16-bit offset checkpoint per 32 values. - -The bin optimizer scores fixed-width identifiers. It restricts bin counts to powers of two. - -The bulk decoder uses one narrow-offset specialization for bins with widths of at most 57 bits. - -| Input | Default bytes | Pco bytes | Fixed-bin bytes | Encode MB/s | Decode MB/s | Scalar ns | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| Synthetic uniform f64 | 14,104,090 | 13,768,605 | 14,037,682 | 1,194 | 10,534 | 48 | -| GloVe ALP child | 6,274,834 | 5,287,510 | 6,177,168 | 539 | 6,600 | 25 | -| CMS ALP child | 4,146,124 | 3,261,563 | 3,515,002 | 1,280 | 13,554 | 23 | - -The GloVe fixed-bin tree saves only 1.6 percent against default. - -The CMS fixed-bin tree saves 15.2 percent against default. It remains 7.8 percent larger than Compact Pco. - -The first CMS decoder reached 8,057 MB/s. The specialized decoder increased throughput by 68.2 percent. - -The complete CMS default tree decodes at 14,575 MB/s. The fixed-bin child is 6.1 percent slower. - -The benchmark now wraps the codec in a serialized Vortex array. - -This wrapper permits complete ALP decode and scalar measurements without a storage-format commitment. - -| Input | Default decode MB/s | Compact decode MB/s | Fixed-bin decode MB/s | Fused decode MB/s | -| --- | ---: | ---: | ---: | ---: | -| GloVe | 17,523 | 1,853 | 4,991 | 4,984 | -| CMS Payments | 14,575 | 5,588 | 9,342 | 9,791 | -| Food | 24,000 | 5,569 | 9,431 | 11,195 | - -| Input | Default scalar ns | Compact scalar ns | Fixed-bin scalar ns | -| --- | ---: | ---: | ---: | -| GloVe | 824 | 21,374 | 535 | -| CMS Payments | 903 | 14,473 | 383 | -| Food | 535 | 13,569 | 140 | - -The fused path writes final floats during range decode. It retains the ALP patch step. - -Fusion increased CMS decode by 4.8 percent and Food decode by 18.7 percent. It did not improve GloVe decode. - -The fixed-bin tree fails the default decode gate on all three inputs. - -CMS and Food show that fixed bins can trade size for faster access than Pco. - -CMS uses 7.8 percent more bytes than Compact. Generic decode is 67.2 percent faster, and scalar access is 37.8 times faster. - -Food uses 5.3 percent more bytes than Compact. Generic decode is 69.4 percent faster, and scalar access is 97.3 times faster. - -The fused Food decode is 2.0 times as fast as Compact. The generic composition already gives most of the benefit. - -GloVe uses 16.8 percent more bytes than Compact because Pco also uses `IntMult(10)`. - -The evidence does not justify a fused ALP and fixed-bin array. - -The temporary Compact selector increased GloVe encode time by 7.8 percent when it rejected RangePacked. - -The selector also increased aggregate CMS bytes by 1.1 percent after it replaced Pco. - -RangePacked therefore remains outside all writer selectors. - -### Composable fixed-bin prototype - -The prior RangePacked prototypes fused bins, offset streams, checkpoints, and reconstruction into one codec. - -The current prototype separates the model from storage. - -`RangeDecomposition` fits any number of bins through 64. - -It produces three logical components: - -- A small array of bin starts. -- One fixed-width bin code per value. -- One integer offset from the selected start per value. - -The manual benchmark builds this exact tree: - -`IntMult(base=1, Dict(BitPacked(codes), starts), BlockResidual(offsets))`. - -The first IntMult decode materialized every dictionary reference before addition. - -A generic dictionary-add path increased decode throughput from 12.06 GB/s to 12.62 GB/s. - -A packed-code specialization increased median decode throughput to 14.27 GB/s. - -The specialization unpacks codes and adds starts directly into the decoded offset buffer. - -It does not multiply because the IntMult base equals one. - -The benchmark uses two million `u64` values in ten separated clusters. - -| Operation | Result | -| --- | ---: | -| Encode | 1.20 GB/s | -| Decode | 14.27 GB/s | -| Scalar access | 332 ns | - -A separate base-ten IntMult benchmark uses two million `i32` values and primitive children. - -| Operation | Result | -| --- | ---: | -| Split encode | 2.88 GB/s | -| Decode | 30.67 GB/s | -| Scalar access | 125 ns | - -These values establish low transform overhead and bounded scalar access. - -They do not establish a Default win because the range benchmark lacks incumbent size and throughput comparisons. - -The next experiment must compare complete trees on the real Pco gap columns. - -### Decomposed fixed-bin real-data results - -The benchmark now builds `IntMult(base=1, Dict(BitPacked(codes), starts), offsets)` on four real float columns. - -It tests `BlockResidual` and `FoR(BitPacked)` as the offset child. - -The GloVe benchmark reader flattens numeric `List`, `LargeList`, and `FixedSizeList` columns. - -| Input | Default bytes | Compact bytes | Decomposed BR bytes | Decomposed FoR bytes | Default decode MB/s | Decomposed BR decode MB/s | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| Euro subjectivity | 14,234,326 | 6,860,457 | 13,000,391 | 14,006,288 | 16,915 | 4,217 | -| Food volume | 7,529,728 | 5,368,140 | 7,223,283 | 10,504,832 | 24,104 | 10,060 | -| GloVe embeddings | 6,274,834 | 5,287,510 | 7,105,298 | 8,160,260 | 17,388 | 7,125 | -| CMS payment | 11,064,308 | 9,288,800 | 11,073,385 | 12,735,216 | 22,542 | 6,570 | - -The `BlockResidual` form saves 8.7 percent on Euro and 4.1 percent on Food against Default. - -It grows GloVe by 13.2 percent and matches Default size on CMS payment. - -Its decode throughput falls by 58 to 75 percent against Default on all four columns. - -The `FoR(BitPacked)` form encodes and decodes faster than the `BlockResidual` form. - -Its global offset range loses more size on every input. - -The offset stream interleaves values from bins with different widths. - -A generic offset child then pays for the widest offsets in each local block. - -The fused prototype avoids this cost because each value uses the width for its selected bin. - -The tested composition therefore does not pass the Default evidence bar. - -A competitive fixed-bin design requires a reusable primitive for code-dependent widths or grouped offsets with rank data. - -Earlier grouped and checkpoint prototypes did not pass the Default decode gate. - -The generic fixed-bin composition remains paused for Default. - -### Pure IntMult real-data results - -The next prototype applies ALP first and splits its integer child with one global IntMult base. - -Default compresses the quotient and remainder independently through normal integer recursion. - -The benchmark tests bases 5, 10, 100, and 1,000. - -| Input | Default bytes | BlockResidual bytes | Best IntMult bytes | Default decode MB/s | Best IntMult decode MB/s | Best IntMult encode MB/s | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| GloVe embeddings | 6,274,834 | 6,298,751 | 6,665,400 at base 1,000 | 17,504 | 10,188 | 445 | -| CMS payment | 11,064,308 | 10,716,519 | 10,442,347 at base 10 | 22,715 | 10,212 | 412 | - -GloVe grows by 6.2 percent against Default. - -CMS shrinks by 5.6 percent against Default and 2.6 percent against `ALP(BlockResidual)`. - -The CMS IntMult tree decodes 55 percent slower than Default and 51 percent slower than `ALP(BlockResidual)`. - -Its encode throughput falls 77 percent against the direct `ALP(BlockResidual)` candidate. - -CMS scalar access remains close to Default, but it remains slower than the BlockResidual candidate. - -Pure IntMult with generic children therefore does not pass the Default evidence bar. - -Pco gains more from its bin and entropy backend than from the multiplication split alone on GloVe. - -The CMS standard-deviation payment column provides a no-Delta IntMult target. - -Pco uses `IntMult(10)` on seven of eight chunks. The remaining chunk uses classic bins. - -| Tree | Bytes | Encode MB/s | Decode MB/s | Scalar ns | -| --- | ---: | ---: | ---: | ---: | -| Default | 10,494,404 | Not isolated | 24,593 | 866 | -| `ALP(BlockResidual)` | 10,220,895 | 2,124 | 21,889 | 551 | -| Best IntMult, base 1,000 | 10,300,986 | 530 | 10,704 | 727 | -| Compact Pco | 9,325,132 | Not isolated | 1,977 | 25,987 | - -The IntMult tree saves 1.8 percent against Default. It remains larger than the BlockResidual tree. - -Its encode throughput is four times lower than BlockResidual. Its decode throughput is more than twice as low. - -This result removes Delta as a confounder. Pure IntMult remains outside Default. - -### Equal-width prefix bins - -The equal-width prototype uses only `IntMult`, `Dict`, and `BitPacked`. - -The base is a power of two. A dictionary stores at most 64 observed quotients. - -The remainder uses one fixed suffix width for every value. - -This tree provides fast rejection when more than 64 quotients appear in the sample. - -| Input | Best suffix width | Candidate bytes | Default bytes | Encode MB/s | Decode MB/s | -| --- | ---: | ---: | ---: | ---: | ---: | -| GloVe embeddings | 20 | 6,909,476 | 6,274,834 | 1,348 | 7,538 | -| CMS payment | 44 | 11,984,848 | 11,064,308 | 2,487 | 8,709 | - -The encoder is fast because it builds one dictionary and two packed streams. - -The fixed suffix grows both columns by 8 to 10 percent against Default. - -The generic decoder is also more than twice as slow as Default. - -Equal-width prefix bins do not justify a Default scheme. - -### Frequency-ranked dictionary codes - -Dictionary codes previously followed hash-table iteration order. - -The prototype assigns the lowest codes to the most frequent values. It then compresses both children through normal recursion. - -This transform uses existing `Dict`, `BitPacked`, `Sparse`, and BlockResidual arrays. It adds no array format. - -Integer statistics already store frequency counts. Float statistics now retain counts during the existing distinct-value pass. - -| Column and tree | Bytes | Decode MB/s | Scalar ns | Candidate construction MB/s | -| --- | ---: | ---: | ---: | ---: | -| Bimbo `Venta_hoy`, old Default | 3,544,154 | 26,689 | 595 | Not isolated | -| Bimbo `Venta_hoy`, ranked proposed Default | 3,197,655 | 25,490 | 577 | 1,480 | -| Bimbo `Dev_proxima`, ranked prior Default | 280,812 | 26,902 | 3,834 | Not isolated | -| Bimbo `Dev_proxima`, ranked Dict with BlockResidual descendants | 234,832 | 27,134 | 3,823 | 1,367 | - -Frequency ranking reduces `Venta_hoy` by 9.8 percent against the old Default. - -The complete proposed tree keeps decode and scalar throughput close to the old tree. - -The `Dev_proxima` gain depends on BlockResidual below the dictionary code child. - -The former ancestor exclusion prevents that useful composition. The corpus trial therefore removes the integer and float Dict exclusions. - -The scheme retains exclusions for string and binary dictionaries. - -The current eight-dataset numeric trial compares the ranked prior Default with the ranked proposed Default. - -| Dataset | Size change | Encode throughput change | Decode throughput change | -| --- | ---: | ---: | ---: | -| Arade | -3.19 percent | -5.17 percent | +1.93 percent | -| Bimbo | -3.57 percent | -0.37 percent | +0.23 percent | -| CMSprovider 1 | -2.26 percent | -6.51 percent | -6.34 percent | -| CMSprovider 2 | -3.99 percent | -6.97 percent | -9.06 percent | -| Euro2016 | -11.47 percent | -3.63 percent | +11.68 percent | -| Food | -7.14 percent | -3.91 percent | -10.18 percent | -| HashTags | -5.68 percent | +0.43 percent | +5.46 percent | -| GloVe | 0.00 percent | +0.11 percent | +1.85 percent | -| Geometric mean | -4.72 percent | -3.29 percent | -0.80 percent | - -Every dataset remains inside the 20-percent throughput gate. - -Frequency ranking needs no separate selector. Dictionary analysis already computes the required distinct values. - -Cheap rejection remains useful guidance for other schemes. A costly scheme can enter Default only with stronger corpus evidence. - -### BlockResidual patch positions - -The patch prototype replaces sorted ALP patch indices and chunk offsets with BlockResidual when it reduces their byte size. - -It leaves patch values unchanged. - -| Input and tree | Original bytes | Candidate bytes | Original decode MB/s | Candidate decode MB/s | Original scalar ns | Candidate scalar ns | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| GloVe Default | 6,274,834 | 6,071,376 | 18,077 | 17,178 | 812 | 2,076 | -| GloVe `ALP(BlockResidual)` | 6,298,751 | 6,095,293 | 19,233 | 18,735 | 503 | 1,766 | -| CMS Default | 11,064,308 | 11,040,803 | 24,168 | 23,111 | 934 | 1,551 | -| CMS `ALP(BlockResidual)` | 10,716,519 | 10,693,014 | 20,678 | 19,982 | 656 | 1,241 | -| Euro subjectivity Default | 14,234,326 | 11,211,722 | 21,055 | 16,583 | 631 | 9,801 | - -Food volume has no ALP patches on this input. - -BlockResidual saves 3.2 percent on GloVe patch trees and 21.2 percent on Euro Default. - -CMS saves only 0.2 percent. - -Patch lookup uses binary search over the index array. - -Compressed indices require one generic scalar read for each comparison. - -This path increases scalar latency by 1.7 to 15.5 times across the measured trees. - -Direct BlockResidual patch positions therefore do not pass the random-access gate. - -A future general patch backend can retain this size opportunity through a fast sorted-search kernel or a bitmap with bounded rank data. - -Do not add a BlockResidual special case to ALP or IntMult patch handling. - -### Direct unsigned BlockResidual decode - -The array decoder now writes unpacked unsigned values directly into the final output buffer. - -The prior path used a temporary 1,024-value block for u8, u16, and u64 values. - -| Type | Prior decode GB/s | Direct decode GB/s | Change | -| --- | ---: | ---: | ---: | -| u8 | 30.14 | 40.37 | +34.0 percent | -| u16 | 32.55 | 45.86 | +40.9 percent | -| u32 | 46.61 | 47.23 | +1.3 percent | -| u64 | 35.56 | 39.37 | +10.7 percent | - -u32 already used the direct path. - -The signed prototype unpacked into the final unsigned buffer and flipped the sign bit in place. - -That extra pass reduced i16 throughput from about 31.4 GB/s to 27.4 GB/s. - -It also reduced throughput for i8, i32, and i64. - -The implementation retains the prior signed path. - -Current i16 BlockResidual decode reaches 31.4 GB/s. The comparable `FoR(BitPacked)` tree reaches 42.0 GB/s. - -At that stage, this result retained the direct i8 and i16 selector exclusions. - -At that stage, the u16 result lacked real selected-tree evidence. - -Later absolute-throughput and complete-file results re-enabled all narrow integer widths. - -The faster unsigned path also improves the BlockResidual patch-position bulk decoder. - -GloVe patch decode now stays within 2 percent of Default. Euro patch decode remains 12 percent slower than Default. - -Scalar patch lookup remains the blocking cost. - -### Nullable and Delta-heavy fixed-bin cases - -The optimized nullable prototype stores only valid values in the range stream. - -It stores canonical validity bits and one 32-bit rank checkpoint per 256 logical values. - -Scalar access tests one validity bit. It then scans at most four words to find the dense value index. - -The complete Euro subjectivity tree uses `OrderedFloat(RangePacked)`. - -| Input | Default bytes | Compact bytes | Fixed-bin bytes | Default decode MB/s | Compact decode MB/s | Fixed-bin decode MB/s | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| Euro subjectivity | 14,234,326 | 6,860,457 | 7,270,094 | 16,822 | 2,475 | 2,487 | -| Euro polarity | 13,505,943 | 11,638,679 | 12,796,702 | 11,517 | 1,487 | 4,767 | -| Arade F8 | 6,380,852 | 5,617,973 | 6,257,548 | 17,944 | 4,371 | 8,350 | - -Euro subjectivity uses classic Pco bins without Delta. Fixed bins use 6.0 percent more bytes and match Compact decode. - -Its scalar access takes 215 ns. Default takes 657 ns, and Compact takes 19,045 ns. - -The complete subjectivity candidate encodes at 440 MB/s. - -Euro polarity mixes lookback Delta and no-Delta chunks. Fixed bins use 10.0 percent more bytes and decode 3.2 times faster. - -Arade F8 uses consecutive Delta in Pco. Fixed bins use 11.4 percent more bytes and decode 1.9 times faster. - -These results do not support Delta in the native candidate. - -Fixed bins retain much of Pco's size benefit without Delta. They also preserve bounded scalar access. - -A ten-percent Compact size allowance includes CMS, Food, Euro subjectivity, and Euro polarity. - -Arade F8 remains outside that threshold. GloVe also remains outside because `IntMult(10)` gives Pco another size advantage. - -### Alternate fixed-bin layouts - -The byte-aligned prototype rounds every bin offset width to a byte boundary. - -Euro2016 `subjectivity_confidence` provides the comparison. - -| Backend | Bytes | Encode MB/s | Decode MB/s | Scalar ns | -| --- | ---: | ---: | ---: | ---: | -| Serial bit-packed bins | 7,238,842 | 541.8 | 3,824.4 | 31.1 | -| Serial byte-aligned bins | 7,835,311 | 474.4 | 3,528.3 | 25.0 | - -Byte alignment improves scalar access. It worsens size, encode throughput, and bulk decode throughput. - -Cross-byte offset extraction is not the main bulk decode cost. - -The grouped-bin prototype stores bit-sliced identifiers and one offset stream per bin. - -It uses rank checkpoints every 256 values. Scalar access scans at most four 64-bit words. - -| Input and backend | Bytes | Encode MB/s | Decode MB/s | Scalar ns | -| --- | ---: | ---: | ---: | ---: | -| Euro serial bins | 7,238,842 | 555.2 | 3,949.5 | 32.1 | -| Euro grouped bins | 7,257,069 | 461.0 | 5,665.8 | 25.0 | -| CMS serial bins | 10,650,059 | 1,256.0 | 13,699.0 | 36.8 | -| CMS grouped bins | 10,670,330 | 929.9 | 6,283.7 | 30.3 | - -Grouped bins improve Euro bulk decode by 43 percent. The complete Default tree still decodes about four times faster. - -### Precomputed offset masks - -The serial decoder previously constructed one low-bit mask for each nonzero offset. - -The 8-lane decoder now stores one mask per bin in its 1 KiB decode table. - -For offset widths from 41 through 57 bits, each lane performs one load, one shift, and one mask operation. - -This path removes the unpredictable zero-width branch and the per-lane mask construction. - -The narrow path retains its zero-width branch because an unconditional load reduced Euro throughput. - -| Input | Fixed-bin bytes | Direct decode MB/s | Complete decode MB/s | Fused decode MB/s | Default decode MB/s | -| --- | ---: | ---: | ---: | ---: | ---: | -| Euro subjectivity | 7,270,094 | 15,932 | 5,121 | 5,965 | 17,829 to 20,600 | -| Food volume | 5,653,643 | 14,244 | 9,025 | 12,114 | 24,903 | -| CMS `LINE_SRVC_CNT` | 2,463,791 | 14,031 | 5,731 | 4,071 | 26,249 | -| CMS average payment | 10,845,743 | 15,390 | 3,545 | 3,746 | 23,861 | - -Euro direct decode previously reached 3,721 to 3,979 MB/s. - -The precomputed mask increases its direct throughput by about four times. - -The 500-iteration rerun produced 15,932 MB/s. - -Food and CMS `LINE_SRVC_CNT` use offset widths of at most 40 bits, so they do not use the new path. - -CMS average payment uses the new path, but BlockResidual gives a smaller complete tree. - -The optimized codec establishes a much higher fixed-bin decode ceiling. - -The complete dense candidate still fails the Default decode gate because validity checks and float reconstruction dominate its final path. - -An in-place reverse validity expansion reduced complete Euro decode to 3,116 MB/s. - -Its reverse bit iteration cost exceeded the allocation and copy cost that it removed. - -The implementation retains the forward expansion with a separate logical output vector. - -### Full-position null storage - -The full-position mode stores physical values at null positions instead of a dense valid-value stream. - -The validity child remains present, but the rank child is absent. - -The missing rank child identifies full-position storage without a metadata flag. - -Bulk decode then writes one value per logical position and skips validity expansion. - -Scalar access uses the logical index directly. - -| Input | Mode | Bytes | Encode MB/s | Fused decode MB/s | Scalar ns | -| --- | --- | ---: | ---: | ---: | ---: | -| Euro subjectivity | Dense | 7,270,094 | 461 | 5,965 | 232 | -| Euro subjectivity | Full-position | 8,180,506 | 654 | 15,180 | 206 | -| CMS `LINE_SRVC_CNT` | Dense | 2,463,791 | 875 | 4,071 | 465 | -| CMS `LINE_SRVC_CNT` | Full-position | 2,470,956 | 942 | 10,623 | 445 | - -The Euro result uses 200 complete-tree decode iterations. - -Default uses 14,234,326 bytes and decodes at 20,924 MB/s on the same run. - -Full-position RangePacked reduces Euro size by 42.5 percent and reduces decode throughput by 27.5 percent. - -Compact uses 6,860,457 bytes and decodes at 2,574 MB/s. - -The full-position candidate retains 82.1 percent of the Compact size gain and decodes 5.9 times faster than Compact. - -CMS `LINE_SRVC_CNT` still favors `ALP(BlockResidual)` for Default. - -That tree uses 2,725,541 bytes and decodes at 23,493 MB/s. - -The Euro candidate now warrants a complete corpus and selector-cost evaluation. - -CMS uses similar offset widths across its bins. The grouped scatter path then loses more than half of serial-bin throughput. - -Grouped bins do not generalize. The prototype remains outside production code. - -### Fused parent kernels and the experimental selector - -RangePacked now provides parent kernels for `OrderedFloat(RangePacked)` and `ALP(RangePacked)`. - -The session registry selects these kernels from the child encoding and the parent encoding. - -The outer arrays remain generic composable arrays. - -On Euro subjectivity, normal recursive execution reaches 14,956 MB/s. - -The manual fused benchmark reaches 15,492 MB/s on the same run. - -The registered kernel is within 3.5 percent of the manual helper. - -An experimental `OrderedFloatRangePackedScheme` remains outside `ALL_SCHEMES`. - -The scheme tests two complete trees on the existing stratified one-percent sample: - -- `OrderedFloat(RangePacked)` -- `ALP(RangePacked)` - -The full encoder builds only the smaller sampled tree. - -The selector divides the sample ratio by a provisional 1.20 decode cost factor. - -This factor does not produce the intended complete-tree choices on CMS or Food. - -The one-percent sample overstates the RangePacked advantage against BlockResidual on those columns. - -| Input | Default bytes | Candidate bytes | Default encode MB/s | Candidate encode MB/s | Default decode MB/s | Candidate decode MB/s | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| Euro subjectivity | 14,234,326 | 8,180,506 | 362.4 | 376.4 | 21,795 | 14,849 | -| CMS `LINE_SRVC_CNT` | 2,725,541 | 2,436,604 | 1,054.8 | 630.5 | 24,227 | 10,354 | -| Food volume | 6,599,221 | 5,653,643 | 491.3 | 538.2 | 19,747 | 11,920 | - -Euro is the target win. - -CMS and Food do not justify their decode losses at the current size gains. - -The rejected-candidate cost is also material. - -On uniform random floats, the candidate reduces total encode throughput by 6.8 percent. - -On widened `f32` values, FloatQuant still wins, but the candidate reduces encode throughput by 18.8 percent. - -These results justify a cheap rejection test, if that test retains the Euro model. - -Fast rejection remains an advantage, not an admission rule. - -A candidate with costly analysis needs stronger corpus evidence. - -### RangePacked estimate and rejection cost - -A focused Samply profile attributed 11.79 percent of CPU time to the rejected RangePacked callback. - -Bin construction consumed almost all of that time. Exact size estimation alone did not reduce the cost. - -The estimator now fits the bins and counts exact child-buffer bytes. It does not construct the packed payload. - -A coarse model uses the existing one-percent sample. It compares fixed prefix bins with a single global range. - -The model also compares 64-value local ranges with the global range model. - -A large local advantage identifies inputs that favor BlockResidual. The selector then skips RangePacked before ALP analysis. - -The locality test retains Euro and rejects the random walk, Food, and CMS candidates. - -| Input | Default encode MB/s | Candidate encode MB/s | Write change | Selected RangePacked | -| --- | ---: | ---: | ---: | --- | -| Uniform floats | 639.5 | 629.7 | -1.5 percent | No | -| Widened `f32` values | 680.3 | 662.9 | -2.6 percent | No | -| Random walk | 581.0 | 576.2 | -0.8 percent | No | -| Nonzero-secondary FloatQuant | 677.8 | 659.8 | -2.7 percent | No | -| Quantized `f32` | 1,410.8 | 1,338.7 | -5.1 percent | No | -| Euro subjectivity | 357.8 | 359.9 | +0.6 percent | Yes | - -These focused paired results contain benchmark noise. The complete corpus will determine the final policy. - -On Euro, RangePacked uses 8,180,506 bytes instead of 14,234,326 bytes. - -Its decode throughput is 15,179 MB/s instead of 19,716 MB/s. - -Food and CMS now retain their existing BlockResidual trees. Their prior RangePacked choices lost too much decode throughput. - -### Broad RangePacked selector pass - -The next pass covered seven Public BI files and the GloVe file. - -Only Euro and HashTags selected RangePacked. The other six files retained every existing tree. - -| Dataset | Default bytes | Candidate bytes | Default encode MB/s | Candidate encode MB/s | Default decode MB/s | Candidate decode MB/s | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| Arade | 26,078,866 | 26,078,866 | 549.8 | 539.6 | 21,853 | 23,772 | -| Bimbo | 10,573,121 | 10,573,121 | 767.8 | 764.8 | 11,181 | 11,342 | -| CMS provider 1 | 70,490,567 | 70,490,567 | 492.0 | 480.3 | 16,387 | 16,394 | -| CMS provider 2 | 70,640,453 | 70,640,453 | 483.9 | 475.0 | 15,641 | 15,623 | -| Euro2016 | 39,571,884 | 33,518,064 | 506.8 | 493.7 | 20,356 | 18,969 | -| Food | 12,297,725 | 12,297,725 | 542.7 | 541.8 | 19,633 | 19,650 | -| HashTags | 21,283,222 | 20,147,808 | 709.8 | 631.6 | 22,303 | 22,207 | -| GloVe | 6,274,834 | 6,274,834 | 363.0 | 342.8 | 19,341 | 19,440 | - -The geometric mean gives 2.7 percent less size and 3.3 percent less write throughput. - -The geometric-mean read throughput increases by 0.4 percent. This small change is within benchmark noise. - -HashTags `interaction#received_at` is the second selected column. - -RangePacked uses 2,128,525 bytes instead of 3,263,939 bytes. It reduces size by 34.8 percent. - -Its focused decode throughput is 14,658 MB/s instead of 15,678 MB/s. - -Its focused write throughput is 231 MB/s instead of 1,254 MB/s. - -The direct RangePacked tree encoder reaches 660 MB/s. Repeated scheme analysis accounts for part of the complete write cost. - -This HashTags trade needs the final threshold review. It does not yet justify default inclusion. - -The ALP RangePacked branch selected no column in this pass. Its rejected analysis contributes to the GloVe and synthetic costs. - -An OrderedFloat-only estimator reduced rejected write costs on every affected control. - -| Input | General selector loss | OrderedFloat-only loss | -| --- | ---: | ---: | -| Widened `f32` | 2.6 percent | 0.5 percent | -| Nonzero-secondary FloatQuant | 2.7 percent | 1.3 percent | -| Quantized `f32` | 5.1 percent | 2.4 percent | -| GloVe | 5.6 percent | 1.3 percent | - -The same estimator increased HashTags write throughput from 231 to 294 MB/s. - -A direct OrderedFloat encoder increased it again to 379 MB/s. The Default baseline reached 1,316 MB/s. - -The direct form kept the Euro write result neutral. It also retained both selected sizes. - -The Default candidate can therefore use a specialized OrderedFloat selector. - -The ALP candidate remains experimental until independent columns justify its analysis cost. - -This split does not reject all selectors with expensive analysis. It reflects the current selection evidence for these two transforms. - -### Corrected RangePacked bundle - -The first controlled RangePacked pass exposed another constant-sample error. - -A constant HashTags sample produced an infinite coarse range ratio. - -The exact candidate then lost to the complete incumbent, but it blocked the existing Sparse tree. - -The prefilter now rejects non-finite range ratios. - -| Dataset | Current bytes | RangePacked bytes | Size change | Write time change | Read time change | -| --- | ---: | ---: | ---: | ---: | ---: | -| Euro2016 | 159,053,012 | 153,090,172 | -3.75 percent | +1.60 percent | -1.30 percent | -| HashTags | 179,382,828 | 178,141,132 | -0.69 percent | +0.89 percent | +0.69 percent | - -The corrected 16-file geometric mean reduces size by 0.28 percent. - -The timing differences in this two-file repeat remain within benchmark noise. - -The focused Euro tree still decodes about 30 percent slower than its incumbent. - -RangePacked therefore remains an experimental Default option. - -### Decomposed fixed-bin bundle - -The storage prototype now uses only composable arrays: - -`OrderedFloat(IntMult(Dict(BitPacked(codes), starts), BlockResidual(offsets)))` - -The scheme permits 1 through 64 bins. - -The `IntMult` base is one, so decode uses addition without multiplication. - -The exact sample estimator constructs this fixed child tree. - -The writer therefore avoids generic integer recursion and its depth limit. - -The first decomposed HashTags tree used 1,991,606 bytes. - -The incumbent ALP-RD tree used 3,263,939 bytes. - -This change reduced the column size by 39.0 percent. - -The generic child execution reached 4,476 MB/s. - -A nullable dictionary fast path reached 8,383 MB/s after it removed validity work from normal payloads. - -An outer in-place decoder now preserves direct BlockResidual unpack. - -It combines bin addition and ordered-float restoration in one pass. - -The final decomposed tree reached 10,148 to 10,389 MB/s across two focused runs. - -The direct fused RangePacked prototype reached 14,658 MB/s. - -The ALP-RD incumbent reached 15,578 to 16,060 MB/s. - -Scalar access used 312 to 323 ns for the decomposed tree and 150 to 158 ns for ALP-RD. - -The decomposed tree retains bounded random access, but the constant factor remains material. - -The matched 16-dataset compression pass produced these geometric-mean changes: - -| Metric | Change versus Current Default | -| --- | ---: | -| File size | -0.056 percent | -| Write time | +1.172 percent | -| Read time | -0.268 percent | - -HashTags file size fell from 179,382,860 bytes to 178,180,812 bytes. - -Its matched write time fell by 0.82 percent, and its matched read time fell by 2.38 percent. - -Those complete-file timing changes conflict with the slower selected column. - -Treat them as aggregate noise until a larger repeated pass confirms them. - -The first two million rows selected the new tree only for HashTags `interaction#received_at`. - -The initial full files also changed size for Bimbo, CMSprovider, Euro2016, and Taxi. - -A later chunk-level trace identified completed incumbent cascades as the cause. - -The current broad size gain does not justify Default inclusion by itself. - -The prototype temporarily added IntMult to a draft edition for file tests. - -The focused branch removed IntMult and that edition membership. - -Fast rejection remains a cost advantage, not an admission rule. - -A scheme without fast rejection can enter Default when corpus evidence justifies its analysis cost. - -### Completed incumbent cascade comparison - -The first decomposed selector compared its complete fixed tree against each incumbent root estimate. - -Several incumbents compressed their children after root selection. - -A full Bimbo chunk trace found eight false fixed-bin selections in `Dev_proxima`. - -The fixed-bin trees used 5.3 to 239.7 percent more bytes than the completed incumbent trees. - -The selector now compares each retained candidate against the incumbent sample cascade. - -The coarse model runs first. Clear losses do not pay for a second sample compression. - -This comparison preserves the current cascade depth and excludes only the fixed-bin scheme. - -The corrected Bimbo file uses 391,860,906 bytes in both configurations. - -Its matched write throughput changes from 515.6 to 509.6 MB/s. - -Its matched read throughput changes from 11,143.6 to 11,130.7 MB/s. - -The focused HashTags file retains the useful selection: - -| Metric | Current | Fixed-bin candidate | Change | -| --- | ---: | ---: | ---: | -| File size | 21,252,695 bytes | 20,718,724 bytes | -2.51 percent | -| Write throughput | 626.6 MB/s | 610.3 MB/s | -2.60 percent | -| Read throughput | 21,294.0 MB/s | 20,913.0 MB/s | -1.79 percent | - -The matched 16-dataset file pass selected the fixed-bin tree only on HashTags. - -| Metric | Geometric-mean change | -| --- | ---: | -| File size | -0.027 percent | -| Write time | +0.301 percent | -| Read time | -0.298 percent | - -The read result is benchmark noise because only one file changes its encoded tree. - -The corpus size result does not justify Default inclusion with the current decode and scalar costs. - -Fast rejection remains an advantage. It does not replace evidence from complete candidate trees. - -### Complete bundle controls - -The compression benchmark exposes four numeric bundles: - -- `prior-default` -- `block-residual` -- `current-default` -- `range-packed` - -All bundles use the same file strategy construction. - -The complete pass covered 16 datasets with three iterations per operation. - -The BlockResidual bundle produced this geometric-mean change against Prior Default: - -| Scope | Size change | Write throughput change | Read throughput change | -| --- | ---: | ---: | ---: | -| Numeric files | -2.6 percent | +0.0 percent | +2.0 percent | -| Real files | -2.5 percent | +0.0 percent | +0.9 percent | -| All files | -1.6 percent | -0.9 percent | +1.3 percent | - -This result supports BlockResidual as the primary Default bundle. - -The first FloatQuant pass exposed two selector errors. - -HashTags produced a constant FloatQuant sample and an estimated ratio of 8,193. - -Full analysis rejected the transform. The failed winner then blocked a compact Sparse tree. - -FloatQuant now rejects a constant sample. The main compressor already handles true constant arrays. - -Food selected FloatQuant from a 1.94 sample ratio. Its full ratio was 1.73. - -ALP had a 1.80 sample ratio and a 2.51 full ratio on the same chunk. - -A 1.10 FloatQuant factor retained every focused FloatQuant selection test and rejected this Food choice. - -The corrected FloatQuant bundle produced this change against the BlockResidual bundle: - -| Scope | Size change | Write throughput change | Read throughput change | -| --- | ---: | ---: | ---: | -| 16-file corpus | 0.0 percent | -0.9 percent | -0.8 percent | - -Every file size matched exactly. - -No selected-tree evidence supports a read difference. Treat the measured read change as noise. - -The write result measures FloatQuant analysis without a size win in this corpus. - -FloatQuant remains a production array and a BtrBlocks candidate. - -Default inclusion requires evidence that justifies its analysis cost. - -### Real Pco-gap FloatQuant pass - -The current benchmark tested 33 real float columns across five inputs: - -- CMS Open Payments. -- California Housing. -- NYC Taxi. -- CMS Provider 1. -- CMS Provider 2. - -The pass compared `block-residual-bundle` with `proposed-default`. - -FloatQuant selected no column. - -Every proposed file size matched the BlockResidual bundle. - -The geometric-mean encode throughput changed by +0.12 percent. - -The geometric-mean decode throughput changed by -0.06 percent. - -Both throughput changes are benchmark noise because the selected trees are identical. - -Compact retains material size gains on many float columns in these files. - -The Pco mode profile attributes those gaps to classic bins, IntMult, and FloatMult. - -This corpus does not expose a FloatQuant win under the current factor. - -The complete synthetic `f32` and `f64` trees produce strict size and throughput wins. - -The complete corpus shows no selected FloatQuant tree and no stable analysis-cost signal. - -Retain FloatQuant as an opportunistic option under the calibrated 1.10 factor. - -### Wider-integer BlockResidual factor sweep - -The final sweep compares wider-integer access cost factors of 1.00, 1.02, and 1.05. - -Each compression pass uses six complete Public BI files and three iterations per operation. - -The access pass requests about 100 rows through cached file handles. Each pattern runs for five seconds. - -| Factor | Size geometric mean | Total bytes | Compression time geometric mean | Decompression time geometric mean | Correlated access | Uniform access | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| 1.02 | -0.25 percent | +0.44 percent | -0.46 percent | +2.17 percent | +3.93 percent | +1.88 percent | -| 1.05 | +0.92 percent | +1.49 percent | -0.05 percent | +5.04 percent | +1.21 percent | +0.03 percent | - -The first candidate access passes contain large Bimbo and CMS correlated outliers. - -Reverse-order repeats remove those outliers. The table uses the repeated correlated results. - -The 1.02 factor diverts enough Food trees to reduce geometric-mean size by 0.25 percent. - -It increases total bytes by 0.44 percent. It also reduces bulk decode and file-access throughput. - -The 1.05 factor increases both geometric-mean size and total bytes. It provides no stable access gain. - -Retain the 1.00 wider-integer factor. The raw ratio floor and patch cost remain sufficient. - -### Final selector factors - -The integer BlockResidual scheme keeps a 1.05 raw-ratio floor. It applies a 1.12 access cost factor to 8-bit estimates. - -This policy resolves the CMS threshold miss and produces a 2.94 percent complete-corpus size gain. - -The Ordered BlockResidual scheme keeps its 1.05 raw floor and 1.02 decode factor. - -The nonlinear patch cost rejects the known dense-patch and slow HashTags trees. - -The 1.02 and 1.05 trials provide no evidence for a wider-integer factor. - -The FloatQuant scheme keeps its 1.10 factor. - -The Food sample mismatch requires a factor above 1.078 to retain its smaller ALP tree. - -The 1.10 value adds a small margin and retains every strict synthetic `f32` and `f64` win. - -The 33-column Pco-gap pass exposes no missed real FloatQuant candidate. - -The 8-bit factor rejects the weakest measured gain. It retains three real quantized GloVe variants with 16 to 22 percent savings. - -### Default bundle options - -The final numeric pass uses complete files and three iterations per operation. - -It covers Taxi, GloVe, Arade, Bimbo, CMSprovider, Euro2016, Food, and HashTags. - -The BlockResidual bundle produces these changes against Prior Default: - -| Dataset | Size | Write throughput | Read throughput | -| --- | ---: | ---: | ---: | -| Taxi | -6.35 percent | -0.81 percent | +7.23 percent | -| GloVe | 0.00 percent | -11.09 percent | +0.15 percent | -| Arade | -2.39 percent | -1.71 percent | +5.70 percent | -| Bimbo | -8.90 percent | -1.56 percent | -5.49 percent | -| CMSprovider | -7.94 percent | -1.40 percent | -8.51 percent | -| Euro2016 | -2.71 percent | +8.26 percent | -1.98 percent | -| Food | -7.79 percent | +1.95 percent | +6.08 percent | -| HashTags | -3.22 percent | +10.67 percent | -7.45 percent | -| Geometric mean | -4.96 percent | +0.34 percent | -0.71 percent | - -Total bytes decrease by 6.18 percent across the eight files. - -Total write time increases by 0.49 percent. Total read time increases by 3.39 percent. - -The GloVe write result conflicts with adjacent runs that reverse the difference. - -GloVe retains the same tree and size. Treat its write result as benchmark noise. - -The remaining eight files contain two TPC-H comment layouts and six nested wide-table cases. - -Both TPC-H files become 3.45 percent smaller through BlockResidual descendants. - -All six wide-table files retain identical sizes. - -Across all 16 files, the BlockResidual bundle produces these changes: - -| Metric | Geometric-mean change | Aggregate change | -| --- | ---: | ---: | -| Size | -2.94 percent | -5.30 percent | -| Write throughput | -1.22 percent | -0.60 percent | -| Read throughput | -1.21 percent | -2.62 percent | - -The evidence supports three bundle options: - -| Option | Schemes | Scope | Comparison | Size | Write throughput | Read throughput | -| --- | --- | --- | --- | ---: | ---: | ---: | -| Conservative | BlockResidual | 16 files | Prior Default | -2.94 percent | -1.22 percent | -1.21 percent | -| Opportunistic | BlockResidual and FloatQuant | 16 files | Conservative | 0.00 percent | Noise | Noise | -| Fixed-bin experiment | Opportunistic plus RangePacked | 8 numeric files | Opportunistic | -0.054 percent | -1.03 percent | Noise | - -The opportunistic option is the leading Default bundle. - -It adds strict synthetic FloatQuant wins without a measured corpus size regression. - -The opportunistic option selects no FloatQuant tree in these eight files. - -Focused synthetic `f32` and `f64` inputs still produce strict FloatQuant size and throughput wins. - -The fixed-bin option changes only HashTags `interaction#received_at`. - -Its numeric subset is 5.99 percent smaller, but the selected tree has material access costs. - -The tree decodes at about 10.1 GB/s instead of 15.6 to 16.1 GB/s. - -Scalar access takes 312 to 323 ns instead of 150 to 158 ns. - -The absolute decode rate remains valid for Default consideration. - -RangePacked remains an active experiment until the broad corpus establishes its coverage and total cost. - -### Default bundle random access - -The random-access benchmark now writes one Vortex file for each numeric bundle. - -The cached-handle pass uses correlated and uniform patterns with about 100 requested rows. - -| Dataset and pattern | Prior Default latency | Current Default latency | Change | -| --- | ---: | ---: | ---: | -| Taxi correlated | 0.609 ms | 0.624 ms | +2.45 percent | -| Taxi uniform | 2.760 ms | 2.807 ms | +1.68 percent | -| Nested lists correlated | 0.154 ms | 0.148 ms | -3.95 percent | -| Nested lists uniform | 0.922 ms | 0.922 ms | -0.06 percent | -| Nested structs correlated | 0.216 ms | 0.220 ms | +1.94 percent | -| Nested structs uniform | 0.532 ms | 0.544 ms | +2.20 percent | -| Geometric mean | — | — | +0.69 percent | - -The six standard patterns remain within four percent of Prior Default. - -Taxi file size decreases by 6.35 percent. Nested-list file size decreases by 0.71 percent. - -The nested-struct files have identical sizes. - -The legacy six-index Taxi case improves from 1.587 ms to 1.325 ms. - -This small target is more sensitive to run noise than the 100-row patterns. - -BlockResidual, Current Default, and RangePacked remain within 2.5 percent on both Taxi 100-row patterns. - -The Public BI pass uses six complete files and cached handles. Each pattern requests about 100 rows. - -The first pass exposed repeated BlockResidual payload validation during each slice operation. - -The source array already passed validation. The slice only changes its logical bounds. - -The slice path now constructs the narrowed array without repeated payload validation. - -The valid patch-position branch now avoids a non-inlined fallible call for each patch. - -A focused CMS correlated pass decreased from 1.127 ms to 0.964 ms after both changes. - -The Prior Default control changed from 0.991 ms to 0.954 ms across the same runs. - -The CMS gap therefore decreased from 13.7 percent to 1.1 percent. - -Two new passes use five-second target windows. The second pass uses reverse dataset order. - -| Dataset | File size | Correlated latency | Uniform latency | -| --- | ---: | ---: | ---: | -| Arade | -2.39 percent | -15.75 to -6.11 percent | -5.11 to +2.75 percent | -| Bimbo | -7.54 percent | -8.28 to +1.96 percent | -1.09 to +7.99 percent | -| CMSprovider | -7.86 percent | -2.22 to +9.22 percent | -6.64 to +11.82 percent | -| Euro2016 | -2.44 percent | +1.55 to +2.64 percent | -1.76 to +3.54 percent | -| Food | -6.19 percent | -29.89 to -15.70 percent | -2.83 to +6.97 percent | -| HashTags | -2.56 percent | +3.23 to +6.85 percent | +8.34 to +13.26 percent | - -The file-size geometric mean still decreases by 4.86 percent. Aggregate bytes still decrease by 5.81 percent. - -Across both passes, correlated latency decreases by 5.05 percent geometrically. Uniform latency increases by 2.91 percent. - -The combined latency geometric mean decreases by 1.15 percent. - -The individual file values remain noisy. The aggregate result removes the earlier random-access trade for the conservative bundle. - -The selector factors remain unchanged. - -### Broad fixed-bin coverage pass - -The Pco mode scan covers 53 float columns across 12 real datasets. - -It uses 524,288 rows per large column. This limit includes two complete Pco chunks. - -Pco selects classic bins without Delta on 41 chunks. It selects classic bins with consecutive Delta on another 17 chunks. - -Four columns use direct classic bins without Delta: - -| Dataset and column | Compact gap | Maximum Pco bins | -| --- | ---: | ---: | -| HashTags `interaction#received_at` | 65.2 percent | 36 | -| Taxi `airport_fee` | 53.6 percent | 4 | -| Euro2016 `subjectivity_confidence` | 51.1 percent | 39 | -| Taxi `tips` | 45.4 percent | 17 | - -The current corpus therefore contains the direct fixed-bin distribution. - -Another 21 columns use classic bins only below an outer transform such as ALP. - -Fifteen of these columns have a Compact gap of at least ten percent. - -Twelve also use at most 64 Pco bins. Six have a Compact gap of at least 20 percent. - -The complete bundle pass uses 18 files and three iterations per operation. - -It adds the CMS Payments and Twitter Pco inputs to the standard 16-file corpus. - -| Fixed-bin change against opportunistic | Geometric mean | Aggregate | -| --- | ---: | ---: | -| Size | -0.024 percent | -0.034 percent | -| Write throughput | -0.45 percent | Noise | -| Read throughput | -0.65 percent | Noise | - -Only HashTags changes its file size. The other 17 files retain identical sizes. - -The HashTags file becomes 0.429 percent smaller. - -Its selected `interaction#received_at` tree changes from 3,263,939 bytes to 1,991,606 bytes. - -The column becomes 39.0 percent smaller. Decode throughput changes from 15.79 GB/s to 11.44 GB/s. - -Scalar access changes from 165 ns to 322 ns. - -Fresh files from the same run produce this cached file-access result: - -| Pattern | Opportunistic | Fixed-bin | Change | -| --- | ---: | ---: | ---: | -| Correlated | 1.978 ms | 2.009 ms | +1.58 percent | -| Uniform | 12.142 ms | 12.018 ms | -1.02 percent | - -The selected tree passes the absolute decode and bounded-access bar. - -The current direct scheme remains niche. Its full-suite size gain does not justify Default inclusion alone. - -The nested Pco evidence motivated the pure integer fixed-bin prototype. - -The experiment added a pure integer scheme that constructs this tree: - -`IntMult(base=1, Dict(BitPacked(codes), starts), BlockResidual(offsets))` - -ALP selects it through normal child recursion. - -Keep the black-box RangePacked array outside Default. - -### Pure integer fixed-bin result - -The prototype constructs this complete integer tree: - -`IntMult(base=1, Dict(BitPacked(codes), starts), BlockResidual(offsets))` - -It supports every signed and unsigned integer type. - -Signed inputs flip the sign bit before the bin fit. The tree restores starts and offsets with modular addition. - -Tests cover every integer width, nullable values, signed-domain boundaries, and composition below ALP. - -The bin trainer previously used one fixed stride. Periodic inputs could alias that stride and hide complete clusters. - -A deterministic offset within each sample stratum now covers periodic phases. One regression test covers four interleaved clusters. - -The first selector version examined every integer recursion site. - -Across nine available datasets, this version reduced write throughput by 6.85 percent geometric mean. - -An ALP-child gate reduced the write loss to 2.78 percent geometric mean. - -The focused pass used 524,288 rows per dataset. - -It covered Arade, Bimbo, both CMS Provider files, Euro2016, Food, HashTags, CMS Payments, and GloVe embeddings. - -The fixed-bin scheme selected no tree. Every encoded size matched the Opportunistic bundle exactly. - -This result agrees with the earlier forced-tree measurements. - -One generic offset child mixes residuals from bins with different local widths. - -BlockResidual then pays for local width variation and occasional cross-bin outliers. - -The Pco bin model retains one offset width per bin. The generic tree loses that advantage. - -The pure integer fixed-bin scheme therefore does not enter Default. - -Keep the implementation as an experimental reference. Do not spend more selector time on this tree without new evidence. - -The direct OrderedFloat fixed-bin scheme remains a separate experiment. - -It still reduces HashTags `interaction#received_at` by 39.0 percent. Its complete file gain remains 0.429 percent. - -### Compact and Parquet size constraint - -The format pass uses three iterations per operation. - -| Scope and comparison for Current Default | Size | Write time | Read time | -| --- | ---: | ---: | ---: | -| 8 numeric files versus Compact | +38.66 percent | -13.42 percent | -66.63 percent | -| 8 numeric files versus Parquet with Zstd | -1.11 percent | -65.37 percent | -96.08 percent | -| Complete 16 files versus Compact | +21.85 percent | -11.69 percent | -55.35 percent | -| Complete 16 files versus Parquet with Zstd | -1.67 percent | -34.80 percent | -85.82 percent | - -Current Default remains 1.67 percent smaller than Parquet with Zstd across the complete corpus. - -It writes 1.5 times faster and reads 7.1 times faster than Parquet with Zstd. - -Compact remains 17.9 percent smaller than Current Default across the complete corpus. - -Current Default writes 13.2 percent faster and reads 2.2 times faster than Compact. - -Quick rejection reduces analysis cost and strengthens a candidate. - -It is not an admission rule. A costly candidate requires stronger size and throughput evidence. - -### Multi-reference block prototype - -The prototype stores up to four quantile references per 1,024-value block. - -Two-bit FastLanes identifiers select one reference for each value. FastLanes stores the residuals. - -The final experiment uses packed `u8` identifiers and a direct four-entry reference lookup. - -| Euro candidate | Bytes | Encode MB/s | Decode MB/s | Scalar ns | -| --- | ---: | ---: | ---: | ---: | -| Default ALP | 14,234,326 | Not isolated | 21,759.7 | 523.6 | -| One-reference BlockResidual | 13,293,263 | 2,171.1 | 24,382.6 | 198.9 | -| Four references with patches | 11,353,335 | 1,026.5 | 11,215.0 | 19.2 | -| Four references without patches | 13,117,584 | 1,357.9 | 12,960.4 | 7.4 | -| Compact Pco | 6,860,457 | Not isolated | 2,467.8 | 18,027.2 | - -Packed `u8` identifiers materially improve the multi-reference decoder. - -The patched form is 20.2 percent smaller than Default. Its decode throughput is 48.5 percent lower. - -The patch-free form is 7.8 percent smaller. Its decode throughput is 40.4 percent lower. - -Patch application is not the main cost. Reference-ID expansion and per-value reference lookup dominate. - -The one-reference BlockResidual tree is both faster and simpler. The multi-reference prototype is rejected for Default. - -### Centered block residual prototype - -Euro2016 `subjectivity_confidence` contains a large cluster near `1.0`. - -About 36.5 percent of the first two million values equal `1.0`. Many other values sit close to that value. - -The prototype selects one median reference per 1,024-value block. It ZigZag-encodes each ordered-float distance from that reference. - -BlockResidual stores the transformed distances. The decoder fuses residual reconstruction, inverse ZigZag, and the ordered-float inverse. - -Null values use the block reference as their payload. They produce zero residuals and keep logical positions. - -The prototype tested extra patch costs from 16 through 96 bits. A 32-bit cost gives the best measured size and speed tradeoff. - -| Euro subjectivity tree | Bytes | Encode MB/s | Decode MB/s | Scalar ns | -| --- | ---: | ---: | ---: | ---: | -| Current Default | 14,234,326 | 358 | 22,664 | 511 | -| `OrderedFloat(BlockResidual)` | 13,293,263 | 2,171 | 24,128 | 196 | -| Centered block residual | 13,071,137 | 1,068 | 14,028 | Direct codec: 12 | -| Compact Pco | 6,860,457 | 424 | 2,711 | 17,582 | - -The centered form uses 1.7 percent fewer bytes than ordinary BlockResidual. - -Its decode throughput is 41.9 percent lower than ordinary BlockResidual. Its encode throughput is also about half as fast. - -The result rejects centered BlockResidual for Default. The extra transform does not recover enough of the Compact gap. - -Ordinary BlockResidual gives a strict size and throughput win on this column. The final selector calibration must include this case. - -### Pcodec corpus gap analysis - -The focused benchmark reads the first two million numeric rows from each source. - -Air Quality, r/place, and the Twitter graph contain no float columns in the benchmark schema. - -The float gap corpus includes California Housing, NYC Taxi, CMS Payments, and four Public BI datasets. - -Thirty float columns use at least ten percent fewer bytes with Compact Pco than with the prior default. - -Pco uses four principal mechanisms on these columns: - -- Entropy bins on ALP integer children. -- First-order Delta plus bins on raw floats or ALP integer children. -- FloatMult plus bins on Taxi tips. -- IntMult splits on selected integer children. - -No Pcodec paper gap used lookback Delta. Some Public BI columns used lookback Delta. - -GloVe, CMS Payments, and Food include important no-Delta wins. - -GloVe uses `IntMult(10)` plus entropy bins. CMS Payments and Food use classic entropy bins. - -Arade relies mainly on first-order consecutive Delta. Two columns combine `IntMult` with Delta. - -The current CMS source revision differs from the paper source snapshot. - -The Twitter source uses the first two million official edges. The paper used an unspecified ID sort. - -## General real-float follow-up - -Target float columns where Compact Pco materially beats the complete default cascade. - -The default baseline includes ALP, ALP-RD, and compression of their children. - -Use a ten-percent Pco size advantage as the initial corpus filter. This filter is not a product threshold. - -Measure gap recovery as `(default bytes - candidate bytes) / (default bytes - Pco bytes)`. - -Retain a prototype only if it recovers a material gap across unrelated real datasets. - -The retained schemes cover lower-precision values in wider floats and locally narrow ordered or integer ranges. - -The multi-reference estimator tested one, two, and four quantile references per block on thirty gap columns. - -The estimate includes reference identifiers, packed residuals, patch positions, patch highs, and block metadata. - -One reference won 25 columns. Four references won five columns. Two references won no columns. - -The best estimate recovered 29.4 percent of the aggregate Pco size gap. - -It reduced default bytes by 7.9 percent across the gap columns. - -The extra references did not justify a new array. Generic integer BlockResidual became the next prototype. - -Test these secondary candidates after the quotient and remainder prototype: - -- Use ordered-bit XOR suffixes instead of arithmetic residuals. -- Use sign and exponent prefixes as fixed reference candidates. -- Use block-local ALP-RD split widths. -- Use independent entropy microblocks only in Compact. - -Classify the other gaps by exponent count, prefix count, local range, low-bit entropy, and outlier rate. - -Require size, compression, decode, and scalar-access results on the same two-million-row inputs. - -Do not add a general codec to the default until it wins across several unrelated real datasets. - -## Removed candidates - -### FloatMult - -The useful exact-multiple form duplicated an ALP candidate that the upstream Rust search omitted. - -The upstream ALP fix now includes equal exponent and factor pairs such as `{0,0}`. - -The two-child form with nonzero adjustments decoded 64 percent slower than the prior default. - -Its scalar access took more than seven times longer in the large-row test. - -Vortex will take the corrected ALP dependency after its release. This release does not block other work. - -### RangeEntropy overlap - -`RangeEntropy` and `OrderedFloat(BlockResidual)` both compress ordered unsigned latents. - -`BlockResidual` models one local range per block. It uses a minimum, packed residuals, and sparse high-bit patches. - -`RangeEntropy` models many ranges across the input. It entropy-codes range identifiers and stores fixed-width offsets inside each range. - -The entropy model can represent separated clusters better than one local reference. - -That benefit requires bin search, model construction, two encoded streams, and ANS state transitions. - -The measured size gains did not compensate for those costs under the default writer priorities. - -| Dataset | Default bytes | RangeEntropy bytes | Compact bytes | -| --- | ---: | ---: | ---: | -| California Housing | 339,682 | 319,705 | 230,197 | -| April 2023 HVFHV Taxi | 28,540,793 | 23,331,922 | 21,127,765 | -| Air Quality | 6,135,414 | 5,149,775 | 2,311,526 | - -RangeEntropy reduced Taxi size by 18.3 percent and Air Quality size by 16.1 percent. - -It remained larger than Compact on every measured dataset. - -RangeEntropy reduced compression throughput by 42 to 49 percent. - -Its decode path was 5.7 to 17.2 times slower than the lightweight default. - -Pco already occupies the compact, slower-encode design point with equal or better measured size. - -RangeEntropy therefore has no current production role in the default or Compact compressor. - -### BitSplit - -The prototype used one prefix dictionary per block and fixed-width suffixes. - -It handled separated clusters better than multiple residual references. - -It remained 1.4 percent larger than ALP-RD on the four-cluster input. - -Its full decode was much slower than the current ALP-RD path. - -The experimental branch retains the prototype for possible ALP-RD decomposition research. - -## Revalidation after the ALP release - -The workspace now uses ALP 0.0.3. - -The corrected ALP selected `ALP(FoR(BitPacked))` for the integer-valued float control. - -The control used 5,002,240 bytes with or without the new schemes. - -This round completed these steps: - -- Replaced the recursive FloatQuant child search with one fixed sample tree. -- Added scalar-access benchmarks on two million values. -- Added nonzero-secondary FloatQuant benchmarks. -- Added all six Pcodec paper datasets. -- Fixed the high-value defects from three adversarial reviews. -- Extended BlockResidual to every integer type. -- Added native-width residual pack and unpack operations. -- Added integer BlockResidual to `single_encoding_throughput`. -- Measured the Pco gap across thirty float columns. -- Rejected the multi-reference residual design. -- Added the outer-sample exclusion and width-specific factors. -- Added the patch-count decode cost. -- Initially excluded direct 8-bit and 16-bit candidates. -- Excluded BlockResidual from dictionary-code children. -- Added fused f32 OrderedFloat with BlockResidual decode and selection. -- Added u32, i32, f32, and patch-density benchmarks. -- Added complete-compressor patch-density datasets and a column filter to the profiling benchmark. -- Added complete temporal, FSST, Sparse, RunEnd, list, and ALP composition benchmarks. -- Added selection tests for BlockResidual under ALP, Sparse, and RunEnd. -- Added exact allocation-free BlockResidual size estimates. -- Replaced nine BlockResidual child arrays with one aligned parent payload buffer. -- Removed eight file segments from each serialized BlockResidual array. -- Revalidated size and throughput across eight file datasets. -- Added an all-valid fast path for locality sample copies. -- Rejected a four-block short-column estimate after a real mis-ranking. -- Added patch-density statistics to the BlockResidual profile. -- Replaced the global patch cost experiment with a nonlinear selector cost. -- Removed the full patch scan from scalar access. -- Made null payload bits neutral for the new default candidates. -- Excluded integer BlockResidual from the CUDA-compatible preset. -- Added the real GloVe embeddings dataset to the compression corpus. -- Added native `f32` FloatQuant selection and scheme benchmarks. -- Replaced the full primary buffer with fused FloatQuant and FastLanes packing. -- Replaced the generic FloatQuant sample with a direct fixed-tree estimator. -- Added fixed-tree analysis for nonzero FloatQuant secondary values. -- Added direct paired FastLanes packing for both FloatQuant children. -- Added fused two-child FloatQuant decode and width-sweep benchmarks. -- Validated selected and rejected two-child FloatQuant paths. -- Added two-child FloatQuant benchmarks for `f16`, `f32`, and `f64`. -- Confirmed strict complete-tree wins for the selected `f32` and `f64` cases. -- Confirmed that the `f16` selector retains its smaller dictionary tree. -- Added Compact as a first-class format in the compression benchmark. -- Compared the prior Default, proposed Default, Compact, and Parquet across all 16 local datasets. -- Attributed the largest Compact numeric gaps to Pco modes at the column level. -- Rejected byte-aligned fixed bins after a direct size and throughput comparison. -- Rejected grouped fixed bins after Euro2016 and CMS comparisons. -- Rejected packed multi-reference blocks after patched and patch-free comparisons. -- Rejected three bounded IntMult remainder layouts on GloVe. -- Rejected the dense-remainder IntMult layout on CMS Payments. -- Rejected pure IntMult on a no-Delta CMS standard-deviation column. -- Rejected centered block residuals after a complete Euro subjectivity comparison. -- Tested dictionary codes in descending value frequency, then removed the policy with IntMult. -- Tested float frequency counts, then restored the existing distinct-value statistics. -- Reopened BlockResidual descendants for integer and float dictionary codes. -- Validated the ranked Dict policy across eight numeric datasets. -- Audited constructor and deserialization validation for all three production arrays. -- Added round-trip tests for every supported integer and float type. -- Added single-encoding benchmarks for every supported integer and float type. -- Added packed-child IntMult benchmarks for every unsigned integer type. -- Added fused IntMult decode for aligned, patch-free BitPacked children. -- Added `f16` support to OrderedFloat, FloatQuant, and both Default schemes. -- Verified zero-secondary and nonzero-secondary `f16` FloatQuant trees. -- Added exact RangePacked size estimates without packed payload construction. -- Added a coarse RangePacked rejection model over the existing one-percent sample. -- Added a locality rejection test that retains Euro and rejects clear BlockResidual fits. -- Updated stale golden trees after the BlockResidual payload and selector changes. -- Added four numeric bundle controls to the complete compression benchmark. -- Tested explicit winner result events, then removed the unused trace event. -- Added optional chunk-level encoding trees to the focused compressor benchmark. -- Rejected constant FloatQuant samples before exact sample encoding. -- Added and calibrated a 1.10 FloatQuant selection factor. -- Replaced FloatQuant sample packing with an exact buffer-size estimate. -- Added isolated FloatQuant rejection controls for all float widths. -- Added adjusted-ratio near-miss controls for `f32` and `f64`. -- Re-enabled 16-bit BlockResidual after absolute-throughput and corpus revalidation. -- Enabled an 8-bit BlockResidual selector trial with selected and rejected synthetic controls. -- Removed the Food and HashTags FloatQuant size regressions. -- Revalidated the BlockResidual and FloatQuant bundles across 16 files. -- Rejected constant RangePacked samples before exact sample encoding. -- Revalidated RangePacked against the corrected current bundle. -- Compared retained fixed-bin candidates against the completed incumbent sample cascade. -- Removed every known Bimbo, CMSprovider, Euro2016, and Taxi fixed-bin false win. -- Tested FloatQuant on 33 real float columns from five Pco-gap inputs. -- Revalidated the final bundles across eight numeric files and all 16 corpus files. -- Revalidated Current Default against Compact and Parquet with Zstd across all 16 files. -- Added numeric bundle controls to the random-access benchmark. -- Compared cached random access across Taxi and two nested datasets. -- Added `i8` and `u8` support to the focused Parquet and scalar benchmarks. -- Validated 8-bit selection with quantized real GloVe values. -- Added local Parquet inputs to the complete compression benchmark. -- Added local Parquet inputs to the random-access benchmark. -- Measured 8-bit BlockResidual file access on accepted and boundary cases. -- Measured complete-file random access for six Public BI datasets. -- Added a fused `f16` decoder for `OrderedFloat(BlockResidual)`. -- Added a fused implicit-zero `f64` decoder for `FloatQuant(FoR(BitPacked))`. -- Rejected 1.02 and 1.05 wider-integer BlockResidual access cost factors. -- Added a fast Pco mode-only profile path. -- Profiled Pco modes across 53 float columns from 12 real datasets. -- Compared opportunistic and fixed-bin bundles across 18 complete files. -- Measured the selected HashTags fixed-bin tree and fresh cached file access. -- Added the pure integer fixed-bin scheme for ALP child recursion. -- Fixed alias errors in the deterministic fixed-bin sample. -- Rejected the integer fixed-bin tree after a focused nine-dataset pass. - -The Pco mode profile and the quotient and remainder experiments are complete. - -The composable IntMult follow-up is complete. The tested IntMult trees remain outside Default. - -The nonzero-secondary FloatQuant implementation and focused validation are complete. - -The specialized OrderedFloat and ALP trees now use registered fused parent kernels. - -The pure integer fixed-bin experiment is complete. - -It selected no integer tree and reduced write throughput. The focused branch no longer contains either fixed-bin scheme. - -## Final calibration - -The final pass compared 17 datasets with five timed iterations per operation. - -The corpus included GloVe `f32` embeddings and OpenAI-on-C4 `f64` embeddings. - -| Bundle or format | Aggregate bytes | Encode time | Decode time | -| --- | ---: | ---: | ---: | -| Prior Default | 3,025,406,960 | 10.928 seconds | 0.424 seconds | -| Opportunistic | 2,672,321,904 | 10.487 seconds | 0.422 seconds | -| Compact | 2,027,179,944 | 11.655 seconds | 1.082 seconds | -| Parquet with Zstd | 2,718,861,135 | 24.830 seconds | 5.635 seconds | - -Against Prior Default, Opportunistic reduced aggregate size by 11.67 percent. - -It reduced aggregate encode time by 4.04 percent and aggregate decode time by 0.56 percent. - -The geometric means improved by 5.69 percent for size, 10.49 percent for encode time, and 4.99 percent for decode time. - -Opportunistic produced 1.71 percent fewer aggregate bytes than Parquet with Zstd. - -Its geometric-mean size ratio against Parquet with Zstd was 0.980. - -The focused cached-access corpus included five tabular datasets and the OpenAI vector dataset. - -Opportunistic reduced geometric-mean scalar latency by 10.97 percent and aggregate scalar latency by 10.20 percent. - -OpenAI-on-C4 selected `FloatQuant(FoR(BitPacked))`. - -That tree reduced size by 40.5 percent and improved bulk decode time against Prior Default. - -GloVe retained its prior tree. Its result still identifies an entropy gap inside the ALP integer child. - -The fixed-bin bundle reduced aggregate size by only 0.025 percent against Opportunistic. - -It changed only the HashTags file in the complete corpus, where it reduced file size by 0.43 percent. - -The selected HashTags column was 35.0 percent smaller, but that narrow win did not justify the scheme cost. - -## Final production decision - -Use the Opportunistic bundle in Default. - -The bundle contains these schemes: - -- `BlockResidualScheme` for integers. -- `OrderedBlockResidualScheme` for floating-point values. -- `FloatQuantScheme` for floating-point values. - -Retain these selector controls: - -- A 1.05 minimum ratio for integer BlockResidual. -- A 1.05 minimum ratio and 1.02 decode factor for ordered BlockResidual. -- A 1.10 FloatQuant factor. -- A 1.12 factor for 8-bit BlockResidual. -- The nonlinear BlockResidual patch cost. -- The one-block rejection for integer BlockResidual. - -Remove RangePacked and both fixed-bin schemes from the focused branch. - -Keep RangeEntropy, BitSplit, and alternate residual models on the experimental branch. - -Use ordinary child arrays for patches and exception payloads. - -Test these items in future research: - -1. Test nonzero-secondary FloatQuant on more real distributions. -2. Add more real float datasets where Pco beats ALP and ALP-RD. -3. Test a true ALP-RD child composition when a real selected tree exposes one. -4. Revisit fixed bins only when the corpus includes more applicable distributions. - -## Pull request structure - -Prepare one pull request for the three arrays, their schemes, fused FastLanes paths, and benchmark support. - -Split the pull request only when review reveals a clear independent boundary. - -Keep entropy, bit-split, and alternate residual models on `wm/pcodec-entropy-experiments`. - -## Adversarial review checkpoint - -The review followed the merge from `origin/develop`. - -It preserved the frozen August editions and added `core:2026.08.4` as a draft. - -It added the array conformance suite to BlockResidual, OrderedFloat, and FloatQuant. - -It added empty-buffer validation to OrderedFloat and FloatQuant deserialization. - -It added signed coverage for the fused nonzero-secondary FloatQuant encoder. - -It changed invalid FloatQuant references from a debug panic to an error. - -It removed the Pco probe, the custom Pco API, abandoned compressor hooks, and public BlockResidual codec internals. - -It removed unrelated dictionary frequency ranking and float frequency statistics. - -It retained the three production candidates, their selectors, and their focused benchmarks. - -## References - -- [PCodec paper](https://arxiv.org/html/2502.06112v2) -- [PCodec repository](https://github.com/pcodec/pcodec) From 815e0ce27ca36085ff4fc78edf81dc51f0418eda Mon Sep 17 00:00:00 2001 From: Will Manning Date: Sat, 22 Aug 2026 08:49:26 -0400 Subject: [PATCH 78/78] fix: stage numeric encodings in preview edition Signed-off-by: Will Manning --- vortex-bench/src/lib.rs | 4 ++-- vortex/src/editions/core/mod.rs | 2 -- vortex/src/editions/mod.rs | 6 +++--- vortex/src/editions/preview/mod.rs | 2 ++ .../{core/v2026_08_4.rs => preview/v2026_08.rs} | 10 +++++----- vortex/src/editions/tests.rs | 14 +++++++------- 6 files changed, 19 insertions(+), 19 deletions(-) rename vortex/src/editions/{core/v2026_08_4.rs => preview/v2026_08.rs} (64%) diff --git a/vortex-bench/src/lib.rs b/vortex-bench/src/lib.rs index 8f3b2ec1fba..bb5d8619d5f 100644 --- a/vortex-bench/src/lib.rs +++ b/vortex-bench/src/lib.rs @@ -74,8 +74,8 @@ pub use datasets::BenchmarkDataset; pub use output::BenchmarkOutput; pub use output::create_output_writer; use vortex::VortexSessionDefault; -use vortex::editions::CORE_2026_08_4; use vortex::editions::EditionSessionExt; +use vortex::editions::PREVIEW_2026_08_0; pub use vortex::error::vortex_panic; use vortex::io::session::RuntimeSessionExt; use vortex::session::VortexSession; @@ -87,7 +87,7 @@ static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; pub static SESSION: LazyLock = LazyLock::new(|| { let session = VortexSession::default().with_tokio(); session - .enable_edition(CORE_2026_08_4) + .enable_edition(PREVIEW_2026_08_0) .unwrap_or_else(|error| { vortex_panic!("numeric benchmark edition is not registered: {error}") }); diff --git a/vortex/src/editions/core/mod.rs b/vortex/src/editions/core/mod.rs index f344baf6136..84a0dcb8a05 100644 --- a/vortex/src/editions/core/mod.rs +++ b/vortex/src/editions/core/mod.rs @@ -12,7 +12,6 @@ pub mod v2025_10; pub mod v2026_08; pub mod v2026_08_2; pub mod v2026_08_3; -pub mod v2026_08_4; pub use v2025_05::CORE_2025_05_0; pub use v2025_06::CORE_2025_06_0; @@ -21,4 +20,3 @@ pub use v2026_08::CORE_2026_08_0; pub use v2026_08::CORE_2026_08_1; pub use v2026_08_2::CORE_2026_08_2; pub use v2026_08_3::CORE_2026_08_3; -pub use v2026_08_4::CORE_2026_08_4; diff --git a/vortex/src/editions/mod.rs b/vortex/src/editions/mod.rs index d770f14c165..1037bf097fd 100644 --- a/vortex/src/editions/mod.rs +++ b/vortex/src/editions/mod.rs @@ -42,18 +42,18 @@ pub use self::core::CORE_2026_08_0; pub use self::core::CORE_2026_08_1; pub use self::core::CORE_2026_08_2; pub use self::core::CORE_2026_08_3; -pub use self::core::CORE_2026_08_4; pub use self::preview::PREVIEW_2025_05_0; pub use self::preview::PREVIEW_2026_02_0; pub use self::preview::PREVIEW_2026_04_0; pub use self::preview::PREVIEW_2026_06_0; +pub use self::preview::PREVIEW_2026_08_0; /// The `core` edition enabled for writing by the default Vortex session. pub const DEFAULT_CORE_EDITION: EditionId = CORE_2026_08_1; /// The `preview` edition enabled for writing by the default Vortex session when the /// `unstable_encodings` feature is selected. -pub const DEFAULT_PREVIEW_EDITION: EditionId = PREVIEW_2026_06_0; +pub const DEFAULT_PREVIEW_EDITION: EditionId = PREVIEW_2026_08_0; /// The first-party Vortex edition declarations. pub static EDITION_DECLARATIONS: &[&EditionDeclaration] = &[ @@ -64,11 +64,11 @@ pub static EDITION_DECLARATIONS: &[&EditionDeclaration] = &[ &core::v2026_08::DECLARATION_1, &core::v2026_08_2::DECLARATION, &core::v2026_08_3::DECLARATION, - &core::v2026_08_4::DECLARATION, &preview::v2025_05::DECLARATION, &preview::v2026_02::DECLARATION, &preview::v2026_04::DECLARATION, &preview::v2026_06::DECLARATION, + &preview::v2026_08::DECLARATION, ]; /// Register the Vortex edition declarations with the session's [`EditionSession`]. diff --git a/vortex/src/editions/preview/mod.rs b/vortex/src/editions/preview/mod.rs index cba3ba1dc6c..07f07c5431e 100644 --- a/vortex/src/editions/preview/mod.rs +++ b/vortex/src/editions/preview/mod.rs @@ -10,8 +10,10 @@ pub mod v2025_05; pub mod v2026_02; pub mod v2026_04; pub mod v2026_06; +pub mod v2026_08; pub use v2025_05::PREVIEW_2025_05_0; pub use v2026_02::PREVIEW_2026_02_0; pub use v2026_04::PREVIEW_2026_04_0; pub use v2026_06::PREVIEW_2026_06_0; +pub use v2026_08::PREVIEW_2026_08_0; diff --git a/vortex/src/editions/core/v2026_08_4.rs b/vortex/src/editions/preview/v2026_08.rs similarity index 64% rename from vortex/src/editions/core/v2026_08_4.rs rename to vortex/src/editions/preview/v2026_08.rs index c91d31d691f..97d92410520 100644 --- a/vortex/src/editions/core/v2026_08_4.rs +++ b/vortex/src/editions/preview/v2026_08.rs @@ -1,20 +1,20 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! The August 2026 draft core edition with numeric array encodings. +//! The August 2026 `preview` encoding cohort. use vortex_edition::Edition; use vortex_edition::EditionDeclaration; use vortex_edition::EditionId; use vortex_edition::EditionMember; -/// The fifth August 2026 edition of the `core` family. -pub const CORE_2026_08_4: EditionId = EditionId::new("core", 2026, 8, 4); +/// The August 2026 draft edition of the `preview` family. +pub const PREVIEW_2026_08_0: EditionId = EditionId::new("preview", 2026, 8, 0); -/// The declaration of [`CORE_2026_08_4`] and its new components. +/// The declaration of [`PREVIEW_2026_08_0`] and the encodings that join the family at it. pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { - id: CORE_2026_08_4, + id: PREVIEW_2026_08_0, min_vortex_version: None, }, added: &[ diff --git a/vortex/src/editions/tests.rs b/vortex/src/editions/tests.rs index c8ef1622e20..53ac1149ee3 100644 --- a/vortex/src/editions/tests.rs +++ b/vortex/src/editions/tests.rs @@ -53,11 +53,11 @@ use super::CORE_2026_08_0; use super::CORE_2026_08_1; use super::CORE_2026_08_2; use super::CORE_2026_08_3; -use super::CORE_2026_08_4; use super::DEFAULT_CORE_EDITION; use super::DEFAULT_PREVIEW_EDITION; use super::EDITION_DECLARATIONS; use super::PREVIEW_2026_06_0; +use super::PREVIEW_2026_08_0; fn session() -> Result { let session = EditionSession::empty(); @@ -195,15 +195,15 @@ fn core_2026_08_3_adds_variants() { } #[test] -fn core_2026_08_4_adds_numeric_arrays() { +fn preview_2026_08_adds_numeric_arrays() { let session = session().unwrap_or_else(|e| panic!("registering editions: {e}")); assert!( session - .find(&CORE_2026_08_4) - .unwrap_or_else(|| panic!("{CORE_2026_08_4} is not registered")) + .find(&PREVIEW_2026_08_0) + .unwrap_or_else(|| panic!("{PREVIEW_2026_08_0} is not registered")) .is_draft() ); - let arrays = session.components_in(&CORE_2026_08_4, ComponentKind::Array); + let arrays = session.components_in(&PREVIEW_2026_08_0, ComponentKind::Array); for id in [ "vortex.block_residual", "vortex.float_quant", @@ -725,7 +725,7 @@ async fn numeric_draft_writer_round_trips_float_quant() -> VortexResult<()> { let session = VortexSession::default(); session - .enable_edition(CORE_2026_08_4) + .enable_edition(PREVIEW_2026_08_0) .map_err(|error| vortex_err!("{error}"))?; let values = (0u32..65_536) .map(|index| { @@ -765,7 +765,7 @@ async fn numeric_draft_writer_round_trips_ordered_block_residual() -> VortexResu let session = VortexSession::default(); session - .enable_edition(CORE_2026_08_4) + .enable_edition(PREVIEW_2026_08_0) .map_err(|error| vortex_err!("{error}"))?; let mut state = 0x4d59_5df4_d0f3_3173_u64; let mut value = 0.0_f64;