Move array serde from VTable to explicit ArrayPlugin impls

docs/developer-guide/internals/vtables.md

@@ -121,7 +121,9 @@ needs:
 But operations like deserialization need to construct a new instance from raw bytes -- there
 is no instance yet. Vortex vtables colocate instance logic (e.g. `serialize`) with type-level
 logic (e.g. `deserialize`) in the same struct, since the vtable itself is a lightweight value
-(often zero-sized) that can be default-constructed.
+(often zero-sized) that can be default-constructed. Array vtables are the exception: their serde
+lives on a separate `ArrayPlugin` implementation, usually on the vtable type itself, so a vtable
+without one is in-memory only.
 
 **No associated type access.** A `dyn Trait` erases all associated types. With Vortex vtables,
 the typed form `Foo<V>` preserves full access to `V::Metadata` and other associated types,
@@ -176,10 +178,11 @@ Methods that exist only for erased dispatch (e.g. `as_any`, `metadata_any`,
 
 ## Registration and Deserialization
 
-Vtables are registered in the session by their ID. When a serialized value is encountered
-(e.g. an extension dtype in a file footer), the session's registry resolves the ID to the
-`FooPlugin` and calls `deserialize` to reconstruct the value. The result is returned as the
-erased form (`FooRef`) so it can be stored without generic parameters.
+Vtables are registered in the session by their ID. An array vtable registers through its
+`ArrayPlugin` implementation, and a vtable without one cannot be registered. When a serialized
+value is encountered (e.g. an extension dtype in a file footer), the session's registry resolves
+the ID to the `FooPlugin` and calls `deserialize` to reconstruct the value. The result is returned
+as the erased form (`FooRef`) so it can be stored without generic parameters.
 
 This pattern -- register plugin by ID, deserialize via plugin, store as erased ref -- is
 consistent across all vtable-backed types in Vortex.

encodings/alp/src/alp/array.rs → encodings/alp/src/alp/array/mod.rs

@@ -7,7 +7,6 @@ use std::fmt::Formatter;
 use std::hash::Hash;
 use std::hash::Hasher;
 
-use prost::Message;
 use vortex_array::Array;
 use vortex_array::ArrayEq;
 use vortex_array::ArrayHash;
@@ -21,7 +20,6 @@ use vortex_array::ExecutionCtx;
 use vortex_array::ExecutionResult;
 use vortex_array::IntoArray;
 use vortex_array::TypedArrayRef;
-use vortex_array::VortexSessionExecute;
 use vortex_array::array_slots;
 use vortex_array::arrays::Primitive;
 use vortex_array::buffer::BufferHandle;
@@ -34,7 +32,6 @@ use vortex_array::patches::PatchesData;
 use vortex_array::patches::PatchesMetadata;
 use vortex_array::require_child;
 use vortex_array::require_patches;
-use vortex_array::serde::ArrayChildren;
 use vortex_array::smallvec::smallvec;
 use vortex_array::vtable::VTable;
 use vortex_array::vtable::ValidityChild;
@@ -45,14 +42,16 @@ use vortex_error::vortex_bail;
 use vortex_error::vortex_ensure;
 use vortex_error::vortex_ensure_eq;
 use vortex_error::vortex_panic;
-use vortex_session::VortexSession;
 use vortex_session::registry::CachedId;
 
 use crate::ALPFloat;
 use crate::alp::Exponents;
 use crate::alp::decompress::execute_decompress;
 use crate::alp::rules::RULES;
 
+mod plugin;
+pub(crate) use plugin::ALPPatchedPlugin;
+
 /// A [`ALP`]-encoded Vortex array.
 pub type ALPArray = Array<ALP>;
 
@@ -113,79 +112,6 @@ impl VTable for ALP {
         vortex_array::vtable::with_empty_buffers(self, array, buffers)
     }
 
-    fn serialize(
-        array: ArrayView<'_, Self>,
-        _session: &VortexSession,
-    ) -> VortexResult<Option<Vec<u8>>> {
-        let exponents = array.exponents();
-        Ok(Some(
-            ALPMetadata {
-                exp_e: exponents.e as u32,
-                exp_f: exponents.f as u32,
-                patches: array
-                    .patches()
-                    .map(|p| p.to_metadata(array.len(), array.dtype()))
-                    .transpose()?,
-            }
-            .encode_to_vec(),
-        ))
-    }
-
-    fn deserialize(
-        &self,
-        dtype: &DType,
-        len: usize,
-        metadata: &[u8],
-        _buffers: &[BufferHandle],
-        children: &dyn ArrayChildren,
-        session: &VortexSession,
-    ) -> VortexResult<ArrayParts<Self>> {
-        let metadata = ALPMetadata::decode(metadata)?;
-        let encoded_ptype = match &dtype {
-            DType::Primitive(PType::F32, n) => DType::Primitive(PType::I32, *n),
-            DType::Primitive(PType::F64, n) => DType::Primitive(PType::I64, *n),
-            d => vortex_bail!("expected type: f32 or f64 but instead got {}", d),
-        };
-        let encoded = children.get(0, &encoded_ptype, len)?;
-
-        let patches = metadata
-            .patches
-            .map(|p| {
-                let indices = children.get(1, &p.indices_dtype()?, p.len()?)?;
-                let values = children.get(2, dtype, p.len()?)?;
-                let chunk_offsets = p
-                    .chunk_offsets_dtype()?
-                    .map(|dtype| children.get(3, &dtype, usize::try_from(p.chunk_offsets_len())?))
-                    .transpose()?;
-
-                Patches::from_metadata(
-                    len,
-                    &p,
-                    indices,
-                    values,
-                    chunk_offsets,
-                    &mut session.create_execution_ctx(),
-                )
-            })
-            .transpose()?;
-
-        let slots = ALPData::make_slots(&encoded, patches.as_ref());
-        let data = ALPData::new(
-            Exponents {
-                e: u8::try_from(metadata.exp_e)?,
-                f: u8::try_from(metadata.exp_f)?,
-            },
-            patches,
-        );
-        Ok(ArrayParts::new(
-            self.clone(),
-            dtype.clone(),
-            len,
-            data,
-            slots,
-        ))
-    }
-
     fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String {
         ALPSlots::NAMES[idx].to_string()
     }

encodings/alp/src/alp/plugin.rs → encodings/alp/src/alp/array/plugin.rs

@@ -1,29 +1,137 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
-//! A custom [`ArrayPlugin`] that lets you load in and deserialize an `ALP` array with interior
-//! patches as a `PatchedArray` that wraps a patchless `ALP` array.
+//! [`ArrayPlugin`] implementations for [`ALP`].
 //!
-//! This enables zero-cost backward compatibility with previously written datasets.
+//! [`ALP`] itself reads and writes the current `vortex.alp` form. [`ALPPatchedPlugin`] reads an
+//! ALP node with interior patches as a `PatchedArray` wrapping a patchless ALP array, which keeps
+//! previously written datasets readable at no cost.
 
+use prost::Message;
 use vortex_array::Array;
 use vortex_array::ArrayDeserialization;
 use vortex_array::ArrayId;
+use vortex_array::ArrayParts;
 use vortex_array::ArrayPlugin;
 use vortex_array::ArrayRef;
 use vortex_array::ArraySerialization;
-use vortex_array::ArrayVTable;
 use vortex_array::IntoArray;
+use vortex_array::VTable;
 use vortex_array::VortexSessionExecute;
 use vortex_array::arrays::Patched;
+use vortex_array::dtype::DType;
+use vortex_array::dtype::PType;
+use vortex_array::patches::Patches;
 use vortex_error::VortexResult;
+use vortex_error::vortex_bail;
 use vortex_error::vortex_ensure_eq;
 use vortex_error::vortex_err;
 use vortex_session::VortexSession;
 
-use crate::ALP;
-use crate::ALPArrayExt;
-use crate::ALPArrayOwnedExt;
+use super::ALP;
+use super::ALPArrayExt;
+use super::ALPArrayOwnedExt;
+use super::ALPData;
+use super::ALPMetadata;
+use crate::alp::Exponents;
+
+impl ArrayPlugin for ALP {
+    fn id(&self) -> ArrayId {
+        VTable::id(self)
+    }
+
+    fn serialize(
+        &self,
+        array: &ArrayRef,
+        _session: &VortexSession,
+    ) -> VortexResult<Option<ArraySerialization>> {
+        vortex_ensure_eq!(
+            array.encoding_id(),
+            ArrayPlugin::id(self),
+            "ALP plugin cannot serialize an array of another encoding",
+        );
+        let array = array.as_::<ALP>();
+        let exponents = array.exponents();
+        let metadata = ALPMetadata {
+            exp_e: exponents.e as u32,
+            exp_f: exponents.f as u32,
+            patches: array
+                .patches()
+                .map(|p| p.to_metadata(array.len(), array.dtype()))
+                .transpose()?,
+        }
+        .encode_to_vec();
+        Ok(Some(ArraySerialization::from_array(
+            ArrayPlugin::id(self),
+            array.array(),
+            metadata,
+        )))
+    }
+
+    fn deserialize(
+        &self,
+        parts: ArrayDeserialization<'_>,
+        session: &VortexSession,
+    ) -> VortexResult<ArrayRef> {
+        vortex_ensure_eq!(
+            parts.serialized_id,
+            ArrayPlugin::id(self),
+            "ALP plugin does not recognize the serialized ID",
+        );
+        let ArrayDeserialization {
+            dtype,
+            len,
+            metadata,
+            children,
+            ..
+        } = parts;
+        let metadata = ALPMetadata::decode(metadata)?;
+        let encoded_ptype = match &dtype {
+            DType::Primitive(PType::F32, n) => DType::Primitive(PType::I32, *n),
+            DType::Primitive(PType::F64, n) => DType::Primitive(PType::I64, *n),
+            d => vortex_bail!("expected type: f32 or f64 but instead got {}", d),
+        };
+        let encoded = children.get(0, &encoded_ptype, len)?;
+
+        let patches = metadata
+            .patches
+            .map(|p| {
+                let indices = children.get(1, &p.indices_dtype()?, p.len()?)?;
+                let values = children.get(2, dtype, p.len()?)?;
+                let chunk_offsets = p
+                    .chunk_offsets_dtype()?
+                    .map(|dtype| children.get(3, &dtype, usize::try_from(p.chunk_offsets_len())?))
+                    .transpose()?;
+
+                Patches::from_metadata(
+                    len,
+                    &p,
+                    indices,
+                    values,
+                    chunk_offsets,
+                    &mut session.create_execution_ctx(),
+                )
+            })
+            .transpose()?;
+
+        let slots = ALPData::make_slots(&encoded, patches.as_ref());
+        let data = ALPData::new(
+            Exponents {
+                e: u8::try_from(metadata.exp_e)?,
+                f: u8::try_from(metadata.exp_f)?,
+            },
+            patches,
+        );
+        Ok(Array::try_from_parts(ArrayParts::new(
+            self.clone(),
+            dtype.clone(),
+            len,
+            data,
+            slots,
+        ))?
+        .into_array())
+    }
+}
 
 /// Custom deserialization plugin that converts an ALP array with interior
 /// patches into a PatchedArray holding an ALP array.
@@ -35,15 +143,7 @@ impl ArrayPlugin for ALPPatchedPlugin {
         // We reuse the existing `ALP` ID so that we can take over its
         // deserialization pathway.
         // TODO(joe): dedup method name
-        ArrayVTable::id(&ALP)
-    }
-
-    fn read_ids(&self) -> Vec<ArrayId> {
-        vec![self.id()]
-    }
-
-    fn write_ids(&self) -> Vec<ArrayId> {
-        vec![self.id()]
+        VTable::id(&ALP)
     }
 
     fn serialize(
@@ -65,16 +165,9 @@ impl ArrayPlugin for ALPPatchedPlugin {
             self.id(),
             "ALP plugin does not recognize serialized ID"
         );
-        let alp_array = Array::<ALP>::try_from_parts(ArrayVTable::deserialize(
-            &ALP,
-            parts.dtype,
-            parts.len,
-            parts.metadata,
-            parts.buffers,
-            parts.children,
-            session,
-        )?)
-        .map_err(|_| vortex_err!("ALP plugin should only deserialize vortex.alp"))?;
+        let alp_array = ArrayPlugin::deserialize(&ALP, parts, session)?
+            .try_downcast::<ALP>()
+            .map_err(|_| vortex_err!("ALP plugin returned an array that is not vortex.alp"))?;
 
         // Check if there are interior patches to externalize.
         let Some(patches) = alp_array.patches() else {
@@ -97,7 +190,7 @@ impl ArrayPlugin for ALPPatchedPlugin {
 
     fn is_supported_encoding(&self, id: &ArrayId) -> bool {
         // TODO(joe): dedup method name
-        id == ArrayVTable::id(&Patched) || id == ArrayVTable::id(&ALP)
+        id == VTable::id(&Patched) || id == VTable::id(&ALP)
     }
 }
 

encodings/alp/src/alp/mod.rs

@@ -15,10 +15,9 @@ mod compress;
 pub(crate) mod compute;
 mod decompress;
 mod ops;
-mod plugin;
 mod rules;
 
-pub(crate) use plugin::ALPPatchedPlugin;
+pub(crate) use array::ALPPatchedPlugin;
 
 pub(crate) fn initialize(session: &VortexSession) {
     rules::initialize(session);
@@ -30,7 +29,8 @@ mod tests {
 
     use prost::Message;
     use rstest::rstest;
-    use vortex_array::Array;
+    use vortex_array::ArrayDeserialization;
+    use vortex_array::ArrayPlugin;
     use vortex_array::IntoArray;
     use vortex_array::VortexSessionExecute;
     use vortex_array::arrays::PrimitiveArray;
@@ -39,7 +39,6 @@ mod tests {
     use vortex_array::dtype::PType;
     use vortex_array::patches::PatchesMetadata;
     use vortex_array::test_harness::check_metadata;
-    use vortex_array::vtable::VTable;
     use vortex_error::VortexResult;
     use vortex_session::VortexSession;
 
@@ -81,16 +80,20 @@ mod tests {
         let encoded = alp_encode(values.as_view(), Some(Exponents { e: 0, f: 0 }), &mut ctx)?;
         let sliced = <ALP as SliceKernel>::slice(encoded.as_view(), start..4096, &mut ctx)?
             .expect("ALP slices");
-        let metadata = ALP::serialize(sliced.as_::<ALP>(), &session)?.expect("ALP serializes");
-        let parts = ALP.deserialize(
-            sliced.dtype(),
-            sliced.len(),
-            &metadata,
-            &[],
-            &sliced.children(),
+        let serialization =
+            ArrayPlugin::serialize(&ALP, &sliced, &session)?.expect("ALP serializes");
+        let read = ArrayPlugin::deserialize(
+            &ALP,
+            ArrayDeserialization::new(
+                serialization.serialized_id,
+                sliced.dtype(),
+                sliced.len(),
+                &serialization.metadata,
+                &[],
+                &serialization.children,
+            ),
             &session,
         )?;
-        let read = Array::<ALP>::try_from_parts(parts)?.into_array();
         let expected = values.into_array().slice(start..4096)?;
 
         let patch_index = ((start / 1024 + 1) * 1024).next_multiple_of(7) - start;