diff --git a/benchmarks/compress-bench/src/lib.rs b/benchmarks/compress-bench/src/lib.rs index 6048b0625d0..9fa6928cf0d 100644 --- a/benchmarks/compress-bench/src/lib.rs +++ b/benchmarks/compress-bench/src/lib.rs @@ -15,11 +15,10 @@ pub mod vortex; pub fn chunked_to_vec_record_batch( chunked: ChunkedArray, ) -> anyhow::Result<(Vec, Arc)> { - let chunks_vec = chunked.chunks(); - assert!(!chunks_vec.is_empty(), "empty chunks"); + assert!(chunked.nchunks() > 0, "empty chunks"); - let batches = chunks_vec - .iter() + let batches = chunked + .iter_chunks() .map(|array| { // TODO(connor)[ListView]: The rust Parquet implementation does not support writing // `ListView` to Parquet files yet. diff --git a/encodings/alp/public-api.lock b/encodings/alp/public-api.lock index 7f769a0fc7a..ea0e9d3cdc2 100644 --- a/encodings/alp/public-api.lock +++ b/encodings/alp/public-api.lock @@ -64,10 +64,6 @@ pub fn vortex_alp::ALP::buffer_name(_array: &vortex_alp::ALPArray, _idx: usize) pub fn vortex_alp::ALP::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_alp::ALP::child(array: &vortex_alp::ALPArray, idx: usize) -> vortex_array::array::ArrayRef - -pub fn vortex_alp::ALP::child_name(array: &vortex_alp::ALPArray, idx: usize) -> alloc::string::String - pub fn vortex_alp::ALP::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_alp::ALP::dtype(array: &vortex_alp::ALPArray) -> &vortex_array::dtype::DType @@ -84,15 +80,17 @@ pub fn vortex_alp::ALP::metadata(array: &vortex_alp::ALPArray) -> vortex_error:: pub fn vortex_alp::ALP::nbuffers(_array: &vortex_alp::ALPArray) -> usize -pub fn vortex_alp::ALP::nchildren(array: &vortex_alp::ALPArray) -> usize - pub fn vortex_alp::ALP::reduce_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> pub fn vortex_alp::ALP::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_alp::ALP::slot_name(_array: &vortex_alp::ALPArray, idx: usize) -> alloc::string::String + +pub fn vortex_alp::ALP::slots(array: &vortex_alp::ALPArray) -> &[core::option::Option] + pub fn vortex_alp::ALP::stats(array: &vortex_alp::ALPArray) -> vortex_array::stats::array::StatsSetRef<'_> -pub fn vortex_alp::ALP::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_alp::ALP::with_slots(array: &mut vortex_alp::ALPArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::operations::OperationsVTable for vortex_alp::ALP @@ -220,10 +218,6 @@ pub fn vortex_alp::ALPRD::buffer_name(_array: &vortex_alp::ALPRDArray, _idx: usi pub fn vortex_alp::ALPRD::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_alp::ALPRD::child(array: &vortex_alp::ALPRDArray, idx: usize) -> vortex_array::array::ArrayRef - -pub fn vortex_alp::ALPRD::child_name(array: &vortex_alp::ALPRDArray, idx: usize) -> alloc::string::String - pub fn vortex_alp::ALPRD::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_alp::ALPRD::dtype(array: &vortex_alp::ALPRDArray) -> &vortex_array::dtype::DType @@ -240,15 +234,17 @@ pub fn vortex_alp::ALPRD::metadata(array: &vortex_alp::ALPRDArray) -> vortex_err pub fn vortex_alp::ALPRD::nbuffers(_array: &vortex_alp::ALPRDArray) -> usize -pub fn vortex_alp::ALPRD::nchildren(array: &vortex_alp::ALPRDArray) -> usize - pub fn vortex_alp::ALPRD::reduce_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> pub fn vortex_alp::ALPRD::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_alp::ALPRD::slot_name(_array: &vortex_alp::ALPRDArray, idx: usize) -> alloc::string::String + +pub fn vortex_alp::ALPRD::slots(array: &vortex_alp::ALPRDArray) -> &[core::option::Option] + pub fn vortex_alp::ALPRD::stats(array: &vortex_alp::ALPRDArray) -> vortex_array::stats::array::StatsSetRef<'_> -pub fn vortex_alp::ALPRD::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_alp::ALPRD::with_slots(array: &mut vortex_alp::ALPRDArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::operations::OperationsVTable for vortex_alp::ALPRD diff --git a/encodings/alp/src/alp/array.rs b/encodings/alp/src/alp/array.rs index 965c1622ca8..42a68b84217 100644 --- a/encodings/alp/src/alp/array.rs +++ b/encodings/alp/src/alp/array.rs @@ -28,14 +28,10 @@ use vortex_array::vtable::ArrayId; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityChild; use vortex_array::vtable::ValidityVTableFromChild; -use vortex_array::vtable::patches_child; -use vortex_array::vtable::patches_child_name; -use vortex_array::vtable::patches_nchildren; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; -use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; @@ -59,7 +55,7 @@ impl VTable for ALP { } fn len(array: &ALPArray) -> usize { - array.encoded.len() + array.encoded().len() } fn dtype(array: &ALPArray) -> &DType { @@ -72,14 +68,14 @@ impl VTable for ALP { fn array_hash(array: &ALPArray, state: &mut H, precision: Precision) { array.dtype.hash(state); - array.encoded.array_hash(state, precision); + array.encoded().array_hash(state, precision); array.exponents.hash(state); array.patches.array_hash(state, precision); } fn array_eq(array: &ALPArray, other: &ALPArray, precision: Precision) -> bool { array.dtype == other.dtype - && array.encoded.array_eq(&other.encoded, precision) + && array.encoded().array_eq(other.encoded(), precision) && array.exponents == other.exponents && array.patches.array_eq(&other.patches, precision) } @@ -96,32 +92,41 @@ impl VTable for ALP { None } - fn nchildren(array: &ALPArray) -> usize { - 1 + array.patches().map_or(0, patches_nchildren) + fn slots(array: &ALPArray) -> &[Option] { + &array.slots } - fn child(array: &ALPArray, idx: usize) -> ArrayRef { - match idx { - 0 => array.encoded().clone(), - _ => { - let patches = array - .patches() - .unwrap_or_else(|| vortex_panic!("ALPArray child index {idx} out of bounds")); - patches_child(patches, idx - 1) - } - } + fn slot_name(_array: &ALPArray, idx: usize) -> String { + SLOT_NAMES[idx].to_string() } - fn child_name(array: &ALPArray, idx: usize) -> String { - match idx { - 0 => "encoded".to_string(), - _ => { - if array.patches().is_none() { - vortex_panic!("ALPArray child_name index {idx} out of bounds"); - } - patches_child_name(idx - 1).to_string() + fn with_slots(array: &mut ALPArray, slots: Vec>) -> VortexResult<()> { + vortex_ensure!( + slots.len() == NUM_SLOTS, + "ALPArray expects {} slots, got {}", + NUM_SLOTS, + slots.len() + ); + + // Reconstruct patches from slots + existing metadata + array.patches = match (&slots[PATCH_INDICES_SLOT], &slots[PATCH_VALUES_SLOT]) { + (Some(indices), Some(values)) => { + let old = array + .patches + .as_ref() + .vortex_expect("ALPArray had patch slots but no patches metadata"); + Some(Patches::new( + old.array_len(), + old.offset(), + indices.clone(), + values.clone(), + slots[PATCH_CHUNK_OFFSETS_SLOT].clone(), + )?) } - } + _ => None, + }; + array.slots = slots; + Ok(()) } fn metadata(array: &ALPArray) -> VortexResult { @@ -190,51 +195,6 @@ impl VTable for ALP { ) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { - // Children: encoded, patches (if present): indices, values, chunk_offsets (optional) - let patches_info = array - .patches - .as_ref() - .map(|p| (p.array_len(), p.offset(), p.chunk_offsets().is_some())); - - let expected_children = match &patches_info { - Some((_, _, has_chunk_offsets)) => 1 + 2 + if *has_chunk_offsets { 1 } else { 0 }, - None => 1, - }; - - vortex_ensure!( - children.len() == expected_children, - "ALPArray expects {} children, got {}", - expected_children, - children.len() - ); - - let mut children_iter = children.into_iter(); - array.encoded = children_iter - .next() - .ok_or_else(|| vortex_err!("Expected encoded child"))?; - - if let Some((array_len, offset, _has_chunk_offsets)) = patches_info { - let indices = children_iter - .next() - .ok_or_else(|| vortex_err!("Expected patch indices child"))?; - let values = children_iter - .next() - .ok_or_else(|| vortex_err!("Expected patch values child"))?; - let chunk_offsets = children_iter.next(); - - array.patches = Some(Patches::new( - array_len, - offset, - indices, - values, - chunk_offsets, - )?); - } - - Ok(()) - } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { // TODO(joe): take by value Ok(ExecutionStep::Done( @@ -260,9 +220,21 @@ impl VTable for ALP { } } +pub(super) const ENCODED_SLOT: usize = 0; +pub(super) const PATCH_INDICES_SLOT: usize = 1; +pub(super) const PATCH_VALUES_SLOT: usize = 2; +pub(super) const PATCH_CHUNK_OFFSETS_SLOT: usize = 3; +pub(super) const NUM_SLOTS: usize = 4; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = [ + "encoded", + "patch_indices", + "patch_values", + "patch_chunk_offsets", +]; + #[derive(Clone, Debug)] pub struct ALPArray { - encoded: ArrayRef, + slots: Vec>, patches: Option, dtype: DType, exponents: Exponents, @@ -431,9 +403,11 @@ impl ALPArray { _ => unreachable!(), }; + let slots = Self::make_slots(&encoded, &patches); + Ok(Self { dtype, - encoded, + slots, exponents, patches, stats_set: Default::default(), @@ -450,21 +424,42 @@ impl ALPArray { patches: Option, dtype: DType, ) -> Self { + let slots = Self::make_slots(&encoded, &patches); + Self { dtype, - encoded, + slots, exponents, patches, stats_set: Default::default(), } } + fn make_slots(encoded: &ArrayRef, patches: &Option) -> Vec> { + let (patch_indices, patch_values, patch_chunk_offsets) = match patches { + Some(p) => ( + Some(p.indices().clone()), + Some(p.values().clone()), + p.chunk_offsets().clone(), + ), + None => (None, None, None), + }; + vec![ + Some(encoded.clone()), + patch_indices, + patch_values, + patch_chunk_offsets, + ] + } + pub fn ptype(&self) -> PType { self.dtype.as_ptype() } pub fn encoded(&self) -> &ArrayRef { - &self.encoded + self.slots[ENCODED_SLOT] + .as_ref() + .vortex_expect("ALPArray encoded slot") } #[inline] @@ -479,7 +474,10 @@ impl ALPArray { /// Consumes the array and returns its parts. #[inline] pub fn into_parts(self) -> (ArrayRef, Exponents, Option, DType) { - (self.encoded, self.exponents, self.patches, self.dtype) + let encoded = self.slots[ENCODED_SLOT] + .clone() + .vortex_expect("ALPArray encoded slot"); + (encoded, self.exponents, self.patches, self.dtype) } } diff --git a/encodings/alp/src/alp_rd/array.rs b/encodings/alp/src/alp_rd/array.rs index df72cc189d3..3dfce9e5b9a 100644 --- a/encodings/alp/src/alp_rd/array.rs +++ b/encodings/alp/src/alp_rd/array.rs @@ -32,10 +32,8 @@ use vortex_array::vtable::ArrayId; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityChild; use vortex_array::vtable::ValidityVTableFromChild; -use vortex_array::vtable::patches_child; -use vortex_array::vtable::patches_child_name; -use vortex_array::vtable::patches_nchildren; use vortex_buffer::Buffer; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -76,7 +74,7 @@ impl VTable for ALPRD { } fn len(array: &ALPRDArray) -> usize { - array.left_parts.len() + array.left_parts().len() } fn dtype(array: &ALPRDArray) -> &DType { @@ -89,20 +87,20 @@ impl VTable for ALPRD { fn array_hash(array: &ALPRDArray, state: &mut H, precision: Precision) { array.dtype.hash(state); - array.left_parts.array_hash(state, precision); + array.left_parts().array_hash(state, precision); array.left_parts_dictionary.array_hash(state, precision); - array.right_parts.array_hash(state, precision); + array.right_parts().array_hash(state, precision); array.right_bit_width.hash(state); array.left_parts_patches.array_hash(state, precision); } fn array_eq(array: &ALPRDArray, other: &ALPRDArray, precision: Precision) -> bool { array.dtype == other.dtype - && array.left_parts.array_eq(&other.left_parts, precision) + && array.left_parts().array_eq(other.left_parts(), precision) && array .left_parts_dictionary .array_eq(&other.left_parts_dictionary, precision) - && array.right_parts.array_eq(&other.right_parts, precision) + && array.right_parts().array_eq(other.right_parts(), precision) && array.right_bit_width == other.right_bit_width && array .left_parts_patches @@ -121,36 +119,6 @@ impl VTable for ALPRD { None } - fn nchildren(array: &ALPRDArray) -> usize { - 2 + array.left_parts_patches().map_or(0, patches_nchildren) - } - - fn child(array: &ALPRDArray, idx: usize) -> ArrayRef { - match idx { - 0 => array.left_parts().clone(), - 1 => array.right_parts().clone(), - _ => { - let patches = array - .left_parts_patches() - .unwrap_or_else(|| vortex_panic!("ALPRDArray child index {idx} out of bounds")); - patches_child(patches, idx - 2) - } - } - } - - fn child_name(array: &ALPRDArray, idx: usize) -> String { - match idx { - 0 => "left_parts".to_string(), - 1 => "right_parts".to_string(), - _ => { - if array.left_parts_patches().is_none() { - vortex_panic!("ALPRDArray child_name index {idx} out of bounds"); - } - patches_child_name(idx - 2).to_string() - } - } - } - fn metadata(array: &ALPRDArray) -> VortexResult { let dict = array .left_parts_dictionary() @@ -162,7 +130,7 @@ impl VTable for ALPRD { right_bit_width: array.right_bit_width() as u32, dict_len: array.left_parts_dictionary().len() as u32, dict, - left_parts_ptype: array.left_parts.dtype().as_ptype() as i32, + left_parts_ptype: array.left_parts().dtype().as_ptype() as i32, patches: array .left_parts_patches() .map(|p| p.to_metadata(array.len(), array.left_parts().dtype())) @@ -255,44 +223,41 @@ impl VTable for ALPRD { ) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { - // Children: left_parts, right_parts, patches (if present): indices, values - let patches_info = array - .left_parts_patches - .as_ref() - .map(|p| (p.array_len(), p.offset())); + fn slots(array: &ALPRDArray) -> &[Option] { + &array.slots + } - let expected_children = if patches_info.is_some() { 4 } else { 2 }; + fn slot_name(_array: &ALPRDArray, idx: usize) -> String { + SLOT_NAMES[idx].to_string() + } + fn with_slots(array: &mut ALPRDArray, slots: Vec>) -> VortexResult<()> { vortex_ensure!( - children.len() == expected_children, - "ALPRDArray expects {} children, got {}", - expected_children, - children.len() + slots.len() == NUM_SLOTS, + "ALPRDArray expects {} slots, got {}", + NUM_SLOTS, + slots.len() ); - let mut children_iter = children.into_iter(); - array.left_parts = children_iter - .next() - .ok_or_else(|| vortex_err!("Expected left_parts child"))?; - array.right_parts = children_iter - .next() - .ok_or_else(|| vortex_err!("Expected right_parts child"))?; - - if let Some((array_len, offset)) = patches_info { - let indices = children_iter - .next() - .ok_or_else(|| vortex_err!("Expected patch indices child"))?; - let values = children_iter - .next() - .ok_or_else(|| vortex_err!("Expected patch values child"))?; - - array.left_parts_patches = Some(Patches::new( - array_len, offset, indices, values, - None, // chunk_offsets not currently supported for ALPRD - )?); - } - + // Reconstruct patches from slots + existing metadata + array.left_parts_patches = + match (&slots[LP_PATCH_INDICES_SLOT], &slots[LP_PATCH_VALUES_SLOT]) { + (Some(indices), Some(values)) => { + let old = array + .left_parts_patches + .as_ref() + .vortex_expect("ALPRDArray had patch slots but no patches metadata"); + Some(Patches::new( + old.array_len(), + old.offset(), + indices.clone(), + values.clone(), + slots[LP_PATCH_CHUNK_OFFSETS_SLOT].clone(), + )?) + } + _ => None, + }; + array.slots = slots; Ok(()) } @@ -356,13 +321,26 @@ impl VTable for ALPRD { } } +pub(super) const LEFT_PARTS_SLOT: usize = 0; +pub(super) const RIGHT_PARTS_SLOT: usize = 1; +pub(super) const LP_PATCH_INDICES_SLOT: usize = 2; +pub(super) const LP_PATCH_VALUES_SLOT: usize = 3; +pub(super) const LP_PATCH_CHUNK_OFFSETS_SLOT: usize = 4; +pub(super) const NUM_SLOTS: usize = 5; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = [ + "left_parts", + "right_parts", + "patch_indices", + "patch_values", + "patch_chunk_offsets", +]; + #[derive(Clone, Debug)] pub struct ALPRDArray { dtype: DType, - left_parts: ArrayRef, + slots: Vec>, left_parts_patches: Option, left_parts_dictionary: Buffer, - right_parts: ArrayRef, right_bit_width: u8, stats_set: ArrayStats, } @@ -429,11 +407,12 @@ impl ALPRDArray { }) .transpose()?; + let slots = Self::make_slots(&left_parts, &right_parts, &left_parts_patches); + Ok(Self { dtype, - left_parts, + slots, left_parts_dictionary, - right_parts, right_bit_width, left_parts_patches, stats_set: Default::default(), @@ -450,17 +429,40 @@ impl ALPRDArray { right_bit_width: u8, left_parts_patches: Option, ) -> Self { + let slots = Self::make_slots(&left_parts, &right_parts, &left_parts_patches); + Self { dtype, - left_parts, + slots, left_parts_patches, left_parts_dictionary, - right_parts, right_bit_width, stats_set: Default::default(), } } + fn make_slots( + left_parts: &ArrayRef, + right_parts: &ArrayRef, + patches: &Option, + ) -> Vec> { + let (pi, pv, pco) = match patches { + Some(p) => ( + Some(p.indices().clone()), + Some(p.values().clone()), + p.chunk_offsets().clone(), + ), + None => (None, None, None), + }; + vec![ + Some(left_parts.clone()), + Some(right_parts.clone()), + pi, + pv, + pco, + ] + } + /// Returns true if logical type of the array values is f32. /// /// Returns false if the logical type of the array values is f64. @@ -474,12 +476,16 @@ impl ALPRDArray { /// These are bit-packed and dictionary encoded, and cannot directly be interpreted without /// the metadata of this array. pub fn left_parts(&self) -> &ArrayRef { - &self.left_parts + self.slots[LEFT_PARTS_SLOT] + .as_ref() + .vortex_expect("ALPRDArray left_parts slot") } /// The rightmost (least significant) bits of the floating point values stored in the array. pub fn right_parts(&self) -> &ArrayRef { - &self.right_parts + self.slots[RIGHT_PARTS_SLOT] + .as_ref() + .vortex_expect("ALPRDArray right_parts slot") } #[inline] @@ -499,6 +505,18 @@ impl ALPRDArray { } pub fn replace_left_parts_patches(&mut self, patches: Option) { + // Update both the patches and the corresponding slots to keep them in sync. + let (pi, pv, pco) = match &patches { + Some(p) => ( + Some(p.indices().clone()), + Some(p.values().clone()), + p.chunk_offsets().clone(), + ), + None => (None, None, None), + }; + self.slots[LP_PATCH_INDICES_SLOT] = pi; + self.slots[LP_PATCH_VALUES_SLOT] = pv; + self.slots[LP_PATCH_CHUNK_OFFSETS_SLOT] = pco; self.left_parts_patches = patches; } } diff --git a/encodings/bytebool/public-api.lock b/encodings/bytebool/public-api.lock index 192d025cf34..c0328bb55c7 100644 --- a/encodings/bytebool/public-api.lock +++ b/encodings/bytebool/public-api.lock @@ -46,10 +46,6 @@ pub fn vortex_bytebool::ByteBool::buffer_name(_array: &vortex_bytebool::ByteBool pub fn vortex_bytebool::ByteBool::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_bytebool::ByteBool::child(array: &vortex_bytebool::ByteBoolArray, idx: usize) -> vortex_array::array::ArrayRef - -pub fn vortex_bytebool::ByteBool::child_name(_array: &vortex_bytebool::ByteBoolArray, idx: usize) -> alloc::string::String - pub fn vortex_bytebool::ByteBool::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_bytebool::ByteBool::dtype(array: &vortex_bytebool::ByteBoolArray) -> &vortex_array::dtype::DType @@ -66,15 +62,17 @@ pub fn vortex_bytebool::ByteBool::metadata(_array: &vortex_bytebool::ByteBoolArr pub fn vortex_bytebool::ByteBool::nbuffers(_array: &vortex_bytebool::ByteBoolArray) -> usize -pub fn vortex_bytebool::ByteBool::nchildren(array: &vortex_bytebool::ByteBoolArray) -> usize - pub fn vortex_bytebool::ByteBool::reduce_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> pub fn vortex_bytebool::ByteBool::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_bytebool::ByteBool::slot_name(_array: &vortex_bytebool::ByteBoolArray, idx: usize) -> alloc::string::String + +pub fn vortex_bytebool::ByteBool::slots(array: &vortex_bytebool::ByteBoolArray) -> &[core::option::Option] + pub fn vortex_bytebool::ByteBool::stats(array: &vortex_bytebool::ByteBoolArray) -> vortex_array::stats::array::StatsSetRef<'_> -pub fn vortex_bytebool::ByteBool::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_bytebool::ByteBool::with_slots(array: &mut vortex_bytebool::ByteBoolArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::operations::OperationsVTable for vortex_bytebool::ByteBool diff --git a/encodings/bytebool/src/array.rs b/encodings/bytebool/src/array.rs index 4b6a0e7cb58..a8c956f1306 100644 --- a/encodings/bytebool/src/array.rs +++ b/encodings/bytebool/src/array.rs @@ -26,11 +26,9 @@ use vortex_array::vtable::OperationsVTable; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityHelper; use vortex_array::vtable::ValidityVTableFromValidityHelper; -use vortex_array::vtable::validity_nchildren; use vortex_array::vtable::validity_to_child; use vortex_buffer::BitBuffer; use vortex_buffer::ByteBuffer; -use vortex_error::VortexExpect as _; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -98,25 +96,6 @@ impl VTable for ByteBool { } } - fn nchildren(array: &ByteBoolArray) -> usize { - validity_nchildren(array.validity()) - } - - fn child(array: &ByteBoolArray, idx: usize) -> ArrayRef { - match idx { - 0 => validity_to_child(array.validity(), array.len()) - .vortex_expect("ByteBoolArray validity child out of bounds"), - _ => vortex_panic!("ByteBoolArray child index {idx} out of bounds"), - } - } - - fn child_name(_array: &ByteBoolArray, idx: usize) -> String { - match idx { - 0 => "validity".to_string(), - _ => vortex_panic!("ByteBoolArray child_name index {idx} out of bounds"), - } - } - fn metadata(_array: &ByteBoolArray) -> VortexResult { Ok(EmptyMetadata) } @@ -159,19 +138,26 @@ impl VTable for ByteBool { Ok(ByteBoolArray::new(buffer, validity)) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { + fn slots(array: &ByteBoolArray) -> &[Option] { + &array.slots + } + + fn slot_name(_array: &ByteBoolArray, idx: usize) -> String { + SLOT_NAMES[idx].to_string() + } + + fn with_slots(array: &mut ByteBoolArray, slots: Vec>) -> VortexResult<()> { vortex_ensure!( - children.len() <= 1, - "ByteBoolArray expects at most 1 child (validity), got {}", - children.len() + slots.len() == NUM_SLOTS, + "ByteBoolArray expects {} slots, got {}", + NUM_SLOTS, + slots.len() ); - - array.validity = if children.is_empty() { - Validity::from(array.dtype.nullability()) - } else { - Validity::Array(children.into_iter().next().vortex_expect("checked")) + array.validity = match &slots[VALIDITY_SLOT] { + Some(arr) => Validity::Array(arr.clone()), + None => Validity::from(array.dtype.nullability()), }; - + array.slots = slots; Ok(()) } @@ -201,11 +187,16 @@ impl VTable for ByteBool { } } +pub(super) const VALIDITY_SLOT: usize = 0; +pub(super) const NUM_SLOTS: usize = 1; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; + #[derive(Clone, Debug)] pub struct ByteBoolArray { dtype: DType, buffer: BufferHandle, validity: Validity, + pub(super) slots: Vec>, stats_set: ArrayStats, } @@ -217,6 +208,10 @@ impl ByteBool { } impl ByteBoolArray { + fn make_slots(validity: &Validity, len: usize) -> Vec> { + vec![validity_to_child(validity, len)] + } + pub fn new(buffer: BufferHandle, validity: Validity) -> Self { let length = buffer.len(); if let Some(vlen) = validity.maybe_len() @@ -228,10 +223,12 @@ impl ByteBoolArray { vlen ); } + let slots = Self::make_slots(&validity, length); Self { dtype: DType::Bool(validity.nullability()), buffer, validity, + slots, stats_set: Default::default(), } } @@ -288,6 +285,17 @@ impl From>> for ByteBoolArray { #[cfg(test)] mod tests { + use vortex_array::ArrayContext; + use vortex_array::IntoArray; + use vortex_array::assert_arrays_eq; + use vortex_array::serde::ArrayParts; + use vortex_array::serde::SerializeOptions; + use vortex_array::session::ArraySession; + use vortex_array::session::ArraySessionExt; + use vortex_buffer::ByteBufferMut; + use vortex_session::VortexSession; + use vortex_session::registry::ReadContext; + use super::*; #[test] @@ -320,4 +328,32 @@ mod tests { } assert_eq!(arr.len(), 2); } + + #[test] + fn test_nullable_bytebool_serde_roundtrip() { + let array = ByteBoolArray::from(vec![Some(true), None, Some(false), None]); + let dtype = array.dtype().clone(); + let len = array.len(); + let session = VortexSession::empty().with::(); + session.arrays().register(ByteBool::ID, ByteBool); + + let ctx = ArrayContext::empty(); + let serialized = array + .clone() + .into_array() + .serialize(&ctx, &SerializeOptions::default()) + .unwrap(); + + let mut concat = ByteBufferMut::empty(); + for buf in serialized { + concat.extend_from_slice(buf.as_ref()); + } + + let parts = ArrayParts::try_from(concat.freeze()).unwrap(); + let decoded = parts + .decode(&dtype, len, &ReadContext::new(ctx.to_ids()), &session) + .unwrap(); + + assert_arrays_eq!(decoded, array); + } } diff --git a/encodings/datetime-parts/public-api.lock b/encodings/datetime-parts/public-api.lock index 475c7da5168..2d1eacce8c1 100644 --- a/encodings/datetime-parts/public-api.lock +++ b/encodings/datetime-parts/public-api.lock @@ -54,10 +54,6 @@ pub fn vortex_datetime_parts::DateTimeParts::buffer_name(_array: &vortex_datetim pub fn vortex_datetime_parts::DateTimeParts::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_datetime_parts::DateTimeParts::child(array: &vortex_datetime_parts::DateTimePartsArray, idx: usize) -> vortex_array::array::ArrayRef - -pub fn vortex_datetime_parts::DateTimeParts::child_name(_array: &vortex_datetime_parts::DateTimePartsArray, idx: usize) -> alloc::string::String - pub fn vortex_datetime_parts::DateTimeParts::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_datetime_parts::DateTimeParts::dtype(array: &vortex_datetime_parts::DateTimePartsArray) -> &vortex_array::dtype::DType @@ -74,15 +70,17 @@ pub fn vortex_datetime_parts::DateTimeParts::metadata(array: &vortex_datetime_pa pub fn vortex_datetime_parts::DateTimeParts::nbuffers(_array: &vortex_datetime_parts::DateTimePartsArray) -> usize -pub fn vortex_datetime_parts::DateTimeParts::nchildren(_array: &vortex_datetime_parts::DateTimePartsArray) -> usize - pub fn vortex_datetime_parts::DateTimeParts::reduce_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> pub fn vortex_datetime_parts::DateTimeParts::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_datetime_parts::DateTimeParts::slot_name(_array: &vortex_datetime_parts::DateTimePartsArray, idx: usize) -> alloc::string::String + +pub fn vortex_datetime_parts::DateTimeParts::slots(array: &vortex_datetime_parts::DateTimePartsArray) -> &[core::option::Option] + pub fn vortex_datetime_parts::DateTimeParts::stats(array: &vortex_datetime_parts::DateTimePartsArray) -> vortex_array::stats::array::StatsSetRef<'_> -pub fn vortex_datetime_parts::DateTimeParts::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_datetime_parts::DateTimeParts::with_slots(array: &mut vortex_datetime_parts::DateTimePartsArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::operations::OperationsVTable for vortex_datetime_parts::DateTimeParts diff --git a/encodings/datetime-parts/src/array.rs b/encodings/datetime-parts/src/array.rs index b50ff8db0a5..5e15fe24e68 100644 --- a/encodings/datetime-parts/src/array.rs +++ b/encodings/datetime-parts/src/array.rs @@ -83,7 +83,7 @@ impl VTable for DateTimeParts { } fn len(array: &DateTimePartsArray) -> usize { - array.days.len() + array.days().len() } fn dtype(array: &DateTimePartsArray) -> &DType { @@ -100,9 +100,9 @@ impl VTable for DateTimeParts { precision: Precision, ) { array.dtype.hash(state); - array.days.array_hash(state, precision); - array.seconds.array_hash(state, precision); - array.subseconds.array_hash(state, precision); + array.days().array_hash(state, precision); + array.seconds().array_hash(state, precision); + array.subseconds().array_hash(state, precision); } fn array_eq( @@ -111,9 +111,9 @@ impl VTable for DateTimeParts { precision: Precision, ) -> bool { array.dtype == other.dtype - && array.days.array_eq(&other.days, precision) - && array.seconds.array_eq(&other.seconds, precision) - && array.subseconds.array_eq(&other.subseconds, precision) + && array.days().array_eq(other.days(), precision) + && array.seconds().array_eq(other.seconds(), precision) + && array.subseconds().array_eq(other.subseconds(), precision) } fn nbuffers(_array: &DateTimePartsArray) -> usize { @@ -128,28 +128,6 @@ impl VTable for DateTimeParts { vortex_panic!("DateTimePartsArray buffer_name index {idx} out of bounds") } - fn nchildren(_array: &DateTimePartsArray) -> usize { - 3 - } - - fn child(array: &DateTimePartsArray, idx: usize) -> ArrayRef { - match idx { - 0 => array.days().clone(), - 1 => array.seconds().clone(), - 2 => array.subseconds().clone(), - _ => vortex_panic!("DateTimePartsArray child index {idx} out of bounds"), - } - } - - fn child_name(_array: &DateTimePartsArray, idx: usize) -> String { - match idx { - 0 => "days".to_string(), - 1 => "seconds".to_string(), - 2 => "subseconds".to_string(), - _ => vortex_panic!("DateTimePartsArray child_name index {idx} out of bounds"), - } - } - fn metadata(array: &DateTimePartsArray) -> VortexResult { Ok(ProstMetadata(DateTimePartsMetadata { days_ptype: PType::try_from(array.days().dtype())? as i32, @@ -207,18 +185,25 @@ impl VTable for DateTimeParts { DateTimePartsArray::try_new(dtype.clone(), days, seconds, subseconds) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { - vortex_ensure!( - children.len() == 3, - "DateTimePartsArray expects exactly 3 children (days, seconds, subseconds), got {}", - children.len() - ); + fn slots(array: &DateTimePartsArray) -> &[Option] { + &array.slots + } - let mut children_iter = children.into_iter(); - array.days = children_iter.next().vortex_expect("checked"); - array.seconds = children_iter.next().vortex_expect("checked"); - array.subseconds = children_iter.next().vortex_expect("checked"); + fn slot_name(_array: &DateTimePartsArray, idx: usize) -> String { + SLOT_NAMES[idx].to_string() + } + fn with_slots( + array: &mut DateTimePartsArray, + slots: Vec>, + ) -> VortexResult<()> { + vortex_ensure!( + slots.len() == NUM_SLOTS, + "DateTimePartsArray expects exactly {} slots, got {}", + NUM_SLOTS, + slots.len() + ); + array.slots = slots; Ok(()) } @@ -246,12 +231,16 @@ impl VTable for DateTimeParts { } } +pub(super) const DAYS_SLOT: usize = 0; +pub(super) const SECONDS_SLOT: usize = 1; +pub(super) const SUBSECONDS_SLOT: usize = 2; +pub(super) const NUM_SLOTS: usize = 3; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["days", "seconds", "subseconds"]; + #[derive(Clone, Debug)] pub struct DateTimePartsArray { dtype: DType, - days: ArrayRef, - seconds: ArrayRef, - subseconds: ArrayRef, + pub(super) slots: Vec>, stats_set: ArrayStats, } @@ -303,9 +292,7 @@ impl DateTimePartsArray { Ok(Self { dtype, - days, - seconds, - subseconds, + slots: vec![Some(days), Some(seconds), Some(subseconds)], stats_set: Default::default(), }) } @@ -318,9 +305,7 @@ impl DateTimePartsArray { ) -> Self { Self { dtype, - days, - seconds, - subseconds, + slots: vec![Some(days), Some(seconds), Some(subseconds)], stats_set: Default::default(), } } @@ -328,22 +313,34 @@ impl DateTimePartsArray { pub fn into_parts(self) -> DateTimePartsArrayParts { DateTimePartsArrayParts { dtype: self.dtype, - days: self.days, - seconds: self.seconds, - subseconds: self.subseconds, + days: self.slots[DAYS_SLOT] + .clone() + .vortex_expect("DateTimePartsArray days slot"), + seconds: self.slots[SECONDS_SLOT] + .clone() + .vortex_expect("DateTimePartsArray seconds slot"), + subseconds: self.slots[SUBSECONDS_SLOT] + .clone() + .vortex_expect("DateTimePartsArray subseconds slot"), } } pub fn days(&self) -> &ArrayRef { - &self.days + self.slots[DAYS_SLOT] + .as_ref() + .vortex_expect("DateTimePartsArray days slot") } pub fn seconds(&self) -> &ArrayRef { - &self.seconds + self.slots[SECONDS_SLOT] + .as_ref() + .vortex_expect("DateTimePartsArray seconds slot") } pub fn subseconds(&self) -> &ArrayRef { - &self.subseconds + self.slots[SUBSECONDS_SLOT] + .as_ref() + .vortex_expect("DateTimePartsArray subseconds slot") } } diff --git a/encodings/datetime-parts/src/compute/rules.rs b/encodings/datetime-parts/src/compute/rules.rs index 6ac0de3c290..730324bff5b 100644 --- a/encodings/datetime-parts/src/compute/rules.rs +++ b/encodings/datetime-parts/src/compute/rules.rs @@ -113,12 +113,11 @@ impl ArrayParentReduceRule for DTPComparisonPushDownRule { return Ok(None); } - let children = parent.children(); let days = child.days(); // Build new children: replace DTP with days, replace constant timestamps with days constants - let mut new_children = Vec::with_capacity(children.len()); - for (idx, c) in children.iter().enumerate() { + let mut new_children = Vec::with_capacity(parent.nchildren()); + for (idx, c) in parent.iter_children().enumerate() { if idx == child_idx { // This is the DTP child - replace with days new_children.push(days.clone()); diff --git a/encodings/decimal-byte-parts/public-api.lock b/encodings/decimal-byte-parts/public-api.lock index cf9c86b6d1a..c7d08fd89db 100644 --- a/encodings/decimal-byte-parts/public-api.lock +++ b/encodings/decimal-byte-parts/public-api.lock @@ -54,10 +54,6 @@ pub fn vortex_decimal_byte_parts::DecimalByteParts::buffer_name(_array: &vortex_ pub fn vortex_decimal_byte_parts::DecimalByteParts::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_decimal_byte_parts::DecimalByteParts::child(array: &vortex_decimal_byte_parts::DecimalBytePartsArray, idx: usize) -> vortex_array::array::ArrayRef - -pub fn vortex_decimal_byte_parts::DecimalByteParts::child_name(_array: &vortex_decimal_byte_parts::DecimalBytePartsArray, idx: usize) -> alloc::string::String - pub fn vortex_decimal_byte_parts::DecimalByteParts::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_decimal_byte_parts::DecimalByteParts::dtype(array: &vortex_decimal_byte_parts::DecimalBytePartsArray) -> &vortex_array::dtype::DType @@ -74,15 +70,17 @@ pub fn vortex_decimal_byte_parts::DecimalByteParts::metadata(array: &vortex_deci pub fn vortex_decimal_byte_parts::DecimalByteParts::nbuffers(_array: &vortex_decimal_byte_parts::DecimalBytePartsArray) -> usize -pub fn vortex_decimal_byte_parts::DecimalByteParts::nchildren(_array: &vortex_decimal_byte_parts::DecimalBytePartsArray) -> usize - pub fn vortex_decimal_byte_parts::DecimalByteParts::reduce_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> pub fn vortex_decimal_byte_parts::DecimalByteParts::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_decimal_byte_parts::DecimalByteParts::slot_name(_array: &vortex_decimal_byte_parts::DecimalBytePartsArray, idx: usize) -> alloc::string::String + +pub fn vortex_decimal_byte_parts::DecimalByteParts::slots(array: &vortex_decimal_byte_parts::DecimalBytePartsArray) -> &[core::option::Option] + pub fn vortex_decimal_byte_parts::DecimalByteParts::stats(array: &vortex_decimal_byte_parts::DecimalBytePartsArray) -> vortex_array::stats::array::StatsSetRef<'_> -pub fn vortex_decimal_byte_parts::DecimalByteParts::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_decimal_byte_parts::DecimalByteParts::with_slots(array: &mut vortex_decimal_byte_parts::DecimalBytePartsArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::operations::OperationsVTable for vortex_decimal_byte_parts::DecimalByteParts diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs index 1c0c043f83f..f749e6388c6 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs @@ -39,19 +39,21 @@ impl CompareKernel for DecimalByteParts { }; let nullability = lhs.dtype.nullability() | rhs.dtype().nullability(); - let scalar_type = lhs.msp.dtype().with_nullability(nullability); + let scalar_type = lhs.msp().dtype().with_nullability(nullability); let rhs_decimal = rhs_const .as_decimal() .decimal_value() .vortex_expect("checked for null in entry func"); - match decimal_value_wrapper_to_primitive(rhs_decimal, lhs.msp.as_primitive_typed().ptype()) - { + match decimal_value_wrapper_to_primitive( + rhs_decimal, + lhs.msp().as_primitive_typed().ptype(), + ) { Ok(value) => { let encoded_scalar = Scalar::try_new(scalar_type, Some(value))?; let encoded_const = ConstantArray::new(encoded_scalar, rhs.len()); - lhs.msp + lhs.msp() .binary(encoded_const.into_array(), Operator::from(operator)) .map(Some) } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/filter.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/filter.rs index a25521db8e7..3bb0e801a06 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/filter.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/filter.rs @@ -12,7 +12,7 @@ use crate::DecimalBytePartsArray; impl FilterReduce for DecimalByteParts { fn filter(array: &DecimalBytePartsArray, mask: &Mask) -> VortexResult> { - DecimalBytePartsArray::try_new(array.msp.filter(mask.clone())?, *array.decimal_dtype()) + DecimalBytePartsArray::try_new(array.msp().filter(mask.clone())?, *array.decimal_dtype()) .map(|d| Some(d.into_array())) } } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mask.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mask.rs index 8bc05af2ff0..152c8b7e11c 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mask.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mask.rs @@ -15,9 +15,9 @@ use crate::DecimalBytePartsArray; impl MaskReduce for DecimalByteParts { fn mask(array: &DecimalBytePartsArray, mask: &ArrayRef) -> VortexResult> { let masked_msp = MaskExpr.try_new_array( - array.msp.len(), + array.msp().len(), EmptyOptions, - [array.msp.clone(), mask.clone()], + [array.msp().clone(), mask.clone()], )?; Ok(Some( DecimalBytePartsArray::try_new(masked_msp, *array.decimal_dtype())?.into_array(), diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/take.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/take.rs index 580fa514558..31fa44e80c5 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/take.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/take.rs @@ -17,7 +17,10 @@ impl TakeExecute for DecimalByteParts { indices: &ArrayRef, _ctx: &mut ExecutionCtx, ) -> VortexResult> { - DecimalBytePartsArray::try_new(array.msp.take(indices.to_array())?, *array.decimal_dtype()) - .map(|a| Some(a.into_array())) + DecimalBytePartsArray::try_new( + array.msp().take(indices.to_array())?, + *array.decimal_dtype(), + ) + .map(|a| Some(a.into_array())) } } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs index cd56c256383..13830bdfaaa 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs @@ -70,7 +70,7 @@ impl VTable for DecimalByteParts { } fn len(array: &DecimalBytePartsArray) -> usize { - array.msp.len() + array.msp().len() } fn dtype(array: &DecimalBytePartsArray) -> &DType { @@ -87,7 +87,7 @@ impl VTable for DecimalByteParts { precision: Precision, ) { array.dtype.hash(state); - array.msp.array_hash(state, precision); + array.msp().array_hash(state, precision); } fn array_eq( @@ -95,7 +95,7 @@ impl VTable for DecimalByteParts { other: &DecimalBytePartsArray, precision: Precision, ) -> bool { - array.dtype == other.dtype && array.msp.array_eq(&other.msp, precision) + array.dtype == other.dtype && array.msp().array_eq(other.msp(), precision) } fn nbuffers(_array: &DecimalBytePartsArray) -> usize { @@ -110,27 +110,9 @@ impl VTable for DecimalByteParts { vortex_panic!("DecimalBytePartsArray buffer_name index {idx} out of bounds") } - fn nchildren(_array: &DecimalBytePartsArray) -> usize { - 1 - } - - fn child(array: &DecimalBytePartsArray, idx: usize) -> ArrayRef { - match idx { - 0 => array.msp.clone(), - _ => vortex_panic!("DecimalBytePartsArray child index {idx} out of bounds"), - } - } - - fn child_name(_array: &DecimalBytePartsArray, idx: usize) -> String { - match idx { - 0 => "msp".to_string(), - _ => vortex_panic!("DecimalBytePartsArray child_name index {idx} out of bounds"), - } - } - fn metadata(array: &DecimalBytePartsArray) -> VortexResult { Ok(ProstMetadata(DecimalBytesPartsMetadata { - zeroth_child_ptype: PType::try_from(array.msp.dtype())? as i32, + zeroth_child_ptype: PType::try_from(array.msp().dtype())? as i32, lower_part_count: 0, })) } @@ -172,13 +154,25 @@ impl VTable for DecimalByteParts { DecimalBytePartsArray::try_new(msp, *decimal_dtype) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { + fn slots(array: &DecimalBytePartsArray) -> &[Option] { + &array.slots + } + + fn slot_name(_array: &DecimalBytePartsArray, idx: usize) -> String { + SLOT_NAMES[idx].to_string() + } + + fn with_slots( + array: &mut DecimalBytePartsArray, + slots: Vec>, + ) -> VortexResult<()> { vortex_ensure!( - children.len() == 1, - "DecimalBytePartsArray expects exactly 1 child (msp), got {}", - children.len() + slots.len() == NUM_SLOTS, + "DecimalBytePartsArray expects exactly {} slots, got {}", + NUM_SLOTS, + slots.len() ); - array.msp = children.into_iter().next().vortex_expect("checked"); + array.slots = slots; Ok(()) } @@ -204,6 +198,10 @@ impl VTable for DecimalByteParts { } } +pub(super) const MSP_SLOT: usize = 0; +pub(super) const NUM_SLOTS: usize = 1; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["msp"]; + /// This array encodes decimals as between 1-4 columns of primitive typed children. /// The most significant part (msp) sorting the most significant decimal bits. /// This array must be signed and is nullable iff the decimal is nullable. @@ -211,7 +209,7 @@ impl VTable for DecimalByteParts { /// e.g. for a decimal i128 \[ 127..64 | 64..0 \] msp = 127..64 and lower_part\[0\] = 64..0 #[derive(Clone, Debug)] pub struct DecimalBytePartsArray { - msp: ArrayRef, + pub(super) slots: Vec>, // NOTE: the lower_parts is currently unused, we reserve this field so that it is properly // read/written during serde, but provide no constructor to initialize this to anything // other than the empty Vec. @@ -234,7 +232,7 @@ impl DecimalBytePartsArray { let nullable = msp.dtype().nullability(); Ok(Self { - msp, + slots: vec![Some(msp)], _lower_parts: Vec::new(), dtype: DType::Decimal(decimal_dtype, nullable), stats_set: Default::default(), @@ -244,7 +242,7 @@ impl DecimalBytePartsArray { pub(crate) unsafe fn new_unchecked(msp: ArrayRef, decimal_dtype: DecimalDType) -> Self { let nullable = msp.dtype().nullability(); Self { - msp, + slots: vec![Some(msp)], _lower_parts: Vec::new(), dtype: DType::Decimal(decimal_dtype, nullable), stats_set: Default::default(), @@ -254,7 +252,9 @@ impl DecimalBytePartsArray { /// If `_lower_parts` is supported check all calls use this correctly. pub fn into_parts(self) -> DecimalBytePartsArrayParts { DecimalBytePartsArrayParts { - msp: self.msp, + msp: self.slots[MSP_SLOT] + .clone() + .vortex_expect("DecimalBytePartsArray msp slot"), dtype: self.dtype, } } @@ -266,7 +266,9 @@ impl DecimalBytePartsArray { } pub(crate) fn msp(&self) -> &ArrayRef { - &self.msp + self.slots[MSP_SLOT] + .as_ref() + .vortex_expect("DecimalBytePartsArray msp slot") } } @@ -283,7 +285,7 @@ fn to_canonical_decimal( ctx: &mut ExecutionCtx, ) -> VortexResult { // TODO(joe): support parts len != 1 - let prim = array.msp.clone().execute::(ctx)?; + let prim = array.msp().clone().execute::(ctx)?; // Depending on the decimal type and the min/max of the primitive array we can choose // the correct buffer size @@ -304,7 +306,7 @@ fn to_canonical_decimal( impl OperationsVTable for DecimalByteParts { fn scalar_at(array: &DecimalBytePartsArray, index: usize) -> VortexResult { // TODO(joe): support parts len != 1 - let scalar = array.msp.scalar_at(index)?; + let scalar = array.msp().scalar_at(index)?; // Note. values in msp, can only be signed integers upto size i64. let primitive_scalar = scalar.as_primitive(); @@ -320,7 +322,7 @@ impl OperationsVTable for DecimalByteParts { impl ValidityChild for DecimalByteParts { fn validity_child(array: &DecimalBytePartsArray) -> &ArrayRef { // validity stored in 0th child - &array.msp + array.msp() } } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/rules.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/rules.rs index a658d15d1a4..137865ddc64 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/rules.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/rules.rs @@ -43,7 +43,7 @@ impl ArrayParentReduceRule for DecimalBytePartsFilterPushDownR return Ok(None); } - let new_msp = child.msp.filter(parent.filter_mask().clone())?; + let new_msp = child.msp().filter(parent.filter_mask().clone())?; let new_child = DecimalBytePartsArray::try_new(new_msp, *child.decimal_dtype())?.into_array(); Ok(Some(new_child)) diff --git a/encodings/fastlanes/public-api.lock b/encodings/fastlanes/public-api.lock index 45bd7d15a42..1e25b3b6b0b 100644 --- a/encodings/fastlanes/public-api.lock +++ b/encodings/fastlanes/public-api.lock @@ -168,10 +168,6 @@ pub fn vortex_fastlanes::BitPacked::buffer_name(_array: &vortex_fastlanes::BitPa pub fn vortex_fastlanes::BitPacked::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_fastlanes::BitPacked::child(array: &vortex_fastlanes::BitPackedArray, idx: usize) -> vortex_array::array::ArrayRef - -pub fn vortex_fastlanes::BitPacked::child_name(array: &vortex_fastlanes::BitPackedArray, idx: usize) -> alloc::string::String - pub fn vortex_fastlanes::BitPacked::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_fastlanes::BitPacked::dtype(array: &vortex_fastlanes::BitPackedArray) -> &vortex_array::dtype::DType @@ -188,15 +184,17 @@ pub fn vortex_fastlanes::BitPacked::metadata(array: &vortex_fastlanes::BitPacked pub fn vortex_fastlanes::BitPacked::nbuffers(_array: &vortex_fastlanes::BitPackedArray) -> usize -pub fn vortex_fastlanes::BitPacked::nchildren(array: &vortex_fastlanes::BitPackedArray) -> usize - pub fn vortex_fastlanes::BitPacked::reduce_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> pub fn vortex_fastlanes::BitPacked::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_fastlanes::BitPacked::slot_name(_array: &vortex_fastlanes::BitPackedArray, idx: usize) -> alloc::string::String + +pub fn vortex_fastlanes::BitPacked::slots(array: &vortex_fastlanes::BitPackedArray) -> &[core::option::Option] + pub fn vortex_fastlanes::BitPacked::stats(array: &vortex_fastlanes::BitPackedArray) -> vortex_array::stats::array::StatsSetRef<'_> -pub fn vortex_fastlanes::BitPacked::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_fastlanes::BitPacked::with_slots(array: &mut vortex_fastlanes::BitPackedArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::operations::OperationsVTable for vortex_fastlanes::BitPacked @@ -316,10 +314,6 @@ pub fn vortex_fastlanes::Delta::buffer_name(_array: &vortex_fastlanes::DeltaArra pub fn vortex_fastlanes::Delta::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_fastlanes::Delta::child(array: &vortex_fastlanes::DeltaArray, idx: usize) -> vortex_array::array::ArrayRef - -pub fn vortex_fastlanes::Delta::child_name(_array: &vortex_fastlanes::DeltaArray, idx: usize) -> alloc::string::String - pub fn vortex_fastlanes::Delta::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_fastlanes::Delta::dtype(array: &vortex_fastlanes::DeltaArray) -> &vortex_array::dtype::DType @@ -334,15 +328,17 @@ pub fn vortex_fastlanes::Delta::metadata(array: &vortex_fastlanes::DeltaArray) - pub fn vortex_fastlanes::Delta::nbuffers(_array: &vortex_fastlanes::DeltaArray) -> usize -pub fn vortex_fastlanes::Delta::nchildren(_array: &vortex_fastlanes::DeltaArray) -> usize - pub fn vortex_fastlanes::Delta::reduce_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> pub fn vortex_fastlanes::Delta::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_fastlanes::Delta::slot_name(_array: &vortex_fastlanes::DeltaArray, idx: usize) -> alloc::string::String + +pub fn vortex_fastlanes::Delta::slots(array: &vortex_fastlanes::DeltaArray) -> &[core::option::Option] + pub fn vortex_fastlanes::Delta::stats(array: &vortex_fastlanes::DeltaArray) -> vortex_array::stats::array::StatsSetRef<'_> -pub fn vortex_fastlanes::Delta::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_fastlanes::Delta::with_slots(array: &mut vortex_fastlanes::DeltaArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::operations::OperationsVTable for vortex_fastlanes::Delta @@ -454,10 +450,6 @@ pub fn vortex_fastlanes::FoR::buffer_name(_array: &vortex_fastlanes::FoRArray, _ pub fn vortex_fastlanes::FoR::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_fastlanes::FoR::child(array: &vortex_fastlanes::FoRArray, idx: usize) -> vortex_array::array::ArrayRef - -pub fn vortex_fastlanes::FoR::child_name(_array: &vortex_fastlanes::FoRArray, idx: usize) -> alloc::string::String - pub fn vortex_fastlanes::FoR::deserialize(bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_fastlanes::FoR::dtype(array: &vortex_fastlanes::FoRArray) -> &vortex_array::dtype::DType @@ -474,15 +466,17 @@ pub fn vortex_fastlanes::FoR::metadata(array: &vortex_fastlanes::FoRArray) -> vo pub fn vortex_fastlanes::FoR::nbuffers(_array: &vortex_fastlanes::FoRArray) -> usize -pub fn vortex_fastlanes::FoR::nchildren(_array: &vortex_fastlanes::FoRArray) -> usize - pub fn vortex_fastlanes::FoR::reduce_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> pub fn vortex_fastlanes::FoR::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_fastlanes::FoR::slot_name(_array: &vortex_fastlanes::FoRArray, idx: usize) -> alloc::string::String + +pub fn vortex_fastlanes::FoR::slots(array: &vortex_fastlanes::FoRArray) -> &[core::option::Option] + pub fn vortex_fastlanes::FoR::stats(array: &vortex_fastlanes::FoRArray) -> vortex_array::stats::array::StatsSetRef<'_> -pub fn vortex_fastlanes::FoR::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_fastlanes::FoR::with_slots(array: &mut vortex_fastlanes::FoRArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::operations::OperationsVTable for vortex_fastlanes::FoR @@ -576,10 +570,6 @@ pub fn vortex_fastlanes::RLE::buffer_name(_array: &vortex_fastlanes::RLEArray, _ pub fn vortex_fastlanes::RLE::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_fastlanes::RLE::child(array: &vortex_fastlanes::RLEArray, idx: usize) -> vortex_array::array::ArrayRef - -pub fn vortex_fastlanes::RLE::child_name(_array: &vortex_fastlanes::RLEArray, idx: usize) -> alloc::string::String - pub fn vortex_fastlanes::RLE::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_fastlanes::RLE::dtype(array: &vortex_fastlanes::RLEArray) -> &vortex_array::dtype::DType @@ -596,15 +586,17 @@ pub fn vortex_fastlanes::RLE::metadata(array: &vortex_fastlanes::RLEArray) -> vo pub fn vortex_fastlanes::RLE::nbuffers(_array: &vortex_fastlanes::RLEArray) -> usize -pub fn vortex_fastlanes::RLE::nchildren(_array: &vortex_fastlanes::RLEArray) -> usize - pub fn vortex_fastlanes::RLE::reduce_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> pub fn vortex_fastlanes::RLE::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_fastlanes::RLE::slot_name(_array: &vortex_fastlanes::RLEArray, idx: usize) -> alloc::string::String + +pub fn vortex_fastlanes::RLE::slots(array: &vortex_fastlanes::RLEArray) -> &[core::option::Option] + pub fn vortex_fastlanes::RLE::stats(array: &vortex_fastlanes::RLEArray) -> vortex_array::stats::array::StatsSetRef<'_> -pub fn vortex_fastlanes::RLE::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_fastlanes::RLE::with_slots(array: &mut vortex_fastlanes::RLEArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::operations::OperationsVTable for vortex_fastlanes::RLE diff --git a/encodings/fastlanes/src/bitpacking/array/mod.rs b/encodings/fastlanes/src/bitpacking/array/mod.rs index dfe02730cb0..6113f815148 100644 --- a/encodings/fastlanes/src/bitpacking/array/mod.rs +++ b/encodings/fastlanes/src/bitpacking/array/mod.rs @@ -11,6 +11,7 @@ use vortex_array::dtype::PType; use vortex_array::patches::Patches; use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; +use vortex_array::vtable::validity_to_child; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -23,6 +24,18 @@ use crate::bitpack_compress::bitpack_encode; use crate::unpack_iter::BitPacked; use crate::unpack_iter::BitUnpackedChunks; +pub(super) const PATCH_INDICES_SLOT: usize = 0; +pub(super) const PATCH_VALUES_SLOT: usize = 1; +pub(super) const PATCH_CHUNK_OFFSETS_SLOT: usize = 2; +pub(super) const VALIDITY_SLOT: usize = 3; +pub(super) const NUM_SLOTS: usize = 4; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = [ + "patch_indices", + "patch_values", + "patch_chunk_offsets", + "validity", +]; + pub struct BitPackedArrayParts { pub offset: u16, pub bit_width: u8, @@ -34,6 +47,7 @@ pub struct BitPackedArrayParts { #[derive(Clone, Debug)] pub struct BitPackedArray { + pub(super) slots: Vec>, /// The offset within the first block (created with a slice). /// 0 <= offset < 1024 pub(super) offset: u16, @@ -76,7 +90,10 @@ impl BitPackedArray { len: usize, offset: u16, ) -> Self { + let slots = Self::make_slots(&patches, &validity, len); + Self { + slots, offset, len, dtype, @@ -88,6 +105,23 @@ impl BitPackedArray { } } + fn make_slots( + patches: &Option, + validity: &Validity, + len: usize, + ) -> Vec> { + let (pi, pv, pco) = match patches { + Some(p) => ( + Some(p.indices().clone()), + Some(p.values().clone()), + p.chunk_offsets().clone(), + ), + None => (None, None, None), + }; + let validity_slot = validity_to_child(validity, len); + vec![pi, pv, pco, validity_slot] + } + /// A safe constructor for a `BitPackedArray` from its components: /// /// * `packed` is ByteBuffer holding the compressed data that was packed with FastLanes @@ -249,6 +283,18 @@ impl BitPackedArray { } pub fn replace_patches(&mut self, patches: Option) { + // Update both the patches and the corresponding slots to keep them in sync. + let (pi, pv, pco) = match &patches { + Some(p) => ( + Some(p.indices().clone()), + Some(p.values().clone()), + p.chunk_offsets().clone(), + ), + None => (None, None, None), + }; + self.slots[PATCH_INDICES_SLOT] = pi; + self.slots[PATCH_VALUES_SLOT] = pv; + self.slots[PATCH_CHUNK_OFFSETS_SLOT] = pco; self.patches = patches; } diff --git a/encodings/fastlanes/src/bitpacking/vtable/mod.rs b/encodings/fastlanes/src/bitpacking/vtable/mod.rs index c414dfa5cc0..2bc8a5bb501 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/mod.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/mod.rs @@ -27,11 +27,6 @@ use vortex_array::vtable; use vortex_array::vtable::ArrayId; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityVTableFromValidityHelper; -use vortex_array::vtable::patches_child; -use vortex_array::vtable::patches_child_name; -use vortex_array::vtable::patches_nchildren; -use vortex_array::vtable::validity_nchildren; -use vortex_array::vtable::validity_to_child; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; @@ -43,6 +38,12 @@ use vortex_session::VortexSession; use crate::BitPackedArray; use crate::bitpack_decompress::unpack_array; use crate::bitpack_decompress::unpack_into_primitive_builder; +use crate::bitpacking::array::NUM_SLOTS; +use crate::bitpacking::array::PATCH_CHUNK_OFFSETS_SLOT; +use crate::bitpacking::array::PATCH_INDICES_SLOT; +use crate::bitpacking::array::PATCH_VALUES_SLOT; +use crate::bitpacking::array::SLOT_NAMES; +use crate::bitpacking::array::VALIDITY_SLOT; use crate::bitpacking::vtable::kernels::PARENT_KERNELS; use crate::bitpacking::vtable::rules::RULES; mod kernels; @@ -128,36 +129,6 @@ impl VTable for BitPacked { } } - fn nchildren(array: &BitPackedArray) -> usize { - array.patches().map_or(0, patches_nchildren) + validity_nchildren(&array.validity) - } - - fn child(array: &BitPackedArray, idx: usize) -> ArrayRef { - let pc = array.patches().map_or(0, patches_nchildren); - if idx < pc { - patches_child( - array - .patches() - .vortex_expect("BitPackedArray child index out of bounds"), - idx, - ) - } else if idx < pc + validity_nchildren(&array.validity) { - validity_to_child(&array.validity, array.len) - .vortex_expect("BitPackedArray child index out of bounds") - } else { - vortex_panic!("BitPackedArray child index {idx} out of bounds") - } - } - - fn child_name(array: &BitPackedArray, idx: usize) -> String { - let pc = array.patches().map_or(0, patches_nchildren); - if idx < pc { - patches_child_name(idx).to_string() - } else { - "validity".to_string() - } - } - fn reduce_parent( array: &Self::Array, parent: &ArrayRef, @@ -166,72 +137,47 @@ impl VTable for BitPacked { RULES.evaluate(array, parent, child_idx) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { - // Children: patches (if present): indices, values, chunk_offsets; then validity (if present) - let patches_info = array - .patches() - .map(|p| (p.offset(), p.chunk_offsets().is_some())); - - let mut child_idx = 0; - let patches = if let Some((patch_offset, has_chunk_offsets)) = patches_info { - let patch_indices = children - .get(child_idx) - .ok_or_else(|| vortex_err!("Expected patch_indices child at index {}", child_idx))? - .clone(); - child_idx += 1; - - let patch_values = children - .get(child_idx) - .ok_or_else(|| vortex_err!("Expected patch_values child at index {}", child_idx))? - .clone(); - child_idx += 1; - - let patch_chunk_offsets = if has_chunk_offsets { - let offsets = children - .get(child_idx) - .ok_or_else(|| { - vortex_err!("Expected patch_chunk_offsets child at index {}", child_idx) - })? - .clone(); - child_idx += 1; - Some(offsets) - } else { - None - }; - - Some(Patches::new( - array.len(), - patch_offset, - patch_indices, - patch_values, - patch_chunk_offsets, - )?) - } else { - None - }; + fn slots(array: &BitPackedArray) -> &[Option] { + &array.slots + } - let validity = if child_idx < children.len() { - Validity::Array(children[child_idx].clone()) - } else { - Validity::from(array.dtype().nullability()) - }; + fn slot_name(_array: &BitPackedArray, idx: usize) -> String { + SLOT_NAMES[idx].to_string() + } - let expected_children = child_idx - + if matches!(validity, Validity::Array(_)) { - 1 - } else { - 0 - }; + fn with_slots(array: &mut BitPackedArray, slots: Vec>) -> VortexResult<()> { vortex_ensure!( - children.len() == expected_children, - "Expected {} children, got {}", - expected_children, - children.len() + slots.len() == NUM_SLOTS, + "BitPackedArray expects {} slots, got {}", + NUM_SLOTS, + slots.len() ); - array.patches = patches; - array.validity = validity; + // Reconstruct patches from slots + existing metadata + array.patches = match (&slots[PATCH_INDICES_SLOT], &slots[PATCH_VALUES_SLOT]) { + (Some(indices), Some(values)) => { + let old = array + .patches + .as_ref() + .vortex_expect("BitPackedArray had patch slots but no patches metadata"); + Some(Patches::new( + array.len, + old.offset(), + indices.clone(), + values.clone(), + slots[PATCH_CHUNK_OFFSETS_SLOT].clone(), + )?) + } + _ => None, + }; + + // Reconstruct validity from slot + array.validity = match &slots[VALIDITY_SLOT] { + Some(arr) => Validity::Array(arr.clone()), + None => Validity::from(array.dtype.nullability()), + }; + array.slots = slots; Ok(()) } diff --git a/encodings/fastlanes/src/delta/array/mod.rs b/encodings/fastlanes/src/delta/array/mod.rs index 0cefe4c9fa9..0ceb68bb520 100644 --- a/encodings/fastlanes/src/delta/array/mod.rs +++ b/encodings/fastlanes/src/delta/array/mod.rs @@ -11,12 +11,18 @@ use vortex_array::dtype::DType; use vortex_array::dtype::PType; use vortex_array::match_each_unsigned_integer_ptype; use vortex_array::stats::ArrayStats; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; pub mod delta_compress; pub mod delta_decompress; +pub(super) const BASES_SLOT: usize = 0; +pub(super) const DELTAS_SLOT: usize = 1; +pub(super) const NUM_SLOTS: usize = 2; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["bases", "deltas"]; + /// A FastLanes-style delta-encoded array of primitive values. /// /// A [`DeltaArray`] comprises a sequence of _chunks_ each representing exactly 1,024 @@ -61,8 +67,7 @@ pub struct DeltaArray { pub(super) offset: usize, pub(super) len: usize, pub(super) dtype: DType, - pub(super) bases: ArrayRef, - pub(super) deltas: ArrayRef, + pub(super) slots: Vec>, pub(super) stats_set: ArrayStats, } @@ -132,24 +137,28 @@ impl DeltaArray { offset: usize, logical_len: usize, ) -> Self { + let dtype = bases.dtype().with_nullability(deltas.dtype().nullability()); Self { offset, len: logical_len, - dtype: bases.dtype().with_nullability(deltas.dtype().nullability()), - bases, - deltas, + dtype, + slots: vec![Some(bases), Some(deltas)], stats_set: Default::default(), } } #[inline] pub fn bases(&self) -> &ArrayRef { - &self.bases + self.slots[BASES_SLOT] + .as_ref() + .vortex_expect("DeltaArray bases slot") } #[inline] pub fn deltas(&self) -> &ArrayRef { - &self.deltas + self.slots[DELTAS_SLOT] + .as_ref() + .vortex_expect("DeltaArray deltas slot") } pub(crate) fn lanes(&self) -> usize { @@ -178,11 +187,11 @@ impl DeltaArray { } pub(crate) fn bases_len(&self) -> usize { - self.bases.len() + self.bases().len() } pub(crate) fn deltas_len(&self) -> usize { - self.deltas.len() + self.deltas().len() } pub(crate) fn stats_set(&self) -> &ArrayStats { diff --git a/encodings/fastlanes/src/delta/vtable/mod.rs b/encodings/fastlanes/src/delta/vtable/mod.rs index 790dedc6873..9dc326f4bcd 100644 --- a/encodings/fastlanes/src/delta/vtable/mod.rs +++ b/encodings/fastlanes/src/delta/vtable/mod.rs @@ -29,6 +29,8 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::DeltaArray; +use crate::delta::array::NUM_SLOTS; +use crate::delta::array::SLOT_NAMES; use crate::delta::array::delta_decompress::delta_decompress; mod operations; @@ -99,26 +101,6 @@ impl VTable for Delta { None } - fn nchildren(_array: &DeltaArray) -> usize { - 2 - } - - fn child(array: &DeltaArray, idx: usize) -> ArrayRef { - match idx { - 0 => array.bases().clone(), - 1 => array.deltas().clone(), - _ => vortex_panic!("DeltaArray child index {idx} out of bounds"), - } - } - - fn child_name(_array: &DeltaArray, idx: usize) -> String { - match idx { - 0 => "bases".to_string(), - 1 => "deltas".to_string(), - _ => vortex_panic!("DeltaArray child name index {idx} out of bounds"), - } - } - fn reduce_parent( array: &Self::Array, parent: &ArrayRef, @@ -127,20 +109,22 @@ impl VTable for Delta { rules::RULES.evaluate(array, parent, child_idx) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { - // DeltaArray children order (from visit_children): - // 1. bases - // 2. deltas + fn slots(array: &DeltaArray) -> &[Option] { + &array.slots + } + + fn slot_name(_array: &DeltaArray, idx: usize) -> String { + SLOT_NAMES[idx].to_string() + } + fn with_slots(array: &mut DeltaArray, slots: Vec>) -> VortexResult<()> { vortex_ensure!( - children.len() == 2, - "Expected 2 children for Delta encoding, got {}", - children.len() + slots.len() == NUM_SLOTS, + "DeltaArray expects exactly {} slots, got {}", + NUM_SLOTS, + slots.len() ); - - array.bases = children[0].clone(); - array.deltas = children[1].clone(); - + array.slots = slots; Ok(()) } diff --git a/encodings/fastlanes/src/for/array/mod.rs b/encodings/fastlanes/src/for/array/mod.rs index 0f90e6fcd08..22f7b61bb0f 100644 --- a/encodings/fastlanes/src/for/array/mod.rs +++ b/encodings/fastlanes/src/for/array/mod.rs @@ -5,19 +5,24 @@ use vortex_array::ArrayRef; use vortex_array::dtype::PType; use vortex_array::scalar::Scalar; use vortex_array::stats::ArrayStats; +use vortex_error::VortexExpect as _; use vortex_error::VortexResult; use vortex_error::vortex_bail; pub mod for_compress; pub mod for_decompress; +pub(super) const ENCODED_SLOT: usize = 0; +pub(super) const NUM_SLOTS: usize = 1; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["encoded"]; + /// Frame of Reference (FoR) encoded array. /// /// This encoding stores values as offsets from a reference value, which can significantly reduce /// storage requirements when values are clustered around a specific point. #[derive(Clone, Debug)] pub struct FoRArray { - pub(super) encoded: ArrayRef, + pub(super) slots: Vec>, pub(super) reference: Scalar, pub(super) stats_set: ArrayStats, } @@ -34,7 +39,7 @@ impl FoRArray { )?; Ok(Self { - encoded, + slots: vec![Some(encoded)], reference, stats_set: Default::default(), }) @@ -42,7 +47,7 @@ impl FoRArray { pub(crate) unsafe fn new_unchecked(encoded: ArrayRef, reference: Scalar) -> Self { Self { - encoded, + slots: vec![Some(encoded)], reference, stats_set: Default::default(), } @@ -55,7 +60,9 @@ impl FoRArray { #[inline] pub fn encoded(&self) -> &ArrayRef { - &self.encoded + self.slots[ENCODED_SLOT] + .as_ref() + .vortex_expect("FoRArray encoded slot") } #[inline] diff --git a/encodings/fastlanes/src/for/vtable/mod.rs b/encodings/fastlanes/src/for/vtable/mod.rs index 5eb73711ae4..72c8eae73c0 100644 --- a/encodings/fastlanes/src/for/vtable/mod.rs +++ b/encodings/fastlanes/src/for/vtable/mod.rs @@ -28,6 +28,8 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::FoRArray; +use crate::r#for::array::NUM_SLOTS; +use crate::r#for::array::SLOT_NAMES; use crate::r#for::array::for_decompress::decompress; use crate::r#for::vtable::kernels::PARENT_KERNELS; use crate::r#for::vtable::rules::PARENT_RULES; @@ -86,36 +88,22 @@ impl VTable for FoR { None } - fn nchildren(_array: &FoRArray) -> usize { - 1 + fn slots(array: &FoRArray) -> &[Option] { + &array.slots } - fn child(array: &FoRArray, idx: usize) -> ArrayRef { - match idx { - 0 => array.encoded().clone(), - _ => vortex_panic!("FoRArray child index {idx} out of bounds"), - } - } - - fn child_name(_array: &FoRArray, idx: usize) -> String { - match idx { - 0 => "encoded".to_string(), - _ => vortex_panic!("FoRArray child name index {idx} out of bounds"), - } + fn slot_name(_array: &FoRArray, idx: usize) -> String { + SLOT_NAMES[idx].to_string() } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { - // FoRArray children order (from visit_children): - // 1. encoded - + fn with_slots(array: &mut FoRArray, slots: Vec>) -> VortexResult<()> { vortex_ensure!( - children.len() == 1, - "Expected 1 child for FoR encoding, got {}", - children.len() + slots.len() == NUM_SLOTS, + "FoRArray expects exactly {} slots, got {}", + NUM_SLOTS, + slots.len() ); - - array.encoded = children[0].clone(); - + array.slots = slots; Ok(()) } diff --git a/encodings/fastlanes/src/for/vtable/rules.rs b/encodings/fastlanes/src/for/vtable/rules.rs index edeffda8bd7..3ce7d8905b3 100644 --- a/encodings/fastlanes/src/for/vtable/rules.rs +++ b/encodings/fastlanes/src/for/vtable/rules.rs @@ -38,7 +38,7 @@ impl ArrayParentReduceRule for FoRFilterPushDownRule { ) -> VortexResult> { let new_array = unsafe { FoRArray::new_unchecked( - child.encoded.filter(parent.filter_mask().clone())?, + child.encoded().filter(parent.filter_mask().clone())?, child.reference.clone(), ) }; diff --git a/encodings/fastlanes/src/rle/array/mod.rs b/encodings/fastlanes/src/rle/array/mod.rs index 77c584ce331..cccd2f2edb1 100644 --- a/encodings/fastlanes/src/rle/array/mod.rs +++ b/encodings/fastlanes/src/rle/array/mod.rs @@ -6,6 +6,7 @@ use vortex_array::DynArray; use vortex_array::dtype::DType; use vortex_array::dtype::PType; use vortex_array::stats::ArrayStats; +use vortex_error::VortexExpect as _; use vortex_error::VortexResult; use vortex_error::vortex_ensure; @@ -14,24 +15,16 @@ use crate::FL_CHUNK_SIZE; pub mod rle_compress; pub mod rle_decompress; +pub(super) const VALUES_SLOT: usize = 0; +pub(super) const INDICES_SLOT: usize = 1; +pub(super) const VALUES_IDX_OFFSETS_SLOT: usize = 2; +pub(super) const NUM_SLOTS: usize = 3; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["values", "indices", "values_idx_offsets"]; + #[derive(Clone, Debug)] pub struct RLEArray { pub(super) dtype: DType, - /// Run value in the dictionary. - pub(super) values: ArrayRef, - /// Chunk-local indices from all chunks. The start of each chunk is looked up in `values_idx_offsets`. - pub(super) indices: ArrayRef, - /// Index start positions of each value chunk. - /// - /// # Example - /// ``` - /// // Chunk 0: [10, 20] (starts at index 0) - /// // Chunk 1: [30, 40] (starts at index 2) - /// let values = [10, 20, 30, 40]; // Global values array - /// let values_idx_offsets = [0, 2]; // Chunk 0 starts at index 0, Chunk 1 starts at index 2 - /// ``` - pub(super) values_idx_offsets: ArrayRef, - + pub(super) slots: Vec>, pub(super) stats_set: ArrayStats, // Offset relative to the start of the chunk. pub(super) offset: usize, @@ -109,9 +102,7 @@ impl RLEArray { Ok(Self { dtype, - values, - indices, - values_idx_offsets, + slots: vec![Some(values), Some(indices), Some(values_idx_offsets)], stats_set: ArrayStats::default(), offset, length, @@ -137,9 +128,7 @@ impl RLEArray { ) -> Self { Self { dtype, - values, - indices, - values_idx_offsets, + slots: vec![Some(values), Some(indices), Some(values_idx_offsets)], stats_set: ArrayStats::default(), offset, length, @@ -163,17 +152,23 @@ impl RLEArray { #[inline] pub fn values(&self) -> &ArrayRef { - &self.values + self.slots[VALUES_SLOT] + .as_ref() + .vortex_expect("RLEArray values slot must be populated") } #[inline] pub fn indices(&self) -> &ArrayRef { - &self.indices + self.slots[INDICES_SLOT] + .as_ref() + .vortex_expect("RLEArray indices slot must be populated") } #[inline] pub fn values_idx_offsets(&self) -> &ArrayRef { - &self.values_idx_offsets + self.slots[VALUES_IDX_OFFSETS_SLOT] + .as_ref() + .vortex_expect("RLEArray values_idx_offsets slot must be populated") } /// Values index offset relative to the first chunk. @@ -186,14 +181,14 @@ impl RLEArray { reason = "expect is safe here as scalar_at returns a valid primitive" )] pub(crate) fn values_idx_offset(&self, chunk_idx: usize) -> usize { - self.values_idx_offsets + self.values_idx_offsets() .scalar_at(chunk_idx) .expect("index must be in bounds") .as_primitive() .as_::() .expect("index must be of type usize") - self - .values_idx_offsets + .values_idx_offsets() .scalar_at(0) .expect("index must be in bounds") .as_primitive() diff --git a/encodings/fastlanes/src/rle/vtable/mod.rs b/encodings/fastlanes/src/rle/vtable/mod.rs index 0df4126cbf5..458ccb3ffc6 100644 --- a/encodings/fastlanes/src/rle/vtable/mod.rs +++ b/encodings/fastlanes/src/rle/vtable/mod.rs @@ -110,28 +110,6 @@ impl VTable for RLE { None } - fn nchildren(_array: &RLEArray) -> usize { - 3 - } - - fn child(array: &RLEArray, idx: usize) -> ArrayRef { - match idx { - 0 => array.values().clone(), - 1 => array.indices().clone(), - 2 => array.values_idx_offsets().clone(), - _ => vortex_panic!("RLEArray child index {idx} out of bounds"), - } - } - - fn child_name(_array: &RLEArray, idx: usize) -> String { - match idx { - 0 => "values".to_string(), - 1 => "indices".to_string(), - 2 => "values_idx_offsets".to_string(), - _ => vortex_panic!("RLEArray child name index {idx} out of bounds"), - } - } - fn reduce_parent( array: &Self::Array, parent: &ArrayRef, @@ -140,22 +118,22 @@ impl VTable for RLE { RULES.evaluate(array, parent, child_idx) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { - // RLEArray children order (from visit_children): - // 1. values - // 2. indices - // 3. values_idx_offsets + fn slots(array: &RLEArray) -> &[Option] { + &array.slots + } + + fn slot_name(_array: &RLEArray, idx: usize) -> String { + crate::rle::array::SLOT_NAMES[idx].to_string() + } + fn with_slots(array: &mut RLEArray, slots: Vec>) -> VortexResult<()> { vortex_ensure!( - children.len() == 3, - "Expected 3 children for RLE encoding, got {}", - children.len() + slots.len() == crate::rle::array::NUM_SLOTS, + "RLEArray expects {} slots, got {}", + crate::rle::array::NUM_SLOTS, + slots.len() ); - - array.values = children[0].clone(); - array.indices = children[1].clone(); - array.values_idx_offsets = children[2].clone(); - + array.slots = slots; Ok(()) } diff --git a/encodings/fsst/public-api.lock b/encodings/fsst/public-api.lock index c7f958d609c..ef2d59b47c8 100644 --- a/encodings/fsst/public-api.lock +++ b/encodings/fsst/public-api.lock @@ -56,10 +56,6 @@ pub fn vortex_fsst::FSST::buffer_name(_array: &vortex_fsst::FSSTArray, idx: usiz pub fn vortex_fsst::FSST::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_fsst::FSST::child(array: &vortex_fsst::FSSTArray, idx: usize) -> vortex_array::array::ArrayRef - -pub fn vortex_fsst::FSST::child_name(_array: &vortex_fsst::FSSTArray, idx: usize) -> alloc::string::String - pub fn vortex_fsst::FSST::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_fsst::FSST::dtype(array: &vortex_fsst::FSSTArray) -> &vortex_array::dtype::DType @@ -76,15 +72,17 @@ pub fn vortex_fsst::FSST::metadata(array: &vortex_fsst::FSSTArray) -> vortex_err pub fn vortex_fsst::FSST::nbuffers(_array: &vortex_fsst::FSSTArray) -> usize -pub fn vortex_fsst::FSST::nchildren(array: &vortex_fsst::FSSTArray) -> usize - pub fn vortex_fsst::FSST::reduce_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> pub fn vortex_fsst::FSST::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_fsst::FSST::slot_name(_array: &vortex_fsst::FSSTArray, idx: usize) -> alloc::string::String + +pub fn vortex_fsst::FSST::slots(array: &vortex_fsst::FSSTArray) -> &[core::option::Option] + pub fn vortex_fsst::FSST::stats(array: &vortex_fsst::FSSTArray) -> vortex_array::stats::array::StatsSetRef<'_> -pub fn vortex_fsst::FSST::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_fsst::FSST::with_slots(array: &mut vortex_fsst::FSSTArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::operations::OperationsVTable for vortex_fsst::FSST diff --git a/encodings/fsst/src/array.rs b/encodings/fsst/src/array.rs index 9228b34085c..b2c37008e92 100644 --- a/encodings/fsst/src/array.rs +++ b/encodings/fsst/src/array.rs @@ -40,10 +40,10 @@ use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityChild; use vortex_array::vtable::ValidityHelper; use vortex_array::vtable::ValidityVTableFromChild; -use vortex_array::vtable::validity_nchildren; use vortex_array::vtable::validity_to_child; use vortex_buffer::Buffer; use vortex_buffer::ByteBuffer; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -102,7 +102,7 @@ impl VTable for FSST { array.symbols.array_hash(state, precision); array.symbol_lengths.array_hash(state, precision); array.codes.as_ref().array_hash(state, precision); - array.uncompressed_lengths.array_hash(state, precision); + array.uncompressed_lengths().array_hash(state, precision); } fn array_eq(array: &FSSTArray, other: &FSSTArray, precision: Precision) -> bool { @@ -116,8 +116,8 @@ impl VTable for FSST { .as_ref() .array_eq(other.codes.as_ref(), precision) && array - .uncompressed_lengths - .array_eq(&other.uncompressed_lengths, precision) + .uncompressed_lengths() + .array_eq(other.uncompressed_lengths(), precision) } fn nbuffers(_array: &FSSTArray) -> usize { @@ -142,29 +142,6 @@ impl VTable for FSST { } } - fn nchildren(array: &FSSTArray) -> usize { - 2 + validity_nchildren(array.codes.validity()) - } - - fn child(array: &FSSTArray, idx: usize) -> ArrayRef { - match idx { - 0 => array.uncompressed_lengths().clone(), - 1 => array.codes.offsets().clone(), - 2 => validity_to_child(array.codes.validity(), array.codes.len()) - .unwrap_or_else(|| vortex_panic!("FSSTArray child index {idx} out of bounds")), - _ => vortex_panic!("FSSTArray child index {idx} out of bounds"), - } - } - - fn child_name(_array: &FSSTArray, idx: usize) -> String { - match idx { - 0 => "uncompressed_lengths".to_string(), - 1 => "codes_offsets".to_string(), - 2 => "validity".to_string(), - _ => vortex_panic!("FSSTArray child_name index {idx} out of bounds"), - } - } - fn metadata(array: &FSSTArray) -> VortexResult { Ok(ProstMetadata(FSSTMetadata { uncompressed_lengths_ptype: array.uncompressed_lengths().dtype().as_ptype().into(), @@ -308,34 +285,38 @@ impl VTable for FSST { ); } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { + fn slots(array: &FSSTArray) -> &[Option] { + &array.slots + } + + fn slot_name(_array: &FSSTArray, idx: usize) -> String { + SLOT_NAMES[idx].to_string() + } + + fn with_slots(array: &mut FSSTArray, slots: Vec>) -> VortexResult<()> { vortex_ensure!( - children.len() == 2, - "FSSTArray expects 2 children, got {}", - children.len() + slots.len() == NUM_SLOTS, + "FSSTArray expects {} slots, got {}", + NUM_SLOTS, + slots.len() ); - let mut children_iter = children.into_iter(); - let codes = children_iter - .next() - .ok_or_else(|| vortex_err!("FSSTArray with_children missing codes"))?; - - let codes = codes - .as_opt::() - .ok_or_else(|| { - vortex_err!( - "Expected VarBinArray for codes, got {}", - codes.encoding_id() - ) - })? - .clone(); - let uncompressed_lengths = children_iter - .next() - .ok_or_else(|| vortex_err!("FSSTArray with_children missing uncompressed_lengths"))?; - - array.codes = codes; - array.uncompressed_lengths = uncompressed_lengths; - + // Reconstruct codes VarBinArray from new offsets + existing bytes + new validity + let codes_offsets = slots[CODES_OFFSETS_SLOT] + .clone() + .vortex_expect("FSSTArray requires codes_offsets slot"); + let codes_validity = match &slots[CODES_VALIDITY_SLOT] { + Some(arr) => Validity::Array(arr.clone()), + None => Validity::from(array.codes.dtype().nullability()), + }; + array.codes = VarBinArray::try_new_from_handle( + codes_offsets, + array.codes.bytes_handle().clone(), + array.codes.dtype().clone(), + codes_validity, + )?; + array.codes_array = array.codes.clone().into_array(); + array.slots = slots; Ok(()) } @@ -361,6 +342,13 @@ impl VTable for FSST { } } +pub(crate) const UNCOMPRESSED_LENGTHS_SLOT: usize = 0; +pub(crate) const CODES_OFFSETS_SLOT: usize = 1; +pub(crate) const CODES_VALIDITY_SLOT: usize = 2; +pub(crate) const NUM_SLOTS: usize = 3; +pub(crate) const SLOT_NAMES: [&str; NUM_SLOTS] = + ["uncompressed_lengths", "codes_offsets", "codes_validity"]; + #[derive(Clone)] pub struct FSSTArray { dtype: DType, @@ -370,7 +358,7 @@ pub struct FSSTArray { /// NOTE(ngates): this === codes, but is stored as an ArrayRef so we can return &ArrayRef! codes_array: ArrayRef, /// Lengths of the original values before compression, can be compressed. - uncompressed_lengths: ArrayRef, + slots: Vec>, stats_set: ArrayStats, /// Memoized compressor used for push-down of compute by compressing the RHS. @@ -384,7 +372,7 @@ impl Debug for FSSTArray { .field("symbols", &self.symbols) .field("symbol_lengths", &self.symbol_lengths) .field("codes", &self.codes) - .field("uncompressed_lengths", &self.uncompressed_lengths) + .field("uncompressed_lengths", self.uncompressed_lengths()) .finish() } } @@ -459,6 +447,8 @@ impl FSSTArray { }) as Box Compressor + Send>)); let codes_array = codes.clone().into_array(); + let codes_offsets_slot = Some(codes.offsets().clone()); + let codes_validity_slot = validity_to_child(codes.validity(), codes.len()); Self { dtype, @@ -466,7 +456,11 @@ impl FSSTArray { symbol_lengths, codes, codes_array, - uncompressed_lengths, + slots: vec![ + Some(uncompressed_lengths), + codes_offsets_slot, + codes_validity_slot, + ], stats_set: Default::default(), compressor, } @@ -495,13 +489,15 @@ impl FSSTArray { /// Get the uncompressed length for each element in the array. pub fn uncompressed_lengths(&self) -> &ArrayRef { - &self.uncompressed_lengths + self.slots[UNCOMPRESSED_LENGTHS_SLOT] + .as_ref() + .vortex_expect("FSSTArray uncompressed_lengths slot") } /// Get the DType of the uncompressed lengths array #[inline] pub fn uncompressed_lengths_dtype(&self) -> &DType { - self.uncompressed_lengths.dtype() + self.uncompressed_lengths().dtype() } /// Build a [`Decompressor`][fsst::Decompressor] that can be used to decompress values from diff --git a/encodings/pco/public-api.lock b/encodings/pco/public-api.lock index b82093e811b..ec61483b6e8 100644 --- a/encodings/pco/public-api.lock +++ b/encodings/pco/public-api.lock @@ -38,10 +38,6 @@ pub fn vortex_pco::Pco::buffer_name(array: &vortex_pco::PcoArray, idx: usize) -> pub fn vortex_pco::Pco::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_pco::Pco::child(array: &vortex_pco::PcoArray, idx: usize) -> vortex_array::array::ArrayRef - -pub fn vortex_pco::Pco::child_name(_array: &vortex_pco::PcoArray, idx: usize) -> alloc::string::String - pub fn vortex_pco::Pco::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_pco::Pco::dtype(array: &vortex_pco::PcoArray) -> &vortex_array::dtype::DType @@ -56,15 +52,17 @@ pub fn vortex_pco::Pco::metadata(array: &vortex_pco::PcoArray) -> vortex_error:: pub fn vortex_pco::Pco::nbuffers(array: &vortex_pco::PcoArray) -> usize -pub fn vortex_pco::Pco::nchildren(array: &vortex_pco::PcoArray) -> usize - pub fn vortex_pco::Pco::reduce_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> pub fn vortex_pco::Pco::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_pco::Pco::slot_name(_array: &vortex_pco::PcoArray, idx: usize) -> alloc::string::String + +pub fn vortex_pco::Pco::slots(array: &vortex_pco::PcoArray) -> &[core::option::Option] + pub fn vortex_pco::Pco::stats(array: &vortex_pco::PcoArray) -> vortex_array::stats::array::StatsSetRef<'_> -pub fn vortex_pco::Pco::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_pco::Pco::with_slots(array: &mut vortex_pco::PcoArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::operations::OperationsVTable for vortex_pco::Pco diff --git a/encodings/pco/src/array.rs b/encodings/pco/src/array.rs index 85ba2bbe6cb..1b536e4f17f 100644 --- a/encodings/pco/src/array.rs +++ b/encodings/pco/src/array.rs @@ -45,18 +45,15 @@ use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityHelper; use vortex_array::vtable::ValiditySliceHelper; use vortex_array::vtable::ValidityVTableFromValiditySliceHelper; -use vortex_array::vtable::validity_nchildren; use vortex_array::vtable::validity_to_child; use vortex_buffer::BufferMut; use vortex_buffer::ByteBuffer; use vortex_buffer::ByteBufferMut; use vortex_error::VortexError; -use vortex_error::VortexExpect as _; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_err; -use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::PcoChunkInfo; @@ -170,22 +167,6 @@ impl VTable for Pco { } } - fn nchildren(array: &PcoArray) -> usize { - validity_nchildren(&array.unsliced_validity) - } - - fn child(array: &PcoArray, idx: usize) -> ArrayRef { - validity_to_child(&array.unsliced_validity, array.unsliced_n_rows) - .unwrap_or_else(|| vortex_panic!("PcoArray child index {idx} out of bounds")) - } - - fn child_name(_array: &PcoArray, idx: usize) -> String { - match idx { - 0 => "validity".to_string(), - _ => vortex_panic!("PcoArray child_name index {idx} out of bounds"), - } - } - fn metadata(array: &PcoArray) -> VortexResult { Ok(ProstMetadata(array.metadata.clone())) } @@ -248,20 +229,26 @@ impl VTable for Pco { )) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { - vortex_ensure!( - children.len() <= 1, - "PcoArray expects 0 or 1 children, got {}", - children.len() - ); + fn slots(array: &PcoArray) -> &[Option] { + &array.slots + } - if children.is_empty() { - array.unsliced_validity = Validity::from(array.dtype.nullability()); - } else { - array.unsliced_validity = - Validity::Array(children.into_iter().next().vortex_expect("validity child")); - } + fn slot_name(_array: &PcoArray, idx: usize) -> String { + SLOT_NAMES[idx].to_string() + } + fn with_slots(array: &mut PcoArray, slots: Vec>) -> VortexResult<()> { + vortex_ensure!( + slots.len() == NUM_SLOTS, + "PcoArray expects {} slots, got {}", + NUM_SLOTS, + slots.len() + ); + array.unsliced_validity = match &slots[VALIDITY_SLOT] { + Some(arr) => Validity::Array(arr.clone()), + None => Validity::from(array.dtype.nullability()), + }; + array.slots = slots; Ok(()) } @@ -315,6 +302,10 @@ impl Pco { pub const ID: ArrayId = ArrayId::new_ref("vortex.pco"); } +pub(super) const VALIDITY_SLOT: usize = 0; +pub(super) const NUM_SLOTS: usize = 1; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; + #[derive(Clone, Debug)] pub struct PcoArray { pub(crate) chunk_metas: Vec, @@ -323,6 +314,7 @@ pub struct PcoArray { dtype: DType, pub(crate) unsliced_validity: Validity, unsliced_n_rows: usize, + pub(super) slots: Vec>, stats_set: ArrayStats, slice_start: usize, slice_stop: usize, @@ -337,6 +329,8 @@ impl PcoArray { len: usize, validity: Validity, ) -> Self { + let validity_slot = validity_to_child(&validity, len); + Self { chunk_metas, pages, @@ -344,6 +338,7 @@ impl PcoArray { dtype, unsliced_validity: validity, unsliced_n_rows: len, + slots: vec![validity_slot], stats_set: Default::default(), slice_start: 0, slice_stop: len, diff --git a/encodings/runend/public-api.lock b/encodings/runend/public-api.lock index a78046051d0..1262e6e1f63 100644 --- a/encodings/runend/public-api.lock +++ b/encodings/runend/public-api.lock @@ -66,10 +66,6 @@ pub fn vortex_runend::RunEnd::buffer_name(_array: &vortex_runend::RunEndArray, i pub fn vortex_runend::RunEnd::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_runend::RunEnd::child(array: &vortex_runend::RunEndArray, idx: usize) -> vortex_array::array::ArrayRef - -pub fn vortex_runend::RunEnd::child_name(_array: &vortex_runend::RunEndArray, idx: usize) -> alloc::string::String - pub fn vortex_runend::RunEnd::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_runend::RunEnd::dtype(array: &vortex_runend::RunEndArray) -> &vortex_array::dtype::DType @@ -86,15 +82,17 @@ pub fn vortex_runend::RunEnd::metadata(array: &vortex_runend::RunEndArray) -> vo pub fn vortex_runend::RunEnd::nbuffers(_array: &vortex_runend::RunEndArray) -> usize -pub fn vortex_runend::RunEnd::nchildren(_array: &vortex_runend::RunEndArray) -> usize - pub fn vortex_runend::RunEnd::reduce_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> pub fn vortex_runend::RunEnd::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_runend::RunEnd::slot_name(_array: &vortex_runend::RunEndArray, idx: usize) -> alloc::string::String + +pub fn vortex_runend::RunEnd::slots(array: &vortex_runend::RunEndArray) -> &[core::option::Option] + pub fn vortex_runend::RunEnd::stats(array: &vortex_runend::RunEndArray) -> vortex_array::stats::array::StatsSetRef<'_> -pub fn vortex_runend::RunEnd::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_runend::RunEnd::with_slots(array: &mut vortex_runend::RunEndArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::operations::OperationsVTable for vortex_runend::RunEnd diff --git a/encodings/runend/src/array.rs b/encodings/runend/src/array.rs index 6dd3fdc521d..32849511715 100644 --- a/encodings/runend/src/array.rs +++ b/encodings/runend/src/array.rs @@ -74,7 +74,7 @@ impl VTable for RunEnd { } fn dtype(array: &RunEndArray) -> &DType { - array.values.dtype() + array.values().dtype() } fn stats(array: &RunEndArray) -> StatsSetRef<'_> { @@ -82,15 +82,15 @@ impl VTable for RunEnd { } fn array_hash(array: &RunEndArray, state: &mut H, precision: Precision) { - array.ends.array_hash(state, precision); - array.values.array_hash(state, precision); + array.ends().array_hash(state, precision); + array.values().array_hash(state, precision); array.offset.hash(state); array.length.hash(state); } fn array_eq(array: &RunEndArray, other: &RunEndArray, precision: Precision) -> bool { - array.ends.array_eq(&other.ends, precision) - && array.values.array_eq(&other.values, precision) + array.ends().array_eq(other.ends(), precision) + && array.values().array_eq(other.values(), precision) && array.offset == other.offset && array.length == other.length } @@ -107,26 +107,6 @@ impl VTable for RunEnd { vortex_panic!("RunEndArray buffer_name index {idx} out of bounds") } - fn nchildren(_array: &RunEndArray) -> usize { - 2 - } - - fn child(array: &RunEndArray, idx: usize) -> ArrayRef { - match idx { - 0 => array.ends().clone(), - 1 => array.values().clone(), - _ => vortex_panic!("RunEndArray child index {idx} out of bounds"), - } - } - - fn child_name(_array: &RunEndArray, idx: usize) -> String { - match idx { - 0 => "ends".to_string(), - 1 => "values".to_string(), - _ => vortex_panic!("RunEndArray child_name index {idx} out of bounds"), - } - } - fn metadata(array: &RunEndArray) -> VortexResult { Ok(ProstMetadata(RunEndMetadata { ends_ptype: PType::try_from(array.ends().dtype()).vortex_expect("Must be a valid PType") @@ -172,17 +152,22 @@ impl VTable for RunEnd { ) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { - vortex_ensure!( - children.len() == 2, - "RunEndArray expects 2 children, got {}", - children.len() - ); + fn slots(array: &RunEndArray) -> &[Option] { + &array.slots + } - let mut children_iter = children.into_iter(); - array.ends = children_iter.next().vortex_expect("ends child"); - array.values = children_iter.next().vortex_expect("values child"); + fn slot_name(_array: &RunEndArray, idx: usize) -> String { + SLOT_NAMES[idx].to_string() + } + fn with_slots(array: &mut RunEndArray, slots: Vec>) -> VortexResult<()> { + vortex_ensure!( + slots.len() == NUM_SLOTS, + "RunEndArray expects exactly {} slots, got {}", + NUM_SLOTS, + slots.len() + ); + array.slots = slots; Ok(()) } @@ -208,10 +193,14 @@ impl VTable for RunEnd { } } +pub(super) const ENDS_SLOT: usize = 0; +pub(super) const VALUES_SLOT: usize = 1; +pub(super) const NUM_SLOTS: usize = 2; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["ends", "values"]; + #[derive(Clone, Debug)] pub struct RunEndArray { - ends: ArrayRef, - values: ArrayRef, + pub(super) slots: Vec>, offset: usize, length: usize, stats_set: ArrayStats, @@ -362,8 +351,7 @@ impl RunEndArray { Self::validate(&ends, &values, offset, length)?; Ok(Self { - ends, - values, + slots: vec![Some(ends), Some(values)], offset, length, stats_set: Default::default(), @@ -385,8 +373,7 @@ impl RunEndArray { length: usize, ) -> Self { Self { - ends, - values, + slots: vec![Some(ends), Some(values)], offset, length, stats_set: Default::default(), @@ -437,7 +424,9 @@ impl RunEndArray { /// at `ends[i]` (inclusive) and terminating at `ends[i+1]` (exclusive). #[inline] pub fn ends(&self) -> &ArrayRef { - &self.ends + self.slots[ENDS_SLOT] + .as_ref() + .vortex_expect("RunEndArray ends slot") } /// The scalar values. @@ -446,15 +435,21 @@ impl RunEndArray { /// at `ends[i]` (inclusive) and terminates at `ends[i+1]` (exclusive). #[inline] pub fn values(&self) -> &ArrayRef { - &self.values + self.slots[VALUES_SLOT] + .as_ref() + .vortex_expect("RunEndArray values slot") } /// Split an `RunEndArray` into parts. #[inline] pub fn into_parts(self) -> RunEndArrayParts { RunEndArrayParts { - ends: self.ends, - values: self.values, + ends: self.slots[ENDS_SLOT] + .clone() + .vortex_expect("RunEndArray ends slot"), + values: self.slots[VALUES_SLOT] + .clone() + .vortex_expect("RunEndArray values slot"), } } } diff --git a/encodings/runend/src/rules.rs b/encodings/runend/src/rules.rs index ead3cc5018e..06f56fbafb3 100644 --- a/encodings/runend/src/rules.rs +++ b/encodings/runend/src/rules.rs @@ -41,7 +41,7 @@ impl ArrayParentReduceRule for RunEndScalarFnRule { parent: &ScalarFnArray, child_idx: usize, ) -> VortexResult> { - for (idx, child) in parent.children().iter().enumerate() { + for (idx, child) in parent.iter_children().enumerate() { if idx == child_idx { // Skip ourselves continue; diff --git a/encodings/sequence/public-api.lock b/encodings/sequence/public-api.lock index 2696a1d1457..fe2b5ef8ce5 100644 --- a/encodings/sequence/public-api.lock +++ b/encodings/sequence/public-api.lock @@ -54,10 +54,6 @@ pub fn vortex_sequence::Sequence::buffer_name(_array: &vortex_sequence::Sequence pub fn vortex_sequence::Sequence::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_sequence::Sequence::child(_array: &vortex_sequence::SequenceArray, idx: usize) -> vortex_array::array::ArrayRef - -pub fn vortex_sequence::Sequence::child_name(_array: &vortex_sequence::SequenceArray, idx: usize) -> alloc::string::String - pub fn vortex_sequence::Sequence::deserialize(bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_sequence::Sequence::dtype(array: &vortex_sequence::SequenceArray) -> &vortex_array::dtype::DType @@ -74,15 +70,17 @@ pub fn vortex_sequence::Sequence::metadata(array: &vortex_sequence::SequenceArra pub fn vortex_sequence::Sequence::nbuffers(_array: &vortex_sequence::SequenceArray) -> usize -pub fn vortex_sequence::Sequence::nchildren(_array: &vortex_sequence::SequenceArray) -> usize - pub fn vortex_sequence::Sequence::reduce_parent(array: &vortex_sequence::SequenceArray, parent: &vortex_array::array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> pub fn vortex_sequence::Sequence::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_sequence::Sequence::slot_name(_array: &vortex_sequence::SequenceArray, idx: usize) -> alloc::string::String + +pub fn vortex_sequence::Sequence::slots(array: &vortex_sequence::SequenceArray) -> &[core::option::Option] + pub fn vortex_sequence::Sequence::stats(array: &vortex_sequence::SequenceArray) -> vortex_array::stats::array::StatsSetRef<'_> -pub fn vortex_sequence::Sequence::with_children(_array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_sequence::Sequence::with_slots(array: &mut vortex_sequence::SequenceArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::operations::OperationsVTable for vortex_sequence::Sequence diff --git a/encodings/sequence/src/array.rs b/encodings/sequence/src/array.rs index ea716ced1ea..65659165fce 100644 --- a/encodings/sequence/src/array.rs +++ b/encodings/sequence/src/array.rs @@ -72,6 +72,8 @@ pub struct SequenceArrayParts { pub nullability: Nullability, } +pub(super) const SLOT_NAMES: [&str; 0] = []; + #[derive(Clone, Debug)] /// An array representing the equation `A[i] = base + i * multiplier`. pub struct SequenceArray { @@ -79,6 +81,7 @@ pub struct SequenceArray { multiplier: PValue, dtype: DType, pub(crate) len: usize, + pub(super) slots: Vec>, stats_set: ArrayStats, } @@ -167,6 +170,7 @@ impl SequenceArray { multiplier, dtype, len: length, + slots: vec![], stats_set: ArrayStats::from(stats_set), } } @@ -288,18 +292,6 @@ impl VTable for Sequence { vortex_panic!("SequenceArray buffer_name index {idx} out of bounds") } - fn nchildren(_array: &SequenceArray) -> usize { - 0 - } - - fn child(_array: &SequenceArray, idx: usize) -> ArrayRef { - vortex_panic!("SequenceArray child index {idx} out of bounds") - } - - fn child_name(_array: &SequenceArray, idx: usize) -> String { - vortex_panic!("SequenceArray child_name index {idx} out of bounds") - } - fn metadata(array: &SequenceArray) -> VortexResult { Ok(SequenceMetadata { base: array.base(), @@ -372,12 +364,22 @@ impl VTable for Sequence { ) } - fn with_children(_array: &mut Self::Array, children: Vec) -> VortexResult<()> { + fn slots(array: &SequenceArray) -> &[Option] { + &array.slots + } + + fn slot_name(_array: &SequenceArray, idx: usize) -> String { + let _ = SLOT_NAMES; + vortex_panic!("SequenceArray has no slots, requested index {idx}") + } + + fn with_slots(array: &mut SequenceArray, slots: Vec>) -> VortexResult<()> { vortex_ensure!( - children.is_empty(), - "SequenceArray expects 0 children, got {}", - children.len() + slots.is_empty(), + "SequenceArray expects 0 slots, got {}", + slots.len() ); + array.slots = slots; Ok(()) } diff --git a/encodings/sparse/public-api.lock b/encodings/sparse/public-api.lock index 9193dec8147..2301248559b 100644 --- a/encodings/sparse/public-api.lock +++ b/encodings/sparse/public-api.lock @@ -70,10 +70,6 @@ pub fn vortex_sparse::Sparse::buffer_name(_array: &vortex_sparse::SparseArray, i pub fn vortex_sparse::Sparse::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_sparse::Sparse::child(array: &vortex_sparse::SparseArray, idx: usize) -> vortex_array::array::ArrayRef - -pub fn vortex_sparse::Sparse::child_name(_array: &vortex_sparse::SparseArray, idx: usize) -> alloc::string::String - pub fn vortex_sparse::Sparse::deserialize(bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_sparse::Sparse::dtype(array: &vortex_sparse::SparseArray) -> &vortex_array::dtype::DType @@ -90,15 +86,17 @@ pub fn vortex_sparse::Sparse::metadata(array: &vortex_sparse::SparseArray) -> vo pub fn vortex_sparse::Sparse::nbuffers(_array: &vortex_sparse::SparseArray) -> usize -pub fn vortex_sparse::Sparse::nchildren(array: &vortex_sparse::SparseArray) -> usize - pub fn vortex_sparse::Sparse::reduce_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> pub fn vortex_sparse::Sparse::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_sparse::Sparse::slot_name(_array: &vortex_sparse::SparseArray, idx: usize) -> alloc::string::String + +pub fn vortex_sparse::Sparse::slots(array: &vortex_sparse::SparseArray) -> &[core::option::Option] + pub fn vortex_sparse::Sparse::stats(array: &vortex_sparse::SparseArray) -> vortex_array::stats::array::StatsSetRef<'_> -pub fn vortex_sparse::Sparse::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_sparse::Sparse::with_slots(array: &mut vortex_sparse::SparseArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::operations::OperationsVTable for vortex_sparse::Sparse diff --git a/encodings/sparse/src/lib.rs b/encodings/sparse/src/lib.rs index 4b59797eca1..358a8d9bd96 100644 --- a/encodings/sparse/src/lib.rs +++ b/encodings/sparse/src/lib.rs @@ -33,9 +33,6 @@ use vortex_array::vtable; use vortex_array::vtable::ArrayId; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityVTable; -use vortex_array::vtable::patches_child; -use vortex_array::vtable::patches_child_name; -use vortex_array::vtable::patches_nchildren; use vortex_buffer::Buffer; use vortex_buffer::ByteBufferMut; use vortex_error::VortexExpect as _; @@ -127,18 +124,6 @@ impl VTable for Sparse { } } - fn nchildren(array: &SparseArray) -> usize { - patches_nchildren(array.patches()) - } - - fn child(array: &SparseArray, idx: usize) -> ArrayRef { - patches_child(array.patches(), idx) - } - - fn child_name(_array: &SparseArray, idx: usize) -> String { - patches_child_name(idx).to_string() - } - fn metadata(array: &SparseArray) -> VortexResult { let patches = array.patches().to_metadata(array.len(), array.dtype())?; @@ -216,26 +201,38 @@ impl VTable for Sparse { ) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { - vortex_ensure_eq!( - children.len(), - 2, - "SparseArray expects 2 children, got {}", - children.len() + fn slots(array: &SparseArray) -> &[Option] { + &array.slots + } + + fn slot_name(_array: &SparseArray, idx: usize) -> String { + SLOT_NAMES[idx].to_string() + } + + fn with_slots(array: &mut SparseArray, slots: Vec>) -> VortexResult<()> { + vortex_ensure!( + slots.len() == NUM_SLOTS, + "SparseArray expects {} slots, got {}", + NUM_SLOTS, + slots.len() ); - let mut children_iter = children.into_iter(); - let patch_indices = children_iter.next().vortex_expect("patch_indices child"); - let patch_values = children_iter.next().vortex_expect("patch_values child"); + // Reconstruct patches from slots + existing metadata + let indices = slots[PATCH_INDICES_SLOT] + .clone() + .vortex_expect("SparseArray requires patch_indices slot"); + let values = slots[PATCH_VALUES_SLOT] + .clone() + .vortex_expect("SparseArray requires patch_values slot"); array.patches = Patches::new( array.patches.array_len(), array.patches.offset(), - patch_indices, - patch_values, - array.patches.chunk_offsets().clone(), + indices, + values, + slots[PATCH_CHUNK_OFFSETS_SLOT].clone(), )?; - + array.slots = slots; Ok(()) } @@ -261,8 +258,16 @@ impl VTable for Sparse { } } +pub(crate) const PATCH_INDICES_SLOT: usize = 0; +pub(crate) const PATCH_VALUES_SLOT: usize = 1; +pub(crate) const PATCH_CHUNK_OFFSETS_SLOT: usize = 2; +pub(crate) const NUM_SLOTS: usize = 3; +pub(crate) const SLOT_NAMES: [&str; NUM_SLOTS] = + ["patch_indices", "patch_values", "patch_chunk_offsets"]; + #[derive(Clone, Debug)] pub struct SparseArray { + pub(crate) slots: Vec>, patches: Patches, fill_value: Scalar, stats_set: ArrayStats, @@ -307,9 +312,13 @@ impl SparseArray { } } + // TODO(0ax1): handle chunk offsets + let patches = Patches::new(len, 0, indices, values, None)?; + let slots = Self::make_slots(&patches); + Ok(Self { - // TODO(0ax1): handle chunk offsets - patches: Patches::new(len, 0, indices, values, None)?, + slots, + patches, fill_value, stats_set: Default::default(), }) @@ -325,7 +334,10 @@ impl SparseArray { fill_value.dtype(), ); + let slots = Self::make_slots(&patches); + Ok(Self { + slots, patches, fill_value, stats_set: Default::default(), @@ -333,13 +345,24 @@ impl SparseArray { } pub(crate) unsafe fn new_unchecked(patches: Patches, fill_value: Scalar) -> Self { + let slots = Self::make_slots(&patches); + Self { + slots, patches, fill_value, stats_set: Default::default(), } } + fn make_slots(patches: &Patches) -> Vec> { + vec![ + Some(patches.indices().clone()), + Some(patches.values().clone()), + patches.chunk_offsets().clone(), + ] + } + #[inline] pub fn patches(&self) -> &Patches { &self.patches diff --git a/encodings/zigzag/public-api.lock b/encodings/zigzag/public-api.lock index 590e897cd8c..31d19183aef 100644 --- a/encodings/zigzag/public-api.lock +++ b/encodings/zigzag/public-api.lock @@ -50,10 +50,6 @@ pub fn vortex_zigzag::ZigZag::buffer_name(_array: &vortex_zigzag::ZigZagArray, i pub fn vortex_zigzag::ZigZag::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_zigzag::ZigZag::child(array: &vortex_zigzag::ZigZagArray, idx: usize) -> vortex_array::array::ArrayRef - -pub fn vortex_zigzag::ZigZag::child_name(_array: &vortex_zigzag::ZigZagArray, idx: usize) -> alloc::string::String - pub fn vortex_zigzag::ZigZag::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_zigzag::ZigZag::dtype(array: &vortex_zigzag::ZigZagArray) -> &vortex_array::dtype::DType @@ -70,15 +66,17 @@ pub fn vortex_zigzag::ZigZag::metadata(_array: &vortex_zigzag::ZigZagArray) -> v pub fn vortex_zigzag::ZigZag::nbuffers(_array: &vortex_zigzag::ZigZagArray) -> usize -pub fn vortex_zigzag::ZigZag::nchildren(_array: &vortex_zigzag::ZigZagArray) -> usize - pub fn vortex_zigzag::ZigZag::reduce_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> pub fn vortex_zigzag::ZigZag::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_zigzag::ZigZag::slot_name(_array: &vortex_zigzag::ZigZagArray, idx: usize) -> alloc::string::String + +pub fn vortex_zigzag::ZigZag::slots(array: &vortex_zigzag::ZigZagArray) -> &[core::option::Option] + pub fn vortex_zigzag::ZigZag::stats(array: &vortex_zigzag::ZigZagArray) -> vortex_array::stats::array::StatsSetRef<'_> -pub fn vortex_zigzag::ZigZag::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_zigzag::ZigZag::with_slots(array: &mut vortex_zigzag::ZigZagArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::operations::OperationsVTable for vortex_zigzag::ZigZag diff --git a/encodings/zigzag/src/array.rs b/encodings/zigzag/src/array.rs index f0b7898b26c..710c3686924 100644 --- a/encodings/zigzag/src/array.rs +++ b/encodings/zigzag/src/array.rs @@ -53,7 +53,7 @@ impl VTable for ZigZag { } fn len(array: &ZigZagArray) -> usize { - array.encoded.len() + array.encoded().len() } fn dtype(array: &ZigZagArray) -> &DType { @@ -66,11 +66,11 @@ impl VTable for ZigZag { fn array_hash(array: &ZigZagArray, state: &mut H, precision: Precision) { array.dtype.hash(state); - array.encoded.array_hash(state, precision); + array.encoded().array_hash(state, precision); } fn array_eq(array: &ZigZagArray, other: &ZigZagArray, precision: Precision) -> bool { - array.dtype == other.dtype && array.encoded.array_eq(&other.encoded, precision) + array.dtype == other.dtype && array.encoded().array_eq(other.encoded(), precision) } fn nbuffers(_array: &ZigZagArray) -> usize { @@ -85,24 +85,6 @@ impl VTable for ZigZag { vortex_panic!("ZigZagArray buffer_name index {idx} out of bounds") } - fn nchildren(_array: &ZigZagArray) -> usize { - 1 - } - - fn child(array: &ZigZagArray, idx: usize) -> ArrayRef { - match idx { - 0 => array.encoded().clone(), - _ => vortex_panic!("ZigZagArray child index {idx} out of bounds"), - } - } - - fn child_name(_array: &ZigZagArray, idx: usize) -> String { - match idx { - 0 => "encoded".to_string(), - _ => vortex_panic!("ZigZagArray child_name index {idx} out of bounds"), - } - } - fn metadata(_array: &ZigZagArray) -> VortexResult { Ok(EmptyMetadata) } @@ -139,13 +121,22 @@ impl VTable for ZigZag { ZigZagArray::try_new(encoded) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { + fn slots(array: &ZigZagArray) -> &[Option] { + &array.slots + } + + fn slot_name(_array: &ZigZagArray, idx: usize) -> String { + SLOT_NAMES[idx].to_string() + } + + fn with_slots(array: &mut ZigZagArray, slots: Vec>) -> VortexResult<()> { vortex_ensure!( - children.len() == 1, - "ZigZagArray expects exactly 1 child (encoded), got {}", - children.len() + slots.len() == NUM_SLOTS, + "ZigZagArray expects exactly {} slots, got {}", + NUM_SLOTS, + slots.len() ); - array.encoded = children.into_iter().next().vortex_expect("checked"); + array.slots = slots; Ok(()) } @@ -173,10 +164,14 @@ impl VTable for ZigZag { } } +pub(super) const ENCODED_SLOT: usize = 0; +pub(super) const NUM_SLOTS: usize = 1; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["encoded"]; + #[derive(Clone, Debug)] pub struct ZigZagArray { dtype: DType, - encoded: ArrayRef, + pub(super) slots: Vec>, stats_set: ArrayStats, } @@ -203,7 +198,7 @@ impl ZigZagArray { Ok(Self { dtype, - encoded, + slots: vec![Some(encoded)], stats_set: Default::default(), }) } @@ -213,7 +208,9 @@ impl ZigZagArray { } pub fn encoded(&self) -> &ArrayRef { - &self.encoded + self.slots[ENCODED_SLOT] + .as_ref() + .vortex_expect("ZigZagArray encoded slot") } } diff --git a/encodings/zstd/public-api.lock b/encodings/zstd/public-api.lock index bfe24e26991..d603aa3119c 100644 --- a/encodings/zstd/public-api.lock +++ b/encodings/zstd/public-api.lock @@ -38,10 +38,6 @@ pub fn vortex_zstd::Zstd::buffer_name(array: &vortex_zstd::ZstdArray, idx: usize pub fn vortex_zstd::Zstd::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_zstd::Zstd::child(array: &vortex_zstd::ZstdArray, idx: usize) -> vortex_array::array::ArrayRef - -pub fn vortex_zstd::Zstd::child_name(_array: &vortex_zstd::ZstdArray, idx: usize) -> alloc::string::String - pub fn vortex_zstd::Zstd::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_zstd::Zstd::dtype(array: &vortex_zstd::ZstdArray) -> &vortex_array::dtype::DType @@ -56,15 +52,17 @@ pub fn vortex_zstd::Zstd::metadata(array: &vortex_zstd::ZstdArray) -> vortex_err pub fn vortex_zstd::Zstd::nbuffers(array: &vortex_zstd::ZstdArray) -> usize -pub fn vortex_zstd::Zstd::nchildren(array: &vortex_zstd::ZstdArray) -> usize - pub fn vortex_zstd::Zstd::reduce_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> pub fn vortex_zstd::Zstd::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_zstd::Zstd::slot_name(_array: &vortex_zstd::ZstdArray, idx: usize) -> alloc::string::String + +pub fn vortex_zstd::Zstd::slots(array: &vortex_zstd::ZstdArray) -> &[core::option::Option] + pub fn vortex_zstd::Zstd::stats(array: &vortex_zstd::ZstdArray) -> vortex_array::stats::array::StatsSetRef<'_> -pub fn vortex_zstd::Zstd::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_zstd::Zstd::with_slots(array: &mut vortex_zstd::ZstdArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::operations::OperationsVTable for vortex_zstd::Zstd diff --git a/encodings/zstd/src/array.rs b/encodings/zstd/src/array.rs index 2acd9a56190..cafee6c1f07 100644 --- a/encodings/zstd/src/array.rs +++ b/encodings/zstd/src/array.rs @@ -40,7 +40,6 @@ use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityHelper; use vortex_array::vtable::ValiditySliceHelper; use vortex_array::vtable::ValidityVTableFromValiditySliceHelper; -use vortex_array::vtable::validity_nchildren; use vortex_array::vtable::validity_to_child; use vortex_buffer::Alignment; use vortex_buffer::Buffer; @@ -179,22 +178,6 @@ impl VTable for Zstd { } } - fn nchildren(array: &ZstdArray) -> usize { - validity_nchildren(&array.unsliced_validity) - } - - fn child(array: &ZstdArray, idx: usize) -> ArrayRef { - validity_to_child(&array.unsliced_validity, array.unsliced_n_rows) - .unwrap_or_else(|| vortex_panic!("ZstdArray child index {idx} out of bounds")) - } - - fn child_name(_array: &ZstdArray, idx: usize) -> String { - match idx { - 0 => "validity".to_string(), - _ => vortex_panic!("ZstdArray child_name index {idx} out of bounds"), - } - } - fn metadata(array: &ZstdArray) -> VortexResult { Ok(ProstMetadata(array.metadata.clone())) } @@ -259,19 +242,28 @@ impl VTable for Zstd { )) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { + fn slots(array: &ZstdArray) -> &[Option] { + &array.slots + } + + fn slot_name(_array: &ZstdArray, idx: usize) -> String { + SLOT_NAMES[idx].to_string() + } + + fn with_slots(array: &mut ZstdArray, slots: Vec>) -> VortexResult<()> { vortex_ensure!( - children.len() <= 1, - "ZstdArray expects at most 1 child (validity), got {}", - children.len() + slots.len() == NUM_SLOTS, + "ZstdArray expects {} slots, got {}", + NUM_SLOTS, + slots.len() ); - array.unsliced_validity = if children.is_empty() { - Validity::from(array.dtype.nullability()) - } else { - Validity::Array(children.into_iter().next().vortex_expect("checked")) + array.unsliced_validity = match &slots[VALIDITY_SLOT] { + Some(arr) => Validity::Array(arr.clone()), + None => Validity::from(array.dtype.nullability()), }; + array.slots = slots; Ok(()) } @@ -298,6 +290,10 @@ impl Zstd { pub const ID: ArrayId = ArrayId::new_ref("vortex.zstd"); } +pub(super) const VALIDITY_SLOT: usize = 0; +pub(super) const NUM_SLOTS: usize = 1; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; + #[derive(Clone, Debug)] pub struct ZstdArray { pub(crate) dictionary: Option, @@ -306,6 +302,7 @@ pub struct ZstdArray { dtype: DType, pub(crate) unsliced_validity: Validity, unsliced_n_rows: usize, + pub(super) slots: Vec>, stats_set: ArrayStats, slice_start: usize, slice_stop: usize, @@ -437,6 +434,7 @@ impl ZstdArray { n_rows: usize, validity: Validity, ) -> Self { + let validity_slot = validity_to_child(&validity, n_rows); Self { dictionary, frames, @@ -444,6 +442,7 @@ impl ZstdArray { dtype, unsliced_validity: validity, unsliced_n_rows: n_rows, + slots: vec![validity_slot], stats_set: Default::default(), slice_start: 0, slice_stop: n_rows, diff --git a/encodings/zstd/src/zstd_buffers.rs b/encodings/zstd/src/zstd_buffers.rs index c33403f2d64..35f141adf6e 100644 --- a/encodings/zstd/src/zstd_buffers.rs +++ b/encodings/zstd/src/zstd_buffers.rs @@ -56,7 +56,7 @@ pub struct ZstdBuffersArray { compressed_buffers: Vec, uncompressed_sizes: Vec, buffer_alignments: Vec, - children: Vec, + pub(crate) slots: Vec>, dtype: DType, len: usize, stats_set: ArrayStats, @@ -172,7 +172,7 @@ impl ZstdBuffersArray { compressed_buffers, uncompressed_sizes, buffer_alignments, - children, + slots: children.into_iter().map(Some).collect(), dtype: array.dtype().clone(), len: array.len(), stats_set: Default::default(), @@ -248,14 +248,14 @@ impl ZstdBuffersArray { .find(&self.inner_encoding_id) .ok_or_else(|| vortex_err!("Unknown inner encoding: {}", self.inner_encoding_id))?; - let children = self.children.as_slice(); + let children: Vec = self.slots.iter().flatten().cloned().collect(); inner_vtable.build( self.inner_encoding_id.clone(), &self.dtype, self.len, &self.inner_metadata, buffer_handles, - &children, + &children.as_slice(), session, ) } @@ -360,7 +360,7 @@ impl VTable for ZstdBuffers { array.buffer_alignments.hash(state); array.dtype.hash(state); array.len.hash(state); - for child in &array.children { + for child in array.slots.iter().flatten() { child.array_hash(state, precision); } } @@ -378,11 +378,12 @@ impl VTable for ZstdBuffers { && array.buffer_alignments == other.buffer_alignments && array.dtype == other.dtype && array.len == other.len - && array.children.len() == other.children.len() + && array.slots.len() == other.slots.len() && array - .children + .slots .iter() - .zip(&other.children) + .flatten() + .zip(other.slots.iter().flatten()) .all(|(a, b)| a.array_eq(b, precision)) } @@ -398,16 +399,17 @@ impl VTable for ZstdBuffers { Some(format!("compressed_{idx}")) } - fn nchildren(array: &ZstdBuffersArray) -> usize { - array.children.len() + fn slots(array: &ZstdBuffersArray) -> &[Option] { + &array.slots } - fn child(array: &ZstdBuffersArray, idx: usize) -> ArrayRef { - array.children[idx].clone() + fn slot_name(_array: &ZstdBuffersArray, idx: usize) -> String { + format!("child_{idx}") } - fn child_name(_array: &ZstdBuffersArray, idx: usize) -> String { - format!("child_{idx}") + fn with_slots(array: &mut ZstdBuffersArray, slots: Vec>) -> VortexResult<()> { + array.slots = slots; + Ok(()) } fn metadata(array: &ZstdBuffersArray) -> VortexResult { @@ -442,8 +444,8 @@ impl VTable for ZstdBuffers { ) -> VortexResult { let compressed_buffers: Vec = buffers.to_vec(); - let child_arrays: Vec = (0..children.len()) - .map(|i| children.get(i, dtype, len)) + let child_arrays: Vec> = (0..children.len()) + .map(|i| children.get(i, dtype, len).map(Some)) .collect::>>()?; let array = ZstdBuffersArray { @@ -452,7 +454,7 @@ impl VTable for ZstdBuffers { compressed_buffers, uncompressed_sizes: metadata.0.uncompressed_sizes.clone(), buffer_alignments: metadata.0.buffer_alignments.clone(), - children: child_arrays, + slots: child_arrays, dtype: dtype.clone(), len, stats_set: Default::default(), @@ -462,11 +464,6 @@ impl VTable for ZstdBuffers { Ok(array) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { - array.children = children; - Ok(()) - } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { let session = ctx.session(); let inner_array = array.decompress_and_build_inner(session)?; diff --git a/fuzz/src/array/filter.rs b/fuzz/src/array/filter.rs index 30cef15d909..cb86febffa4 100644 --- a/fuzz/src/array/filter.rs +++ b/fuzz/src/array/filter.rs @@ -93,8 +93,7 @@ pub fn filter_canonical_array(array: &ArrayRef, filter: &[bool]) -> VortexResult DType::Struct(..) => { let struct_array = array.to_struct(); let filtered_children = struct_array - .unmasked_fields() - .iter() + .iter_unmasked_fields() .map(|c| filter_canonical_array(c, filter)) .collect::>>()?; diff --git a/fuzz/src/array/scalar_at.rs b/fuzz/src/array/scalar_at.rs index 8c28f975675..1de5fa762ac 100644 --- a/fuzz/src/array/scalar_at.rs +++ b/fuzz/src/array/scalar_at.rs @@ -76,8 +76,7 @@ pub fn scalar_at_canonical_array(canonical: Canonical, index: usize) -> VortexRe } Canonical::Struct(array) => { let field_scalars: Vec = array - .unmasked_fields() - .iter() + .iter_unmasked_fields() .map(|field| { scalar_at_canonical_array( field diff --git a/fuzz/src/array/slice.rs b/fuzz/src/array/slice.rs index 22b2d5bbd16..b5402cb5d21 100644 --- a/fuzz/src/array/slice.rs +++ b/fuzz/src/array/slice.rs @@ -61,8 +61,7 @@ pub fn slice_canonical_array( DType::Struct(..) => { let struct_array = array.to_struct(); let sliced_children = struct_array - .unmasked_fields() - .iter() + .iter_unmasked_fields() .map(|c| slice_canonical_array(c, start, stop)) .collect::>>()?; StructArray::try_new_with_dtype( diff --git a/fuzz/src/array/take.rs b/fuzz/src/array/take.rs index 22a477d1e0b..2eb27b032fd 100644 --- a/fuzz/src/array/take.rs +++ b/fuzz/src/array/take.rs @@ -106,8 +106,7 @@ pub fn take_canonical_array(array: &ArrayRef, indices: &[Option]) -> Vort DType::Struct(..) => { let struct_array = array.to_struct(); let taken_children = struct_array - .unmasked_fields() - .iter() + .iter_unmasked_fields() .map(|c| take_canonical_array_non_nullable_indices(c, indices_slice_non_opt)) .collect::>>()?; diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index b72ced1ebf3..92a24d8fdac 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -908,9 +908,9 @@ pub fn vortex_array::arrays::Bool::buffer_name(_array: &vortex_array::arrays::Bo pub fn vortex_array::arrays::Bool::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::Bool::child(array: &vortex_array::arrays::BoolArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::Bool::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Bool::child_name(_array: &vortex_array::arrays::BoolArray, _idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Bool::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Bool::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -928,7 +928,7 @@ pub fn vortex_array::arrays::Bool::metadata(array: &vortex_array::arrays::BoolAr pub fn vortex_array::arrays::Bool::nbuffers(_array: &vortex_array::arrays::BoolArray) -> usize -pub fn vortex_array::arrays::Bool::nchildren(array: &vortex_array::arrays::BoolArray) -> usize +pub fn vortex_array::arrays::Bool::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Bool::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -936,9 +936,13 @@ pub fn vortex_array::arrays::Bool::reduce_parent(array: &Self::Array, parent: &v pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Bool::slot_name(_array: &vortex_array::arrays::BoolArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Bool::slots(array: &vortex_array::arrays::BoolArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::BoolArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Bool::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Bool::with_slots(array: &mut vortex_array::arrays::BoolArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::bool::BoolArray @@ -1112,13 +1116,13 @@ pub fn vortex_array::arrays::Chunked::array_hash(array: & pub fn vortex_array::arrays::Chunked::buffer(_array: &vortex_array::arrays::ChunkedArray, idx: usize) -> vortex_array::buffer::BufferHandle -pub fn vortex_array::arrays::Chunked::buffer_name(_array: &vortex_array::arrays::ChunkedArray, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Chunked::buffer_name(_array: &vortex_array::arrays::ChunkedArray, _idx: usize) -> core::option::Option pub fn vortex_array::arrays::Chunked::build(dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::Chunked::child(array: &vortex_array::arrays::ChunkedArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::Chunked::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Chunked::child_name(_array: &vortex_array::arrays::ChunkedArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Chunked::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Chunked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -1136,7 +1140,7 @@ pub fn vortex_array::arrays::Chunked::metadata(_array: &vortex_array::arrays::Ch pub fn vortex_array::arrays::Chunked::nbuffers(_array: &vortex_array::arrays::ChunkedArray) -> usize -pub fn vortex_array::arrays::Chunked::nchildren(array: &vortex_array::arrays::ChunkedArray) -> usize +pub fn vortex_array::arrays::Chunked::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Chunked::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -1144,9 +1148,13 @@ pub fn vortex_array::arrays::Chunked::reduce_parent(array: &Self::Array, parent: pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Chunked::slot_name(_array: &vortex_array::arrays::ChunkedArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Chunked::slots(array: &vortex_array::arrays::ChunkedArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::ChunkedArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Chunked::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Chunked::with_slots(array: &mut vortex_array::arrays::ChunkedArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Chunked @@ -1164,7 +1172,9 @@ pub fn vortex_array::arrays::ChunkedArray::chunk(&self, idx: usize) -> &vortex_a pub fn vortex_array::arrays::ChunkedArray::chunk_offsets(&self) -> vortex_buffer::buffer::Buffer -pub fn vortex_array::arrays::ChunkedArray::chunks(&self) -> &[vortex_array::ArrayRef] +pub fn vortex_array::arrays::ChunkedArray::chunks(&self) -> alloc::vec::Vec + +pub fn vortex_array::arrays::ChunkedArray::iter_chunks(&self) -> impl core::iter::traits::iterator::Iterator + '_ pub fn vortex_array::arrays::ChunkedArray::nchunks(&self) -> usize @@ -1282,9 +1292,9 @@ pub fn vortex_array::arrays::Constant::buffer_name(_array: &vortex_array::arrays pub fn vortex_array::arrays::Constant::build(_dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::Constant::child(_array: &vortex_array::arrays::ConstantArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::Constant::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Constant::child_name(_array: &vortex_array::arrays::ConstantArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Constant::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Constant::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -1302,7 +1312,7 @@ pub fn vortex_array::arrays::Constant::metadata(array: &vortex_array::arrays::Co pub fn vortex_array::arrays::Constant::nbuffers(_array: &vortex_array::arrays::ConstantArray) -> usize -pub fn vortex_array::arrays::Constant::nchildren(_array: &vortex_array::arrays::ConstantArray) -> usize +pub fn vortex_array::arrays::Constant::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Constant::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -1310,9 +1320,13 @@ pub fn vortex_array::arrays::Constant::reduce_parent(array: &Self::Array, parent pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Constant::slot_name(_array: &vortex_array::arrays::ConstantArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Constant::slots(array: &vortex_array::arrays::ConstantArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::ConstantArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Constant::with_children(_array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Constant::with_slots(array: &mut vortex_array::arrays::ConstantArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Constant @@ -1488,9 +1502,9 @@ pub fn vortex_array::arrays::Decimal::buffer_name(_array: &vortex_array::arrays: pub fn vortex_array::arrays::Decimal::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::Decimal::child(array: &vortex_array::arrays::DecimalArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::Decimal::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Decimal::child_name(_array: &vortex_array::arrays::DecimalArray, _idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Decimal::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Decimal::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -1508,7 +1522,7 @@ pub fn vortex_array::arrays::Decimal::metadata(array: &vortex_array::arrays::Dec pub fn vortex_array::arrays::Decimal::nbuffers(_array: &vortex_array::arrays::DecimalArray) -> usize -pub fn vortex_array::arrays::Decimal::nchildren(array: &vortex_array::arrays::DecimalArray) -> usize +pub fn vortex_array::arrays::Decimal::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Decimal::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -1516,9 +1530,13 @@ pub fn vortex_array::arrays::Decimal::reduce_parent(array: &Self::Array, parent: pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Decimal::slot_name(_array: &vortex_array::arrays::DecimalArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Decimal::slots(array: &vortex_array::arrays::DecimalArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::DecimalArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Decimal::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Decimal::with_slots(array: &mut vortex_array::arrays::DecimalArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::decimal::DecimalArray @@ -1696,9 +1714,9 @@ pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: &vortex_array::arra pub fn vortex_array::arrays::dict::Dict::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::dict::Dict::child(array: &vortex_array::arrays::dict::DictArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::dict::Dict::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::dict::Dict::child_name(_array: &vortex_array::arrays::dict::DictArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::dict::Dict::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::dict::Dict::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -1716,7 +1734,7 @@ pub fn vortex_array::arrays::dict::Dict::metadata(array: &vortex_array::arrays:: pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: &vortex_array::arrays::dict::DictArray) -> usize -pub fn vortex_array::arrays::dict::Dict::nchildren(_array: &vortex_array::arrays::dict::DictArray) -> usize +pub fn vortex_array::arrays::dict::Dict::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::dict::Dict::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -1724,9 +1742,13 @@ pub fn vortex_array::arrays::dict::Dict::reduce_parent(array: &Self::Array, pare pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::dict::Dict::slot_name(_array: &vortex_array::arrays::dict::DictArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::dict::Dict::slots(array: &vortex_array::arrays::dict::DictArray) -> &[core::option::Option] + pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::dict::Dict::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut vortex_array::arrays::dict::DictArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::dict::Dict @@ -1800,9 +1822,9 @@ pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: &vortex_array::arra pub fn vortex_array::arrays::dict::Dict::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::dict::Dict::child(array: &vortex_array::arrays::dict::DictArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::dict::Dict::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::dict::Dict::child_name(_array: &vortex_array::arrays::dict::DictArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::dict::Dict::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::dict::Dict::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -1820,7 +1842,7 @@ pub fn vortex_array::arrays::dict::Dict::metadata(array: &vortex_array::arrays:: pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: &vortex_array::arrays::dict::DictArray) -> usize -pub fn vortex_array::arrays::dict::Dict::nchildren(_array: &vortex_array::arrays::dict::DictArray) -> usize +pub fn vortex_array::arrays::dict::Dict::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::dict::Dict::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -1828,9 +1850,13 @@ pub fn vortex_array::arrays::dict::Dict::reduce_parent(array: &Self::Array, pare pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::dict::Dict::slot_name(_array: &vortex_array::arrays::dict::DictArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::dict::Dict::slots(array: &vortex_array::arrays::dict::DictArray) -> &[core::option::Option] + pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::dict::Dict::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut vortex_array::arrays::dict::DictArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::dict::Dict @@ -2096,9 +2122,9 @@ pub fn vortex_array::arrays::Extension::buffer_name(_array: &vortex_array::array pub fn vortex_array::arrays::Extension::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::Extension::child(array: &vortex_array::arrays::ExtensionArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::Extension::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Extension::child_name(_array: &vortex_array::arrays::ExtensionArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Extension::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Extension::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -2116,7 +2142,7 @@ pub fn vortex_array::arrays::Extension::metadata(_array: &vortex_array::arrays:: pub fn vortex_array::arrays::Extension::nbuffers(_array: &vortex_array::arrays::ExtensionArray) -> usize -pub fn vortex_array::arrays::Extension::nchildren(_array: &vortex_array::arrays::ExtensionArray) -> usize +pub fn vortex_array::arrays::Extension::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Extension::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -2124,9 +2150,13 @@ pub fn vortex_array::arrays::Extension::reduce_parent(array: &Self::Array, paren pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Extension::slot_name(_array: &vortex_array::arrays::ExtensionArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Extension::slots(array: &vortex_array::arrays::ExtensionArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::ExtensionArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Extension::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::Array, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityChild for vortex_array::arrays::Extension @@ -2234,7 +2264,7 @@ pub fn vortex_array::arrays::Filter::build(dtype: &vortex_array::dtype::DType, l pub fn vortex_array::arrays::Filter::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Filter::child_name(_array: &Self::Array, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Filter::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Filter::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -2252,7 +2282,7 @@ pub fn vortex_array::arrays::Filter::metadata(array: &Self::Array) -> vortex_err pub fn vortex_array::arrays::Filter::nbuffers(_array: &Self::Array) -> usize -pub fn vortex_array::arrays::Filter::nchildren(_array: &Self::Array) -> usize +pub fn vortex_array::arrays::Filter::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Filter::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -2260,9 +2290,13 @@ pub fn vortex_array::arrays::Filter::reduce_parent(array: &Self::Array, parent: pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Filter::slot_name(_array: &Self::Array, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Filter::slots(array: &Self::Array) -> &[core::option::Option] + pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::FilterArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Filter::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::Array, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Filter @@ -2448,9 +2482,9 @@ pub fn vortex_array::arrays::FixedSizeList::buffer_name(_array: &vortex_array::a pub fn vortex_array::arrays::FixedSizeList::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::FixedSizeList::child(array: &vortex_array::arrays::FixedSizeListArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::FixedSizeList::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::FixedSizeList::child_name(_array: &vortex_array::arrays::FixedSizeListArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::FixedSizeList::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::FixedSizeList::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -2468,7 +2502,7 @@ pub fn vortex_array::arrays::FixedSizeList::metadata(_array: &vortex_array::arra pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: &vortex_array::arrays::FixedSizeListArray) -> usize -pub fn vortex_array::arrays::FixedSizeList::nchildren(array: &vortex_array::arrays::FixedSizeListArray) -> usize +pub fn vortex_array::arrays::FixedSizeList::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::FixedSizeList::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -2476,9 +2510,13 @@ pub fn vortex_array::arrays::FixedSizeList::reduce_parent(array: &Self::Array, p pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: &vortex_array::arrays::FixedSizeListArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::FixedSizeList::slots(array: &vortex_array::arrays::FixedSizeListArray) -> &[core::option::Option] + pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::FixedSizeListArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::FixedSizeList::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut vortex_array::arrays::FixedSizeListArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::fixed_size_list::FixedSizeListArray @@ -2596,9 +2634,9 @@ pub fn vortex_array::arrays::List::buffer_name(_array: &vortex_array::arrays::Li pub fn vortex_array::arrays::List::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::List::child(array: &vortex_array::arrays::ListArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::List::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::List::child_name(_array: &vortex_array::arrays::ListArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::List::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::List::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -2616,7 +2654,7 @@ pub fn vortex_array::arrays::List::metadata(array: &vortex_array::arrays::ListAr pub fn vortex_array::arrays::List::nbuffers(_array: &vortex_array::arrays::ListArray) -> usize -pub fn vortex_array::arrays::List::nchildren(array: &vortex_array::arrays::ListArray) -> usize +pub fn vortex_array::arrays::List::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::List::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -2624,9 +2662,13 @@ pub fn vortex_array::arrays::List::reduce_parent(array: &Self::Array, parent: &v pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::List::slot_name(_array: &vortex_array::arrays::ListArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::List::slots(array: &vortex_array::arrays::ListArray) -> &[core::option::Option] + pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::ListArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::List::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::List::with_slots(array: &mut vortex_array::arrays::ListArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::list::ListArray @@ -2764,9 +2806,9 @@ pub fn vortex_array::arrays::ListView::buffer_name(_array: &vortex_array::arrays pub fn vortex_array::arrays::ListView::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::ListView::child(array: &vortex_array::arrays::ListViewArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::ListView::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::ListView::child_name(_array: &vortex_array::arrays::ListViewArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::ListView::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::ListView::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -2784,7 +2826,7 @@ pub fn vortex_array::arrays::ListView::metadata(array: &vortex_array::arrays::Li pub fn vortex_array::arrays::ListView::nbuffers(_array: &vortex_array::arrays::ListViewArray) -> usize -pub fn vortex_array::arrays::ListView::nchildren(array: &vortex_array::arrays::ListViewArray) -> usize +pub fn vortex_array::arrays::ListView::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::ListView::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -2792,9 +2834,13 @@ pub fn vortex_array::arrays::ListView::reduce_parent(array: &Self::Array, parent pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::ListView::slot_name(_array: &vortex_array::arrays::ListViewArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::ListView::slots(array: &vortex_array::arrays::ListViewArray) -> &[core::option::Option] + pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::ListViewArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::ListView::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::ListView::with_slots(array: &mut vortex_array::arrays::ListViewArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::listview::ListViewArray @@ -2944,7 +2990,7 @@ pub fn vortex_array::arrays::Masked::build(dtype: &vortex_array::dtype::DType, l pub fn vortex_array::arrays::Masked::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Masked::child_name(_array: &Self::Array, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Masked::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Masked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -2970,9 +3016,13 @@ pub fn vortex_array::arrays::Masked::reduce_parent(array: &Self::Array, parent: pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Masked::slot_name(_array: &vortex_array::arrays::MaskedArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Masked::slots(array: &vortex_array::arrays::MaskedArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::MaskedArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Masked::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Masked::with_slots(array: &mut vortex_array::arrays::MaskedArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::masked::MaskedArray @@ -3080,9 +3130,9 @@ pub fn vortex_array::arrays::null::Null::buffer_name(_array: &vortex_array::arra pub fn vortex_array::arrays::null::Null::build(_dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::null::Null::child(_array: &vortex_array::arrays::null::NullArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::null::Null::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::null::Null::child_name(_array: &vortex_array::arrays::null::NullArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::null::Null::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::null::Null::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -3100,7 +3150,7 @@ pub fn vortex_array::arrays::null::Null::metadata(_array: &vortex_array::arrays: pub fn vortex_array::arrays::null::Null::nbuffers(_array: &vortex_array::arrays::null::NullArray) -> usize -pub fn vortex_array::arrays::null::Null::nchildren(_array: &vortex_array::arrays::null::NullArray) -> usize +pub fn vortex_array::arrays::null::Null::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::null::Null::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -3108,9 +3158,13 @@ pub fn vortex_array::arrays::null::Null::reduce_parent(array: &Self::Array, pare pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::null::Null::slot_name(_array: &vortex_array::arrays::null::NullArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::null::Null::slots(array: &vortex_array::arrays::null::NullArray) -> &[core::option::Option] + pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::null::Null::with_children(_array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::null::Null::with_slots(array: &mut vortex_array::arrays::null::NullArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::null::Null @@ -3290,9 +3344,9 @@ pub fn vortex_array::arrays::Primitive::buffer_name(_array: &vortex_array::array pub fn vortex_array::arrays::Primitive::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::Primitive::child(array: &vortex_array::arrays::PrimitiveArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::Primitive::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Primitive::child_name(_array: &vortex_array::arrays::PrimitiveArray, _idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Primitive::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Primitive::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -3310,7 +3364,7 @@ pub fn vortex_array::arrays::Primitive::metadata(_array: &vortex_array::arrays:: pub fn vortex_array::arrays::Primitive::nbuffers(_array: &vortex_array::arrays::PrimitiveArray) -> usize -pub fn vortex_array::arrays::Primitive::nchildren(array: &vortex_array::arrays::PrimitiveArray) -> usize +pub fn vortex_array::arrays::Primitive::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Primitive::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -3318,9 +3372,13 @@ pub fn vortex_array::arrays::Primitive::reduce_parent(array: &Self::Array, paren pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Primitive::slot_name(_array: &vortex_array::arrays::PrimitiveArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Primitive::slots(array: &vortex_array::arrays::PrimitiveArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::PrimitiveArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Primitive::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Primitive::with_slots(array: &mut vortex_array::arrays::PrimitiveArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::primitive::PrimitiveArray @@ -3498,7 +3556,13 @@ pub struct vortex_array::arrays::scalar_fn::ScalarFnArray impl vortex_array::arrays::scalar_fn::ScalarFnArray -pub fn vortex_array::arrays::scalar_fn::ScalarFnArray::children(&self) -> &[vortex_array::ArrayRef] +pub fn vortex_array::arrays::scalar_fn::ScalarFnArray::children(&self) -> alloc::vec::Vec + +pub fn vortex_array::arrays::scalar_fn::ScalarFnArray::get_child(&self, idx: usize) -> &vortex_array::ArrayRef + +pub fn vortex_array::arrays::scalar_fn::ScalarFnArray::iter_children(&self) -> impl core::iter::traits::iterator::Iterator + '_ + +pub fn vortex_array::arrays::scalar_fn::ScalarFnArray::nchildren(&self) -> usize pub fn vortex_array::arrays::scalar_fn::ScalarFnArray::scalar_fn(&self) -> &vortex_array::scalar_fn::ScalarFnRef @@ -3594,13 +3658,13 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::array_hash vortex_array::buffer::BufferHandle -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: &vortex_array::arrays::scalar_fn::ScalarFnArray, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: &vortex_array::arrays::scalar_fn::ScalarFnArray, _idx: usize) -> core::option::Option pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: &vortex_array::arrays::scalar_fn::ScalarFnArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: &vortex_array::arrays::scalar_fn::ScalarFnArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -3618,7 +3682,7 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::metadata(array: &Self::A pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: &vortex_array::arrays::scalar_fn::ScalarFnArray) -> usize -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nchildren(array: &vortex_array::arrays::scalar_fn::ScalarFnArray) -> usize +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -3626,9 +3690,13 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: &Se pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: &vortex_array::arrays::scalar_fn::ScalarFnArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: &vortex_array::arrays::scalar_fn::ScalarFnArray) -> &[core::option::Option] + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut vortex_array::arrays::scalar_fn::ScalarFnArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -3682,7 +3750,7 @@ pub fn vortex_array::arrays::Shared::build(dtype: &vortex_array::dtype::DType, l pub fn vortex_array::arrays::Shared::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Shared::child_name(_array: &Self::Array, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Shared::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Shared::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -3700,7 +3768,7 @@ pub fn vortex_array::arrays::Shared::metadata(_array: &Self::Array) -> vortex_er pub fn vortex_array::arrays::Shared::nbuffers(_array: &Self::Array) -> usize -pub fn vortex_array::arrays::Shared::nchildren(_array: &Self::Array) -> usize +pub fn vortex_array::arrays::Shared::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Shared::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -3708,9 +3776,13 @@ pub fn vortex_array::arrays::Shared::reduce_parent(array: &Self::Array, parent: pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Shared::slot_name(_array: &vortex_array::arrays::SharedArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Shared::slots(array: &vortex_array::arrays::SharedArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::SharedArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Shared::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::Array, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Shared @@ -3800,7 +3872,7 @@ pub fn vortex_array::arrays::slice::Slice::build(dtype: &vortex_array::dtype::DT pub fn vortex_array::arrays::slice::Slice::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::slice::Slice::child_name(_array: &Self::Array, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::slice::Slice::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -3818,7 +3890,7 @@ pub fn vortex_array::arrays::slice::Slice::metadata(array: &Self::Array) -> vort pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: &Self::Array) -> usize -pub fn vortex_array::arrays::slice::Slice::nchildren(_array: &Self::Array) -> usize +pub fn vortex_array::arrays::slice::Slice::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::slice::Slice::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -3826,9 +3898,13 @@ pub fn vortex_array::arrays::slice::Slice::reduce_parent(array: &Self::Array, pa pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::slice::Slice::slot_name(_array: &Self::Array, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::slice::Slice::slots(array: &Self::Array) -> &[core::option::Option] + pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::slice::Slice::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::Array, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::slice::Slice @@ -4056,9 +4132,9 @@ pub fn vortex_array::arrays::Struct::buffer_name(_array: &vortex_array::arrays:: pub fn vortex_array::arrays::Struct::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::Struct::child(array: &vortex_array::arrays::StructArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::Struct::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Struct::child_name(array: &vortex_array::arrays::StructArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Struct::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Struct::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -4076,7 +4152,7 @@ pub fn vortex_array::arrays::Struct::metadata(_array: &vortex_array::arrays::Str pub fn vortex_array::arrays::Struct::nbuffers(_array: &vortex_array::arrays::StructArray) -> usize -pub fn vortex_array::arrays::Struct::nchildren(array: &vortex_array::arrays::StructArray) -> usize +pub fn vortex_array::arrays::Struct::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Struct::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -4084,9 +4160,13 @@ pub fn vortex_array::arrays::Struct::reduce_parent(array: &Self::Array, parent: pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Struct::slot_name(array: &vortex_array::arrays::StructArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Struct::slots(array: &vortex_array::arrays::StructArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::StructArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Struct::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Struct::with_slots(array: &mut vortex_array::arrays::StructArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::struct_::StructArray @@ -4098,6 +4178,8 @@ pub fn vortex_array::arrays::StructArray::into_fields(self) -> alloc::vec::Vec vortex_array::arrays::struct_::StructArrayParts +pub fn vortex_array::arrays::StructArray::iter_unmasked_fields(&self) -> impl core::iter::traits::iterator::Iterator + '_ + pub fn vortex_array::arrays::StructArray::names(&self) -> &vortex_array::dtype::FieldNames pub fn vortex_array::arrays::StructArray::new(names: vortex_array::dtype::FieldNames, fields: impl core::convert::Into>, length: usize, validity: vortex_array::validity::Validity) -> Self @@ -4120,11 +4202,13 @@ pub fn vortex_array::arrays::StructArray::try_new(names: vortex_array::dtype::Fi pub fn vortex_array::arrays::StructArray::try_new_with_dtype(fields: impl core::convert::Into>, dtype: vortex_array::dtype::StructFields, length: usize, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult +pub fn vortex_array::arrays::StructArray::unmasked_field(&self, idx: usize) -> &vortex_array::ArrayRef + pub fn vortex_array::arrays::StructArray::unmasked_field_by_name(&self, name: impl core::convert::AsRef) -> vortex_error::VortexResult<&vortex_array::ArrayRef> pub fn vortex_array::arrays::StructArray::unmasked_field_by_name_opt(&self, name: impl core::convert::AsRef) -> core::option::Option<&vortex_array::ArrayRef> -pub fn vortex_array::arrays::StructArray::unmasked_fields(&self) -> &alloc::sync::Arc<[vortex_array::ArrayRef]> +pub fn vortex_array::arrays::StructArray::unmasked_fields(&self) -> alloc::sync::Arc<[vortex_array::ArrayRef]> pub fn vortex_array::arrays::StructArray::validate(fields: &[vortex_array::ArrayRef], dtype: &vortex_array::dtype::StructFields, length: usize, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> @@ -4272,9 +4356,9 @@ pub fn vortex_array::arrays::VarBin::buffer_name(_array: &vortex_array::arrays:: pub fn vortex_array::arrays::VarBin::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::VarBin::child(array: &vortex_array::arrays::VarBinArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::VarBin::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::VarBin::child_name(_array: &vortex_array::arrays::VarBinArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::VarBin::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBin::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -4292,7 +4376,7 @@ pub fn vortex_array::arrays::VarBin::metadata(array: &vortex_array::arrays::VarB pub fn vortex_array::arrays::VarBin::nbuffers(_array: &vortex_array::arrays::VarBinArray) -> usize -pub fn vortex_array::arrays::VarBin::nchildren(array: &vortex_array::arrays::VarBinArray) -> usize +pub fn vortex_array::arrays::VarBin::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::VarBin::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -4300,9 +4384,13 @@ pub fn vortex_array::arrays::VarBin::reduce_parent(array: &Self::Array, parent: pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBin::slot_name(_array: &vortex_array::arrays::VarBinArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::VarBin::slots(array: &vortex_array::arrays::VarBinArray) -> &[core::option::Option] + pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::VarBinArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::VarBin::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBin::with_slots(array: &mut vortex_array::arrays::VarBinArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::varbin::VarBinArray @@ -4674,9 +4762,9 @@ pub fn vortex_array::arrays::VarBinView::buffer_name(array: &vortex_array::array pub fn vortex_array::arrays::VarBinView::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::VarBinView::child(array: &vortex_array::arrays::VarBinViewArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::VarBinView::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::VarBinView::child_name(_array: &vortex_array::arrays::VarBinViewArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::VarBinView::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBinView::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -4694,7 +4782,7 @@ pub fn vortex_array::arrays::VarBinView::metadata(_array: &vortex_array::arrays: pub fn vortex_array::arrays::VarBinView::nbuffers(array: &vortex_array::arrays::VarBinViewArray) -> usize -pub fn vortex_array::arrays::VarBinView::nchildren(array: &vortex_array::arrays::VarBinViewArray) -> usize +pub fn vortex_array::arrays::VarBinView::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::VarBinView::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -4702,9 +4790,13 @@ pub fn vortex_array::arrays::VarBinView::reduce_parent(array: &Self::Array, pare pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBinView::slot_name(_array: &vortex_array::arrays::VarBinViewArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::VarBinView::slots(array: &vortex_array::arrays::VarBinViewArray) -> &[core::option::Option] + pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::VarBinViewArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::VarBinView::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut vortex_array::arrays::VarBinViewArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::varbinview::VarBinViewArray @@ -4866,7 +4958,7 @@ pub fn vortex_array::arrays::Variant::build(dtype: &vortex_array::dtype::DType, pub fn vortex_array::arrays::Variant::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Variant::child_name(_array: &Self::Array, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Variant::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Variant::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -4884,7 +4976,7 @@ pub fn vortex_array::arrays::Variant::metadata(_array: &Self::Array) -> vortex_e pub fn vortex_array::arrays::Variant::nbuffers(_array: &Self::Array) -> usize -pub fn vortex_array::arrays::Variant::nchildren(_array: &Self::Array) -> usize +pub fn vortex_array::arrays::Variant::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Variant::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -4892,9 +4984,13 @@ pub fn vortex_array::arrays::Variant::reduce_parent(array: &Self::Array, parent: pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Variant::slot_name(_array: &Self::Array, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Variant::slots(array: &Self::Array) -> &[core::option::Option] + pub fn vortex_array::arrays::Variant::stats(array: &Self::Array) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Variant::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::Array, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Variant @@ -5004,9 +5100,9 @@ pub fn vortex_array::arrays::Bool::buffer_name(_array: &vortex_array::arrays::Bo pub fn vortex_array::arrays::Bool::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::Bool::child(array: &vortex_array::arrays::BoolArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::Bool::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Bool::child_name(_array: &vortex_array::arrays::BoolArray, _idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Bool::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Bool::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -5024,7 +5120,7 @@ pub fn vortex_array::arrays::Bool::metadata(array: &vortex_array::arrays::BoolAr pub fn vortex_array::arrays::Bool::nbuffers(_array: &vortex_array::arrays::BoolArray) -> usize -pub fn vortex_array::arrays::Bool::nchildren(array: &vortex_array::arrays::BoolArray) -> usize +pub fn vortex_array::arrays::Bool::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Bool::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -5032,9 +5128,13 @@ pub fn vortex_array::arrays::Bool::reduce_parent(array: &Self::Array, parent: &v pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Bool::slot_name(_array: &vortex_array::arrays::BoolArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Bool::slots(array: &vortex_array::arrays::BoolArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::BoolArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Bool::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Bool::with_slots(array: &mut vortex_array::arrays::BoolArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::BoolArray @@ -5180,13 +5280,13 @@ pub fn vortex_array::arrays::Chunked::array_hash(array: & pub fn vortex_array::arrays::Chunked::buffer(_array: &vortex_array::arrays::ChunkedArray, idx: usize) -> vortex_array::buffer::BufferHandle -pub fn vortex_array::arrays::Chunked::buffer_name(_array: &vortex_array::arrays::ChunkedArray, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Chunked::buffer_name(_array: &vortex_array::arrays::ChunkedArray, _idx: usize) -> core::option::Option pub fn vortex_array::arrays::Chunked::build(dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::Chunked::child(array: &vortex_array::arrays::ChunkedArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::Chunked::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Chunked::child_name(_array: &vortex_array::arrays::ChunkedArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Chunked::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Chunked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -5204,7 +5304,7 @@ pub fn vortex_array::arrays::Chunked::metadata(_array: &vortex_array::arrays::Ch pub fn vortex_array::arrays::Chunked::nbuffers(_array: &vortex_array::arrays::ChunkedArray) -> usize -pub fn vortex_array::arrays::Chunked::nchildren(array: &vortex_array::arrays::ChunkedArray) -> usize +pub fn vortex_array::arrays::Chunked::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Chunked::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -5212,9 +5312,13 @@ pub fn vortex_array::arrays::Chunked::reduce_parent(array: &Self::Array, parent: pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Chunked::slot_name(_array: &vortex_array::arrays::ChunkedArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Chunked::slots(array: &vortex_array::arrays::ChunkedArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::ChunkedArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Chunked::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Chunked::with_slots(array: &mut vortex_array::arrays::ChunkedArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Chunked @@ -5232,7 +5336,9 @@ pub fn vortex_array::arrays::ChunkedArray::chunk(&self, idx: usize) -> &vortex_a pub fn vortex_array::arrays::ChunkedArray::chunk_offsets(&self) -> vortex_buffer::buffer::Buffer -pub fn vortex_array::arrays::ChunkedArray::chunks(&self) -> &[vortex_array::ArrayRef] +pub fn vortex_array::arrays::ChunkedArray::chunks(&self) -> alloc::vec::Vec + +pub fn vortex_array::arrays::ChunkedArray::iter_chunks(&self) -> impl core::iter::traits::iterator::Iterator + '_ pub fn vortex_array::arrays::ChunkedArray::nchunks(&self) -> usize @@ -5348,9 +5454,9 @@ pub fn vortex_array::arrays::Constant::buffer_name(_array: &vortex_array::arrays pub fn vortex_array::arrays::Constant::build(_dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::Constant::child(_array: &vortex_array::arrays::ConstantArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::Constant::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Constant::child_name(_array: &vortex_array::arrays::ConstantArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Constant::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Constant::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -5368,7 +5474,7 @@ pub fn vortex_array::arrays::Constant::metadata(array: &vortex_array::arrays::Co pub fn vortex_array::arrays::Constant::nbuffers(_array: &vortex_array::arrays::ConstantArray) -> usize -pub fn vortex_array::arrays::Constant::nchildren(_array: &vortex_array::arrays::ConstantArray) -> usize +pub fn vortex_array::arrays::Constant::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Constant::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -5376,9 +5482,13 @@ pub fn vortex_array::arrays::Constant::reduce_parent(array: &Self::Array, parent pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Constant::slot_name(_array: &vortex_array::arrays::ConstantArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Constant::slots(array: &vortex_array::arrays::ConstantArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::ConstantArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Constant::with_children(_array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Constant::with_slots(array: &mut vortex_array::arrays::ConstantArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Constant @@ -5490,9 +5600,9 @@ pub fn vortex_array::arrays::Decimal::buffer_name(_array: &vortex_array::arrays: pub fn vortex_array::arrays::Decimal::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::Decimal::child(array: &vortex_array::arrays::DecimalArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::Decimal::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Decimal::child_name(_array: &vortex_array::arrays::DecimalArray, _idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Decimal::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Decimal::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -5510,7 +5620,7 @@ pub fn vortex_array::arrays::Decimal::metadata(array: &vortex_array::arrays::Dec pub fn vortex_array::arrays::Decimal::nbuffers(_array: &vortex_array::arrays::DecimalArray) -> usize -pub fn vortex_array::arrays::Decimal::nchildren(array: &vortex_array::arrays::DecimalArray) -> usize +pub fn vortex_array::arrays::Decimal::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Decimal::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -5518,9 +5628,13 @@ pub fn vortex_array::arrays::Decimal::reduce_parent(array: &Self::Array, parent: pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Decimal::slot_name(_array: &vortex_array::arrays::DecimalArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Decimal::slots(array: &vortex_array::arrays::DecimalArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::DecimalArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Decimal::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Decimal::with_slots(array: &mut vortex_array::arrays::DecimalArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::DecimalArray @@ -5666,9 +5780,9 @@ pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: &vortex_array::arra pub fn vortex_array::arrays::dict::Dict::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::dict::Dict::child(array: &vortex_array::arrays::dict::DictArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::dict::Dict::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::dict::Dict::child_name(_array: &vortex_array::arrays::dict::DictArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::dict::Dict::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::dict::Dict::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -5686,7 +5800,7 @@ pub fn vortex_array::arrays::dict::Dict::metadata(array: &vortex_array::arrays:: pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: &vortex_array::arrays::dict::DictArray) -> usize -pub fn vortex_array::arrays::dict::Dict::nchildren(_array: &vortex_array::arrays::dict::DictArray) -> usize +pub fn vortex_array::arrays::dict::Dict::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::dict::Dict::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -5694,9 +5808,13 @@ pub fn vortex_array::arrays::dict::Dict::reduce_parent(array: &Self::Array, pare pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::dict::Dict::slot_name(_array: &vortex_array::arrays::dict::DictArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::dict::Dict::slots(array: &vortex_array::arrays::dict::DictArray) -> &[core::option::Option] + pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::dict::Dict::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut vortex_array::arrays::dict::DictArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::dict::Dict @@ -5820,9 +5938,9 @@ pub fn vortex_array::arrays::Extension::buffer_name(_array: &vortex_array::array pub fn vortex_array::arrays::Extension::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::Extension::child(array: &vortex_array::arrays::ExtensionArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::Extension::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Extension::child_name(_array: &vortex_array::arrays::ExtensionArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Extension::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Extension::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -5840,7 +5958,7 @@ pub fn vortex_array::arrays::Extension::metadata(_array: &vortex_array::arrays:: pub fn vortex_array::arrays::Extension::nbuffers(_array: &vortex_array::arrays::ExtensionArray) -> usize -pub fn vortex_array::arrays::Extension::nchildren(_array: &vortex_array::arrays::ExtensionArray) -> usize +pub fn vortex_array::arrays::Extension::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Extension::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -5848,9 +5966,13 @@ pub fn vortex_array::arrays::Extension::reduce_parent(array: &Self::Array, paren pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Extension::slot_name(_array: &vortex_array::arrays::ExtensionArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Extension::slots(array: &vortex_array::arrays::ExtensionArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::ExtensionArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Extension::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::Array, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityChild for vortex_array::arrays::Extension @@ -5956,7 +6078,7 @@ pub fn vortex_array::arrays::Filter::build(dtype: &vortex_array::dtype::DType, l pub fn vortex_array::arrays::Filter::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Filter::child_name(_array: &Self::Array, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Filter::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Filter::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -5974,7 +6096,7 @@ pub fn vortex_array::arrays::Filter::metadata(array: &Self::Array) -> vortex_err pub fn vortex_array::arrays::Filter::nbuffers(_array: &Self::Array) -> usize -pub fn vortex_array::arrays::Filter::nchildren(_array: &Self::Array) -> usize +pub fn vortex_array::arrays::Filter::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Filter::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -5982,9 +6104,13 @@ pub fn vortex_array::arrays::Filter::reduce_parent(array: &Self::Array, parent: pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Filter::slot_name(_array: &Self::Array, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Filter::slots(array: &Self::Array) -> &[core::option::Option] + pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::FilterArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Filter::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::Array, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Filter @@ -6086,9 +6212,9 @@ pub fn vortex_array::arrays::FixedSizeList::buffer_name(_array: &vortex_array::a pub fn vortex_array::arrays::FixedSizeList::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::FixedSizeList::child(array: &vortex_array::arrays::FixedSizeListArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::FixedSizeList::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::FixedSizeList::child_name(_array: &vortex_array::arrays::FixedSizeListArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::FixedSizeList::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::FixedSizeList::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -6106,7 +6232,7 @@ pub fn vortex_array::arrays::FixedSizeList::metadata(_array: &vortex_array::arra pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: &vortex_array::arrays::FixedSizeListArray) -> usize -pub fn vortex_array::arrays::FixedSizeList::nchildren(array: &vortex_array::arrays::FixedSizeListArray) -> usize +pub fn vortex_array::arrays::FixedSizeList::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::FixedSizeList::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -6114,9 +6240,13 @@ pub fn vortex_array::arrays::FixedSizeList::reduce_parent(array: &Self::Array, p pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: &vortex_array::arrays::FixedSizeListArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::FixedSizeList::slots(array: &vortex_array::arrays::FixedSizeListArray) -> &[core::option::Option] + pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::FixedSizeListArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::FixedSizeList::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut vortex_array::arrays::FixedSizeListArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::FixedSizeListArray @@ -6232,9 +6362,9 @@ pub fn vortex_array::arrays::List::buffer_name(_array: &vortex_array::arrays::Li pub fn vortex_array::arrays::List::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::List::child(array: &vortex_array::arrays::ListArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::List::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::List::child_name(_array: &vortex_array::arrays::ListArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::List::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::List::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -6252,7 +6382,7 @@ pub fn vortex_array::arrays::List::metadata(array: &vortex_array::arrays::ListAr pub fn vortex_array::arrays::List::nbuffers(_array: &vortex_array::arrays::ListArray) -> usize -pub fn vortex_array::arrays::List::nchildren(array: &vortex_array::arrays::ListArray) -> usize +pub fn vortex_array::arrays::List::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::List::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -6260,9 +6390,13 @@ pub fn vortex_array::arrays::List::reduce_parent(array: &Self::Array, parent: &v pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::List::slot_name(_array: &vortex_array::arrays::ListArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::List::slots(array: &vortex_array::arrays::ListArray) -> &[core::option::Option] + pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::ListArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::List::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::List::with_slots(array: &mut vortex_array::arrays::ListArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::ListArray @@ -6378,9 +6512,9 @@ pub fn vortex_array::arrays::ListView::buffer_name(_array: &vortex_array::arrays pub fn vortex_array::arrays::ListView::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::ListView::child(array: &vortex_array::arrays::ListViewArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::ListView::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::ListView::child_name(_array: &vortex_array::arrays::ListViewArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::ListView::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::ListView::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -6398,7 +6532,7 @@ pub fn vortex_array::arrays::ListView::metadata(array: &vortex_array::arrays::Li pub fn vortex_array::arrays::ListView::nbuffers(_array: &vortex_array::arrays::ListViewArray) -> usize -pub fn vortex_array::arrays::ListView::nchildren(array: &vortex_array::arrays::ListViewArray) -> usize +pub fn vortex_array::arrays::ListView::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::ListView::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -6406,9 +6540,13 @@ pub fn vortex_array::arrays::ListView::reduce_parent(array: &Self::Array, parent pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::ListView::slot_name(_array: &vortex_array::arrays::ListViewArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::ListView::slots(array: &vortex_array::arrays::ListViewArray) -> &[core::option::Option] + pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::ListViewArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::ListView::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::ListView::with_slots(array: &mut vortex_array::arrays::ListViewArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::ListViewArray @@ -6538,7 +6676,7 @@ pub fn vortex_array::arrays::Masked::build(dtype: &vortex_array::dtype::DType, l pub fn vortex_array::arrays::Masked::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Masked::child_name(_array: &Self::Array, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Masked::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Masked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -6564,9 +6702,13 @@ pub fn vortex_array::arrays::Masked::reduce_parent(array: &Self::Array, parent: pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Masked::slot_name(_array: &vortex_array::arrays::MaskedArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Masked::slots(array: &vortex_array::arrays::MaskedArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::MaskedArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Masked::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Masked::with_slots(array: &mut vortex_array::arrays::MaskedArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::MaskedArray @@ -6670,9 +6812,9 @@ pub fn vortex_array::arrays::null::Null::buffer_name(_array: &vortex_array::arra pub fn vortex_array::arrays::null::Null::build(_dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::null::Null::child(_array: &vortex_array::arrays::null::NullArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::null::Null::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::null::Null::child_name(_array: &vortex_array::arrays::null::NullArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::null::Null::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::null::Null::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -6690,7 +6832,7 @@ pub fn vortex_array::arrays::null::Null::metadata(_array: &vortex_array::arrays: pub fn vortex_array::arrays::null::Null::nbuffers(_array: &vortex_array::arrays::null::NullArray) -> usize -pub fn vortex_array::arrays::null::Null::nchildren(_array: &vortex_array::arrays::null::NullArray) -> usize +pub fn vortex_array::arrays::null::Null::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::null::Null::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -6698,9 +6840,13 @@ pub fn vortex_array::arrays::null::Null::reduce_parent(array: &Self::Array, pare pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::null::Null::slot_name(_array: &vortex_array::arrays::null::NullArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::null::Null::slots(array: &vortex_array::arrays::null::NullArray) -> &[core::option::Option] + pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::null::Null::with_children(_array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::null::Null::with_slots(array: &mut vortex_array::arrays::null::NullArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::null::Null @@ -6812,9 +6958,9 @@ pub fn vortex_array::arrays::Primitive::buffer_name(_array: &vortex_array::array pub fn vortex_array::arrays::Primitive::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::Primitive::child(array: &vortex_array::arrays::PrimitiveArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::Primitive::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Primitive::child_name(_array: &vortex_array::arrays::PrimitiveArray, _idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Primitive::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Primitive::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -6832,7 +6978,7 @@ pub fn vortex_array::arrays::Primitive::metadata(_array: &vortex_array::arrays:: pub fn vortex_array::arrays::Primitive::nbuffers(_array: &vortex_array::arrays::PrimitiveArray) -> usize -pub fn vortex_array::arrays::Primitive::nchildren(array: &vortex_array::arrays::PrimitiveArray) -> usize +pub fn vortex_array::arrays::Primitive::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Primitive::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -6840,9 +6986,13 @@ pub fn vortex_array::arrays::Primitive::reduce_parent(array: &Self::Array, paren pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Primitive::slot_name(_array: &vortex_array::arrays::PrimitiveArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Primitive::slots(array: &vortex_array::arrays::PrimitiveArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::PrimitiveArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Primitive::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Primitive::with_slots(array: &mut vortex_array::arrays::PrimitiveArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::PrimitiveArray @@ -6958,7 +7108,13 @@ pub struct vortex_array::arrays::ScalarFnArray impl vortex_array::arrays::scalar_fn::ScalarFnArray -pub fn vortex_array::arrays::scalar_fn::ScalarFnArray::children(&self) -> &[vortex_array::ArrayRef] +pub fn vortex_array::arrays::scalar_fn::ScalarFnArray::children(&self) -> alloc::vec::Vec + +pub fn vortex_array::arrays::scalar_fn::ScalarFnArray::get_child(&self, idx: usize) -> &vortex_array::ArrayRef + +pub fn vortex_array::arrays::scalar_fn::ScalarFnArray::iter_children(&self) -> impl core::iter::traits::iterator::Iterator + '_ + +pub fn vortex_array::arrays::scalar_fn::ScalarFnArray::nchildren(&self) -> usize pub fn vortex_array::arrays::scalar_fn::ScalarFnArray::scalar_fn(&self) -> &vortex_array::scalar_fn::ScalarFnRef @@ -7042,13 +7198,13 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::array_hash vortex_array::buffer::BufferHandle -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: &vortex_array::arrays::scalar_fn::ScalarFnArray, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: &vortex_array::arrays::scalar_fn::ScalarFnArray, _idx: usize) -> core::option::Option pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: &vortex_array::arrays::scalar_fn::ScalarFnArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: &vortex_array::arrays::scalar_fn::ScalarFnArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -7066,7 +7222,7 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::metadata(array: &Self::A pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: &vortex_array::arrays::scalar_fn::ScalarFnArray) -> usize -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nchildren(array: &vortex_array::arrays::scalar_fn::ScalarFnArray) -> usize +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -7074,9 +7230,13 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: &Se pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: &vortex_array::arrays::scalar_fn::ScalarFnArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: &vortex_array::arrays::scalar_fn::ScalarFnArray) -> &[core::option::Option] + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut vortex_array::arrays::scalar_fn::ScalarFnArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -7120,7 +7280,7 @@ pub fn vortex_array::arrays::Shared::build(dtype: &vortex_array::dtype::DType, l pub fn vortex_array::arrays::Shared::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Shared::child_name(_array: &Self::Array, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Shared::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Shared::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -7138,7 +7298,7 @@ pub fn vortex_array::arrays::Shared::metadata(_array: &Self::Array) -> vortex_er pub fn vortex_array::arrays::Shared::nbuffers(_array: &Self::Array) -> usize -pub fn vortex_array::arrays::Shared::nchildren(_array: &Self::Array) -> usize +pub fn vortex_array::arrays::Shared::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Shared::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -7146,9 +7306,13 @@ pub fn vortex_array::arrays::Shared::reduce_parent(array: &Self::Array, parent: pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Shared::slot_name(_array: &vortex_array::arrays::SharedArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Shared::slots(array: &vortex_array::arrays::SharedArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::SharedArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Shared::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::Array, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Shared @@ -7236,7 +7400,7 @@ pub fn vortex_array::arrays::slice::Slice::build(dtype: &vortex_array::dtype::DT pub fn vortex_array::arrays::slice::Slice::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::slice::Slice::child_name(_array: &Self::Array, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::slice::Slice::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -7254,7 +7418,7 @@ pub fn vortex_array::arrays::slice::Slice::metadata(array: &Self::Array) -> vort pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: &Self::Array) -> usize -pub fn vortex_array::arrays::slice::Slice::nchildren(_array: &Self::Array) -> usize +pub fn vortex_array::arrays::slice::Slice::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::slice::Slice::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -7262,9 +7426,13 @@ pub fn vortex_array::arrays::slice::Slice::reduce_parent(array: &Self::Array, pa pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::slice::Slice::slot_name(_array: &Self::Array, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::slice::Slice::slots(array: &Self::Array) -> &[core::option::Option] + pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::slice::Slice::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::Array, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::slice::Slice @@ -7370,9 +7538,9 @@ pub fn vortex_array::arrays::Struct::buffer_name(_array: &vortex_array::arrays:: pub fn vortex_array::arrays::Struct::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::Struct::child(array: &vortex_array::arrays::StructArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::Struct::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Struct::child_name(array: &vortex_array::arrays::StructArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Struct::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Struct::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -7390,7 +7558,7 @@ pub fn vortex_array::arrays::Struct::metadata(_array: &vortex_array::arrays::Str pub fn vortex_array::arrays::Struct::nbuffers(_array: &vortex_array::arrays::StructArray) -> usize -pub fn vortex_array::arrays::Struct::nchildren(array: &vortex_array::arrays::StructArray) -> usize +pub fn vortex_array::arrays::Struct::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Struct::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -7398,9 +7566,13 @@ pub fn vortex_array::arrays::Struct::reduce_parent(array: &Self::Array, parent: pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Struct::slot_name(array: &vortex_array::arrays::StructArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Struct::slots(array: &vortex_array::arrays::StructArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::StructArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Struct::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Struct::with_slots(array: &mut vortex_array::arrays::StructArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::StructArray @@ -7412,6 +7584,8 @@ pub fn vortex_array::arrays::StructArray::into_fields(self) -> alloc::vec::Vec vortex_array::arrays::struct_::StructArrayParts +pub fn vortex_array::arrays::StructArray::iter_unmasked_fields(&self) -> impl core::iter::traits::iterator::Iterator + '_ + pub fn vortex_array::arrays::StructArray::names(&self) -> &vortex_array::dtype::FieldNames pub fn vortex_array::arrays::StructArray::new(names: vortex_array::dtype::FieldNames, fields: impl core::convert::Into>, length: usize, validity: vortex_array::validity::Validity) -> Self @@ -7434,11 +7608,13 @@ pub fn vortex_array::arrays::StructArray::try_new(names: vortex_array::dtype::Fi pub fn vortex_array::arrays::StructArray::try_new_with_dtype(fields: impl core::convert::Into>, dtype: vortex_array::dtype::StructFields, length: usize, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult +pub fn vortex_array::arrays::StructArray::unmasked_field(&self, idx: usize) -> &vortex_array::ArrayRef + pub fn vortex_array::arrays::StructArray::unmasked_field_by_name(&self, name: impl core::convert::AsRef) -> vortex_error::VortexResult<&vortex_array::ArrayRef> pub fn vortex_array::arrays::StructArray::unmasked_field_by_name_opt(&self, name: impl core::convert::AsRef) -> core::option::Option<&vortex_array::ArrayRef> -pub fn vortex_array::arrays::StructArray::unmasked_fields(&self) -> &alloc::sync::Arc<[vortex_array::ArrayRef]> +pub fn vortex_array::arrays::StructArray::unmasked_fields(&self) -> alloc::sync::Arc<[vortex_array::ArrayRef]> pub fn vortex_array::arrays::StructArray::validate(fields: &[vortex_array::ArrayRef], dtype: &vortex_array::dtype::StructFields, length: usize, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> @@ -7610,9 +7786,9 @@ pub fn vortex_array::arrays::VarBin::buffer_name(_array: &vortex_array::arrays:: pub fn vortex_array::arrays::VarBin::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::VarBin::child(array: &vortex_array::arrays::VarBinArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::VarBin::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::VarBin::child_name(_array: &vortex_array::arrays::VarBinArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::VarBin::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBin::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -7630,7 +7806,7 @@ pub fn vortex_array::arrays::VarBin::metadata(array: &vortex_array::arrays::VarB pub fn vortex_array::arrays::VarBin::nbuffers(_array: &vortex_array::arrays::VarBinArray) -> usize -pub fn vortex_array::arrays::VarBin::nchildren(array: &vortex_array::arrays::VarBinArray) -> usize +pub fn vortex_array::arrays::VarBin::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::VarBin::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -7638,9 +7814,13 @@ pub fn vortex_array::arrays::VarBin::reduce_parent(array: &Self::Array, parent: pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBin::slot_name(_array: &vortex_array::arrays::VarBinArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::VarBin::slots(array: &vortex_array::arrays::VarBinArray) -> &[core::option::Option] + pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::VarBinArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::VarBin::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBin::with_slots(array: &mut vortex_array::arrays::VarBinArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::VarBinArray @@ -7826,9 +8006,9 @@ pub fn vortex_array::arrays::VarBinView::buffer_name(array: &vortex_array::array pub fn vortex_array::arrays::VarBinView::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::VarBinView::child(array: &vortex_array::arrays::VarBinViewArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::VarBinView::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::VarBinView::child_name(_array: &vortex_array::arrays::VarBinViewArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::VarBinView::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBinView::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -7846,7 +8026,7 @@ pub fn vortex_array::arrays::VarBinView::metadata(_array: &vortex_array::arrays: pub fn vortex_array::arrays::VarBinView::nbuffers(array: &vortex_array::arrays::VarBinViewArray) -> usize -pub fn vortex_array::arrays::VarBinView::nchildren(array: &vortex_array::arrays::VarBinViewArray) -> usize +pub fn vortex_array::arrays::VarBinView::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::VarBinView::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -7854,9 +8034,13 @@ pub fn vortex_array::arrays::VarBinView::reduce_parent(array: &Self::Array, pare pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBinView::slot_name(_array: &vortex_array::arrays::VarBinViewArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::VarBinView::slots(array: &vortex_array::arrays::VarBinViewArray) -> &[core::option::Option] + pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::VarBinViewArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::VarBinView::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut vortex_array::arrays::VarBinViewArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::VarBinViewArray @@ -8006,7 +8190,7 @@ pub fn vortex_array::arrays::Variant::build(dtype: &vortex_array::dtype::DType, pub fn vortex_array::arrays::Variant::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Variant::child_name(_array: &Self::Array, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Variant::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Variant::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -8024,7 +8208,7 @@ pub fn vortex_array::arrays::Variant::metadata(_array: &Self::Array) -> vortex_e pub fn vortex_array::arrays::Variant::nbuffers(_array: &Self::Array) -> usize -pub fn vortex_array::arrays::Variant::nchildren(_array: &Self::Array) -> usize +pub fn vortex_array::arrays::Variant::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Variant::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -8032,9 +8216,13 @@ pub fn vortex_array::arrays::Variant::reduce_parent(array: &Self::Array, parent: pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Variant::slot_name(_array: &Self::Array, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Variant::slots(array: &Self::Array) -> &[core::option::Option] + pub fn vortex_array::arrays::Variant::stats(array: &Self::Array) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Variant::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::Array, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Variant @@ -20400,7 +20588,7 @@ pub fn vortex_array::vtable::DynVTable::reduce(&self, array: &vortex_array::Arra pub fn vortex_array::vtable::DynVTable::reduce_parent(&self, array: &vortex_array::ArrayRef, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::vtable::DynVTable::with_children(&self, array: &vortex_array::ArrayRef, children: alloc::vec::Vec) -> vortex_error::VortexResult +pub fn vortex_array::vtable::DynVTable::with_slots(&self, array: &vortex_array::ArrayRef, slots: alloc::vec::Vec>) -> vortex_error::VortexResult pub trait vortex_array::vtable::OperationsVTable @@ -20540,9 +20728,13 @@ pub fn vortex_array::vtable::VTable::reduce_parent(array: &Self::Array, parent: pub fn vortex_array::vtable::VTable::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::vtable::VTable::slot_name(array: &Self::Array, idx: usize) -> alloc::string::String + +pub fn vortex_array::vtable::VTable::slots(array: &Self::Array) -> &[core::option::Option] + pub fn vortex_array::vtable::VTable::stats(array: &Self::Array) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::vtable::VTable::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::vtable::VTable::with_slots(array: &mut Self::Array, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Bool @@ -20566,9 +20758,9 @@ pub fn vortex_array::arrays::Bool::buffer_name(_array: &vortex_array::arrays::Bo pub fn vortex_array::arrays::Bool::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::Bool::child(array: &vortex_array::arrays::BoolArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::Bool::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Bool::child_name(_array: &vortex_array::arrays::BoolArray, _idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Bool::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Bool::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -20586,7 +20778,7 @@ pub fn vortex_array::arrays::Bool::metadata(array: &vortex_array::arrays::BoolAr pub fn vortex_array::arrays::Bool::nbuffers(_array: &vortex_array::arrays::BoolArray) -> usize -pub fn vortex_array::arrays::Bool::nchildren(array: &vortex_array::arrays::BoolArray) -> usize +pub fn vortex_array::arrays::Bool::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Bool::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -20594,9 +20786,13 @@ pub fn vortex_array::arrays::Bool::reduce_parent(array: &Self::Array, parent: &v pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Bool::slot_name(_array: &vortex_array::arrays::BoolArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Bool::slots(array: &vortex_array::arrays::BoolArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::BoolArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Bool::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Bool::with_slots(array: &mut vortex_array::arrays::BoolArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Chunked @@ -20616,13 +20812,13 @@ pub fn vortex_array::arrays::Chunked::array_hash(array: & pub fn vortex_array::arrays::Chunked::buffer(_array: &vortex_array::arrays::ChunkedArray, idx: usize) -> vortex_array::buffer::BufferHandle -pub fn vortex_array::arrays::Chunked::buffer_name(_array: &vortex_array::arrays::ChunkedArray, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Chunked::buffer_name(_array: &vortex_array::arrays::ChunkedArray, _idx: usize) -> core::option::Option pub fn vortex_array::arrays::Chunked::build(dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::Chunked::child(array: &vortex_array::arrays::ChunkedArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::Chunked::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Chunked::child_name(_array: &vortex_array::arrays::ChunkedArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Chunked::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Chunked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -20640,7 +20836,7 @@ pub fn vortex_array::arrays::Chunked::metadata(_array: &vortex_array::arrays::Ch pub fn vortex_array::arrays::Chunked::nbuffers(_array: &vortex_array::arrays::ChunkedArray) -> usize -pub fn vortex_array::arrays::Chunked::nchildren(array: &vortex_array::arrays::ChunkedArray) -> usize +pub fn vortex_array::arrays::Chunked::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Chunked::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -20648,9 +20844,13 @@ pub fn vortex_array::arrays::Chunked::reduce_parent(array: &Self::Array, parent: pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Chunked::slot_name(_array: &vortex_array::arrays::ChunkedArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Chunked::slots(array: &vortex_array::arrays::ChunkedArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::ChunkedArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Chunked::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Chunked::with_slots(array: &mut vortex_array::arrays::ChunkedArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Constant @@ -20674,9 +20874,9 @@ pub fn vortex_array::arrays::Constant::buffer_name(_array: &vortex_array::arrays pub fn vortex_array::arrays::Constant::build(_dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::Constant::child(_array: &vortex_array::arrays::ConstantArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::Constant::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Constant::child_name(_array: &vortex_array::arrays::ConstantArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Constant::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Constant::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -20694,7 +20894,7 @@ pub fn vortex_array::arrays::Constant::metadata(array: &vortex_array::arrays::Co pub fn vortex_array::arrays::Constant::nbuffers(_array: &vortex_array::arrays::ConstantArray) -> usize -pub fn vortex_array::arrays::Constant::nchildren(_array: &vortex_array::arrays::ConstantArray) -> usize +pub fn vortex_array::arrays::Constant::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Constant::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -20702,9 +20902,13 @@ pub fn vortex_array::arrays::Constant::reduce_parent(array: &Self::Array, parent pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Constant::slot_name(_array: &vortex_array::arrays::ConstantArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Constant::slots(array: &vortex_array::arrays::ConstantArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::ConstantArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Constant::with_children(_array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Constant::with_slots(array: &mut vortex_array::arrays::ConstantArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Decimal @@ -20728,9 +20932,9 @@ pub fn vortex_array::arrays::Decimal::buffer_name(_array: &vortex_array::arrays: pub fn vortex_array::arrays::Decimal::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::Decimal::child(array: &vortex_array::arrays::DecimalArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::Decimal::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Decimal::child_name(_array: &vortex_array::arrays::DecimalArray, _idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Decimal::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Decimal::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -20748,7 +20952,7 @@ pub fn vortex_array::arrays::Decimal::metadata(array: &vortex_array::arrays::Dec pub fn vortex_array::arrays::Decimal::nbuffers(_array: &vortex_array::arrays::DecimalArray) -> usize -pub fn vortex_array::arrays::Decimal::nchildren(array: &vortex_array::arrays::DecimalArray) -> usize +pub fn vortex_array::arrays::Decimal::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Decimal::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -20756,9 +20960,13 @@ pub fn vortex_array::arrays::Decimal::reduce_parent(array: &Self::Array, parent: pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Decimal::slot_name(_array: &vortex_array::arrays::DecimalArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Decimal::slots(array: &vortex_array::arrays::DecimalArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::DecimalArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Decimal::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Decimal::with_slots(array: &mut vortex_array::arrays::DecimalArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Extension @@ -20782,9 +20990,9 @@ pub fn vortex_array::arrays::Extension::buffer_name(_array: &vortex_array::array pub fn vortex_array::arrays::Extension::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::Extension::child(array: &vortex_array::arrays::ExtensionArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::Extension::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Extension::child_name(_array: &vortex_array::arrays::ExtensionArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Extension::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Extension::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -20802,7 +21010,7 @@ pub fn vortex_array::arrays::Extension::metadata(_array: &vortex_array::arrays:: pub fn vortex_array::arrays::Extension::nbuffers(_array: &vortex_array::arrays::ExtensionArray) -> usize -pub fn vortex_array::arrays::Extension::nchildren(_array: &vortex_array::arrays::ExtensionArray) -> usize +pub fn vortex_array::arrays::Extension::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Extension::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -20810,9 +21018,13 @@ pub fn vortex_array::arrays::Extension::reduce_parent(array: &Self::Array, paren pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Extension::slot_name(_array: &vortex_array::arrays::ExtensionArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Extension::slots(array: &vortex_array::arrays::ExtensionArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::ExtensionArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Extension::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::Array, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Filter @@ -20838,7 +21050,7 @@ pub fn vortex_array::arrays::Filter::build(dtype: &vortex_array::dtype::DType, l pub fn vortex_array::arrays::Filter::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Filter::child_name(_array: &Self::Array, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Filter::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Filter::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -20856,7 +21068,7 @@ pub fn vortex_array::arrays::Filter::metadata(array: &Self::Array) -> vortex_err pub fn vortex_array::arrays::Filter::nbuffers(_array: &Self::Array) -> usize -pub fn vortex_array::arrays::Filter::nchildren(_array: &Self::Array) -> usize +pub fn vortex_array::arrays::Filter::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Filter::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -20864,9 +21076,13 @@ pub fn vortex_array::arrays::Filter::reduce_parent(array: &Self::Array, parent: pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Filter::slot_name(_array: &Self::Array, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Filter::slots(array: &Self::Array) -> &[core::option::Option] + pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::FilterArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Filter::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::Array, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::FixedSizeList @@ -20890,9 +21106,9 @@ pub fn vortex_array::arrays::FixedSizeList::buffer_name(_array: &vortex_array::a pub fn vortex_array::arrays::FixedSizeList::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::FixedSizeList::child(array: &vortex_array::arrays::FixedSizeListArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::FixedSizeList::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::FixedSizeList::child_name(_array: &vortex_array::arrays::FixedSizeListArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::FixedSizeList::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::FixedSizeList::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -20910,7 +21126,7 @@ pub fn vortex_array::arrays::FixedSizeList::metadata(_array: &vortex_array::arra pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: &vortex_array::arrays::FixedSizeListArray) -> usize -pub fn vortex_array::arrays::FixedSizeList::nchildren(array: &vortex_array::arrays::FixedSizeListArray) -> usize +pub fn vortex_array::arrays::FixedSizeList::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::FixedSizeList::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -20918,9 +21134,13 @@ pub fn vortex_array::arrays::FixedSizeList::reduce_parent(array: &Self::Array, p pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: &vortex_array::arrays::FixedSizeListArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::FixedSizeList::slots(array: &vortex_array::arrays::FixedSizeListArray) -> &[core::option::Option] + pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::FixedSizeListArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::FixedSizeList::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut vortex_array::arrays::FixedSizeListArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::List @@ -20944,9 +21164,9 @@ pub fn vortex_array::arrays::List::buffer_name(_array: &vortex_array::arrays::Li pub fn vortex_array::arrays::List::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::List::child(array: &vortex_array::arrays::ListArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::List::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::List::child_name(_array: &vortex_array::arrays::ListArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::List::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::List::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -20964,7 +21184,7 @@ pub fn vortex_array::arrays::List::metadata(array: &vortex_array::arrays::ListAr pub fn vortex_array::arrays::List::nbuffers(_array: &vortex_array::arrays::ListArray) -> usize -pub fn vortex_array::arrays::List::nchildren(array: &vortex_array::arrays::ListArray) -> usize +pub fn vortex_array::arrays::List::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::List::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -20972,9 +21192,13 @@ pub fn vortex_array::arrays::List::reduce_parent(array: &Self::Array, parent: &v pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::List::slot_name(_array: &vortex_array::arrays::ListArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::List::slots(array: &vortex_array::arrays::ListArray) -> &[core::option::Option] + pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::ListArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::List::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::List::with_slots(array: &mut vortex_array::arrays::ListArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::ListView @@ -20998,9 +21222,9 @@ pub fn vortex_array::arrays::ListView::buffer_name(_array: &vortex_array::arrays pub fn vortex_array::arrays::ListView::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::ListView::child(array: &vortex_array::arrays::ListViewArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::ListView::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::ListView::child_name(_array: &vortex_array::arrays::ListViewArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::ListView::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::ListView::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -21018,7 +21242,7 @@ pub fn vortex_array::arrays::ListView::metadata(array: &vortex_array::arrays::Li pub fn vortex_array::arrays::ListView::nbuffers(_array: &vortex_array::arrays::ListViewArray) -> usize -pub fn vortex_array::arrays::ListView::nchildren(array: &vortex_array::arrays::ListViewArray) -> usize +pub fn vortex_array::arrays::ListView::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::ListView::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -21026,9 +21250,13 @@ pub fn vortex_array::arrays::ListView::reduce_parent(array: &Self::Array, parent pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::ListView::slot_name(_array: &vortex_array::arrays::ListViewArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::ListView::slots(array: &vortex_array::arrays::ListViewArray) -> &[core::option::Option] + pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::ListViewArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::ListView::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::ListView::with_slots(array: &mut vortex_array::arrays::ListViewArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Masked @@ -21054,7 +21282,7 @@ pub fn vortex_array::arrays::Masked::build(dtype: &vortex_array::dtype::DType, l pub fn vortex_array::arrays::Masked::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Masked::child_name(_array: &Self::Array, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Masked::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Masked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -21080,9 +21308,13 @@ pub fn vortex_array::arrays::Masked::reduce_parent(array: &Self::Array, parent: pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Masked::slot_name(_array: &vortex_array::arrays::MaskedArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Masked::slots(array: &vortex_array::arrays::MaskedArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::MaskedArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Masked::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Masked::with_slots(array: &mut vortex_array::arrays::MaskedArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Primitive @@ -21106,9 +21338,9 @@ pub fn vortex_array::arrays::Primitive::buffer_name(_array: &vortex_array::array pub fn vortex_array::arrays::Primitive::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::Primitive::child(array: &vortex_array::arrays::PrimitiveArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::Primitive::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Primitive::child_name(_array: &vortex_array::arrays::PrimitiveArray, _idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Primitive::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Primitive::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -21126,7 +21358,7 @@ pub fn vortex_array::arrays::Primitive::metadata(_array: &vortex_array::arrays:: pub fn vortex_array::arrays::Primitive::nbuffers(_array: &vortex_array::arrays::PrimitiveArray) -> usize -pub fn vortex_array::arrays::Primitive::nchildren(array: &vortex_array::arrays::PrimitiveArray) -> usize +pub fn vortex_array::arrays::Primitive::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Primitive::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -21134,9 +21366,13 @@ pub fn vortex_array::arrays::Primitive::reduce_parent(array: &Self::Array, paren pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Primitive::slot_name(_array: &vortex_array::arrays::PrimitiveArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Primitive::slots(array: &vortex_array::arrays::PrimitiveArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::PrimitiveArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Primitive::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Primitive::with_slots(array: &mut vortex_array::arrays::PrimitiveArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Shared @@ -21162,7 +21398,7 @@ pub fn vortex_array::arrays::Shared::build(dtype: &vortex_array::dtype::DType, l pub fn vortex_array::arrays::Shared::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Shared::child_name(_array: &Self::Array, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Shared::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Shared::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -21180,7 +21416,7 @@ pub fn vortex_array::arrays::Shared::metadata(_array: &Self::Array) -> vortex_er pub fn vortex_array::arrays::Shared::nbuffers(_array: &Self::Array) -> usize -pub fn vortex_array::arrays::Shared::nchildren(_array: &Self::Array) -> usize +pub fn vortex_array::arrays::Shared::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Shared::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -21188,9 +21424,13 @@ pub fn vortex_array::arrays::Shared::reduce_parent(array: &Self::Array, parent: pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Shared::slot_name(_array: &vortex_array::arrays::SharedArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Shared::slots(array: &vortex_array::arrays::SharedArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::SharedArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Shared::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::Array, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Struct @@ -21214,9 +21454,9 @@ pub fn vortex_array::arrays::Struct::buffer_name(_array: &vortex_array::arrays:: pub fn vortex_array::arrays::Struct::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::Struct::child(array: &vortex_array::arrays::StructArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::Struct::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Struct::child_name(array: &vortex_array::arrays::StructArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Struct::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Struct::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -21234,7 +21474,7 @@ pub fn vortex_array::arrays::Struct::metadata(_array: &vortex_array::arrays::Str pub fn vortex_array::arrays::Struct::nbuffers(_array: &vortex_array::arrays::StructArray) -> usize -pub fn vortex_array::arrays::Struct::nchildren(array: &vortex_array::arrays::StructArray) -> usize +pub fn vortex_array::arrays::Struct::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Struct::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -21242,9 +21482,13 @@ pub fn vortex_array::arrays::Struct::reduce_parent(array: &Self::Array, parent: pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Struct::slot_name(array: &vortex_array::arrays::StructArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Struct::slots(array: &vortex_array::arrays::StructArray) -> &[core::option::Option] + pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::StructArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Struct::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Struct::with_slots(array: &mut vortex_array::arrays::StructArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::VarBin @@ -21268,9 +21512,9 @@ pub fn vortex_array::arrays::VarBin::buffer_name(_array: &vortex_array::arrays:: pub fn vortex_array::arrays::VarBin::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::VarBin::child(array: &vortex_array::arrays::VarBinArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::VarBin::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::VarBin::child_name(_array: &vortex_array::arrays::VarBinArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::VarBin::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBin::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -21288,7 +21532,7 @@ pub fn vortex_array::arrays::VarBin::metadata(array: &vortex_array::arrays::VarB pub fn vortex_array::arrays::VarBin::nbuffers(_array: &vortex_array::arrays::VarBinArray) -> usize -pub fn vortex_array::arrays::VarBin::nchildren(array: &vortex_array::arrays::VarBinArray) -> usize +pub fn vortex_array::arrays::VarBin::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::VarBin::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -21296,9 +21540,13 @@ pub fn vortex_array::arrays::VarBin::reduce_parent(array: &Self::Array, parent: pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBin::slot_name(_array: &vortex_array::arrays::VarBinArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::VarBin::slots(array: &vortex_array::arrays::VarBinArray) -> &[core::option::Option] + pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::VarBinArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::VarBin::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBin::with_slots(array: &mut vortex_array::arrays::VarBinArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::VarBinView @@ -21322,9 +21570,9 @@ pub fn vortex_array::arrays::VarBinView::buffer_name(array: &vortex_array::array pub fn vortex_array::arrays::VarBinView::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::VarBinView::child(array: &vortex_array::arrays::VarBinViewArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::VarBinView::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::VarBinView::child_name(_array: &vortex_array::arrays::VarBinViewArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::VarBinView::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBinView::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -21342,7 +21590,7 @@ pub fn vortex_array::arrays::VarBinView::metadata(_array: &vortex_array::arrays: pub fn vortex_array::arrays::VarBinView::nbuffers(array: &vortex_array::arrays::VarBinViewArray) -> usize -pub fn vortex_array::arrays::VarBinView::nchildren(array: &vortex_array::arrays::VarBinViewArray) -> usize +pub fn vortex_array::arrays::VarBinView::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::VarBinView::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -21350,9 +21598,13 @@ pub fn vortex_array::arrays::VarBinView::reduce_parent(array: &Self::Array, pare pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBinView::slot_name(_array: &vortex_array::arrays::VarBinViewArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::VarBinView::slots(array: &vortex_array::arrays::VarBinViewArray) -> &[core::option::Option] + pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::VarBinViewArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::VarBinView::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut vortex_array::arrays::VarBinViewArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Variant @@ -21378,7 +21630,7 @@ pub fn vortex_array::arrays::Variant::build(dtype: &vortex_array::dtype::DType, pub fn vortex_array::arrays::Variant::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::Variant::child_name(_array: &Self::Array, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::Variant::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Variant::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -21396,7 +21648,7 @@ pub fn vortex_array::arrays::Variant::metadata(_array: &Self::Array) -> vortex_e pub fn vortex_array::arrays::Variant::nbuffers(_array: &Self::Array) -> usize -pub fn vortex_array::arrays::Variant::nchildren(_array: &Self::Array) -> usize +pub fn vortex_array::arrays::Variant::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::Variant::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -21404,9 +21656,13 @@ pub fn vortex_array::arrays::Variant::reduce_parent(array: &Self::Array, parent: pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Variant::slot_name(_array: &Self::Array, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::Variant::slots(array: &Self::Array) -> &[core::option::Option] + pub fn vortex_array::arrays::Variant::stats(array: &Self::Array) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::Variant::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::Array, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::dict::Dict @@ -21430,9 +21686,9 @@ pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: &vortex_array::arra pub fn vortex_array::arrays::dict::Dict::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::dict::Dict::child(array: &vortex_array::arrays::dict::DictArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::dict::Dict::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::dict::Dict::child_name(_array: &vortex_array::arrays::dict::DictArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::dict::Dict::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::dict::Dict::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -21450,7 +21706,7 @@ pub fn vortex_array::arrays::dict::Dict::metadata(array: &vortex_array::arrays:: pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: &vortex_array::arrays::dict::DictArray) -> usize -pub fn vortex_array::arrays::dict::Dict::nchildren(_array: &vortex_array::arrays::dict::DictArray) -> usize +pub fn vortex_array::arrays::dict::Dict::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::dict::Dict::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -21458,9 +21714,13 @@ pub fn vortex_array::arrays::dict::Dict::reduce_parent(array: &Self::Array, pare pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::dict::Dict::slot_name(_array: &vortex_array::arrays::dict::DictArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::dict::Dict::slots(array: &vortex_array::arrays::dict::DictArray) -> &[core::option::Option] + pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::dict::Dict::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut vortex_array::arrays::dict::DictArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::null::Null @@ -21484,9 +21744,9 @@ pub fn vortex_array::arrays::null::Null::buffer_name(_array: &vortex_array::arra pub fn vortex_array::arrays::null::Null::build(_dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::null::Null::child(_array: &vortex_array::arrays::null::NullArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::null::Null::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::null::Null::child_name(_array: &vortex_array::arrays::null::NullArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::null::Null::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::null::Null::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -21504,7 +21764,7 @@ pub fn vortex_array::arrays::null::Null::metadata(_array: &vortex_array::arrays: pub fn vortex_array::arrays::null::Null::nbuffers(_array: &vortex_array::arrays::null::NullArray) -> usize -pub fn vortex_array::arrays::null::Null::nchildren(_array: &vortex_array::arrays::null::NullArray) -> usize +pub fn vortex_array::arrays::null::Null::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::null::Null::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -21512,9 +21772,13 @@ pub fn vortex_array::arrays::null::Null::reduce_parent(array: &Self::Array, pare pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::null::Null::slot_name(_array: &vortex_array::arrays::null::NullArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::null::Null::slots(array: &vortex_array::arrays::null::NullArray) -> &[core::option::Option] + pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::null::Null::with_children(_array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::null::Null::with_slots(array: &mut vortex_array::arrays::null::NullArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -21534,13 +21798,13 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::array_hash vortex_array::buffer::BufferHandle -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: &vortex_array::arrays::scalar_fn::ScalarFnArray, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: &vortex_array::arrays::scalar_fn::ScalarFnArray, _idx: usize) -> core::option::Option pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: &vortex_array::arrays::scalar_fn::ScalarFnArray, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: &vortex_array::arrays::scalar_fn::ScalarFnArray, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -21558,7 +21822,7 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::metadata(array: &Self::A pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: &vortex_array::arrays::scalar_fn::ScalarFnArray) -> usize -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nchildren(array: &vortex_array::arrays::scalar_fn::ScalarFnArray) -> usize +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -21566,9 +21830,13 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: &Se pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: &vortex_array::arrays::scalar_fn::ScalarFnArray, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: &vortex_array::arrays::scalar_fn::ScalarFnArray) -> &[core::option::Option] + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut vortex_array::arrays::scalar_fn::ScalarFnArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::slice::Slice @@ -21594,7 +21862,7 @@ pub fn vortex_array::arrays::slice::Slice::build(dtype: &vortex_array::dtype::DT pub fn vortex_array::arrays::slice::Slice::child(array: &Self::Array, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::slice::Slice::child_name(_array: &Self::Array, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::slice::Slice::child_name(array: &Self::Array, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult @@ -21612,7 +21880,7 @@ pub fn vortex_array::arrays::slice::Slice::metadata(array: &Self::Array) -> vort pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: &Self::Array) -> usize -pub fn vortex_array::arrays::slice::Slice::nchildren(_array: &Self::Array) -> usize +pub fn vortex_array::arrays::slice::Slice::nchildren(array: &Self::Array) -> usize pub fn vortex_array::arrays::slice::Slice::reduce(array: &Self::Array) -> vortex_error::VortexResult> @@ -21620,9 +21888,13 @@ pub fn vortex_array::arrays::slice::Slice::reduce_parent(array: &Self::Array, pa pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::slice::Slice::slot_name(_array: &Self::Array, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::slice::Slice::slots(array: &Self::Array) -> &[core::option::Option] + pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceArray) -> vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::arrays::slice::Slice::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::Array, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> pub trait vortex_array::vtable::ValidityChild @@ -21956,13 +22228,13 @@ pub enum vortex_array::ExecutionStep pub vortex_array::ExecutionStep::Done(vortex_array::ArrayRef) -pub vortex_array::ExecutionStep::ExecuteChild(usize, vortex_array::DonePredicate) +pub vortex_array::ExecutionStep::ExecuteSlot(usize, vortex_array::DonePredicate) impl vortex_array::ExecutionStep pub fn vortex_array::ExecutionStep::done(result: vortex_array::ArrayRef) -> Self -pub fn vortex_array::ExecutionStep::execute_child(child_idx: usize) -> Self +pub fn vortex_array::ExecutionStep::execute_slot(slot_idx: usize) -> Self impl core::fmt::Debug for vortex_array::ExecutionStep @@ -22050,6 +22322,8 @@ pub fn vortex_array::ArrayAdapter::nchildren(&self) -> usize pub fn vortex_array::ArrayAdapter::nth_child(&self, idx: usize) -> core::option::Option +pub fn vortex_array::ArrayAdapter::slots(&self) -> &[core::option::Option] + impl vortex_array::DynArray for vortex_array::ArrayAdapter pub fn vortex_array::ArrayAdapter::all_invalid(&self) -> vortex_error::VortexResult @@ -22098,8 +22372,6 @@ pub fn vortex_array::ArrayAdapter::validity_mask(&self) -> vortex_error::Vort pub fn vortex_array::ArrayAdapter::vtable(&self) -> &dyn vortex_array::vtable::DynVTable -pub fn vortex_array::ArrayAdapter::with_children(&self, children: alloc::vec::Vec) -> vortex_error::VortexResult - impl vortex_array::scalar_fn::ReduceNode for vortex_array::ArrayAdapter pub fn vortex_array::ArrayAdapter::as_any(&self) -> &dyn core::any::Any @@ -22342,6 +22614,8 @@ pub fn vortex_array::ArrayVisitor::nchildren(&self) -> usize pub fn vortex_array::ArrayVisitor::nth_child(&self, idx: usize) -> core::option::Option +pub fn vortex_array::ArrayVisitor::slots(&self) -> &[core::option::Option] + impl vortex_array::ArrayVisitor for alloc::sync::Arc pub fn alloc::sync::Arc::buffer_handles(&self) -> alloc::vec::Vec @@ -22370,6 +22644,8 @@ pub fn alloc::sync::Arc::nchildren(&self) -> usize pub fn alloc::sync::Arc::nth_child(&self, idx: usize) -> core::option::Option +pub fn alloc::sync::Arc::slots(&self) -> &[core::option::Option] + impl vortex_array::ArrayVisitor for vortex_array::ArrayAdapter pub fn vortex_array::ArrayAdapter::buffer_handles(&self) -> alloc::vec::Vec @@ -22398,6 +22674,8 @@ pub fn vortex_array::ArrayAdapter::nchildren(&self) -> usize pub fn vortex_array::ArrayAdapter::nth_child(&self, idx: usize) -> core::option::Option +pub fn vortex_array::ArrayAdapter::slots(&self) -> &[core::option::Option] + pub trait vortex_array::ArrayVisitorExt: vortex_array::DynArray pub fn vortex_array::ArrayVisitorExt::depth_first_traversal(&self) -> impl core::iter::traits::iterator::Iterator @@ -22482,8 +22760,6 @@ pub fn vortex_array::DynArray::validity_mask(&self) -> vortex_error::VortexResul pub fn vortex_array::DynArray::vtable(&self) -> &dyn vortex_array::vtable::DynVTable -pub fn vortex_array::DynArray::with_children(&self, children: alloc::vec::Vec) -> vortex_error::VortexResult - impl vortex_array::DynArray for alloc::sync::Arc pub fn alloc::sync::Arc::all_invalid(&self) -> vortex_error::VortexResult @@ -22532,8 +22808,6 @@ pub fn alloc::sync::Arc::validity_mask(&self) -> vor pub fn alloc::sync::Arc::vtable(&self) -> &dyn vortex_array::vtable::DynVTable -pub fn alloc::sync::Arc::with_children(&self, children: alloc::vec::Vec) -> vortex_error::VortexResult - impl vortex_array::DynArray for vortex_array::ArrayAdapter pub fn vortex_array::ArrayAdapter::all_invalid(&self) -> vortex_error::VortexResult @@ -22582,8 +22856,6 @@ pub fn vortex_array::ArrayAdapter::validity_mask(&self) -> vortex_error::Vort pub fn vortex_array::ArrayAdapter::vtable(&self) -> &dyn vortex_array::vtable::DynVTable -pub fn vortex_array::ArrayAdapter::with_children(&self, children: alloc::vec::Vec) -> vortex_error::VortexResult - pub trait vortex_array::DynArrayEq: vortex_array::hash::private::SealedEq pub fn vortex_array::DynArrayEq::dyn_array_eq(&self, other: &dyn core::any::Any, precision: vortex_array::Precision) -> bool diff --git a/vortex-array/src/array/mod.rs b/vortex-array/src/array/mod.rs index 79eb980d317..d69c8082481 100644 --- a/vortex-array/src/array/mod.rs +++ b/vortex-array/src/array/mod.rs @@ -162,9 +162,6 @@ pub trait DynArray: /// Returns the statistics of the array. // TODO(ngates): change how this works. It's weird. fn statistics(&self) -> StatsSetRef<'_>; - - /// Replaces the children of the array with the given array references. - fn with_children(&self, children: Vec) -> VortexResult; } impl DynArray for Arc { @@ -274,10 +271,6 @@ impl DynArray for Arc { fn statistics(&self) -> StatsSetRef<'_> { self.as_ref().statistics() } - - fn with_children(&self, children: Vec) -> VortexResult { - self.as_ref().with_children(children) - } } /// A reference counted pointer to a dynamic [`DynArray`] trait object. @@ -354,17 +347,18 @@ impl dyn DynArray + '_ { self.is::() } - /// Returns a new array with the child at `child_idx` replaced by `replacement`. - pub fn with_child(&self, child_idx: usize, replacement: ArrayRef) -> VortexResult { - let mut children: Vec = self.children(); + /// Returns a new array with the slot at `slot_idx` replaced by `replacement`. + pub fn with_slot(&self, slot_idx: usize, replacement: ArrayRef) -> VortexResult { + let slots = self.slots(); vortex_ensure!( - child_idx < children.len(), - "child index {} out of bounds for array with {} children", - child_idx, - children.len() + slot_idx < slots.len(), + "slot index {} out of bounds for array with {} slots", + slot_idx, + slots.len() ); - children[child_idx] = replacement; - self.with_children(children) + let mut slots = slots.to_vec(); + slots[slot_idx] = Some(replacement); + self.vtable().with_slots(&self.to_array(), slots) } } @@ -657,12 +651,6 @@ impl DynArray for ArrayAdapter { fn statistics(&self) -> StatsSetRef<'_> { V::stats(&self.0) } - - fn with_children(&self, children: Vec) -> VortexResult { - let mut this = self.0.clone(); - V::with_children(&mut this, children)?; - Ok(this.into_array()) - } } impl ArrayHash for ArrayAdapter { @@ -733,6 +721,10 @@ impl ArrayVisitor for ArrayAdapter { V::nbuffers(&self.0) } + fn slots(&self) -> &[Option] { + V::slots(&self.0) + } + fn metadata(&self) -> VortexResult>> { V::serialize(V::metadata(&self.0)?) } diff --git a/vortex-array/src/array/visitor.rs b/vortex-array/src/array/visitor.rs index 24440a99ada..012bb1924bc 100644 --- a/vortex-array/src/array/visitor.rs +++ b/vortex-array/src/array/visitor.rs @@ -26,6 +26,9 @@ pub trait ArrayVisitor { /// Returns the names of the children of the array. fn children_names(&self) -> Vec; + /// Returns the slots of the array as a slice. + fn slots(&self) -> &[Option]; + /// Returns the array's children with their names. fn named_children(&self) -> Vec<(String, ArrayRef)>; @@ -74,6 +77,10 @@ impl ArrayVisitor for Arc { self.as_ref().children_names() } + fn slots(&self) -> &[Option] { + self.as_ref().slots() + } + fn named_children(&self) -> Vec<(String, ArrayRef)> { self.as_ref().named_children() } diff --git a/vortex-array/src/arrays/bool/array.rs b/vortex-array/src/arrays/bool/array.rs index 124aa3f2427..730b494b8ba 100644 --- a/vortex-array/src/arrays/bool/array.rs +++ b/vortex-array/src/arrays/bool/array.rs @@ -11,11 +11,15 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; -use crate::arrays::bool; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::stats::ArrayStats; use crate::validity::Validity; +use crate::vtable::validity_to_child; + +pub(super) const VALIDITY_SLOT: usize = 0; +pub(super) const NUM_SLOTS: usize = 1; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; /// A boolean array that stores true/false values in a compact bit-packed format. /// @@ -51,6 +55,7 @@ use crate::validity::Validity; /// ``` #[derive(Clone, Debug)] pub struct BoolArray { + pub(super) slots: Vec>, pub(super) dtype: DType, pub(super) bits: BufferHandle, pub(super) offset: usize, @@ -67,6 +72,10 @@ pub struct BoolArrayParts { } impl BoolArray { + fn make_slots(validity: &Validity, len: usize) -> Vec> { + vec![validity_to_child(validity, len)] + } + /// Constructs a new `BoolArray`. /// /// # Panics @@ -101,6 +110,7 @@ impl BoolArray { let (offset, len, buffer) = bits.into_inner(); Ok(Self { + slots: Self::make_slots(&validity, len), dtype: DType::Bool(validity.nullability()), bits: BufferHandle::new_host(buffer), offset, @@ -138,6 +148,7 @@ impl BoolArray { ); Ok(Self { + slots: Self::make_slots(&validity, len), dtype: DType::Bool(validity.nullability()), bits, offset, @@ -159,6 +170,7 @@ impl BoolArray { let (offset, len, buffer) = bits.into_inner(); Self { + slots: Self::make_slots(&validity, len), dtype: DType::Bool(validity.nullability()), bits: BufferHandle::new_host(buffer), offset, diff --git a/vortex-array/src/arrays/bool/vtable/mod.rs b/vortex-array/src/arrays/bool/vtable/mod.rs index d439ecf0e78..23a1177bc4f 100644 --- a/vortex-array/src/arrays/bool/vtable/mod.rs +++ b/vortex-array/src/arrays/bool/vtable/mod.rs @@ -17,6 +17,9 @@ use crate::IntoArray; use crate::ProstMetadata; use crate::SerializeMetadata; use crate::arrays::BoolArray; +use crate::arrays::bool::array::NUM_SLOTS; +use crate::arrays::bool::array::SLOT_NAMES; +use crate::arrays::bool::array::VALIDITY_SLOT; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::serde::ArrayChildren; @@ -24,8 +27,6 @@ use crate::validity::Validity; use crate::vtable; use crate::vtable::VTable; use crate::vtable::ValidityVTableFromValidityHelper; -use crate::vtable::validity_nchildren; -use crate::vtable::validity_to_child; mod canonical; mod kernel; mod operations; @@ -106,22 +107,6 @@ impl VTable for Bool { } } - fn nchildren(array: &BoolArray) -> usize { - validity_nchildren(&array.validity) - } - - fn child(array: &BoolArray, idx: usize) -> ArrayRef { - match idx { - 0 => validity_to_child(&array.validity, array.len()) - .vortex_expect("BoolArray child index out of bounds"), - _ => vortex_panic!("BoolArray child index {idx} out of bounds"), - } - } - - fn child_name(_array: &BoolArray, _idx: usize) -> String { - "validity".to_string() - } - fn metadata(array: &BoolArray) -> VortexResult { assert!(array.offset < 8, "Offset must be <8, got {}", array.offset); Ok(ProstMetadata(BoolMetadata { @@ -169,19 +154,26 @@ impl VTable for Bool { BoolArray::try_new_from_handle(buffer, metadata.offset as usize, len, validity) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { + fn slots(array: &BoolArray) -> &[Option] { + &array.slots + } + + fn slot_name(_array: &BoolArray, idx: usize) -> String { + SLOT_NAMES[idx].to_string() + } + + fn with_slots(array: &mut BoolArray, slots: Vec>) -> VortexResult<()> { vortex_ensure!( - children.len() <= 1, - "BoolArray can have at most 1 child (validity), got {}", - children.len() + slots.len() == NUM_SLOTS, + "BoolArray expects {} slots, got {}", + NUM_SLOTS, + slots.len() ); - - array.validity = if children.is_empty() { - Validity::from(array.dtype().nullability()) - } else { - Validity::Array(children.into_iter().next().vortex_expect("checked")) + array.validity = match &slots[VALIDITY_SLOT] { + Some(arr) => Validity::Array(arr.clone()), + None => Validity::from(array.dtype().nullability()), }; - + array.slots = slots; Ok(()) } @@ -213,3 +205,47 @@ pub struct Bool; impl Bool { pub const ID: ArrayId = ArrayId::new_ref("vortex.bool"); } + +#[cfg(test)] +mod tests { + use vortex_buffer::ByteBufferMut; + use vortex_session::registry::ReadContext; + + use crate::ArrayContext; + use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::arrays::BoolArray; + use crate::assert_arrays_eq; + use crate::serde::ArrayParts; + use crate::serde::SerializeOptions; + + #[test] + fn test_nullable_bool_serde_roundtrip() { + let array = BoolArray::from_iter([Some(true), None, Some(false), None]); + let dtype = array.dtype().clone(); + let len = array.len(); + + let ctx = ArrayContext::empty(); + let serialized = array + .clone() + .into_array() + .serialize(&ctx, &SerializeOptions::default()) + .unwrap(); + + let mut concat = ByteBufferMut::empty(); + for buf in serialized { + concat.extend_from_slice(buf.as_ref()); + } + let parts = ArrayParts::try_from(concat.freeze()).unwrap(); + let decoded = parts + .decode( + &dtype, + len, + &ReadContext::new(ctx.to_ids()), + &LEGACY_SESSION, + ) + .unwrap(); + + assert_arrays_eq!(decoded, array); + } +} diff --git a/vortex-array/src/arrays/chunked/array.rs b/vortex-array/src/arrays/chunked/array.rs index 98b1c76dd5e..f787b9aa82f 100644 --- a/vortex-array/src/arrays/chunked/array.rs +++ b/vortex-array/src/arrays/chunked/array.rs @@ -10,13 +10,14 @@ use std::fmt::Debug; use futures::stream; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; -use vortex_error::VortexExpect as _; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use crate::ArrayRef; use crate::DynArray; use crate::IntoArray; +use crate::ToCanonical; use crate::arrays::PrimitiveArray; use crate::dtype::DType; use crate::iter::ArrayIterator; @@ -28,12 +29,15 @@ use crate::stream::ArrayStream; use crate::stream::ArrayStreamAdapter; use crate::validity::Validity; +// ChunkedArray has a variable number of slots: +// slots[0] = chunk_offsets +// slots[1..] = chunks + #[derive(Clone, Debug)] pub struct ChunkedArray { pub(super) dtype: DType, pub(super) len: usize, - pub(super) chunk_offsets: PrimitiveArray, - pub(super) chunks: Vec, + pub(super) slots: Vec>, pub(super) stats_set: ArrayStats, } @@ -78,15 +82,19 @@ impl ChunkedArray { unsafe { chunk_offsets_buf.push_unchecked(curr_offset) } } - let chunk_offsets = PrimitiveArray::new(chunk_offsets_buf.freeze(), Validity::NonNullable); + let chunk_offsets = + PrimitiveArray::new(chunk_offsets_buf.freeze(), Validity::NonNullable).into_array(); + + let mut slots = Vec::with_capacity(1 + nchunks); + slots.push(Some(chunk_offsets)); + slots.extend(chunks.into_iter().map(Some)); Self { dtype, len: curr_offset .try_into() .vortex_expect("chunk offset must fit in usize"), - chunk_offsets, - chunks, + slots, stats_set: Default::default(), } } @@ -107,17 +115,26 @@ impl ChunkedArray { #[inline] pub fn chunk(&self, idx: usize) -> &ArrayRef { assert!(idx < self.nchunks(), "chunk index {idx} out of bounds"); - // SAFETY: bounds checked by the assert above. - unsafe { self.chunks.get_unchecked(idx) } + self.slots[idx + 1] + .as_ref() + .vortex_expect("ChunkedArray chunk slot") } pub fn nchunks(&self) -> usize { - self.chunks.len() + self.slots.len().saturating_sub(1) + } + + /// Returns the chunk offsets as a primitive array. + pub(crate) fn chunk_offsets_array(&self) -> PrimitiveArray { + self.slots[0] + .as_ref() + .vortex_expect("ChunkedArray chunk_offsets slot") + .to_primitive() } #[inline] pub fn chunk_offsets(&self) -> Buffer { - self.chunk_offsets.to_buffer() + self.chunk_offsets_array().to_buffer() } pub(crate) fn find_chunk_idx(&self, index: usize) -> VortexResult<(usize, usize)> { @@ -138,22 +155,42 @@ impl ChunkedArray { Ok((index_chunk, index_in_chunk)) } - pub fn chunks(&self) -> &[ArrayRef] { - &self.chunks + /// Returns an iterator over chunk references without allocation. + pub fn iter_chunks(&self) -> impl Iterator + '_ { + self.slots[1..] + .iter() + .map(|s| s.as_ref().vortex_expect("ChunkedArray chunk slot")) + } + + /// Returns the chunks as a vector of owned references. + pub fn chunks(&self) -> Vec { + self.iter_chunks().cloned().collect() } pub fn non_empty_chunks(&self) -> impl Iterator + '_ { - self.chunks().iter().filter(|c| !c.is_empty()) + self.slots[1..] + .iter() + .filter_map(|s| s.as_ref()) + .filter(|c| !c.is_empty()) } pub fn array_iterator(&self) -> impl ArrayIterator + '_ { - ArrayIteratorAdapter::new(self.dtype().clone(), self.chunks().iter().cloned().map(Ok)) + ArrayIteratorAdapter::new( + self.dtype().clone(), + self.slots[1..] + .iter() + .map(|s| Ok(s.as_ref().vortex_expect("ChunkedArray chunk slot").clone())), + ) } pub fn array_stream(&self) -> impl ArrayStream + '_ { ArrayStreamAdapter::new( self.dtype().clone(), - stream::iter(self.chunks().iter().cloned().map(Ok)), + stream::iter( + self.slots[1..] + .iter() + .map(|s| Ok(s.as_ref().vortex_expect("ChunkedArray chunk slot").clone())), + ), ) } @@ -162,7 +199,7 @@ impl ChunkedArray { let mut chunks_to_combine = Vec::new(); let mut new_chunk_n_bytes = 0; let mut new_chunk_n_elements = 0; - for chunk in self.chunks() { + for chunk in self.iter_chunks() { let n_bytes = chunk.nbytes(); let n_elements = chunk.len(); @@ -277,7 +314,7 @@ mod test { let rechunked = chunked.rechunk(1 << 16, 5).unwrap(); assert_eq!(rechunked.nchunks(), 2); - assert!(rechunked.chunks().iter().all(|c| c.len() < 5)); + assert!(rechunked.iter_chunks().all(|c| c.len() < 5)); assert_arrays_eq!(chunked, rechunked); } diff --git a/vortex-array/src/arrays/chunked/compute/aggregate.rs b/vortex-array/src/arrays/chunked/compute/aggregate.rs index e175754bc30..6fa690542ea 100644 --- a/vortex-array/src/arrays/chunked/compute/aggregate.rs +++ b/vortex-array/src/arrays/chunked/compute/aggregate.rs @@ -25,7 +25,7 @@ impl DynAggregateKernel for ChunkedArrayAggregate { }; let mut acc = aggregate_fn.accumulator(chunked.dtype())?; - for chunk in chunked.chunks() { + for chunk in chunked.iter_chunks() { acc.accumulate(chunk, ctx)?; } // Return the partial (not finalized) result, since the outer accumulator diff --git a/vortex-array/src/arrays/chunked/compute/cast.rs b/vortex-array/src/arrays/chunked/compute/cast.rs index a74927199fa..8464ce0fc4d 100644 --- a/vortex-array/src/arrays/chunked/compute/cast.rs +++ b/vortex-array/src/arrays/chunked/compute/cast.rs @@ -14,7 +14,7 @@ use crate::scalar_fn::fns::cast::CastReduce; impl CastReduce for Chunked { fn cast(array: &ChunkedArray, dtype: &DType) -> VortexResult> { let mut cast_chunks = Vec::new(); - for chunk in array.chunks() { + for chunk in array.iter_chunks() { cast_chunks.push(chunk.cast(dtype.clone())?); } diff --git a/vortex-array/src/arrays/chunked/compute/fill_null.rs b/vortex-array/src/arrays/chunked/compute/fill_null.rs index 46aa17622ac..110bf171fc9 100644 --- a/vortex-array/src/arrays/chunked/compute/fill_null.rs +++ b/vortex-array/src/arrays/chunked/compute/fill_null.rs @@ -14,8 +14,7 @@ use crate::scalar_fn::fns::fill_null::FillNullReduce; impl FillNullReduce for Chunked { fn fill_null(array: &ChunkedArray, fill_value: &Scalar) -> VortexResult> { let new_chunks = array - .chunks() - .iter() + .iter_chunks() .map(|c| c.fill_null(fill_value.clone())) .collect::>>()?; diff --git a/vortex-array/src/arrays/chunked/compute/filter.rs b/vortex-array/src/arrays/chunked/compute/filter.rs index 4d2ff8149cb..48c81821fb6 100644 --- a/vortex-array/src/arrays/chunked/compute/filter.rs +++ b/vortex-array/src/arrays/chunked/compute/filter.rs @@ -70,7 +70,7 @@ fn filter_slices( let chunk_filters = chunk_filters(array, slices)?; // Now, apply the chunk filter to every slice. - for (chunk, chunk_filter) in array.chunks().iter().zip(chunk_filters.into_iter()) { + for (chunk, chunk_filter) in array.iter_chunks().zip(chunk_filters.into_iter()) { match chunk_filter { // All => preserve the entire chunk unfiltered. ChunkFilter::All => result.push(chunk.clone()), diff --git a/vortex-array/src/arrays/chunked/compute/mask.rs b/vortex-array/src/arrays/chunked/compute/mask.rs index 441f9a5ce3e..9217e5627f0 100644 --- a/vortex-array/src/arrays/chunked/compute/mask.rs +++ b/vortex-array/src/arrays/chunked/compute/mask.rs @@ -21,8 +21,7 @@ impl MaskKernel for Chunked { ) -> VortexResult> { let chunk_offsets = array.chunk_offsets(); let new_chunks: Vec = array - .chunks() - .iter() + .iter_chunks() .enumerate() .map(|(i, chunk)| { let start: usize = chunk_offsets[i].try_into()?; diff --git a/vortex-array/src/arrays/chunked/compute/rules.rs b/vortex-array/src/arrays/chunked/compute/rules.rs index 808b50c7d94..d20c96d6a3b 100644 --- a/vortex-array/src/arrays/chunked/compute/rules.rs +++ b/vortex-array/src/arrays/chunked/compute/rules.rs @@ -37,13 +37,12 @@ impl ArrayParentReduceRule for ChunkedUnaryScalarFnPushDownRule { parent: &ScalarFnArray, _child_idx: usize, ) -> VortexResult> { - if parent.children().len() != 1 { + if parent.nchildren() != 1 { return Ok(None); } let new_chunks: Vec<_> = array - .chunks - .iter() + .iter_chunks() .map(|chunk| { ScalarFnArray::try_new( parent.scalar_fn().clone(), @@ -73,7 +72,7 @@ impl ArrayParentReduceRule for ChunkedConstantScalarFnPushDownRule { parent: &ScalarFnArray, child_idx: usize, ) -> VortexResult> { - for (idx, child) in parent.children().iter().enumerate() { + for (idx, child) in parent.iter_children().enumerate() { if idx == child_idx { continue; } @@ -83,12 +82,10 @@ impl ArrayParentReduceRule for ChunkedConstantScalarFnPushDownRule { } let new_chunks: Vec<_> = array - .chunks - .iter() + .iter_chunks() .map(|chunk| { let new_children: Vec<_> = parent - .children() - .iter() + .iter_children() .enumerate() .map(|(idx, child)| { if idx == child_idx { diff --git a/vortex-array/src/arrays/chunked/paired_chunks.rs b/vortex-array/src/arrays/chunked/paired_chunks.rs index 1677a489070..fcc4eae0e05 100644 --- a/vortex-array/src/arrays/chunked/paired_chunks.rs +++ b/vortex-array/src/arrays/chunked/paired_chunks.rs @@ -14,16 +14,16 @@ pub(crate) struct AlignedPair { pub pos: Range, } -/// Cursor over a chunk slice that maintains the invariant: `idx` always +/// Cursor over a chunk list that maintains the invariant: `idx` always /// points at a non-empty chunk or is past the end. -struct ChunkCursor<'a> { - chunks: &'a [ArrayRef], +struct ChunkCursor { + chunks: Vec, idx: usize, offset: usize, } -impl<'a> ChunkCursor<'a> { - fn new(chunks: &'a [ArrayRef]) -> Self { +impl ChunkCursor { + fn new(chunks: Vec) -> Self { let mut cursor = Self { chunks, idx: 0, @@ -34,22 +34,21 @@ impl<'a> ChunkCursor<'a> { } fn skip_empty(&mut self) { - while self.idx < self.chunks.len() - && unsafe { self.chunks.get_unchecked(self.idx) }.is_empty() - { + while self.idx < self.chunks.len() && self.chunks[self.idx].is_empty() { self.idx += 1; } } - fn current_chunk(&self) -> Option<&'a ArrayRef> { - (self.idx < self.chunks.len()).then(|| unsafe { self.chunks.get_unchecked(self.idx) }) + fn is_exhausted(&self) -> bool { + self.idx >= self.chunks.len() } - fn remaining(&self, chunk: &ArrayRef) -> usize { - chunk.len() - self.offset + fn remaining(&self) -> usize { + self.chunks[self.idx].len() - self.offset } - fn take(&mut self, chunk: &ArrayRef, n: usize) -> VortexResult { + fn take(&mut self, n: usize) -> VortexResult { + let chunk = &self.chunks[self.idx]; let slice = chunk.slice(self.offset..self.offset + n)?; self.offset += n; if self.offset == chunk.len() { @@ -61,49 +60,43 @@ impl<'a> ChunkCursor<'a> { } } -pub(crate) struct PairedChunks<'a> { - left: ChunkCursor<'a>, - right: ChunkCursor<'a>, +pub(crate) struct PairedChunks { + left: ChunkCursor, + right: ChunkCursor, pos: usize, total_len: usize, } impl ChunkedArray { - pub(crate) fn paired_chunks<'a>(&'a self, other: &'a ChunkedArray) -> PairedChunks<'a> { + pub(crate) fn paired_chunks(&self, other: &ChunkedArray) -> PairedChunks { assert_eq!( self.len(), other.len(), "paired_chunks requires arrays of equal length" ); PairedChunks { - left: ChunkCursor::new(&self.chunks), - right: ChunkCursor::new(&other.chunks), + left: ChunkCursor::new(self.chunks()), + right: ChunkCursor::new(other.chunks()), pos: 0, total_len: self.len(), } } } -impl Iterator for PairedChunks<'_> { +impl Iterator for PairedChunks { type Item = VortexResult; fn next(&mut self) -> Option { - if self.pos >= self.total_len { + if self.pos >= self.total_len || self.left.is_exhausted() || self.right.is_exhausted() { return None; } - let lhs_chunk = self.left.current_chunk()?; - let rhs_chunk = self.right.current_chunk()?; - - let take = self - .left - .remaining(lhs_chunk) - .min(self.right.remaining(rhs_chunk)); + let take = self.left.remaining().min(self.right.remaining()); let (lhs_slice, rhs_slice) = match self .left - .take(lhs_chunk, take) - .and_then(|l| self.right.take(rhs_chunk, take).map(|r| (l, r))) + .take(take) + .and_then(|l| self.right.take(take).map(|r| (l, r))) { Ok(pair) => pair, Err(e) => return Some(Err(e)), diff --git a/vortex-array/src/arrays/chunked/tests.rs b/vortex-array/src/arrays/chunked/tests.rs index fdf1e54907b..a4c169ad474 100644 --- a/vortex-array/src/arrays/chunked/tests.rs +++ b/vortex-array/src/arrays/chunked/tests.rs @@ -8,6 +8,7 @@ use vortex_buffer::buffer; use crate::IntoArray; use crate::accessor::ArrayAccessor; +use crate::arrays::Chunked; use crate::arrays::ChunkedArray; use crate::arrays::ListArray; use crate::arrays::PrimitiveArray; @@ -20,6 +21,7 @@ use crate::dtype::Nullability; use crate::dtype::PType; use crate::dtype::PType::I32; use crate::validity::Validity; +use crate::vtable::VTable; fn chunked_array() -> ChunkedArray { ChunkedArray::try_new( @@ -194,3 +196,33 @@ pub fn pack_nested_lists() { assert_eq!(l1.scalar_at(0).unwrap(), canon_values.scalar_at(0).unwrap()); assert_eq!(l2.scalar_at(0).unwrap(), canon_values.scalar_at(1).unwrap()); } + +#[test] +fn with_slots_updates_nchunks_len_and_offsets() { + let mut array = chunked_array(); + let slots = vec![ + Some(buffer![0u64, 4, 9].into_array()), + Some(buffer![10u64, 11, 12, 13].into_array()), + Some(buffer![14u64, 15, 16, 17, 18].into_array()), + ]; + let expected_nchunks = slots.len() - 1; + let expected_len = array.len(); + + ::with_slots(&mut array, slots).unwrap(); + + assert_eq!(array.nchunks(), expected_nchunks); + assert_eq!(array.len(), expected_len); + assert_eq!(array.chunk_offsets(), buffer![0u64, 4, 9]); + assert_arrays_eq!( + array.chunk(0).clone(), + PrimitiveArray::from_iter([10u64, 11, 12, 13]) + ); + assert_arrays_eq!( + array.chunk(1).clone(), + PrimitiveArray::from_iter([14u64, 15, 16, 17, 18]) + ); + assert_arrays_eq!( + array, + PrimitiveArray::from_iter([10u64, 11, 12, 13, 14, 15, 16, 17, 18]) + ); +} diff --git a/vortex-array/src/arrays/chunked/vtable/canonical.rs b/vortex-array/src/arrays/chunked/vtable/canonical.rs index d165dc0be6e..05b8bdfd42b 100644 --- a/vortex-array/src/arrays/chunked/vtable/canonical.rs +++ b/vortex-array/src/arrays/chunked/vtable/canonical.rs @@ -31,13 +31,14 @@ pub(super) fn _canonicalize( return Ok(Canonical::empty(array.dtype())); } if array.nchunks() == 1 { - return array.chunks()[0].clone().execute::(ctx); + return array.chunk(0).clone().execute::(ctx); } + let owned_chunks: Vec = array.iter_chunks().cloned().collect(); Ok(match array.dtype() { DType::Struct(struct_dtype, _) => { let struct_array = pack_struct_chunks( - array.chunks(), + &owned_chunks, Validity::copy_from_array(&array.clone().into_array())?, struct_dtype, ctx, @@ -45,7 +46,7 @@ pub(super) fn _canonicalize( Canonical::Struct(struct_array) } DType::List(elem_dtype, _) => Canonical::List(swizzle_list_chunks( - array.chunks(), + &owned_chunks, Validity::copy_from_array(&array.clone().into_array())?, elem_dtype, ctx, @@ -79,11 +80,7 @@ fn pack_struct_chunks( for (field_idx, field_dtype) in struct_dtype.fields().enumerate() { let mut field_chunks = Vec::with_capacity(chunks.len()); for struct_array in &executed_chunks { - let field = struct_array - .unmasked_fields() - .get(field_idx) - .vortex_expect("Invalid field index") - .to_array(); + let field = struct_array.unmasked_field(field_idx).to_array(); field_chunks.push(field); } @@ -228,8 +225,8 @@ mod tests { .unwrap() .into_array(); let canonical_struct = chunked.to_struct(); - let canonical_varbin = canonical_struct.unmasked_fields()[0].to_varbinview(); - let original_varbin = struct_array.unmasked_fields()[0].to_varbinview(); + let canonical_varbin = canonical_struct.unmasked_field(0).to_varbinview(); + let original_varbin = struct_array.unmasked_field(0).to_varbinview(); let orig_values = original_varbin .with_iterator(|it| it.map(|a| a.map(|v| v.to_vec())).collect::>()); let canon_values = canonical_varbin diff --git a/vortex-array/src/arrays/chunked/vtable/mod.rs b/vortex-array/src/arrays/chunked/vtable/mod.rs index 49dd7f9f6d7..1770d0467c4 100644 --- a/vortex-array/src/arrays/chunked/vtable/mod.rs +++ b/vortex-array/src/arrays/chunked/vtable/mod.rs @@ -74,8 +74,11 @@ impl VTable for Chunked { fn array_hash(array: &ChunkedArray, state: &mut H, precision: Precision) { array.dtype.hash(state); array.len.hash(state); - array.chunk_offsets.as_ref().array_hash(state, precision); - for chunk in &array.chunks { + array + .chunk_offsets_array() + .as_ref() + .array_hash(state, precision); + for chunk in array.iter_chunks() { chunk.array_hash(state, precision); } } @@ -84,14 +87,13 @@ impl VTable for Chunked { array.dtype == other.dtype && array.len == other.len && array - .chunk_offsets + .chunk_offsets_array() .as_ref() - .array_eq(other.chunk_offsets.as_ref(), precision) - && array.chunks.len() == other.chunks.len() + .array_eq(other.chunk_offsets_array().as_ref(), precision) + && array.nchunks() == other.nchunks() && array - .chunks - .iter() - .zip(&other.chunks) + .iter_chunks() + .zip(other.iter_chunks()) .all(|(a, b)| a.array_eq(b, precision)) } @@ -103,26 +105,8 @@ impl VTable for Chunked { vortex_panic!("ChunkedArray buffer index {idx} out of bounds") } - fn buffer_name(_array: &ChunkedArray, idx: usize) -> Option { - vortex_panic!("ChunkedArray buffer_name index {idx} out of bounds") - } - - fn nchildren(array: &ChunkedArray) -> usize { - 1 + array.chunks().len() - } - - fn child(array: &ChunkedArray, idx: usize) -> ArrayRef { - match idx { - 0 => array.chunk_offsets.clone().into_array(), - n => array.chunks()[n - 1].clone(), - } - } - - fn child_name(_array: &ChunkedArray, idx: usize) -> String { - match idx { - 0 => "chunk_offsets".to_string(), - n => format!("chunks[{}]", n - 1), - } + fn buffer_name(_array: &ChunkedArray, _idx: usize) -> Option { + None } fn metadata(_array: &ChunkedArray) -> VortexResult { @@ -169,7 +153,7 @@ impl VTable for Chunked { let chunk_offsets_buf = chunk_offsets_array.to_buffer::(); // The remaining children contain the actual data of the chunks - let chunks = chunk_offsets_buf + let chunks: Vec = chunk_offsets_buf .iter() .tuple_windows() .enumerate() @@ -188,53 +172,24 @@ impl VTable for Chunked { let len = usize::try_from(*total_len) .map_err(|_| vortex_err!("total length {} exceeds usize range", total_len))?; - // Construct directly using the struct fields to avoid recomputing chunk_offsets + // Construct directly using slots to avoid recomputing chunk_offsets + let mut slots = Vec::with_capacity(1 + chunks.len()); + slots.push(Some(chunk_offsets.into_array())); + slots.extend(chunks.into_iter().map(Some)); Ok(ChunkedArray { dtype: dtype.clone(), len, - chunk_offsets, - chunks, + slots, stats_set: Default::default(), }) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { - // Children: chunk_offsets, then chunks... - vortex_ensure!( - !children.is_empty(), - "Chunked array needs at least one child" - ); - - let nchunks = children.len() - 1; - let chunk_offsets_array = children[0].to_primitive(); - let chunk_offsets_buf = chunk_offsets_array.to_buffer::(); - - vortex_ensure!( - chunk_offsets_buf.len() == nchunks + 1, - "Expected {} chunk offsets, found {}", - nchunks + 1, - chunk_offsets_buf.len() - ); - - let chunks = children.into_iter().skip(1).collect(); - array.chunk_offsets = PrimitiveArray::new(chunk_offsets_buf.clone(), Validity::NonNullable); - array.chunks = chunks; - - let total_len = chunk_offsets_buf - .last() - .ok_or_else(|| vortex_err!("chunk_offsets must not be empty"))?; - array.len = usize::try_from(*total_len) - .map_err(|_| vortex_err!("total length {} exceeds usize range", total_len))?; - - Ok(()) - } - fn append_to_builder( array: &ChunkedArray, builder: &mut dyn ArrayBuilder, ctx: &mut ExecutionCtx, ) -> VortexResult<()> { - for chunk in array.chunks() { + for chunk in array.iter_chunks() { chunk.append_to_builder(builder, ctx)?; } Ok(()) @@ -244,10 +199,86 @@ impl VTable for Chunked { Ok(ExecutionStep::Done(_canonicalize(array, ctx)?.into_array())) } + fn slots(array: &ChunkedArray) -> &[Option] { + &array.slots + } + + fn slot_name(_array: &ChunkedArray, idx: usize) -> String { + match idx { + 0 => "chunk_offsets".to_string(), + n => format!("chunks[{}]", n - 1), + } + } + + fn with_slots(array: &mut ChunkedArray, slots: Vec>) -> VortexResult<()> { + vortex_ensure!(!slots.is_empty(), "Chunked array needs at least one slot"); + + let chunk_offsets = slots[0] + .as_ref() + .ok_or_else(|| vortex_err!("Chunked array chunk_offsets slot must be present"))?; + let chunk_offsets_dtype = DType::Primitive(PType::U64, Nullability::NonNullable); + vortex_ensure!( + chunk_offsets.dtype() == &chunk_offsets_dtype, + MismatchedTypes: &chunk_offsets_dtype, + chunk_offsets.dtype() + ); + + #[cfg(debug_assertions)] + { + let chunk_offsets = chunk_offsets + .as_opt::() + .unwrap_or_else(|| { + vortex_panic!("Chunked array chunk_offsets slot must be primitive") + }); + let chunk_offsets_buf = chunk_offsets.to_buffer::(); + debug_assert_eq!( + chunk_offsets_buf.len(), + slots.len(), + "Expected {} chunk offsets, found {}", + slots.len(), + chunk_offsets_buf.len(), + ); + debug_assert_eq!( + chunk_offsets_buf.last().copied().unwrap_or_default(), + array.len as u64, + "Chunked array replacement changed logical len: expected {}, got {}", + array.len, + chunk_offsets_buf.last().copied().unwrap_or_default(), + ); + + let mut total_len = 0usize; + + for (idx, chunk_slot) in slots[1..].iter().enumerate() { + let chunk = chunk_slot.as_ref().unwrap_or_else(|| { + vortex_panic!("Chunked array chunk slot {idx} must be present") + }); + debug_assert!( + chunk.dtype() == array.dtype(), + "Chunked array chunk slot {} has dtype {:?}, expected {:?}", + idx, + chunk.dtype(), + array.dtype(), + ); + total_len = total_len.checked_add(chunk.len()).unwrap_or_else(|| { + vortex_panic!("Chunked array chunk lengths exceed usize range") + }); + } + + debug_assert_eq!( + total_len, array.len, + "Chunked array replacement changed logical len: expected {}, got {}", + array.len, total_len, + ); + } + + array.slots = slots; + Ok(()) + } + fn reduce(array: &Self::Array) -> VortexResult> { - Ok(match array.chunks.len() { + Ok(match array.nchunks() { 0 => Some(Canonical::empty(array.dtype()).into_array()), - 1 => Some(array.chunks[0].clone()), + 1 => Some(array.chunk(0).clone()), _ => None, }) } diff --git a/vortex-array/src/arrays/chunked/vtable/validity.rs b/vortex-array/src/arrays/chunked/vtable/validity.rs index 04a9cf73956..961339a7be0 100644 --- a/vortex-array/src/arrays/chunked/vtable/validity.rs +++ b/vortex-array/src/arrays/chunked/vtable/validity.rs @@ -15,8 +15,7 @@ use crate::vtable::ValidityVTable; impl ValidityVTable for Chunked { fn validity(array: &ChunkedArray) -> VortexResult { - let validities: Vec = - array.chunks().iter().map(|c| c.validity()).try_collect()?; + let validities: Vec = array.iter_chunks().map(|c| c.validity()).try_collect()?; match validities.first() { // If there are no chunks, return the array's dtype nullability @@ -42,7 +41,7 @@ impl ValidityVTable for Chunked { ChunkedArray::new_unchecked( validities .into_iter() - .zip(array.chunks()) + .zip(array.iter_chunks()) .map(|(v, chunk)| v.to_array(chunk.len())) .collect(), DType::Bool(Nullability::NonNullable), diff --git a/vortex-array/src/arrays/constant/array.rs b/vortex-array/src/arrays/constant/array.rs index e8bcb7d88c6..b6c29077374 100644 --- a/vortex-array/src/arrays/constant/array.rs +++ b/vortex-array/src/arrays/constant/array.rs @@ -1,13 +1,17 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use crate::ArrayRef; use crate::scalar::Scalar; use crate::stats::ArrayStats; +pub(super) const NUM_SLOTS: usize = 0; + #[derive(Clone, Debug)] pub struct ConstantArray { pub(super) scalar: Scalar, pub(super) len: usize, + pub(super) slots: Vec>, pub(super) stats_set: ArrayStats, } @@ -20,6 +24,7 @@ impl ConstantArray { Self { scalar, len, + slots: vec![], stats_set: Default::default(), } } diff --git a/vortex-array/src/arrays/constant/vtable/mod.rs b/vortex-array/src/arrays/constant/vtable/mod.rs index d35b874193b..b3eca6c61de 100644 --- a/vortex-array/src/arrays/constant/vtable/mod.rs +++ b/vortex-array/src/arrays/constant/vtable/mod.rs @@ -17,6 +17,7 @@ use crate::ExecutionStep; use crate::IntoArray; use crate::Precision; use crate::arrays::ConstantArray; +use crate::arrays::constant::array::NUM_SLOTS; use crate::arrays::constant::compute::rules::PARENT_RULES; use crate::arrays::constant::vtable::canonical::constant_canonicalize; use crate::buffer::BufferHandle; @@ -107,16 +108,23 @@ impl VTable for Constant { } } - fn nchildren(_array: &ConstantArray) -> usize { - 0 + fn slots(array: &ConstantArray) -> &[Option] { + &array.slots } - fn child(_array: &ConstantArray, idx: usize) -> ArrayRef { - vortex_panic!("ConstantArray child index {idx} out of bounds") + fn slot_name(_array: &ConstantArray, idx: usize) -> String { + vortex_panic!("ConstantArray slot_name index {idx} out of bounds") } - fn child_name(_array: &ConstantArray, idx: usize) -> String { - vortex_panic!("ConstantArray child_name index {idx} out of bounds") + fn with_slots(array: &mut ConstantArray, slots: Vec>) -> VortexResult<()> { + vortex_ensure!( + slots.len() == NUM_SLOTS, + "ConstantArray expects exactly {} slots, got {}", + NUM_SLOTS, + slots.len() + ); + array.slots = slots; + Ok(()) } fn metadata(array: &ConstantArray) -> VortexResult { @@ -161,15 +169,6 @@ impl VTable for Constant { Ok(ConstantArray::new(metadata.clone(), len)) } - fn with_children(_array: &mut Self::Array, children: Vec) -> VortexResult<()> { - vortex_ensure!( - children.is_empty(), - "ConstantArray has no children, got {}", - children.len() - ); - Ok(()) - } - fn reduce_parent( array: &Self::Array, parent: &ArrayRef, diff --git a/vortex-array/src/arrays/decimal/array.rs b/vortex-array/src/arrays/decimal/array.rs index 9f1be7e487a..66263455c37 100644 --- a/vortex-array/src/arrays/decimal/array.rs +++ b/vortex-array/src/arrays/decimal/array.rs @@ -11,6 +11,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; +use crate::ArrayRef; use crate::ExecutionCtx; use crate::IntoArray; use crate::arrays::PrimitiveArray; @@ -27,6 +28,11 @@ use crate::patches::Patches; use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable::ValidityHelper; +use crate::vtable::validity_to_child; + +pub(super) const VALIDITY_SLOT: usize = 0; +pub(super) const NUM_SLOTS: usize = 1; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; /// A decimal array that stores fixed-precision decimal numbers with configurable scale. /// @@ -87,6 +93,7 @@ use crate::vtable::ValidityHelper; /// ``` #[derive(Clone, Debug)] pub struct DecimalArray { + pub(super) slots: Vec>, pub(super) dtype: DType, pub(super) values: BufferHandle, pub(super) values_type: DecimalType, @@ -102,6 +109,10 @@ pub struct DecimalArrayParts { } impl DecimalArray { + fn make_slots(validity: &Validity, len: usize) -> Vec> { + vec![validity_to_child(validity, len)] + } + /// Creates a new [`DecimalArray`] using a host-native buffer. /// /// # Panics @@ -222,7 +233,9 @@ impl DecimalArray { .vortex_expect("[Debug Assertion]: Invalid `DecimalArray` parameters"); } + let len = values.len() / values_type.byte_width(); Self { + slots: Self::make_slots(&validity, len), values, values_type, dtype: DType::Decimal(decimal_dtype, validity.nullability()), diff --git a/vortex-array/src/arrays/decimal/vtable/mod.rs b/vortex-array/src/arrays/decimal/vtable/mod.rs index 37b62240121..456e8a9b777 100644 --- a/vortex-array/src/arrays/decimal/vtable/mod.rs +++ b/vortex-array/src/arrays/decimal/vtable/mod.rs @@ -3,7 +3,6 @@ use kernel::PARENT_KERNELS; use vortex_buffer::Alignment; -use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -18,6 +17,9 @@ use crate::IntoArray; use crate::ProstMetadata; use crate::SerializeMetadata; use crate::arrays::DecimalArray; +use crate::arrays::decimal::array::NUM_SLOTS; +use crate::arrays::decimal::array::SLOT_NAMES; +use crate::arrays::decimal::array::VALIDITY_SLOT; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::dtype::DecimalType; @@ -28,8 +30,6 @@ use crate::validity::Validity; use crate::vtable; use crate::vtable::VTable; use crate::vtable::ValidityVTableFromValidityHelper; -use crate::vtable::validity_nchildren; -use crate::vtable::validity_to_child; mod kernel; mod operations; mod validity; @@ -114,22 +114,6 @@ impl VTable for Decimal { } } - fn nchildren(array: &DecimalArray) -> usize { - validity_nchildren(&array.validity) - } - - fn child(array: &DecimalArray, idx: usize) -> ArrayRef { - match idx { - 0 => validity_to_child(&array.validity, array.len()) - .vortex_expect("DecimalArray child index out of bounds"), - _ => vortex_panic!("DecimalArray child index {idx} out of bounds"), - } - } - - fn child_name(_array: &DecimalArray, _idx: usize) -> String { - "validity".to_string() - } - fn metadata(array: &DecimalArray) -> VortexResult { Ok(ProstMetadata(DecimalMetadata { values_type: array.values_type() as i32, @@ -187,23 +171,26 @@ impl VTable for Decimal { }) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { + fn slots(array: &DecimalArray) -> &[Option] { + &array.slots + } + + fn slot_name(_array: &DecimalArray, idx: usize) -> String { + SLOT_NAMES[idx].to_string() + } + + fn with_slots(array: &mut DecimalArray, slots: Vec>) -> VortexResult<()> { vortex_ensure!( - children.len() <= 1, - "DecimalArray expects 0 or 1 child (validity), got {}", - children.len() + slots.len() == NUM_SLOTS, + "DecimalArray expects {} slots, got {}", + NUM_SLOTS, + slots.len() ); - - if children.is_empty() { - array.validity = Validity::from(array.dtype.nullability()); - } else { - array.validity = Validity::Array( - children - .into_iter() - .next() - .vortex_expect("children length already validated"), - ); - } + array.validity = match &slots[VALIDITY_SLOT] { + Some(arr) => Validity::Array(arr.clone()), + None => Validity::from(array.dtype.nullability()), + }; + array.slots = slots; Ok(()) } @@ -247,6 +234,7 @@ mod tests { use crate::LEGACY_SESSION; use crate::arrays::Decimal; use crate::arrays::DecimalArray; + use crate::assert_arrays_eq; use crate::dtype::DecimalDType; use crate::serde::ArrayParts; use crate::serde::SerializeOptions; @@ -280,4 +268,38 @@ mod tests { .unwrap(); assert!(decoded.is::()); } + + #[test] + fn test_nullable_decimal_serde_roundtrip() { + let array = DecimalArray::new( + buffer![1234567i32, 0i32, -9999999i32], + DecimalDType::new(7, 3), + Validity::from_iter([true, false, true]), + ); + let dtype = array.dtype().clone(); + let len = array.len(); + + let ctx = ArrayContext::empty(); + let out = array + .clone() + .into_array() + .serialize(&ctx, &SerializeOptions::default()) + .unwrap(); + let mut concat = ByteBufferMut::empty(); + for buf in out { + concat.extend_from_slice(buf.as_ref()); + } + + let parts = ArrayParts::try_from(concat.freeze()).unwrap(); + let decoded = parts + .decode( + &dtype, + len, + &ReadContext::new(ctx.to_ids()), + &LEGACY_SESSION, + ) + .unwrap(); + + assert_arrays_eq!(decoded, array); + } } diff --git a/vortex-array/src/arrays/dict/array.rs b/vortex-array/src/arrays/dict/array.rs index ac9bd1edb2e..20feb57d738 100644 --- a/vortex-array/src/arrays/dict/array.rs +++ b/vortex-array/src/arrays/dict/array.rs @@ -31,10 +31,14 @@ pub struct DictMetadata { pub(super) all_values_referenced: Option, } +pub(super) const CODES_SLOT: usize = 0; +pub(super) const VALUES_SLOT: usize = 1; +pub(super) const NUM_SLOTS: usize = 2; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["codes", "values"]; + #[derive(Debug, Clone)] pub struct DictArray { - pub(super) codes: ArrayRef, - pub(super) values: ArrayRef, + pub(super) slots: Vec>, pub(super) stats_set: ArrayStats, pub(super) dtype: DType, /// Indicates whether all dictionary values are definitely referenced by at least one code. @@ -63,8 +67,7 @@ impl DictArray { .dtype() .union_nullability(codes.dtype().nullability()); Self { - codes, - values, + slots: vec![Some(codes), Some(values)], stats_set: Default::default(), dtype, all_values_referenced: false, @@ -122,20 +125,28 @@ impl DictArray { pub fn into_parts(self) -> DictArrayParts { DictArrayParts { - codes: self.codes, - values: self.values, + codes: self.slots[CODES_SLOT] + .clone() + .vortex_expect("DictArray codes slot"), + values: self.slots[VALUES_SLOT] + .clone() + .vortex_expect("DictArray values slot"), dtype: self.dtype, } } #[inline] pub fn codes(&self) -> &ArrayRef { - &self.codes + self.slots[CODES_SLOT] + .as_ref() + .vortex_expect("DictArray codes slot") } #[inline] pub fn values(&self) -> &ArrayRef { - &self.values + self.slots[VALUES_SLOT] + .as_ref() + .vortex_expect("DictArray values slot") } /// Returns `true` if all dictionary values are definitely referenced by at least one code. diff --git a/vortex-array/src/arrays/dict/compute/rules.rs b/vortex-array/src/arrays/dict/compute/rules.rs index 30f8b4d6c61..939b2e16bce 100644 --- a/vortex-array/src/arrays/dict/compute/rules.rs +++ b/vortex-array/src/arrays/dict/compute/rules.rs @@ -85,8 +85,7 @@ impl ArrayParentReduceRule for DictionaryScalarFnValuesPushDownRule { // Check that all siblings are constant // TODO(ngates): we can also support other dictionaries if the values are the same! if !parent - .children() - .iter() + .iter_children() .enumerate() .all(|(idx, c)| idx == child_idx || c.is::()) { @@ -95,7 +94,9 @@ impl ArrayParentReduceRule for DictionaryScalarFnValuesPushDownRule { // If the scalar function is null-sensitive, then we cannot push it down to values if // we have any nulls in the codes. - if array.codes.dtype().is_nullable() && !array.codes.all_valid()? && sig.is_null_sensitive() + if array.codes().dtype().is_nullable() + && !array.codes().all_valid()? + && sig.is_null_sensitive() { tracing::trace!( "Not pushing down null-sensitive scalar function {} over dictionary with null codes {}", @@ -107,8 +108,8 @@ impl ArrayParentReduceRule for DictionaryScalarFnValuesPushDownRule { // Now we push the parent scalar function into the dictionary values. let values_len = array.values().len(); - let mut new_children = Vec::with_capacity(parent.children().len()); - for (idx, child) in parent.children().iter().enumerate() { + let mut new_children = Vec::with_capacity(parent.nchildren()); + for (idx, child) in parent.iter_children().enumerate() { if idx == child_idx { new_children.push(array.values().clone()); } else { @@ -151,13 +152,13 @@ impl ArrayParentReduceRule for DictionaryScalarFnCodesPullUpRule { child_idx: usize, ) -> VortexResult> { // Don't attempt to pull up if there are less than 2 siblings. - if parent.children().len() < 2 { + if parent.nchildren() < 2 { return Ok(None); } // Check that all siblings are dictionaries, and have the same number of values as us. // This is a cheap first loop. - if !parent.children().iter().enumerate().all(|(idx, c)| { + if !parent.iter_children().enumerate().all(|(idx, c)| { idx == child_idx || c.as_opt::() .is_some_and(|c| c.values().len() == array.values().len()) @@ -167,7 +168,7 @@ impl ArrayParentReduceRule for DictionaryScalarFnCodesPullUpRule { // Now run the slightly more expensive check that all siblings have the same codes as us. // We use the cheaper Precision::Ptr to avoid doing data comparisons. - if !parent.children().iter().enumerate().all(|(idx, c)| { + if !parent.iter_children().enumerate().all(|(idx, c)| { idx == child_idx || c.as_opt::() .is_some_and(|c| c.codes().array_eq(array.codes(), Precision::Value)) @@ -175,8 +176,8 @@ impl ArrayParentReduceRule for DictionaryScalarFnCodesPullUpRule { return Ok(None); } - let mut new_children = Vec::with_capacity(parent.children().len()); - for (idx, child) in parent.children().iter().enumerate() { + let mut new_children = Vec::with_capacity(parent.nchildren()); + for (idx, child) in parent.iter_children().enumerate() { if idx == child_idx { new_children.push(array.values().clone()); } else { @@ -184,10 +185,13 @@ impl ArrayParentReduceRule for DictionaryScalarFnCodesPullUpRule { } } - let new_values = - ScalarFnArray::try_new(parent.scalar_fn().clone(), new_children, array.values.len())? - .into_array() - .optimize()?; + let new_values = ScalarFnArray::try_new( + parent.scalar_fn().clone(), + new_children, + array.values().len(), + )? + .into_array() + .optimize()?; let new_dict = unsafe { DictArray::new_unchecked(array.codes().clone(), new_values) }.into_array(); diff --git a/vortex-array/src/arrays/dict/vtable/mod.rs b/vortex-array/src/arrays/dict/vtable/mod.rs index 2903967c230..3329bb9d54c 100644 --- a/vortex-array/src/arrays/dict/vtable/mod.rs +++ b/vortex-array/src/arrays/dict/vtable/mod.rs @@ -13,6 +13,8 @@ use vortex_session::VortexSession; use super::DictArray; use super::DictMetadata; +use super::array::NUM_SLOTS; +use super::array::SLOT_NAMES; use super::take_canonical; use crate::ArrayRef; use crate::Canonical; @@ -63,7 +65,7 @@ impl VTable for Dict { } fn len(array: &DictArray) -> usize { - array.codes.len() + array.codes().len() } fn dtype(array: &DictArray) -> &DType { @@ -76,14 +78,14 @@ impl VTable for Dict { fn array_hash(array: &DictArray, state: &mut H, precision: Precision) { array.dtype.hash(state); - array.codes.array_hash(state, precision); - array.values.array_hash(state, precision); + array.codes().array_hash(state, precision); + array.values().array_hash(state, precision); } fn array_eq(array: &DictArray, other: &DictArray, precision: Precision) -> bool { array.dtype == other.dtype - && array.codes.array_eq(&other.codes, precision) - && array.values.array_eq(&other.values, precision) + && array.codes().array_eq(other.codes(), precision) + && array.values().array_eq(other.values(), precision) } fn nbuffers(_array: &DictArray) -> usize { @@ -98,26 +100,6 @@ impl VTable for Dict { None } - fn nchildren(_array: &DictArray) -> usize { - 2 - } - - fn child(array: &DictArray, idx: usize) -> ArrayRef { - match idx { - 0 => array.codes().clone(), - 1 => array.values().clone(), - _ => vortex_panic!("DictArray child index {idx} out of bounds"), - } - } - - fn child_name(_array: &DictArray, idx: usize) -> String { - match idx { - 0 => "codes".to_string(), - 1 => "values".to_string(), - _ => vortex_panic!("DictArray child_name index {idx} out of bounds"), - } - } - fn metadata(array: &DictArray) -> VortexResult { Ok(ProstMetadata(DictMetadata { codes_ptype: PType::try_from(array.codes().dtype())? as i32, @@ -177,17 +159,22 @@ impl VTable for Dict { }) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { + fn slots(array: &DictArray) -> &[Option] { + &array.slots + } + + fn slot_name(_array: &DictArray, idx: usize) -> String { + SLOT_NAMES[idx].to_string() + } + + fn with_slots(array: &mut DictArray, slots: Vec>) -> VortexResult<()> { vortex_ensure!( - children.len() == 2, - "DictArray expects exactly 2 children (codes, values), got {}", - children.len() + slots.len() == NUM_SLOTS, + "DictArray expects exactly {} slots, got {}", + NUM_SLOTS, + slots.len() ); - let [codes, values]: [ArrayRef; 2] = children - .try_into() - .map_err(|_| vortex_err!("Failed to convert children to array"))?; - array.codes = codes; - array.values = values; + array.slots = slots; Ok(()) } @@ -246,10 +233,13 @@ pub(super) fn execute_fast_path( } // All codes are null - result is all nulls - if array.codes.all_invalid()? { + if array.codes().all_invalid()? { return Ok(Some( - ConstantArray::new(Scalar::null(array.dtype().as_nullable()), array.codes.len()) - .into_array(), + ConstantArray::new( + Scalar::null(array.dtype().as_nullable()), + array.codes().len(), + ) + .into_array(), )); } diff --git a/vortex-array/src/arrays/extension/array.rs b/vortex-array/src/arrays/extension/array.rs index 799646817a9..e3122613042 100644 --- a/vortex-array/src/arrays/extension/array.rs +++ b/vortex-array/src/arrays/extension/array.rs @@ -9,6 +9,10 @@ use crate::dtype::DType; use crate::dtype::extension::ExtDTypeRef; use crate::stats::ArrayStats; +pub(super) const STORAGE_SLOT: usize = 0; +pub(super) const NUM_SLOTS: usize = 1; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["storage"]; + /// An extension array that wraps another array with additional type information. /// /// **⚠️ Unstable API**: This is an experimental feature that may change significantly @@ -51,11 +55,7 @@ use crate::stats::ArrayStats; pub struct ExtensionArray { /// The storage dtype. This **must** be a [`Extension::DType`] variant. pub(super) dtype: DType, - - /// The backing storage array for this extension array. - pub(super) storage_array: ArrayRef, - - /// The stats for this array. + pub(super) slots: Vec>, pub(super) stats_set: ArrayStats, } @@ -108,7 +108,7 @@ impl ExtensionArray { Self { dtype: DType::Extension(ext_dtype), - storage_array, + slots: vec![Some(storage_array)], stats_set: ArrayStats::default(), } } @@ -123,6 +123,8 @@ impl ExtensionArray { } pub fn storage_array(&self) -> &ArrayRef { - &self.storage_array + self.slots[STORAGE_SLOT] + .as_ref() + .vortex_expect("ExtensionArray storage slot") } } diff --git a/vortex-array/src/arrays/extension/vtable/mod.rs b/vortex-array/src/arrays/extension/vtable/mod.rs index 243a0ef35d8..ee14f7e0824 100644 --- a/vortex-array/src/arrays/extension/vtable/mod.rs +++ b/vortex-array/src/arrays/extension/vtable/mod.rs @@ -8,7 +8,6 @@ mod validity; use std::hash::Hash; use kernel::PARENT_KERNELS; -use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -22,6 +21,8 @@ use crate::ExecutionStep; use crate::IntoArray; use crate::Precision; use crate::arrays::ExtensionArray; +use crate::arrays::extension::array::NUM_SLOTS; +use crate::arrays::extension::array::SLOT_NAMES; use crate::arrays::extension::compute::rules::PARENT_RULES; use crate::buffer::BufferHandle; use crate::dtype::DType; @@ -48,7 +49,7 @@ impl VTable for Extension { } fn len(array: &ExtensionArray) -> usize { - array.storage_array.len() + array.storage_array().len() } fn dtype(array: &ExtensionArray) -> &DType { @@ -65,14 +66,14 @@ impl VTable for Extension { precision: Precision, ) { array.dtype.hash(state); - array.storage_array.array_hash(state, precision); + array.storage_array().array_hash(state, precision); } fn array_eq(array: &ExtensionArray, other: &ExtensionArray, precision: Precision) -> bool { array.dtype == other.dtype && array - .storage_array - .array_eq(&other.storage_array, precision) + .storage_array() + .array_eq(other.storage_array(), precision) } fn nbuffers(_array: &ExtensionArray) -> usize { @@ -87,22 +88,12 @@ impl VTable for Extension { None } - fn nchildren(_array: &ExtensionArray) -> usize { - 1 + fn slots(array: &ExtensionArray) -> &[Option] { + &array.slots } - fn child(array: &ExtensionArray, idx: usize) -> ArrayRef { - match idx { - 0 => array.storage_array.clone(), - _ => vortex_panic!("ExtensionArray child index {idx} out of bounds"), - } - } - - fn child_name(_array: &ExtensionArray, idx: usize) -> String { - match idx { - 0 => "storage".to_string(), - _ => vortex_panic!("ExtensionArray child_name index {idx} out of bounds"), - } + fn slot_name(_array: &ExtensionArray, idx: usize) -> String { + SLOT_NAMES[idx].to_string() } fn metadata(_array: &ExtensionArray) -> VortexResult { @@ -140,16 +131,14 @@ impl VTable for Extension { Ok(ExtensionArray::new(ext_dtype.clone(), storage)) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { + fn with_slots(array: &mut Self::Array, slots: Vec>) -> VortexResult<()> { vortex_ensure!( - children.len() == 1, - "ExtensionArray expects exactly 1 child (storage), got {}", - children.len() + slots.len() == NUM_SLOTS, + "ExtensionArray expects exactly {} slots, got {}", + NUM_SLOTS, + slots.len() ); - array.storage_array = children - .into_iter() - .next() - .vortex_expect("children length already validated"); + array.slots = slots; Ok(()) } diff --git a/vortex-array/src/arrays/extension/vtable/validity.rs b/vortex-array/src/arrays/extension/vtable/validity.rs index 7bdd96f2f78..0e1429b6d10 100644 --- a/vortex-array/src/arrays/extension/vtable/validity.rs +++ b/vortex-array/src/arrays/extension/vtable/validity.rs @@ -8,6 +8,6 @@ use crate::vtable::ValidityChild; impl ValidityChild for Extension { fn validity_child(array: &ExtensionArray) -> &ArrayRef { - &array.storage_array + array.storage_array() } } diff --git a/vortex-array/src/arrays/filter/array.rs b/vortex-array/src/arrays/filter/array.rs index 40857487a65..fdbe28754b4 100644 --- a/vortex-array/src/arrays/filter/array.rs +++ b/vortex-array/src/arrays/filter/array.rs @@ -9,6 +9,10 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::stats::ArrayStats; +pub(super) const CHILD_SLOT: usize = 0; +pub(super) const NUM_SLOTS: usize = 1; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["child"]; + /// Decomposed parts of the filter array. pub struct FilterArrayParts { /// Child array that is filtered by the mask @@ -24,8 +28,8 @@ pub struct FilterArrayParts { /// The resulting array contains only the elements where the mask is true. #[derive(Clone, Debug)] pub struct FilterArray { - /// The source array being filtered. - pub(super) child: ArrayRef, + /// The slots holding child arrays. + pub(super) slots: Vec>, /// The boolean mask selecting which elements to keep. pub(super) mask: Mask, @@ -49,7 +53,7 @@ impl FilterArray { ); Ok(Self { - child: array, + slots: vec![Some(array)], mask, stats: ArrayStats::default(), }) @@ -57,7 +61,9 @@ impl FilterArray { /// The child array being filtered. pub fn child(&self) -> &ArrayRef { - &self.child + self.slots[CHILD_SLOT] + .as_ref() + .vortex_expect("FilterArray child slot") } /// The mask used to filter the child array. @@ -68,7 +74,9 @@ impl FilterArray { /// Consume the array and return its individual components. pub fn into_parts(self) -> FilterArrayParts { FilterArrayParts { - child: self.child, + child: self.slots[CHILD_SLOT] + .clone() + .vortex_expect("FilterArray child slot"), mask: self.mask, } } diff --git a/vortex-array/src/arrays/filter/execute/mod.rs b/vortex-array/src/arrays/filter/execute/mod.rs index da9b059d8ed..6afaf92cbaf 100644 --- a/vortex-array/src/arrays/filter/execute/mod.rs +++ b/vortex-array/src/arrays/filter/execute/mod.rs @@ -60,7 +60,7 @@ pub(super) fn execute_filter_fast_paths( // If the mask selects everything, then we can just fully decompress the whole thing. if true_count == array.mask.len() { - return Ok(Some(array.child.clone())); + return Ok(Some(array.child().clone())); } // Also check if the array itself is completely null, in which case we only care about the total diff --git a/vortex-array/src/arrays/filter/execute/struct_.rs b/vortex-array/src/arrays/filter/execute/struct_.rs index 1cdc4b17c64..15a2512f265 100644 --- a/vortex-array/src/arrays/filter/execute/struct_.rs +++ b/vortex-array/src/arrays/filter/execute/struct_.rs @@ -17,8 +17,7 @@ pub fn filter_struct(array: &StructArray, mask: &Arc) -> StructArray let mask_for_filter = values_to_mask(mask); let fields: Vec = array - .unmasked_fields() - .iter() + .iter_unmasked_fields() .map(|field| { field .filter(mask_for_filter.clone()) diff --git a/vortex-array/src/arrays/filter/rules.rs b/vortex-array/src/arrays/filter/rules.rs index f5683e86126..d54f0c0de8f 100644 --- a/vortex-array/src/arrays/filter/rules.rs +++ b/vortex-array/src/arrays/filter/rules.rs @@ -39,7 +39,7 @@ impl ArrayParentReduceRule for FilterFilterRule { _child_idx: usize, ) -> VortexResult> { let combined_mask = child.mask.intersect_by_rank(&parent.mask); - let new_array = child.child.filter(combined_mask)?; + let new_array = child.child().filter(combined_mask)?; Ok(Some(new_array.into_array())) } @@ -51,7 +51,7 @@ struct TrivialFilterRule; impl ArrayReduceRule for TrivialFilterRule { fn reduce(&self, array: &FilterArray) -> VortexResult> { match array.filter_mask() { - Mask::AllTrue(_) => Ok(Some(array.child.clone())), + Mask::AllTrue(_) => Ok(Some(array.child().clone())), Mask::AllFalse(_) => Ok(Some(Canonical::empty(array.dtype()).into_array())), Mask::Values(_) => Ok(None), } diff --git a/vortex-array/src/arrays/filter/vtable.rs b/vortex-array/src/arrays/filter/vtable.rs index b7099cb7c97..27b321206b4 100644 --- a/vortex-array/src/arrays/filter/vtable.rs +++ b/vortex-array/src/arrays/filter/vtable.rs @@ -5,7 +5,6 @@ use std::fmt::Debug; use std::fmt::Formatter; use std::hash::Hasher; -use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -20,6 +19,8 @@ use crate::DynArray; use crate::IntoArray; use crate::Precision; use crate::arrays::filter::array::FilterArray; +use crate::arrays::filter::array::NUM_SLOTS; +use crate::arrays::filter::array::SLOT_NAMES; use crate::arrays::filter::execute::execute_filter; use crate::arrays::filter::execute::execute_filter_fast_paths; use crate::arrays::filter::rules::PARENT_RULES; @@ -61,7 +62,7 @@ impl VTable for Filter { } fn dtype(array: &FilterArray) -> &DType { - array.child.dtype() + array.child().dtype() } fn stats(array: &FilterArray) -> StatsSetRef<'_> { @@ -69,12 +70,13 @@ impl VTable for Filter { } fn array_hash(array: &FilterArray, state: &mut H, precision: Precision) { - array.child.array_hash(state, precision); + array.child().array_hash(state, precision); array.mask.array_hash(state, precision); } fn array_eq(array: &FilterArray, other: &FilterArray, precision: Precision) -> bool { - array.child.array_eq(&other.child, precision) && array.mask.array_eq(&other.mask, precision) + array.child().array_eq(other.child(), precision) + && array.mask.array_eq(&other.mask, precision) } fn nbuffers(_array: &Self::Array) -> usize { @@ -89,22 +91,12 @@ impl VTable for Filter { None } - fn nchildren(_array: &Self::Array) -> usize { - 1 + fn slots(array: &Self::Array) -> &[Option] { + &array.slots } - fn child(array: &Self::Array, idx: usize) -> ArrayRef { - match idx { - 0 => array.child.clone(), - _ => vortex_panic!("FilterArray child index {idx} out of bounds"), - } - } - - fn child_name(_array: &Self::Array, idx: usize) -> String { - match idx { - 0 => "child".to_string(), - _ => vortex_panic!("FilterArray child_name index {idx} out of bounds"), - } + fn slot_name(_array: &Self::Array, idx: usize) -> String { + SLOT_NAMES[idx].to_string() } fn metadata(array: &Self::Array) -> VortexResult { @@ -135,23 +127,17 @@ impl VTable for Filter { ) -> VortexResult { assert_eq!(len, metadata.0.true_count()); let child = children.get(0, dtype, metadata.0.len())?; - Ok(FilterArray { - child, - mask: metadata.0.clone(), - stats: Default::default(), - }) + FilterArray::try_new(child, metadata.0.clone()) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { + fn with_slots(array: &mut Self::Array, slots: Vec>) -> VortexResult<()> { vortex_ensure!( - children.len() == 1, - "FilterArray expects exactly 1 child, got {}", - children.len() + slots.len() == NUM_SLOTS, + "FilterArray expects exactly {} slots, got {}", + NUM_SLOTS, + slots.len() ); - array.child = children - .into_iter() - .next() - .vortex_expect("children length already validated"); + array.slots = slots; Ok(()) } @@ -166,7 +152,7 @@ impl VTable for Filter { // We rely on the optimization pass that runs prior to this execution for filter pushdown, // so now we can just execute the filter without worrying. Ok(ExecutionStep::Done( - execute_filter(array.child.clone().execute(ctx)?, mask_values).into_array(), + execute_filter(array.child().clone().execute(ctx)?, mask_values).into_array(), )) } @@ -185,13 +171,13 @@ impl VTable for Filter { impl OperationsVTable for Filter { fn scalar_at(array: &FilterArray, index: usize) -> VortexResult { let rank_idx = array.mask.rank(index); - array.child.scalar_at(rank_idx) + array.child().scalar_at(rank_idx) } } impl ValidityVTable for Filter { fn validity(array: &FilterArray) -> VortexResult { - array.child.validity()?.filter(&array.mask) + array.child().validity()?.filter(&array.mask) } } diff --git a/vortex-array/src/arrays/fixed_size_list/array.rs b/vortex-array/src/arrays/fixed_size_list/array.rs index 12a9e044c11..78cff0078b8 100644 --- a/vortex-array/src/arrays/fixed_size_list/array.rs +++ b/vortex-array/src/arrays/fixed_size_list/array.rs @@ -12,6 +12,12 @@ use crate::DynArray; use crate::dtype::DType; use crate::stats::ArrayStats; use crate::validity::Validity; +use crate::vtable::validity_to_child; + +pub(super) const ELEMENTS_SLOT: usize = 0; +pub(super) const VALIDITY_SLOT: usize = 1; +pub(super) const NUM_SLOTS: usize = 2; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["elements", "validity"]; /// The canonical encoding for fixed-size list arrays. /// @@ -67,12 +73,8 @@ pub struct FixedSizeListArray { /// This type **must** be the variant [`DType::FixedSizeList`]. pub(super) dtype: DType, - /// The `elements` data array, where each fixed-size list scalar is a _slice_ of the `elements` - /// array, and each inner list element is a _scalar_ of the `elements` array. - /// - /// The fixed-size list scalars (or the elements of the array) are contiguous (regardless of - /// nullability for easy lookups), each with equal size in memory. - elements: ArrayRef, + /// Slots holding [elements]. + pub(super) slots: Vec>, /// The size of each fixed-size list scalar in the array. /// @@ -158,10 +160,11 @@ impl FixedSizeListArray { .vortex_expect("[Debug Assertion]: Invalid `FixedSizeListArray` parameters"); let nullability = validity.nullability(); + let validity_slot = validity_to_child(&validity, len); Self { dtype: DType::FixedSizeList(Arc::new(elements.dtype().clone()), list_size, nullability), - elements, + slots: vec![Some(elements), validity_slot], list_size, validity, len, @@ -170,7 +173,13 @@ impl FixedSizeListArray { } pub fn into_parts(self) -> (ArrayRef, Validity, DType) { - (self.elements, self.validity, self.dtype) + ( + self.slots[ELEMENTS_SLOT] + .clone() + .vortex_expect("FixedSizeListArray elements slot"), + self.validity, + self.dtype, + ) } /// Validates the components that would be used to create a [`FixedSizeListArray`]. @@ -211,7 +220,9 @@ impl FixedSizeListArray { /// Returns the elements array. pub fn elements(&self) -> &ArrayRef { - &self.elements + self.slots[ELEMENTS_SLOT] + .as_ref() + .vortex_expect("FixedSizeListArray elements slot") } /// The size of each fixed-size list scalar in the array. diff --git a/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs b/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs index d5c6f673671..977a5cfff53 100644 --- a/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs +++ b/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs @@ -3,7 +3,6 @@ use std::hash::Hash; -use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -17,6 +16,9 @@ use crate::ExecutionStep; use crate::IntoArray; use crate::Precision; use crate::arrays::FixedSizeListArray; +use crate::arrays::fixed_size_list::array::NUM_SLOTS; +use crate::arrays::fixed_size_list::array::SLOT_NAMES; +use crate::arrays::fixed_size_list::array::VALIDITY_SLOT; use crate::arrays::fixed_size_list::compute::rules::PARENT_RULES; use crate::buffer::BufferHandle; use crate::dtype::DType; @@ -29,8 +31,6 @@ use crate::vtable; use crate::vtable::ArrayId; use crate::vtable::VTable; use crate::vtable::ValidityVTableFromValidityHelper; -use crate::vtable::validity_nchildren; -use crate::vtable::validity_to_child; mod kernel; mod operations; mod validity; @@ -101,27 +101,6 @@ impl VTable for FixedSizeList { vortex_panic!("FixedSizeListArray buffer_name index {idx} out of bounds") } - fn nchildren(array: &FixedSizeListArray) -> usize { - 1 + validity_nchildren(&array.validity) - } - - fn child(array: &FixedSizeListArray, idx: usize) -> ArrayRef { - match idx { - 0 => array.elements().clone(), - 1 => validity_to_child(&array.validity, array.len()) - .vortex_expect("FixedSizeListArray validity child out of bounds"), - _ => vortex_panic!("FixedSizeListArray child index {idx} out of bounds"), - } - } - - fn child_name(_array: &FixedSizeListArray, idx: usize) -> String { - match idx { - 0 => "elements".to_string(), - 1 => "validity".to_string(), - _ => vortex_panic!("FixedSizeListArray child_name index {idx} out of bounds"), - } - } - fn reduce_parent( array: &Self::Array, parent: &ArrayRef, @@ -196,26 +175,29 @@ impl VTable for FixedSizeList { FixedSizeListArray::try_new(elements, *list_size, validity, len) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { + fn slots(array: &FixedSizeListArray) -> &[Option] { + &array.slots + } + + fn slot_name(_array: &FixedSizeListArray, idx: usize) -> String { + SLOT_NAMES[idx].to_string() + } + + fn with_slots( + array: &mut FixedSizeListArray, + slots: Vec>, + ) -> VortexResult<()> { vortex_ensure!( - children.len() == 1 || children.len() == 2, - "FixedSizeListArray expects 1 or 2 children, got {}", - children.len() + slots.len() == NUM_SLOTS, + "FixedSizeListArray expects exactly {} slots, got {}", + NUM_SLOTS, + slots.len() ); - - let mut iter = children.into_iter(); - let elements = iter - .next() - .vortex_expect("children length already validated"); - let validity = if let Some(validity_array) = iter.next() { - Validity::Array(validity_array) - } else { - Validity::from(array.dtype.nullability()) + array.validity = match &slots[VALIDITY_SLOT] { + Some(arr) => Validity::Array(arr.clone()), + None => Validity::from(array.dtype.nullability()), }; - - let new_array = - FixedSizeListArray::try_new(elements, array.list_size(), validity, array.len())?; - *array = new_array; + array.slots = slots; Ok(()) } diff --git a/vortex-array/src/arrays/list/array.rs b/vortex-array/src/arrays/list/array.rs index d4148c4f480..21b9f276481 100644 --- a/vortex-array/src/arrays/list/array.rs +++ b/vortex-array/src/arrays/list/array.rs @@ -27,6 +27,13 @@ use crate::match_each_native_ptype; use crate::scalar_fn::fns::operators::Operator; use crate::stats::ArrayStats; use crate::validity::Validity; +use crate::vtable::validity_to_child; + +pub(super) const ELEMENTS_SLOT: usize = 0; +pub(super) const OFFSETS_SLOT: usize = 1; +pub(super) const VALIDITY_SLOT: usize = 2; +pub(super) const NUM_SLOTS: usize = 3; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["elements", "offsets", "validity"]; /// A list array that stores variable-length lists of elements, similar to `Vec>`. /// @@ -83,8 +90,7 @@ use crate::validity::Validity; #[derive(Clone, Debug)] pub struct ListArray { pub(super) dtype: DType, - pub(super) elements: ArrayRef, - pub(super) offsets: ArrayRef, + pub(super) slots: Vec>, pub(super) validity: Validity, pub(super) stats_set: ArrayStats, } @@ -147,10 +153,12 @@ impl ListArray { Self::validate(&elements, &offsets, &validity) .vortex_expect("[Debug Assertion]: Invalid `ListViewArray` parameters"); + let len = offsets.len().saturating_sub(1); + let validity_slot = validity_to_child(&validity, len); + Self { dtype: DType::List(Arc::new(elements.dtype().clone()), validity.nullability()), - elements, - offsets, + slots: vec![Some(elements), Some(offsets), validity_slot], validity, stats_set: Default::default(), } @@ -245,8 +253,12 @@ impl ListArray { pub fn into_parts(self) -> ListArrayParts { ListArrayParts { dtype: self.dtype, - elements: self.elements, - offsets: self.offsets, + elements: self.slots[ELEMENTS_SLOT] + .clone() + .vortex_expect("ListArray elements slot"), + offsets: self.slots[OFFSETS_SLOT] + .clone() + .vortex_expect("ListArray offsets slot"), validity: self.validity, } } @@ -293,7 +305,9 @@ impl ListArray { /// Returns the offsets array. pub fn offsets(&self) -> &ArrayRef { - &self.offsets + self.slots[OFFSETS_SLOT] + .as_ref() + .vortex_expect("ListArray offsets slot") } /// Returns the element dtype of the list array. @@ -306,7 +320,9 @@ impl ListArray { /// Returns the elements array. pub fn elements(&self) -> &ArrayRef { - &self.elements + self.slots[ELEMENTS_SLOT] + .as_ref() + .vortex_expect("ListArray elements slot") } // TODO(connor)[ListView]: Create 2 functions `reset_offsets` and `recursive_reset_offsets`, diff --git a/vortex-array/src/arrays/list/vtable/mod.rs b/vortex-array/src/arrays/list/vtable/mod.rs index 13428f3a46b..5ec4a95cec3 100644 --- a/vortex-array/src/arrays/list/vtable/mod.rs +++ b/vortex-array/src/arrays/list/vtable/mod.rs @@ -3,7 +3,6 @@ use std::hash::Hash; -use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -18,6 +17,9 @@ use crate::IntoArray; use crate::Precision; use crate::ProstMetadata; use crate::arrays::ListArray; +use crate::arrays::list::array::NUM_SLOTS; +use crate::arrays::list::array::SLOT_NAMES; +use crate::arrays::list::array::VALIDITY_SLOT; use crate::arrays::list::compute::PARENT_KERNELS; use crate::arrays::list::compute::rules::PARENT_RULES; use crate::arrays::listview::list_view_from_list; @@ -36,8 +38,6 @@ use crate::vtable; use crate::vtable::ArrayId; use crate::vtable::VTable; use crate::vtable::ValidityVTableFromValidityHelper; -use crate::vtable::validity_nchildren; -use crate::vtable::validity_to_child; mod operations; mod validity; vtable!(List); @@ -61,7 +61,7 @@ impl VTable for List { } fn len(array: &ListArray) -> usize { - array.offsets.len().saturating_sub(1) + array.offsets().len().saturating_sub(1) } fn dtype(array: &ListArray) -> &DType { @@ -74,15 +74,15 @@ impl VTable for List { fn array_hash(array: &ListArray, state: &mut H, precision: Precision) { array.dtype.hash(state); - array.elements.array_hash(state, precision); - array.offsets.array_hash(state, precision); + array.elements().array_hash(state, precision); + array.offsets().array_hash(state, precision); array.validity.array_hash(state, precision); } fn array_eq(array: &ListArray, other: &ListArray, precision: Precision) -> bool { array.dtype == other.dtype - && array.elements.array_eq(&other.elements, precision) - && array.offsets.array_eq(&other.offsets, precision) + && array.elements().array_eq(other.elements(), precision) + && array.offsets().array_eq(other.offsets(), precision) && array.validity.array_eq(&other.validity, precision) } @@ -98,29 +98,6 @@ impl VTable for List { vortex_panic!("ListArray buffer_name index {idx} out of bounds") } - fn nchildren(array: &ListArray) -> usize { - 2 + validity_nchildren(&array.validity) - } - - fn child(array: &ListArray, idx: usize) -> ArrayRef { - match idx { - 0 => array.elements().clone(), - 1 => array.offsets().clone(), - 2 => validity_to_child(&array.validity, array.len()) - .vortex_expect("ListArray validity child out of bounds"), - _ => vortex_panic!("ListArray child index {idx} out of bounds"), - } - } - - fn child_name(_array: &ListArray, idx: usize) -> String { - match idx { - 0 => "elements".to_string(), - 1 => "offsets".to_string(), - 2 => "validity".to_string(), - _ => vortex_panic!("ListArray child_name index {idx} out of bounds"), - } - } - fn reduce_parent( array: &Self::Array, parent: &ArrayRef, @@ -186,28 +163,26 @@ impl VTable for List { ListArray::try_new(elements, offsets, validity) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { + fn slots(array: &ListArray) -> &[Option] { + &array.slots + } + + fn slot_name(_array: &ListArray, idx: usize) -> String { + SLOT_NAMES[idx].to_string() + } + + fn with_slots(array: &mut ListArray, slots: Vec>) -> VortexResult<()> { vortex_ensure!( - children.len() == 2 || children.len() == 3, - "ListArray expects 2 or 3 children, got {}", - children.len() + slots.len() == NUM_SLOTS, + "ListArray expects exactly {} slots, got {}", + NUM_SLOTS, + slots.len() ); - - let mut iter = children.into_iter(); - let elements = iter - .next() - .vortex_expect("children length already validated"); - let offsets = iter - .next() - .vortex_expect("children length already validated"); - let validity = if let Some(validity_array) = iter.next() { - Validity::Array(validity_array) - } else { - Validity::from(array.dtype.nullability()) + array.validity = match &slots[VALIDITY_SLOT] { + Some(arr) => Validity::Array(arr.clone()), + None => Validity::from(array.dtype.nullability()), }; - - let new_array = ListArray::try_new(elements, offsets, validity)?; - *array = new_array; + array.slots = slots; Ok(()) } diff --git a/vortex-array/src/arrays/listview/array.rs b/vortex-array/src/arrays/listview/array.rs index 3dfc9be786a..cadaede81c1 100644 --- a/vortex-array/src/arrays/listview/array.rs +++ b/vortex-array/src/arrays/listview/array.rs @@ -21,6 +21,14 @@ use crate::dtype::IntegerPType; use crate::match_each_integer_ptype; use crate::stats::ArrayStats; use crate::validity::Validity; +use crate::vtable::validity_to_child; + +pub(super) const ELEMENTS_SLOT: usize = 0; +pub(super) const OFFSETS_SLOT: usize = 1; +pub(super) const SIZES_SLOT: usize = 2; +pub(super) const VALIDITY_SLOT: usize = 3; +pub(super) const NUM_SLOTS: usize = 4; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["elements", "offsets", "sizes", "validity"]; /// The canonical encoding for variable-length list arrays. /// @@ -91,21 +99,8 @@ pub struct ListViewArray { /// This type **must** be the variant [`DType::List`]. pub(super) dtype: DType, - /// The `elements` data array, where each list scalar is a _slice_ of the `elements` array, and - /// each inner list element is a _scalar_ of the `elements` array. - elements: ArrayRef, - - /// The `offsets` array indicating the start position of each list in elements. - /// - /// Since we also store `sizes`, this `offsets` field is allowed to be stored out-of-order - /// (which is different from [`ListArray`](crate::arrays::ListArray)), - offsets: ArrayRef, - - /// The `sizes` array indicating the length of each list. - /// - /// This field is intended to be paired with a corresponding offset to determine the list scalar - /// we want to access. - sizes: ArrayRef, + /// Slots holding [elements, offsets, sizes]. + pub(super) slots: Vec>, // TODO(connor)[ListView]: Add the n+1 memory allocation optimization. /// A flag denoting if the array is zero-copyable* to a [`ListArray`](crate::arrays::ListArray). @@ -169,11 +164,12 @@ impl ListViewArray { ) -> VortexResult { Self::validate(&elements, &offsets, &sizes, &validity)?; + let len = offsets.len(); + let validity_slot = validity_to_child(&validity, len); + Ok(Self { dtype: DType::List(Arc::new(elements.dtype().clone()), validity.nullability()), - elements, - offsets, - sizes, + slots: vec![Some(elements), Some(offsets), Some(sizes), validity_slot], validity, is_zero_copy_to_list: false, stats_set: Default::default(), @@ -210,11 +206,12 @@ impl ListViewArray { .vortex_expect("Failed to crate `ListViewArray`"); } + let len = offsets.len(); + let validity_slot = validity_to_child(&validity, len); + Self { dtype: DType::List(Arc::new(elements.dtype().clone()), validity.nullability()), - elements, - offsets, - sizes, + slots: vec![Some(elements), Some(offsets), Some(sizes), validity_slot], validity, is_zero_copy_to_list: false, stats_set: Default::default(), @@ -305,9 +302,9 @@ impl ListViewArray { pub unsafe fn with_zero_copy_to_list(mut self, is_zctl: bool) -> Self { if cfg!(debug_assertions) && is_zctl { validate_zctl( - &self.elements, - self.offsets.to_primitive(), - self.sizes.to_primitive(), + self.elements(), + self.offsets().to_primitive(), + self.sizes().to_primitive(), ) .vortex_expect("Failed to validate zero-copy to list flag"); } @@ -334,9 +331,9 @@ impl ListViewArray { /// [`with_zero_copy_to_list`]: Self::with_zero_copy_to_list pub fn verify_is_zero_copy_to_list(&self) -> bool { validate_zctl( - &self.elements, - self.offsets.to_primitive(), - self.sizes.to_primitive(), + self.elements(), + self.offsets().to_primitive(), + self.sizes().to_primitive(), ) .is_ok() } @@ -345,9 +342,15 @@ impl ListViewArray { let dtype = self.dtype.into_list_element_opt().vortex_expect("is list"); ListViewArrayParts { elements_dtype: dtype, - elements: self.elements, - offsets: self.offsets, - sizes: self.sizes, + elements: self.slots[ELEMENTS_SLOT] + .clone() + .vortex_expect("ListViewArray elements slot"), + offsets: self.slots[OFFSETS_SLOT] + .clone() + .vortex_expect("ListViewArray offsets slot"), + sizes: self.slots[SIZES_SLOT] + .clone() + .vortex_expect("ListViewArray sizes slot"), validity: self.validity, } } @@ -365,12 +368,12 @@ impl ListViewArray { ); // Fast path for `PrimitiveArray`. - self.offsets + self.offsets() .as_opt::() .map(|p| match_each_integer_ptype!(p.ptype(), |P| { p.as_slice::

()[index].as_() })) .unwrap_or_else(|| { // Slow path: use `scalar_at` if we can't downcast directly to `PrimitiveArray`. - self.offsets + self.offsets() .scalar_at(index) .vortex_expect("offsets must support scalar_at") .as_primitive() @@ -393,12 +396,12 @@ impl ListViewArray { ); // Fast path for `PrimitiveArray`. - self.sizes + self.sizes() .as_opt::() .map(|p| match_each_integer_ptype!(p.ptype(), |P| { p.as_slice::

()[index].as_() })) .unwrap_or_else(|| { // Slow path: use `scalar_at` if we can't downcast directly to `PrimitiveArray`. - self.sizes + self.sizes() .scalar_at(index) .vortex_expect("sizes must support scalar_at") .as_primitive() @@ -420,17 +423,23 @@ impl ListViewArray { /// Returns the offsets array. pub fn offsets(&self) -> &ArrayRef { - &self.offsets + self.slots[OFFSETS_SLOT] + .as_ref() + .vortex_expect("ListViewArray offsets slot") } /// Returns the sizes array. pub fn sizes(&self) -> &ArrayRef { - &self.sizes + self.slots[SIZES_SLOT] + .as_ref() + .vortex_expect("ListViewArray sizes slot") } /// Returns the elements array. pub fn elements(&self) -> &ArrayRef { - &self.elements + self.slots[ELEMENTS_SLOT] + .as_ref() + .vortex_expect("ListViewArray elements slot") } /// Returns true if the `ListViewArray` is zero-copyable to a diff --git a/vortex-array/src/arrays/listview/vtable/mod.rs b/vortex-array/src/arrays/listview/vtable/mod.rs index a31edbb536f..74cda731579 100644 --- a/vortex-array/src/arrays/listview/vtable/mod.rs +++ b/vortex-array/src/arrays/listview/vtable/mod.rs @@ -3,7 +3,6 @@ use std::hash::Hash; -use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -19,6 +18,9 @@ use crate::Precision; use crate::ProstMetadata; use crate::SerializeMetadata; use crate::arrays::ListViewArray; +use crate::arrays::listview::array::NUM_SLOTS; +use crate::arrays::listview::array::SLOT_NAMES; +use crate::arrays::listview::array::VALIDITY_SLOT; use crate::arrays::listview::compute::rules::PARENT_RULES; use crate::buffer::BufferHandle; use crate::dtype::DType; @@ -33,8 +35,6 @@ use crate::vtable; use crate::vtable::ArrayId; use crate::vtable::VTable; use crate::vtable::ValidityVTableFromValidityHelper; -use crate::vtable::validity_nchildren; -use crate::vtable::validity_to_child; mod operations; mod validity; vtable!(ListView); @@ -111,31 +111,6 @@ impl VTable for ListView { vortex_panic!("ListViewArray buffer_name index {idx} out of bounds") } - fn nchildren(array: &ListViewArray) -> usize { - 3 + validity_nchildren(&array.validity) - } - - fn child(array: &ListViewArray, idx: usize) -> ArrayRef { - match idx { - 0 => array.elements().clone(), - 1 => array.offsets().clone(), - 2 => array.sizes().clone(), - 3 => validity_to_child(&array.validity, array.len()) - .vortex_expect("ListViewArray validity child out of bounds"), - _ => vortex_panic!("ListViewArray child index {idx} out of bounds"), - } - } - - fn child_name(_array: &ListViewArray, idx: usize) -> String { - match idx { - 0 => "elements".to_string(), - 1 => "offsets".to_string(), - 2 => "sizes".to_string(), - 3 => "validity".to_string(), - _ => vortex_panic!("ListViewArray child_name index {idx} out of bounds"), - } - } - fn metadata(array: &ListViewArray) -> VortexResult { Ok(ProstMetadata(ListViewMetadata { elements_len: array.elements().len() as u64, @@ -211,31 +186,26 @@ impl VTable for ListView { ListViewArray::try_new(elements, offsets, sizes, validity) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { + fn slots(array: &ListViewArray) -> &[Option] { + &array.slots + } + + fn slot_name(_array: &ListViewArray, idx: usize) -> String { + SLOT_NAMES[idx].to_string() + } + + fn with_slots(array: &mut ListViewArray, slots: Vec>) -> VortexResult<()> { vortex_ensure!( - children.len() == 3 || children.len() == 4, - "ListViewArray expects 3 or 4 children, got {}", - children.len() + slots.len() == NUM_SLOTS, + "ListViewArray expects exactly {} slots, got {}", + NUM_SLOTS, + slots.len() ); - - let mut iter = children.into_iter(); - let elements = iter - .next() - .vortex_expect("children length already validated"); - let offsets = iter - .next() - .vortex_expect("children length already validated"); - let sizes = iter - .next() - .vortex_expect("children length already validated"); - let validity = if let Some(validity_array) = iter.next() { - Validity::Array(validity_array) - } else { - Validity::from(array.dtype.nullability()) + array.validity = match &slots[VALIDITY_SLOT] { + Some(arr) => Validity::Array(arr.clone()), + None => Validity::from(array.dtype.nullability()), }; - - let new_array = ListViewArray::try_new(elements, offsets, sizes, validity)?; - *array = new_array; + array.slots = slots; Ok(()) } diff --git a/vortex-array/src/arrays/masked/array.rs b/vortex-array/src/arrays/masked/array.rs index 38317caacd7..437be1fc8ce 100644 --- a/vortex-array/src/arrays/masked/array.rs +++ b/vortex-array/src/arrays/masked/array.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; @@ -8,10 +9,16 @@ use crate::ArrayRef; use crate::dtype::DType; use crate::stats::ArrayStats; use crate::validity::Validity; +use crate::vtable::validity_to_child; + +pub(super) const CHILD_SLOT: usize = 0; +pub(super) const VALIDITY_SLOT: usize = 1; +pub(super) const NUM_SLOTS: usize = 2; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["child", "validity"]; #[derive(Clone, Debug)] pub struct MaskedArray { - pub(super) child: ArrayRef, + pub(super) slots: Vec>, pub(super) validity: Validity, pub(super) dtype: DType, pub(super) stats: ArrayStats, @@ -36,9 +43,11 @@ impl MaskedArray { // MaskedArray's nullability is determined solely by its validity, not the child's dtype. // The child can have nullable dtype but must not have any actual null values. let dtype = child.dtype().as_nullable(); + let len = child.len(); + let validity_slot = validity_to_child(&validity, len); Ok(Self { - child, + slots: vec![Some(child), validity_slot], validity, dtype, stats: ArrayStats::default(), @@ -46,6 +55,8 @@ impl MaskedArray { } pub fn child(&self) -> &ArrayRef { - &self.child + self.slots[CHILD_SLOT] + .as_ref() + .vortex_expect("MaskedArray child slot") } } diff --git a/vortex-array/src/arrays/masked/compute/filter.rs b/vortex-array/src/arrays/masked/compute/filter.rs index 31576a9c3ce..66e6e0c02cb 100644 --- a/vortex-array/src/arrays/masked/compute/filter.rs +++ b/vortex-array/src/arrays/masked/compute/filter.rs @@ -18,7 +18,7 @@ impl FilterReduce for Masked { // Filter the child array // The child is guaranteed to have no nulls, so filtering it is straightforward - let filtered_child = array.child.filter(mask.clone())?; + let filtered_child = array.child().filter(mask.clone())?; // Construct new MaskedArray Ok(Some( diff --git a/vortex-array/src/arrays/masked/compute/mask.rs b/vortex-array/src/arrays/masked/compute/mask.rs index b0ad8b00be6..86d6e142afa 100644 --- a/vortex-array/src/arrays/masked/compute/mask.rs +++ b/vortex-array/src/arrays/masked/compute/mask.rs @@ -22,9 +22,9 @@ impl MaskReduce for Masked { .and(Validity::Array(mask.clone()))? .to_array(array.len()); let masked_child = MaskExpr.try_new_array( - array.child.len(), + array.child().len(), EmptyOptions, - [array.child.clone(), combined_mask], + [array.child().clone(), combined_mask], )?; Ok(Some(masked_child)) } diff --git a/vortex-array/src/arrays/masked/compute/slice.rs b/vortex-array/src/arrays/masked/compute/slice.rs index eb81786c91b..870cbec21f9 100644 --- a/vortex-array/src/arrays/masked/compute/slice.rs +++ b/vortex-array/src/arrays/masked/compute/slice.rs @@ -10,21 +10,12 @@ use crate::IntoArray; use crate::arrays::Masked; use crate::arrays::MaskedArray; use crate::arrays::slice::SliceReduce; -use crate::stats::ArrayStats; impl SliceReduce for Masked { fn slice(array: &Self::Array, range: Range) -> VortexResult> { - let child = array.child.slice(range.clone())?; + let child = array.child().slice(range.clone())?; let validity = array.validity.slice(range)?; - Ok(Some( - MaskedArray { - child, - validity, - dtype: array.dtype.clone(), - stats: ArrayStats::default(), - } - .into_array(), - )) + Ok(Some(MaskedArray::try_new(child, validity)?.into_array())) } } diff --git a/vortex-array/src/arrays/masked/compute/take.rs b/vortex-array/src/arrays/masked/compute/take.rs index 2dad6672241..565349c4fc5 100644 --- a/vortex-array/src/arrays/masked/compute/take.rs +++ b/vortex-array/src/arrays/masked/compute/take.rs @@ -19,9 +19,9 @@ impl TakeReduce for Masked { // This is safe because we'll mask out these positions in the validity. let fill_scalar = Scalar::zero_value(indices.dtype()); let filled_take_indices = indices.to_array().fill_null(fill_scalar)?; - array.child.take(filled_take_indices)? + array.child().take(filled_take_indices)? } else { - array.child.take(indices.to_array())? + array.child().take(indices.to_array())? }; // Compute the new validity by taking from array's validity and merging with indices validity diff --git a/vortex-array/src/arrays/masked/vtable/mod.rs b/vortex-array/src/arrays/masked/vtable/mod.rs index 4dbe1670a43..3af1934adeb 100644 --- a/vortex-array/src/arrays/masked/vtable/mod.rs +++ b/vortex-array/src/arrays/masked/vtable/mod.rs @@ -6,7 +6,6 @@ mod validity; use std::hash::Hash; -use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -20,6 +19,9 @@ use crate::IntoArray; use crate::Precision; use crate::arrays::ConstantArray; use crate::arrays::MaskedArray; +use crate::arrays::masked::array::NUM_SLOTS; +use crate::arrays::masked::array::SLOT_NAMES; +use crate::arrays::masked::array::VALIDITY_SLOT; use crate::arrays::masked::compute::rules::PARENT_RULES; use crate::arrays::masked::mask_validity_canonical; use crate::buffer::BufferHandle; @@ -36,8 +38,6 @@ use crate::vtable; use crate::vtable::ArrayId; use crate::vtable::VTable; use crate::vtable::ValidityVTableFromValidityHelper; -use crate::vtable::validity_nchildren; -use crate::vtable::validity_to_child; vtable!(Masked); #[derive(Debug)] @@ -59,7 +59,7 @@ impl VTable for Masked { } fn len(array: &MaskedArray) -> usize { - array.child.len() + array.child().len() } fn dtype(array: &MaskedArray) -> &DType { @@ -71,13 +71,13 @@ impl VTable for Masked { } fn array_hash(array: &MaskedArray, state: &mut H, precision: Precision) { - array.child.array_hash(state, precision); + array.child().array_hash(state, precision); array.validity.array_hash(state, precision); array.dtype.hash(state); } fn array_eq(array: &MaskedArray, other: &MaskedArray, precision: Precision) -> bool { - array.child.array_eq(&other.child, precision) + array.child().array_eq(other.child(), precision) && array.validity.array_eq(&other.validity, precision) && array.dtype == other.dtype } @@ -94,27 +94,6 @@ impl VTable for Masked { None } - fn nchildren(array: &Self::Array) -> usize { - 1 + validity_nchildren(&array.validity) - } - - fn child(array: &Self::Array, idx: usize) -> ArrayRef { - match idx { - 0 => array.child.clone(), - 1 => validity_to_child(&array.validity, array.child.len()) - .vortex_expect("MaskedArray validity child out of bounds"), - _ => vortex_panic!("MaskedArray child index {idx} out of bounds"), - } - } - - fn child_name(_array: &Self::Array, idx: usize) -> String { - match idx { - 0 => "child".to_string(), - 1 => "validity".to_string(), - _ => vortex_panic!("MaskedArray child_name index {idx} out of bounds"), - } - } - fn metadata(_array: &MaskedArray) -> VortexResult { Ok(EmptyMetadata) } @@ -144,18 +123,19 @@ impl VTable for Masked { vortex_bail!("Expected 0 buffer, got {}", buffers.len()); } + vortex_ensure!( + children.len() == 1 || children.len() == 2, + "`MaskedArray::build` expects 1 or 2 children, got {}", + children.len() + ); + let child = children.get(0, &dtype.as_nonnullable(), len)?; - let validity = if children.len() == 1 { - Validity::from(dtype.nullability()) - } else if children.len() == 2 { + let validity = if children.len() == 2 { let validity = children.get(1, &Validity::DTYPE, len)?; Validity::Array(validity) } else { - vortex_bail!( - "`MaskedArray::build` expects 1 or 2 children, got {}", - children.len() - ); + Validity::from(dtype.nullability()) }; MaskedArray::try_new(child, validity) @@ -192,25 +172,26 @@ impl VTable for Masked { PARENT_RULES.evaluate(array, parent, child_idx) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { + fn slots(array: &MaskedArray) -> &[Option] { + &array.slots + } + + fn slot_name(_array: &MaskedArray, idx: usize) -> String { + SLOT_NAMES[idx].to_string() + } + + fn with_slots(array: &mut MaskedArray, slots: Vec>) -> VortexResult<()> { vortex_ensure!( - children.len() == 1 || children.len() == 2, - "MaskedArray expects 1 or 2 children, got {}", - children.len() + slots.len() == NUM_SLOTS, + "MaskedArray expects exactly {} slots, got {}", + NUM_SLOTS, + slots.len() ); - - let mut iter = children.into_iter(); - let child = iter - .next() - .vortex_expect("children length already validated"); - let validity = if let Some(validity_array) = iter.next() { - Validity::Array(validity_array) - } else { - Validity::from(array.dtype.nullability()) + array.validity = match &slots[VALIDITY_SLOT] { + Some(arr) => Validity::Array(arr.clone()), + None => Validity::from(array.dtype.nullability()), }; - - let new_array = MaskedArray::try_new(child, validity)?; - *array = new_array; + array.slots = slots; Ok(()) } } diff --git a/vortex-array/src/arrays/masked/vtable/operations.rs b/vortex-array/src/arrays/masked/vtable/operations.rs index 220afcf291f..fff05a7f893 100644 --- a/vortex-array/src/arrays/masked/vtable/operations.rs +++ b/vortex-array/src/arrays/masked/vtable/operations.rs @@ -12,6 +12,6 @@ use crate::vtable::OperationsVTable; impl OperationsVTable for Masked { fn scalar_at(array: &MaskedArray, index: usize) -> VortexResult { // Invalid indices are handled by the entrypoint function. - Ok(array.child.scalar_at(index)?.into_nullable()) + Ok(array.child().scalar_at(index)?.into_nullable()) } } diff --git a/vortex-array/src/arrays/null/mod.rs b/vortex-array/src/arrays/null/mod.rs index e2ac02f247c..f782202c0df 100644 --- a/vortex-array/src/arrays/null/mod.rs +++ b/vortex-array/src/arrays/null/mod.rs @@ -28,6 +28,8 @@ use crate::vtable::OperationsVTable; use crate::vtable::VTable; use crate::vtable::ValidityVTable; +const NUM_SLOTS: usize = 0; + pub(crate) mod compute; vtable!(Null); @@ -75,16 +77,23 @@ impl VTable for Null { None } - fn nchildren(_array: &NullArray) -> usize { - 0 + fn slots(array: &NullArray) -> &[Option] { + &array.slots } - fn child(_array: &NullArray, idx: usize) -> ArrayRef { - vortex_panic!("NullArray child index {idx} out of bounds") + fn slot_name(_array: &NullArray, idx: usize) -> String { + vortex_panic!("NullArray slot_name index {idx} out of bounds") } - fn child_name(_array: &NullArray, idx: usize) -> String { - vortex_panic!("NullArray child_name index {idx} out of bounds") + fn with_slots(array: &mut NullArray, slots: Vec>) -> VortexResult<()> { + vortex_ensure!( + slots.len() == NUM_SLOTS, + "NullArray expects exactly {} slots, got {}", + NUM_SLOTS, + slots.len() + ); + array.slots = slots; + Ok(()) } fn metadata(_array: &NullArray) -> VortexResult { @@ -115,15 +124,6 @@ impl VTable for Null { Ok(NullArray::new(len)) } - fn with_children(_array: &mut Self::Array, children: Vec) -> VortexResult<()> { - vortex_ensure!( - children.is_empty(), - "NullArray has no children, got {}", - children.len() - ); - Ok(()) - } - fn reduce_parent( array: &Self::Array, parent: &ArrayRef, @@ -167,6 +167,7 @@ impl VTable for Null { #[derive(Clone, Debug)] pub struct NullArray { len: usize, + slots: Vec>, stats_set: ArrayStats, } @@ -181,6 +182,7 @@ impl NullArray { pub fn new(len: usize) -> Self { Self { len, + slots: vec![], stats_set: Default::default(), } } diff --git a/vortex-array/src/arrays/primitive/array/mod.rs b/vortex-array/src/arrays/primitive/array/mod.rs index 37c9b475800..ac9284e4856 100644 --- a/vortex-array/src/arrays/primitive/array/mod.rs +++ b/vortex-array/src/arrays/primitive/array/mod.rs @@ -31,7 +31,13 @@ mod top_value; pub use patch::chunk_range; pub use patch::patch_chunk; +use crate::ArrayRef; use crate::buffer::BufferHandle; +use crate::vtable::validity_to_child; + +pub(super) const VALIDITY_SLOT: usize = 0; +pub(super) const NUM_SLOTS: usize = 1; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; /// A primitive array that stores [native types][crate::dtype::NativePType] in a contiguous buffer /// of memory, along with an optional validity child. @@ -70,6 +76,7 @@ use crate::buffer::BufferHandle; /// ``` #[derive(Clone, Debug)] pub struct PrimitiveArray { + pub(super) slots: Vec>, pub(super) dtype: DType, pub(super) buffer: BufferHandle, pub(super) validity: Validity, @@ -84,6 +91,10 @@ pub struct PrimitiveArrayParts { // TODO(connor): There are a lot of places where we could be using `new_unchecked` in the codebase. impl PrimitiveArray { + fn make_slots(validity: &Validity, len: usize) -> Vec> { + vec![validity_to_child(validity, len)] + } + /// Create a new array from a buffer handle. /// /// # Safety @@ -95,7 +106,9 @@ impl PrimitiveArray { ptype: PType, validity: Validity, ) -> Self { + let len = handle.len() / ptype.byte_width(); Self { + slots: Self::make_slots(&validity, len), buffer: handle, dtype: DType::Primitive(ptype, validity.nullability()), validity, @@ -148,7 +161,9 @@ impl PrimitiveArray { Self::validate(&buffer, &validity) .vortex_expect("[Debug Assertion]: Invalid `PrimitiveArray` parameters"); + let len = buffer.len(); Self { + slots: Self::make_slots(&validity, len), dtype: DType::Primitive(T::PTYPE, validity.nullability()), buffer: BufferHandle::new_host(buffer.into_byte_buffer()), validity, @@ -203,7 +218,9 @@ impl PrimitiveArray { pub fn from_buffer_handle(handle: BufferHandle, ptype: PType, validity: Validity) -> Self { let dtype = DType::Primitive(ptype, validity.nullability()); + let len = handle.len() / ptype.byte_width(); Self { + slots: Self::make_slots(&validity, len), buffer: handle, dtype, validity, diff --git a/vortex-array/src/arrays/primitive/vtable/mod.rs b/vortex-array/src/arrays/primitive/vtable/mod.rs index c48a72217b1..8e5a625498d 100644 --- a/vortex-array/src/arrays/primitive/vtable/mod.rs +++ b/vortex-array/src/arrays/primitive/vtable/mod.rs @@ -2,7 +2,6 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use kernel::PARENT_KERNELS; -use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -14,6 +13,9 @@ use crate::ExecutionCtx; use crate::ExecutionStep; use crate::IntoArray; use crate::arrays::PrimitiveArray; +use crate::arrays::primitive::array::NUM_SLOTS; +use crate::arrays::primitive::array::SLOT_NAMES; +use crate::arrays::primitive::array::VALIDITY_SLOT; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::dtype::PType; @@ -22,8 +24,6 @@ use crate::validity::Validity; use crate::vtable; use crate::vtable::VTable; use crate::vtable::ValidityVTableFromValidityHelper; -use crate::vtable::validity_nchildren; -use crate::vtable::validity_to_child; mod kernel; mod operations; mod validity; @@ -96,22 +96,6 @@ impl VTable for Primitive { } } - fn nchildren(array: &PrimitiveArray) -> usize { - validity_nchildren(&array.validity) - } - - fn child(array: &PrimitiveArray, idx: usize) -> ArrayRef { - match idx { - 0 => validity_to_child(&array.validity, array.len()) - .vortex_expect("PrimitiveArray child index out of bounds"), - _ => vortex_panic!("PrimitiveArray child index {idx} out of bounds"), - } - } - - fn child_name(_array: &PrimitiveArray, _idx: usize) -> String { - "validity".to_string() - } - fn metadata(_array: &PrimitiveArray) -> VortexResult { Ok(EmptyMetadata) } @@ -183,19 +167,26 @@ impl VTable for Primitive { } } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { + fn slots(array: &PrimitiveArray) -> &[Option] { + &array.slots + } + + fn slot_name(_array: &PrimitiveArray, idx: usize) -> String { + SLOT_NAMES[idx].to_string() + } + + fn with_slots(array: &mut PrimitiveArray, slots: Vec>) -> VortexResult<()> { vortex_ensure!( - children.len() <= 1, - "PrimitiveArray can have at most 1 child (validity), got {}", - children.len() + slots.len() == NUM_SLOTS, + "PrimitiveArray expects {} slots, got {}", + NUM_SLOTS, + slots.len() ); - - array.validity = if children.is_empty() { - Validity::from(array.dtype().nullability()) - } else { - Validity::Array(children.into_iter().next().vortex_expect("checked")) + array.validity = match &slots[VALIDITY_SLOT] { + Some(arr) => Validity::Array(arr.clone()), + None => Validity::from(array.dtype().nullability()), }; - + array.slots = slots; Ok(()) } @@ -227,3 +218,52 @@ pub struct Primitive; impl Primitive { pub const ID: ArrayId = ArrayId::new_ref("vortex.primitive"); } + +#[cfg(test)] +mod tests { + use vortex_buffer::ByteBufferMut; + use vortex_buffer::buffer; + use vortex_session::registry::ReadContext; + + use crate::ArrayContext; + use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::arrays::PrimitiveArray; + use crate::assert_arrays_eq; + use crate::serde::ArrayParts; + use crate::serde::SerializeOptions; + use crate::validity::Validity; + + #[test] + fn test_nullable_primitive_serde_roundtrip() { + let array = PrimitiveArray::new( + buffer![1i32, 2, 3, 4], + Validity::from_iter([true, false, true, false]), + ); + let dtype = array.dtype().clone(); + let len = array.len(); + + let ctx = ArrayContext::empty(); + let serialized = array + .clone() + .into_array() + .serialize(&ctx, &SerializeOptions::default()) + .unwrap(); + + let mut concat = ByteBufferMut::empty(); + for buf in serialized { + concat.extend_from_slice(buf.as_ref()); + } + let parts = ArrayParts::try_from(concat.freeze()).unwrap(); + let decoded = parts + .decode( + &dtype, + len, + &ReadContext::new(ctx.to_ids()), + &LEGACY_SESSION, + ) + .unwrap(); + + assert_arrays_eq!(decoded, array); + } +} diff --git a/vortex-array/src/arrays/scalar_fn/array.rs b/vortex-array/src/arrays/scalar_fn/array.rs index 1e3ef1d14ba..c50665a1f90 100644 --- a/vortex-array/src/arrays/scalar_fn/array.rs +++ b/vortex-array/src/arrays/scalar_fn/array.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; @@ -10,12 +11,14 @@ use crate::dtype::DType; use crate::scalar_fn::ScalarFnRef; use crate::stats::ArrayStats; +// ScalarFnArray has a variable number of slots (one per child) + #[derive(Clone, Debug)] pub struct ScalarFnArray { pub(super) scalar_fn: ScalarFnRef, pub(super) dtype: DType, pub(super) len: usize, - pub(super) children: Vec, + pub(super) slots: Vec>, pub(super) stats: ArrayStats, } @@ -30,11 +33,13 @@ impl ScalarFnArray { "ScalarFnArray must have children equal to the array length" ); + let slots = children.into_iter().map(Some).collect(); + Ok(Self { scalar_fn: bound, dtype, len, - children, + slots, stats: Default::default(), }) } @@ -45,8 +50,27 @@ impl ScalarFnArray { &self.scalar_fn } + /// Get a child array by index. + pub fn get_child(&self, idx: usize) -> &ArrayRef { + self.slots[idx] + .as_ref() + .vortex_expect("ScalarFnArray child slot") + } + + /// Get the number of children. + pub fn nchildren(&self) -> usize { + self.slots.len() + } + + /// Iterate over the children arrays without allocation. + pub fn iter_children(&self) -> impl Iterator + '_ { + self.slots + .iter() + .map(|s| s.as_ref().vortex_expect("ScalarFnArray child slot")) + } + /// Get the children arrays of this scalar function array. - pub fn children(&self) -> &[ArrayRef] { - &self.children + pub fn children(&self) -> Vec { + self.iter_children().cloned().collect() } } diff --git a/vortex-array/src/arrays/scalar_fn/rules.rs b/vortex-array/src/arrays/scalar_fn/rules.rs index 00318c4fbd7..ef2e1f2142f 100644 --- a/vortex-array/src/arrays/scalar_fn/rules.rs +++ b/vortex-array/src/arrays/scalar_fn/rules.rs @@ -60,7 +60,7 @@ impl ArrayReduceRule for ScalarFnPackToStructRule { Ok(Some( StructArray::try_new( pack_options.names.clone(), - array.children.clone(), + array.children(), array.len, validity, )? @@ -73,7 +73,7 @@ impl ArrayReduceRule for ScalarFnPackToStructRule { struct ScalarFnConstantRule; impl ArrayReduceRule for ScalarFnConstantRule { fn reduce(&self, array: &ScalarFnArray) -> VortexResult> { - if !array.children.iter().all(|c| c.is::()) { + if !array.iter_children().all(|c| c.is::()) { return Ok(None); } if array.is_empty() { @@ -120,11 +120,11 @@ impl ReduceNode for ScalarFnArray { } fn child(&self, idx: usize) -> ReduceNodeRef { - Arc::new(self.children()[idx].clone()) + Arc::new(self.get_child(idx).clone()) } fn child_count(&self) -> usize { - self.children.len() + self.nchildren() } } @@ -194,15 +194,13 @@ impl ArrayParentReduceRule for ScalarFnUnaryFilterPushDownRule { // If we only have one non-constant child, then it is _always_ cheaper to push down the // filter over the children of the scalar function array. if child - .children - .iter() + .iter_children() .filter(|c| !c.is::()) .count() == 1 { let new_children: Vec<_> = child - .children - .iter() + .iter_children() .map(|c| match c.as_opt::() { Some(array) => { Ok(ConstantArray::new(array.scalar().clone(), parent.len()).into_array()) diff --git a/vortex-array/src/arrays/scalar_fn/slice.rs b/vortex-array/src/arrays/scalar_fn/slice.rs index b2389475b86..d8c816a2aee 100644 --- a/vortex-array/src/arrays/scalar_fn/slice.rs +++ b/vortex-array/src/arrays/scalar_fn/slice.rs @@ -14,8 +14,7 @@ use crate::arrays::slice::SliceReduce; impl SliceReduce for ScalarFnVTable { fn slice(array: &Self::Array, range: Range) -> VortexResult> { let children: Vec<_> = array - .children() - .iter() + .iter_children() .map(|c| c.slice(range.clone())) .collect::>()?; @@ -24,7 +23,7 @@ impl SliceReduce for ScalarFnVTable { scalar_fn: array.scalar_fn.clone(), dtype: array.dtype.clone(), len: range.len(), - children, + slots: children.into_iter().map(Some).collect(), stats: Default::default(), } .into_array(), diff --git a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs index 9d8131c7612..8451eac9f17 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs @@ -75,7 +75,7 @@ impl VTable for ScalarFnVTable { array.len.hash(state); array.dtype.hash(state); array.scalar_fn.hash(state); - for child in &array.children { + for child in array.iter_children() { child.array_hash(state, precision); } } @@ -90,7 +90,7 @@ impl VTable for ScalarFnVTable { if array.scalar_fn != other.scalar_fn { return false; } - for (child, other_child) in array.children.iter().zip(other.children.iter()) { + for (child, other_child) in array.iter_children().zip(other.iter_children()) { if !child.array_eq(other_child, precision) { return false; } @@ -106,29 +106,12 @@ impl VTable for ScalarFnVTable { vortex_panic!("ScalarFnArray buffer index {idx} out of bounds") } - fn buffer_name(_array: &ScalarFnArray, idx: usize) -> Option { - vortex_panic!("ScalarFnArray buffer_name index {idx} out of bounds") - } - - fn nchildren(array: &ScalarFnArray) -> usize { - array.children.len() - } - - fn child(array: &ScalarFnArray, idx: usize) -> ArrayRef { - array.children[idx].clone() - } - - fn child_name(array: &ScalarFnArray, idx: usize) -> String { - array - .scalar_fn - .signature() - .child_name(idx) - .as_ref() - .to_string() + fn buffer_name(_array: &ScalarFnArray, _idx: usize) -> Option { + None } fn metadata(array: &Self::Array) -> VortexResult { - let child_dtypes = array.children().iter().map(|c| c.dtype().clone()).collect(); + let child_dtypes = array.iter_children().map(|c| c.dtype().clone()).collect(); Ok(ScalarFnMetadata { scalar_fn: array.scalar_fn.clone(), child_dtypes, @@ -178,25 +161,32 @@ impl VTable for ScalarFnVTable { scalar_fn: metadata.scalar_fn.clone(), dtype: dtype.clone(), len, - children, + slots: children.into_iter().map(Some).collect(), stats: Default::default(), }) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { - vortex_ensure!( - children.len() == array.children.len(), - "ScalarFnArray expects {} children, got {}", - array.children.len(), - children.len() - ); - array.children = children; + fn slots(array: &ScalarFnArray) -> &[Option] { + &array.slots + } + + fn slot_name(array: &ScalarFnArray, idx: usize) -> String { + array + .scalar_fn + .signature() + .child_name(idx) + .as_ref() + .to_string() + } + + fn with_slots(array: &mut ScalarFnArray, slots: Vec>) -> VortexResult<()> { + array.slots = slots; Ok(()) } fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { ctx.log(format_args!("scalar_fn({}): executing", array.scalar_fn)); - let args = VecExecutionArgs::new(array.children.clone(), array.len); + let args = VecExecutionArgs::new(array.children(), array.len); array.scalar_fn.execute(&args, ctx).map(ExecutionStep::Done) } @@ -236,7 +226,7 @@ pub trait ScalarFnArrayExt: scalar_fn::ScalarFnVTable { scalar_fn, dtype, len, - children, + slots: children.into_iter().map(Some).collect(), stats: Default::default(), } .into_array()) diff --git a/vortex-array/src/arrays/scalar_fn/vtable/operations.rs b/vortex-array/src/arrays/scalar_fn/vtable/operations.rs index 628c829a4c3..cc704697009 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/operations.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/operations.rs @@ -18,7 +18,7 @@ use crate::vtable::OperationsVTable; impl OperationsVTable for ScalarFnVTable { fn scalar_at(array: &ScalarFnArray, index: usize) -> VortexResult { let inputs: Vec<_> = array - .children + .children() .iter() .map(|child| Ok(ConstantArray::new(child.scalar_at(index)?, 1).into_array())) .collect::>()?; diff --git a/vortex-array/src/arrays/scalar_fn/vtable/validity.rs b/vortex-array/src/arrays/scalar_fn/vtable/validity.rs index 8da32d352ec..dfd19b5f25f 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/validity.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/validity.rs @@ -52,8 +52,7 @@ fn execute_expr(expr: &Expression, row_count: usize) -> VortexResult { impl ValidityVTable for ScalarFnVTable { fn validity(array: &ScalarFnArray) -> VortexResult { let inputs: Vec<_> = array - .children - .iter() + .iter_children() .map(|child| { if let Some(scalar) = child.as_constant() { return Ok(lit(scalar)); diff --git a/vortex-array/src/arrays/shared/array.rs b/vortex-array/src/arrays/shared/array.rs index c69ae39395d..758a093666f 100644 --- a/vortex-array/src/arrays/shared/array.rs +++ b/vortex-array/src/arrays/shared/array.rs @@ -7,6 +7,7 @@ use std::sync::OnceLock; use async_lock::Mutex as AsyncMutex; use vortex_error::SharedVortexResult; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::ArrayRef; @@ -15,15 +16,19 @@ use crate::IntoArray; use crate::dtype::DType; use crate::stats::ArrayStats; +pub(super) const SOURCE_SLOT: usize = 0; +pub(super) const NUM_SLOTS: usize = 1; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["source"]; + /// A lazily-executing array wrapper with a one-way transition from source to cached form. /// /// Before materialization, operations delegate to the source array. /// After materialization (via `get_or_compute`), operations delegate to the cached result. #[derive(Debug, Clone)] pub struct SharedArray { - source: ArrayRef, - cached: Arc>>, - async_compute_lock: Arc>, + pub(super) slots: Vec>, + pub(super) cached: Arc>>, + pub(super) async_compute_lock: Arc>, pub(super) dtype: DType, pub(super) stats: ArrayStats, } @@ -32,13 +37,20 @@ impl SharedArray { pub fn new(source: ArrayRef) -> Self { Self { dtype: source.dtype().clone(), - source, + slots: vec![Some(source)], cached: Arc::new(OnceLock::new()), async_compute_lock: Arc::new(AsyncMutex::new(())), stats: ArrayStats::default(), } } + /// Returns the source array reference. + pub(super) fn source(&self) -> &ArrayRef { + self.slots[SOURCE_SLOT] + .as_ref() + .vortex_expect("SharedArray source slot") + } + /// Returns the current array reference. /// /// After materialization, returns the cached result. Otherwise, returns the source. @@ -46,7 +58,7 @@ impl SharedArray { pub(super) fn current_array_ref(&self) -> &ArrayRef { match self.cached.get() { Some(Ok(arr)) => arr, - _ => &self.source, + _ => self.source(), } } @@ -59,7 +71,7 @@ impl SharedArray { ) -> VortexResult { let result = self .cached - .get_or_init(|| f(&self.source).map(|c| c.into_array()).map_err(Arc::new)); + .get_or_init(|| f(self.source()).map(|c| c.into_array()).map_err(Arc::new)); result.clone().map_err(Into::into) } @@ -82,7 +94,7 @@ impl SharedArray { return result.clone().map_err(Into::into); } - let computed = f(self.source.clone()) + let computed = f(self.source().clone()) .await .map(|c| c.into_array()) .map_err(Arc::new); @@ -90,11 +102,4 @@ impl SharedArray { let result = self.cached.get_or_init(|| computed); result.clone().map_err(Into::into) } - - pub(super) fn set_source(&mut self, source: ArrayRef) { - self.dtype = source.dtype().clone(); - self.source = source; - self.cached = Arc::new(OnceLock::new()); - self.async_compute_lock = Arc::new(AsyncMutex::new(())); - } } diff --git a/vortex-array/src/arrays/shared/vtable.rs b/vortex-array/src/arrays/shared/vtable.rs index 5880ebae568..427639e83fc 100644 --- a/vortex-array/src/arrays/shared/vtable.rs +++ b/vortex-array/src/arrays/shared/vtable.rs @@ -2,9 +2,12 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::hash::Hash; +use std::sync::Arc; +use std::sync::OnceLock; -use vortex_error::VortexExpect; +use async_lock::Mutex as AsyncMutex; use vortex_error::VortexResult; +use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; @@ -15,6 +18,8 @@ use crate::ExecutionCtx; use crate::ExecutionStep; use crate::Precision; use crate::arrays::SharedArray; +use crate::arrays::shared::array::NUM_SLOTS; +use crate::arrays::shared::array::SLOT_NAMES; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::hash::ArrayEq; @@ -84,22 +89,25 @@ impl VTable for Shared { None } - fn nchildren(_array: &Self::Array) -> usize { - 1 + fn slots(array: &SharedArray) -> &[Option] { + &array.slots } - fn child(array: &Self::Array, idx: usize) -> ArrayRef { - match idx { - 0 => array.current_array_ref().clone(), - _ => vortex_panic!("SharedArray child index {idx} out of bounds"), - } + fn slot_name(_array: &SharedArray, idx: usize) -> String { + SLOT_NAMES[idx].to_string() } - fn child_name(_array: &Self::Array, idx: usize) -> String { - match idx { - 0 => "source".to_string(), - _ => vortex_panic!("SharedArray child_name index {idx} out of bounds"), - } + fn with_slots(array: &mut Self::Array, slots: Vec>) -> VortexResult<()> { + vortex_ensure!( + slots.len() == NUM_SLOTS, + "SharedArray expects exactly {} slots, got {}", + NUM_SLOTS, + slots.len() + ); + array.slots = slots; + array.cached = Arc::new(OnceLock::new()); + array.async_compute_lock = Arc::new(AsyncMutex::new(())); + Ok(()) } fn metadata(_array: &Self::Array) -> VortexResult { @@ -131,20 +139,6 @@ impl VTable for Shared { Ok(SharedArray::new(child)) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { - vortex_error::vortex_ensure!( - children.len() == 1, - "SharedArray expects exactly 1 child, got {}", - children.len() - ); - let child = children - .into_iter() - .next() - .vortex_expect("children length already validated"); - array.set_source(child); - Ok(()) - } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { array .get_or_compute(|source| source.clone().execute::(ctx)) diff --git a/vortex-array/src/arrays/slice/array.rs b/vortex-array/src/arrays/slice/array.rs index 8a0dbbb642c..cf6db0d9be8 100644 --- a/vortex-array/src/arrays/slice/array.rs +++ b/vortex-array/src/arrays/slice/array.rs @@ -10,9 +10,13 @@ use vortex_error::vortex_panic; use crate::ArrayRef; use crate::stats::ArrayStats; +pub(super) const CHILD_SLOT: usize = 0; +pub(super) const NUM_SLOTS: usize = 1; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["child"]; + #[derive(Clone, Debug)] pub struct SliceArray { - pub(super) child: ArrayRef, + pub(super) slots: Vec>, pub(super) range: Range, pub(super) stats: ArrayStats, } @@ -32,7 +36,7 @@ impl SliceArray { ); } Ok(Self { - child, + slots: vec![Some(child)], range, stats: ArrayStats::default(), }) @@ -49,13 +53,17 @@ impl SliceArray { /// The child array being sliced. pub fn child(&self) -> &ArrayRef { - &self.child + self.slots[CHILD_SLOT] + .as_ref() + .vortex_expect("SliceArray child slot") } /// Consume the slice array and return its components. pub fn into_parts(self) -> SliceArrayParts { SliceArrayParts { - child: self.child, + child: self.slots[CHILD_SLOT] + .clone() + .vortex_expect("SliceArray child slot"), range: self.range, } } diff --git a/vortex-array/src/arrays/slice/vtable.rs b/vortex-array/src/arrays/slice/vtable.rs index 367b5c5560c..b659970059b 100644 --- a/vortex-array/src/arrays/slice/vtable.rs +++ b/vortex-array/src/arrays/slice/vtable.rs @@ -7,7 +7,6 @@ use std::hash::Hash; use std::hash::Hasher; use std::ops::Range; -use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -21,6 +20,8 @@ use crate::ArrayRef; use crate::Canonical; use crate::DynArray; use crate::Precision; +use crate::arrays::slice::array::NUM_SLOTS; +use crate::arrays::slice::array::SLOT_NAMES; use crate::arrays::slice::array::SliceArray; use crate::arrays::slice::rules::PARENT_RULES; use crate::buffer::BufferHandle; @@ -60,7 +61,7 @@ impl VTable for Slice { } fn dtype(array: &SliceArray) -> &DType { - array.child.dtype() + array.child().dtype() } fn stats(array: &SliceArray) -> StatsSetRef<'_> { @@ -68,13 +69,13 @@ impl VTable for Slice { } fn array_hash(array: &SliceArray, state: &mut H, precision: Precision) { - array.child.array_hash(state, precision); + array.child().array_hash(state, precision); array.range.start.hash(state); array.range.end.hash(state); } fn array_eq(array: &SliceArray, other: &SliceArray, precision: Precision) -> bool { - array.child.array_eq(&other.child, precision) && array.range == other.range + array.child().array_eq(other.child(), precision) && array.range == other.range } fn nbuffers(_array: &Self::Array) -> usize { @@ -89,22 +90,12 @@ impl VTable for Slice { None } - fn nchildren(_array: &Self::Array) -> usize { - 1 + fn slots(array: &Self::Array) -> &[Option] { + &array.slots } - fn child(array: &Self::Array, idx: usize) -> ArrayRef { - match idx { - 0 => array.child.clone(), - _ => vortex_panic!("SliceArray child index {idx} out of bounds"), - } - } - - fn child_name(_array: &Self::Array, idx: usize) -> String { - match idx { - 0 => "child".to_string(), - _ => vortex_panic!("SliceArray child_name index {idx} out of bounds"), - } + fn slot_name(_array: &Self::Array, idx: usize) -> String { + SLOT_NAMES[idx].to_string() } fn metadata(array: &Self::Array) -> VortexResult { @@ -135,32 +126,26 @@ impl VTable for Slice { ) -> VortexResult { assert_eq!(len, metadata.0.len()); let child = children.get(0, dtype, metadata.0.end)?; - Ok(SliceArray { - child, - range: metadata.0.clone(), - stats: Default::default(), - }) + SliceArray::try_new(child, metadata.0.clone()) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { + fn with_slots(array: &mut Self::Array, slots: Vec>) -> VortexResult<()> { vortex_ensure!( - children.len() == 1, - "SliceArray expects exactly 1 child, got {}", - children.len() + slots.len() == NUM_SLOTS, + "SliceArray expects exactly {} slots, got {}", + NUM_SLOTS, + slots.len() ); - array.child = children - .into_iter() - .next() - .vortex_expect("children length already validated"); + array.slots = slots; Ok(()) } fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { // Execute the child to get canonical form, then slice it - let Some(canonical) = array.child.as_opt::() else { + let Some(canonical) = array.child().as_opt::() else { // If the child is not canonical, recurse. return array - .child + .child() .clone() .execute::(ctx)? .slice(array.slice_range().clone()) @@ -184,13 +169,13 @@ impl VTable for Slice { } impl OperationsVTable for Slice { fn scalar_at(array: &SliceArray, index: usize) -> VortexResult { - array.child.scalar_at(array.range.start + index) + array.child().scalar_at(array.range.start + index) } } impl ValidityVTable for Slice { fn validity(array: &SliceArray) -> VortexResult { - array.child.validity()?.slice(array.range.clone()) + array.child().validity()?.slice(array.range.clone()) } } diff --git a/vortex-array/src/arrays/struct_/array.rs b/vortex-array/src/arrays/struct_/array.rs index a351cc84a5d..7756e3afe72 100644 --- a/vortex-array/src/arrays/struct_/array.rs +++ b/vortex-array/src/arrays/struct_/array.rs @@ -20,6 +20,11 @@ use crate::dtype::StructFields; use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable::ValidityHelper; +use crate::vtable::validity_to_child; + +// StructArray has a variable number of slots: [validity?, field_0, ..., field_N] +pub(super) const VALIDITY_SLOT: usize = 0; +pub(super) const FIELDS_OFFSET: usize = 1; /// A struct array that stores multiple named fields as columns, similar to a database row. /// @@ -143,7 +148,7 @@ use crate::vtable::ValidityHelper; pub struct StructArray { pub(super) len: usize, pub(super) dtype: DType, - pub(super) fields: Arc<[ArrayRef]>, + pub(super) slots: Vec>, pub(super) validity: Validity, pub(super) stats_set: ArrayStats, } @@ -155,9 +160,23 @@ pub struct StructArrayParts { } impl StructArray { - /// Return the struct fields without the validity of the struct applied - pub fn unmasked_fields(&self) -> &Arc<[ArrayRef]> { - &self.fields + /// Return a single struct field by index without the validity of the struct applied. + pub fn unmasked_field(&self, idx: usize) -> &ArrayRef { + self.slots[FIELDS_OFFSET + idx] + .as_ref() + .vortex_expect("StructArray field slot") + } + + /// Return an iterator over the struct fields without the validity of the struct applied. + pub fn iter_unmasked_fields(&self) -> impl Iterator + '_ { + self.slots[FIELDS_OFFSET..] + .iter() + .map(|s| s.as_ref().vortex_expect("StructArray field slot")) + } + + /// Return the struct fields without the validity of the struct applied. + pub fn unmasked_fields(&self) -> Arc<[ArrayRef]> { + self.iter_unmasked_fields().cloned().collect() } /// Return the struct field without the validity of the struct applied @@ -174,7 +193,11 @@ impl StructArray { /// Return the struct field without the validity of the struct applied pub fn unmasked_field_by_name_opt(&self, name: impl AsRef) -> Option<&ArrayRef> { let name = name.as_ref(); - self.struct_fields().find(name).map(|idx| &self.fields[idx]) + self.struct_fields().find(name).map(|idx| { + self.slots[FIELDS_OFFSET + idx] + .as_ref() + .vortex_expect("StructArray field slot") + }) } pub fn names(&self) -> &FieldNames { @@ -278,10 +301,15 @@ impl StructArray { Self::validate(&fields, &dtype, length, &validity) .vortex_expect("[Debug Assertion]: Invalid `StructArray` parameters"); + let validity_slot = validity_to_child(&validity, length); + let slots = once(validity_slot) + .chain(fields.iter().map(|f| Some(f.clone()))) + .collect(); + Self { len: length, dtype: DType::Struct(dtype, validity.nullability()), - fields, + slots, validity, stats_set: Default::default(), } @@ -355,9 +383,15 @@ impl StructArray { pub fn into_parts(self) -> StructArrayParts { let struct_fields = self.dtype.into_struct_fields(); + let fields: Arc<[ArrayRef]> = self + .slots + .into_iter() + .skip(FIELDS_OFFSET) + .map(|s| s.vortex_expect("StructArray field slot")) + .collect(); StructArrayParts { struct_fields, - fields: self.fields, + fields, validity: self.validity, } } @@ -407,7 +441,6 @@ impl StructArray { let mut children = Vec::with_capacity(projection.len()); let mut names = Vec::with_capacity(projection.len()); - let fields = self.unmasked_fields(); for f_name in projection.iter() { let idx = self .names() @@ -416,7 +449,12 @@ impl StructArray { .ok_or_else(|| vortex_err!("Unknown field {f_name}"))?; names.push(self.names()[idx].clone()); - children.push(fields[idx].clone()); + children.push( + self.slots[FIELDS_OFFSET + idx] + .as_ref() + .vortex_expect("StructArray field slot") + .clone(), + ); } StructArray::try_new( @@ -439,17 +477,21 @@ impl StructArray { .iter() .position(|field_name| field_name.as_ref() == name.as_ref())?; - let field = self.fields[position].clone(); - let new_fields: Arc<[ArrayRef]> = self - .fields + let slot_position = FIELDS_OFFSET + position; + let field = self.slots[slot_position] + .as_ref() + .vortex_expect("StructArray field slot") + .clone(); + let new_slots: Vec> = self + .slots .iter() .enumerate() - .filter(|(i, _)| *i != position) - .map(|(_, f)| f.clone()) + .filter(|(i, _)| *i != slot_position) + .map(|(_, s)| s.clone()) .collect(); if let Ok(new_dtype) = struct_dtype.without_field(position) { - self.fields = new_fields; + self.slots = new_slots; self.dtype = DType::Struct(new_dtype, self.dtype.nullability()); return Some(field); } @@ -465,7 +507,11 @@ impl StructArray { let types = struct_dtype.fields().chain(once(array.dtype().clone())); let new_fields = StructFields::new(names.collect(), types.collect()); - let children: Arc<[ArrayRef]> = self.fields.iter().cloned().chain(once(array)).collect(); + let children: Arc<[ArrayRef]> = self.slots[FIELDS_OFFSET..] + .iter() + .map(|s| s.as_ref().vortex_expect("StructArray field slot").clone()) + .chain(once(array)) + .collect(); Self::try_new_with_dtype(children, new_fields, self.len, self.validity.clone()) } diff --git a/vortex-array/src/arrays/struct_/compute/cast.rs b/vortex-array/src/arrays/struct_/compute/cast.rs index 291830556c9..272e6965164 100644 --- a/vortex-array/src/arrays/struct_/compute/cast.rs +++ b/vortex-array/src/arrays/struct_/compute/cast.rs @@ -38,10 +38,7 @@ impl CastKernel for Struct { let mut cast_fields = Vec::with_capacity(target_sdtype.nfields()); if fields_match_order { - for (field, target_type) in array - .unmasked_fields() - .iter() - .zip_eq(target_sdtype.fields()) + for (field, target_type) in array.iter_unmasked_fields().zip_eq(target_sdtype.fields()) { let cast_field = field.cast(target_type)?; cast_fields.push(cast_field); @@ -66,8 +63,7 @@ impl CastKernel for Struct { } Some(src_field_idx) => { // Field exists in source field. Cast it to the target type. - let cast_field = - array.unmasked_fields()[src_field_idx].cast(target_type)?; + let cast_field = array.unmasked_field(src_field_idx).cast(target_type)?; cast_fields.push(cast_field); } } @@ -213,7 +209,7 @@ mod tests { .unwrap(); assert_eq!(result.dtype(), &target_dtype); assert_eq!(result.len(), 3); - assert_eq!(result.to_struct().unmasked_fields().len(), 2); + assert_eq!(result.to_struct().struct_fields().nfields(), 2); } #[test] @@ -242,6 +238,6 @@ mod tests { .unwrap(); assert_eq!(result.dtype(), &target_dtype); assert_eq!(result.len(), 3); - assert_eq!(result.to_struct().unmasked_fields().len(), 3); + assert_eq!(result.to_struct().struct_fields().nfields(), 3); } } diff --git a/vortex-array/src/arrays/struct_/compute/mask.rs b/vortex-array/src/arrays/struct_/compute/mask.rs index c10e1974f0c..e8896da3cf7 100644 --- a/vortex-array/src/arrays/struct_/compute/mask.rs +++ b/vortex-array/src/arrays/struct_/compute/mask.rs @@ -14,7 +14,7 @@ use crate::vtable::ValidityHelper; impl MaskReduce for Struct { fn mask(array: &StructArray, mask: &ArrayRef) -> VortexResult> { StructArray::try_new_with_dtype( - array.unmasked_fields().clone(), + array.unmasked_fields().iter().cloned().collect::>(), array.struct_fields().clone(), array.len(), array diff --git a/vortex-array/src/arrays/struct_/compute/slice.rs b/vortex-array/src/arrays/struct_/compute/slice.rs index beb38e8c74c..72edae26307 100644 --- a/vortex-array/src/arrays/struct_/compute/slice.rs +++ b/vortex-array/src/arrays/struct_/compute/slice.rs @@ -16,8 +16,7 @@ use crate::vtable::ValidityHelper; impl SliceReduce for Struct { fn slice(array: &Self::Array, range: Range) -> VortexResult> { let fields: Vec<_> = array - .unmasked_fields() - .iter() + .iter_unmasked_fields() .map(|field| field.slice(range.clone())) .try_collect()?; diff --git a/vortex-array/src/arrays/struct_/compute/take.rs b/vortex-array/src/arrays/struct_/compute/take.rs index 6694d64e6e6..2809ea9e2ac 100644 --- a/vortex-array/src/arrays/struct_/compute/take.rs +++ b/vortex-array/src/arrays/struct_/compute/take.rs @@ -20,7 +20,7 @@ impl TakeReduce for Struct { // an out of bounds element. if array.is_empty() { return StructArray::try_new_with_dtype( - array.unmasked_fields().clone(), + array.iter_unmasked_fields().cloned().collect::>(), array.struct_fields().clone(), indices.len(), Validity::AllInvalid, @@ -39,8 +39,7 @@ impl TakeReduce for Struct { StructArray::try_new_with_dtype( array - .unmasked_fields() - .iter() + .iter_unmasked_fields() .map(|field| field.take(inner_indices.clone())) .collect::, _>>()?, array.struct_fields().clone(), diff --git a/vortex-array/src/arrays/struct_/compute/zip.rs b/vortex-array/src/arrays/struct_/compute/zip.rs index 425d07a28a3..91f6b9f370b 100644 --- a/vortex-array/src/arrays/struct_/compute/zip.rs +++ b/vortex-array/src/arrays/struct_/compute/zip.rs @@ -34,9 +34,8 @@ impl ZipKernel for Struct { ); let fields = if_true - .unmasked_fields() - .iter() - .zip(if_false.unmasked_fields().iter()) + .iter_unmasked_fields() + .zip(if_false.iter_unmasked_fields()) .map(|(t, f)| ArrayBuiltins::zip(mask, t.clone(), f.clone())) .collect::>>()?; diff --git a/vortex-array/src/arrays/struct_/tests.rs b/vortex-array/src/arrays/struct_/tests.rs index 7981fd0458c..7a261810e61 100644 --- a/vortex-array/src/arrays/struct_/tests.rs +++ b/vortex-array/src/arrays/struct_/tests.rs @@ -45,13 +45,13 @@ fn test_project() { assert_eq!(struct_b.len(), 5); - let bools = &struct_b.fields[0]; + let bools = struct_b.unmasked_field(0); assert_arrays_eq!( bools, BoolArray::from_iter([true, true, true, false, false]) ); - let prims = &struct_b.fields[1]; + let prims = struct_b.unmasked_field(1); assert_arrays_eq!(prims, PrimitiveArray::from_iter([0i64, 1, 2, 3, 4])); } @@ -76,14 +76,14 @@ fn test_remove_column() { assert_arrays_eq!(removed, PrimitiveArray::from_iter([0i64, 1, 2, 3, 4])); assert_eq!(struct_a.names(), &["ys"]); - assert_eq!(struct_a.fields.len(), 1); + assert_eq!(struct_a.struct_fields().nfields(), 1); assert_eq!(struct_a.len(), 5); assert_eq!( - struct_a.fields[0].dtype(), + struct_a.unmasked_field(0).dtype(), &DType::Primitive(PType::U64, Nullability::NonNullable) ); assert_arrays_eq!( - struct_a.fields[0], + struct_a.unmasked_field(0), PrimitiveArray::from_iter([4u64, 5, 6, 7, 8]) ); @@ -126,7 +126,7 @@ fn test_duplicate_field_names() { ); // Verify the third field (second "value") can be accessed by index - let third_field = &struct_array.unmasked_fields()[2]; + let third_field = struct_array.unmasked_field(2); assert_arrays_eq!(third_field, PrimitiveArray::from_iter([100i32, 200, 300])); } diff --git a/vortex-array/src/arrays/struct_/vtable/mod.rs b/vortex-array/src/arrays/struct_/vtable/mod.rs index 39df6e8783f..9815fadb5c6 100644 --- a/vortex-array/src/arrays/struct_/vtable/mod.rs +++ b/vortex-array/src/arrays/struct_/vtable/mod.rs @@ -1,14 +1,11 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::sync::Arc; - use itertools::Itertools; use kernel::PARENT_KERNELS; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; -use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; @@ -18,6 +15,8 @@ use crate::ExecutionCtx; use crate::ExecutionStep; use crate::IntoArray; use crate::arrays::StructArray; +use crate::arrays::struct_::array::FIELDS_OFFSET; +use crate::arrays::struct_::array::VALIDITY_SLOT; use crate::arrays::struct_::compute::rules::PARENT_RULES; use crate::buffer::BufferHandle; use crate::dtype::DType; @@ -26,8 +25,6 @@ use crate::validity::Validity; use crate::vtable; use crate::vtable::VTable; use crate::vtable::ValidityVTableFromValidityHelper; -use crate::vtable::validity_nchildren; -use crate::vtable::validity_to_child; mod kernel; mod operations; mod validity; @@ -66,7 +63,7 @@ impl VTable for Struct { fn array_hash(array: &StructArray, state: &mut H, precision: Precision) { array.len.hash(state); array.dtype.hash(state); - for field in array.fields.iter() { + for field in array.iter_unmasked_fields() { field.array_hash(state, precision); } array.validity.array_hash(state, precision); @@ -75,11 +72,10 @@ impl VTable for Struct { fn array_eq(array: &StructArray, other: &StructArray, precision: Precision) -> bool { array.len == other.len && array.dtype == other.dtype - && array.fields.len() == other.fields.len() + && array.slots.len() == other.slots.len() && array - .fields - .iter() - .zip(other.fields.iter()) + .iter_unmasked_fields() + .zip(other.iter_unmasked_fields()) .all(|(a, b)| a.array_eq(b, precision)) && array.validity.array_eq(&other.validity, precision) } @@ -96,29 +92,6 @@ impl VTable for Struct { vortex_panic!("StructArray buffer_name index {idx} out of bounds") } - fn nchildren(array: &StructArray) -> usize { - validity_nchildren(&array.validity) + array.unmasked_fields().len() - } - - fn child(array: &StructArray, idx: usize) -> ArrayRef { - let vc = validity_nchildren(&array.validity); - if idx < vc { - validity_to_child(&array.validity, array.len()) - .vortex_expect("StructArray validity child out of bounds") - } else { - array.unmasked_fields()[idx - vc].clone() - } - } - - fn child_name(array: &StructArray, idx: usize) -> String { - let vc = validity_nchildren(&array.validity); - if idx < vc { - "validity".to_string() - } else { - array.names()[idx - vc].as_ref().to_string() - } - } - fn metadata(_array: &StructArray) -> VortexResult { Ok(EmptyMetadata) } @@ -151,7 +124,6 @@ impl VTable for Struct { let (validity, non_data_children) = if children.len() == struct_dtype.nfields() { (Validity::from(*nullability), 0_usize) } else if children.len() == struct_dtype.nfields() + 1 { - // Validity is the first child if it exists. let validity = children.get(0, &Validity::DTYPE, len)?; (Validity::Array(validity), 1_usize) } else { @@ -163,7 +135,7 @@ impl VTable for Struct { ); }; - let children: Vec<_> = (0..struct_dtype.nfields()) + let field_children: Vec<_> = (0..struct_dtype.nfields()) .map(|i| { let child_dtype = struct_dtype .field_by_index(i) @@ -172,38 +144,27 @@ impl VTable for Struct { }) .try_collect()?; - StructArray::try_new_with_dtype(children, struct_dtype.clone(), len, validity) + StructArray::try_new_with_dtype(field_children, struct_dtype.clone(), len, validity) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { - let DType::Struct(struct_dtype, _nullability) = &array.dtype else { - vortex_bail!("Expected struct dtype, found {:?}", array.dtype) - }; + fn slots(array: &StructArray) -> &[Option] { + &array.slots + } - // First child is validity (if present), followed by fields - let (validity, non_data_children) = if children.len() == struct_dtype.nfields() { - (array.validity.clone(), 0_usize) - } else if children.len() == struct_dtype.nfields() + 1 { - (Validity::Array(children[0].clone()), 1_usize) + fn slot_name(array: &StructArray, idx: usize) -> String { + if idx == VALIDITY_SLOT { + "validity".to_string() } else { - vortex_bail!( - "Expected {} or {} children, found {}", - struct_dtype.nfields(), - struct_dtype.nfields() + 1, - children.len() - ); - }; - - let fields: Arc<[ArrayRef]> = children.into_iter().skip(non_data_children).collect(); - vortex_ensure!( - fields.len() == struct_dtype.nfields(), - "Expected {} field children, found {}", - struct_dtype.nfields(), - fields.len() - ); + array.names()[idx - FIELDS_OFFSET].to_string() + } + } - array.fields = fields; - array.validity = validity; + fn with_slots(array: &mut StructArray, slots: Vec>) -> VortexResult<()> { + array.validity = match &slots[VALIDITY_SLOT] { + Some(arr) => Validity::Array(arr.clone()), + None => Validity::from(array.dtype.nullability()), + }; + array.slots = slots; Ok(()) } diff --git a/vortex-array/src/arrays/struct_/vtable/operations.rs b/vortex-array/src/arrays/struct_/vtable/operations.rs index 5db676dca76..326cb624a4a 100644 --- a/vortex-array/src/arrays/struct_/vtable/operations.rs +++ b/vortex-array/src/arrays/struct_/vtable/operations.rs @@ -12,8 +12,7 @@ use crate::vtable::OperationsVTable; impl OperationsVTable for Struct { fn scalar_at(array: &StructArray, index: usize) -> VortexResult { let field_scalars: VortexResult> = array - .unmasked_fields() - .iter() + .iter_unmasked_fields() .map(|field| field.scalar_at(index)) .collect(); // SAFETY: The vtable guarantees index is in-bounds and non-null before this is called. diff --git a/vortex-array/src/arrays/varbin/array.rs b/vortex-array/src/arrays/varbin/array.rs index 5c08545d6ba..4c54af7f7c7 100644 --- a/vortex-array/src/arrays/varbin/array.rs +++ b/vortex-array/src/arrays/varbin/array.rs @@ -19,12 +19,18 @@ use crate::dtype::Nullability; use crate::match_each_integer_ptype; use crate::stats::ArrayStats; use crate::validity::Validity; +use crate::vtable::validity_to_child; + +pub(super) const OFFSETS_SLOT: usize = 0; +pub(super) const VALIDITY_SLOT: usize = 1; +pub(super) const NUM_SLOTS: usize = 2; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["offsets", "validity"]; #[derive(Clone, Debug)] pub struct VarBinArray { pub(super) dtype: DType, pub(super) bytes: BufferHandle, - pub(super) offsets: ArrayRef, + pub(super) slots: Vec>, pub(super) validity: Validity, pub(super) stats_set: ArrayStats, } @@ -154,10 +160,13 @@ impl VarBinArray { Self::validate(&offsets, &bytes, &dtype, &validity) .vortex_expect("[Debug Assertion]: Invalid `VarBinArray` parameters"); + let len = offsets.len().saturating_sub(1); + let validity_slot = validity_to_child(&validity, len); + Self { dtype, bytes, - offsets, + slots: vec![Some(offsets), validity_slot], validity, stats_set: Default::default(), } @@ -259,7 +268,9 @@ impl VarBinArray { #[inline] pub fn offsets(&self) -> &ArrayRef { - &self.offsets + self.slots[OFFSETS_SLOT] + .as_ref() + .vortex_expect("VarBinArray offsets slot") } /// Access the value bytes child buffer @@ -371,7 +382,10 @@ impl VarBinArray { /// Consumes self, returning a tuple containing the `DType`, the `bytes` array, /// the `offsets` array, and the `validity`. pub fn into_parts(self) -> (DType, BufferHandle, ArrayRef, Validity) { - (self.dtype, self.bytes, self.offsets, self.validity) + let offsets = self.slots[OFFSETS_SLOT] + .clone() + .vortex_expect("VarBinArray offsets slot"); + (self.dtype, self.bytes, offsets, self.validity) } } diff --git a/vortex-array/src/arrays/varbin/vtable/mod.rs b/vortex-array/src/arrays/varbin/vtable/mod.rs index a366cd6d0c1..4fbe79309dd 100644 --- a/vortex-array/src/arrays/varbin/vtable/mod.rs +++ b/vortex-array/src/arrays/varbin/vtable/mod.rs @@ -4,7 +4,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; -use vortex_error::vortex_err; +use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use crate::ArrayRef; @@ -15,6 +15,9 @@ use crate::IntoArray; use crate::ProstMetadata; use crate::SerializeMetadata; use crate::arrays::VarBinArray; +use crate::arrays::varbin::array::NUM_SLOTS; +use crate::arrays::varbin::array::SLOT_NAMES; +use crate::arrays::varbin::array::VALIDITY_SLOT; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::dtype::Nullability; @@ -25,8 +28,6 @@ use crate::vtable; use crate::vtable::ArrayId; use crate::vtable::VTable; use crate::vtable::ValidityVTableFromValidityHelper; -use crate::vtable::validity_nchildren; -use crate::vtable::validity_to_child; mod canonical; mod kernel; mod operations; @@ -105,27 +106,6 @@ impl VTable for VarBin { } } - fn nchildren(array: &VarBinArray) -> usize { - 1 + validity_nchildren(&array.validity) - } - - fn child(array: &VarBinArray, idx: usize) -> ArrayRef { - match idx { - 0 => array.offsets().clone(), - 1 => validity_to_child(&array.validity, array.len()) - .vortex_expect("VarBinArray validity child out of bounds"), - _ => vortex_panic!("VarBinArray child index {idx} out of bounds"), - } - } - - fn child_name(_array: &VarBinArray, idx: usize) -> String { - match idx { - 0 => "offsets".to_string(), - 1 => "validity".to_string(), - _ => vortex_panic!("VarBinArray child_name index {idx} out of bounds"), - } - } - fn metadata(array: &VarBinArray) -> VortexResult { Ok(ProstMetadata(VarBinMetadata { offsets_ptype: PType::try_from(array.offsets().dtype()) @@ -179,26 +159,26 @@ impl VTable for VarBin { VarBinArray::try_new(offsets, bytes, dtype.clone(), validity) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { - match children.len() { - 1 => { - let [offsets]: [ArrayRef; 1] = children - .try_into() - .map_err(|_| vortex_err!("Failed to convert children to array"))?; - array.offsets = offsets; - } - 2 => { - let [offsets, validity]: [ArrayRef; 2] = children - .try_into() - .map_err(|_| vortex_err!("Failed to convert children to array"))?; - array.offsets = offsets; - array.validity = Validity::Array(validity); - } - _ => vortex_bail!( - "VarBinArray expects 1 or 2 children (offsets, validity?), got {}", - children.len() - ), - } + fn slots(array: &VarBinArray) -> &[Option] { + &array.slots + } + + fn slot_name(_array: &VarBinArray, idx: usize) -> String { + SLOT_NAMES[idx].to_string() + } + + fn with_slots(array: &mut VarBinArray, slots: Vec>) -> VortexResult<()> { + vortex_ensure!( + slots.len() == NUM_SLOTS, + "VarBinArray expects exactly {} slots, got {}", + NUM_SLOTS, + slots.len() + ); + array.validity = match &slots[VALIDITY_SLOT] { + Some(arr) => Validity::Array(arr.clone()), + None => Validity::from(array.dtype.nullability()), + }; + array.slots = slots; Ok(()) } diff --git a/vortex-array/src/arrays/varbinview/array.rs b/vortex-array/src/arrays/varbinview/array.rs index 0c1276c928f..32451b91cd5 100644 --- a/vortex-array/src/arrays/varbinview/array.rs +++ b/vortex-array/src/arrays/varbinview/array.rs @@ -13,6 +13,7 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_error::vortex_panic; +use crate::ArrayRef; use crate::arrays::varbinview::BinaryView; use crate::buffer::BufferHandle; use crate::builders::ArrayBuilder; @@ -21,6 +22,11 @@ use crate::dtype::DType; use crate::dtype::Nullability; use crate::stats::ArrayStats; use crate::validity::Validity; +use crate::vtable::validity_to_child; + +pub(super) const VALIDITY_SLOT: usize = 0; +pub(super) const NUM_SLOTS: usize = 1; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; /// A variable-length binary view array that stores strings and binary data efficiently. /// @@ -83,6 +89,7 @@ use crate::validity::Validity; /// ``` #[derive(Clone, Debug)] pub struct VarBinViewArray { + pub(super) slots: Vec>, pub(super) dtype: DType, pub(super) buffers: Arc<[BufferHandle]>, pub(super) views: BufferHandle, @@ -98,6 +105,10 @@ pub struct VarBinViewArrayParts { } impl VarBinViewArray { + fn make_slots(validity: &Validity, len: usize) -> Vec> { + vec![validity_to_child(validity, len)] + } + /// Creates a new [`VarBinViewArray`]. /// /// # Panics @@ -244,7 +255,9 @@ impl VarBinViewArray { dtype: DType, validity: Validity, ) -> Self { + let len = views.len() / size_of::(); Self { + slots: Self::make_slots(&validity, len), views, buffers, dtype, diff --git a/vortex-array/src/arrays/varbinview/vtable/mod.rs b/vortex-array/src/arrays/varbinview/vtable/mod.rs index 0c4decaa772..3edeb143b8b 100644 --- a/vortex-array/src/arrays/varbinview/vtable/mod.rs +++ b/vortex-array/src/arrays/varbinview/vtable/mod.rs @@ -7,9 +7,9 @@ use std::sync::Arc; use kernel::PARENT_KERNELS; use vortex_buffer::Buffer; -use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; +use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; @@ -22,6 +22,9 @@ use crate::IntoArray; use crate::Precision; use crate::arrays::VarBinViewArray; use crate::arrays::varbinview::BinaryView; +use crate::arrays::varbinview::array::NUM_SLOTS; +use crate::arrays::varbinview::array::SLOT_NAMES; +use crate::arrays::varbinview::array::VALIDITY_SLOT; use crate::arrays::varbinview::compute::rules::PARENT_RULES; use crate::buffer::BufferHandle; use crate::dtype::DType; @@ -34,8 +37,6 @@ use crate::vtable; use crate::vtable::ArrayId; use crate::vtable::VTable; use crate::vtable::ValidityVTableFromValidityHelper; -use crate::vtable::validity_nchildren; -use crate::vtable::validity_to_child; mod kernel; mod operations; mod validity; @@ -121,25 +122,6 @@ impl VTable for VarBinView { } } - fn nchildren(array: &VarBinViewArray) -> usize { - validity_nchildren(&array.validity) - } - - fn child(array: &VarBinViewArray, idx: usize) -> ArrayRef { - match idx { - 0 => validity_to_child(&array.validity, array.len()) - .vortex_expect("VarBinViewArray validity child out of bounds"), - _ => vortex_panic!("VarBinViewArray child index {idx} out of bounds"), - } - } - - fn child_name(_array: &VarBinViewArray, idx: usize) -> String { - match idx { - 0 => "validity".to_string(), - _ => vortex_panic!("VarBinViewArray child_name index {idx} out of bounds"), - } - } - fn metadata(_array: &VarBinViewArray) -> VortexResult { Ok(EmptyMetadata) } @@ -209,20 +191,26 @@ impl VTable for VarBinView { VarBinViewArray::try_new(views, Arc::from(data_buffers), dtype.clone(), validity) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { - match children.len() { - 0 => {} - 1 => { - let [validity]: [ArrayRef; 1] = children - .try_into() - .map_err(|_| vortex_err!("Failed to convert children to array"))?; - array.validity = Validity::Array(validity); - } - _ => vortex_bail!( - "VarBinViewArray expects 0 or 1 children (validity?), got {}", - children.len() - ), - } + fn slots(array: &VarBinViewArray) -> &[Option] { + &array.slots + } + + fn slot_name(_array: &VarBinViewArray, idx: usize) -> String { + SLOT_NAMES[idx].to_string() + } + + fn with_slots(array: &mut VarBinViewArray, slots: Vec>) -> VortexResult<()> { + vortex_ensure!( + slots.len() == NUM_SLOTS, + "VarBinViewArray expects {} slots, got {}", + NUM_SLOTS, + slots.len() + ); + array.validity = match &slots[VALIDITY_SLOT] { + Some(arr) => Validity::Array(arr.clone()), + None => Validity::from(array.dtype.nullability()), + }; + array.slots = slots; Ok(()) } @@ -247,3 +235,53 @@ impl VTable for VarBinView { Ok(ExecutionStep::Done(array.clone().into_array())) } } + +#[cfg(test)] +mod tests { + use vortex_buffer::ByteBufferMut; + use vortex_session::registry::ReadContext; + + use super::*; + use crate::ArrayContext; + use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::assert_arrays_eq; + use crate::serde::ArrayParts; + use crate::serde::SerializeOptions; + + #[test] + fn test_nullable_varbinview_serde_roundtrip() { + let array = VarBinViewArray::from_iter_nullable_str([ + Some("hello"), + None, + Some("world"), + None, + Some("a moderately long string for testing"), + ]); + let dtype = array.dtype().clone(); + let len = array.len(); + + let ctx = ArrayContext::empty(); + let serialized = array + .clone() + .into_array() + .serialize(&ctx, &SerializeOptions::default()) + .unwrap(); + + let mut concat = ByteBufferMut::empty(); + for buf in serialized { + concat.extend_from_slice(buf.as_ref()); + } + let parts = ArrayParts::try_from(concat.freeze()).unwrap(); + let decoded = parts + .decode( + &dtype, + len, + &ReadContext::new(ctx.to_ids()), + &LEGACY_SESSION, + ) + .unwrap(); + + assert_arrays_eq!(decoded, array); + } +} diff --git a/vortex-array/src/arrays/variant/mod.rs b/vortex-array/src/arrays/variant/mod.rs index 054cdedb9b8..b631ed97f5f 100644 --- a/vortex-array/src/arrays/variant/mod.rs +++ b/vortex-array/src/arrays/variant/mod.rs @@ -3,11 +3,16 @@ mod vtable; +use vortex_error::VortexExpect; + pub use self::vtable::Variant; use crate::ArrayRef; use crate::dtype::DType; use crate::dtype::Nullability; +pub(super) const NUM_SLOTS: usize = 1; +pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["child"]; + /// The canonical in-memory representation of variant (semi-structured) data. /// /// Wraps a single child array that contains the actual variant-encoded data @@ -15,7 +20,7 @@ use crate::dtype::Nullability; #[derive(Clone, Debug)] pub struct VariantArray { dtype: DType, - child: ArrayRef, + slots: [Option; NUM_SLOTS], } impl VariantArray { @@ -23,12 +28,14 @@ impl VariantArray { pub fn new(child: ArrayRef, nullability: Nullability) -> Self { Self { dtype: DType::Variant(nullability), - child, + slots: [Some(child)], } } /// Returns a reference to the underlying child array. pub fn child(&self) -> &ArrayRef { - &self.child + self.slots[0] + .as_ref() + .vortex_expect("VariantArray child slot") } } diff --git a/vortex-array/src/arrays/variant/vtable/mod.rs b/vortex-array/src/arrays/variant/vtable/mod.rs index 71ae0665212..145ac7031d8 100644 --- a/vortex-array/src/arrays/variant/vtable/mod.rs +++ b/vortex-array/src/arrays/variant/vtable/mod.rs @@ -9,6 +9,7 @@ use std::hash::Hasher; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; +use vortex_error::vortex_err; use vortex_error::vortex_panic; use crate::ArrayEq; @@ -20,6 +21,8 @@ use crate::ExecutionStep; use crate::IntoArray; use crate::Precision; use crate::arrays::VariantArray; +use crate::arrays::variant::NUM_SLOTS; +use crate::arrays::variant::SLOT_NAMES; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::serde::ArrayChildren; @@ -51,7 +54,7 @@ impl VTable for Variant { } fn len(array: &Self::Array) -> usize { - array.child.len() + array.child().len() } fn dtype(array: &Self::Array) -> &DType { @@ -59,15 +62,15 @@ impl VTable for Variant { } fn stats(array: &Self::Array) -> StatsSetRef<'_> { - array.child.statistics() + array.child().statistics() } fn array_hash(array: &Self::Array, state: &mut H, precision: Precision) { - array.child.array_hash(state, precision); + array.child().array_hash(state, precision); } fn array_eq(array: &Self::Array, other: &Self::Array, precision: Precision) -> bool { - array.child.array_eq(&other.child, precision) + array.child().array_eq(other.child(), precision) } fn nbuffers(_array: &Self::Array) -> usize { @@ -82,21 +85,14 @@ impl VTable for Variant { None } - fn nchildren(_array: &Self::Array) -> usize { - 1 + fn slots(array: &Self::Array) -> &[Option] { + &array.slots } - fn child(array: &Self::Array, idx: usize) -> ArrayRef { - match idx { - 0 => array.child.clone(), - _ => vortex_panic!("VariantArray child index {idx} out of bounds"), - } - } - - fn child_name(_array: &Self::Array, idx: usize) -> String { - match idx { - 0 => "child".to_string(), - _ => vortex_panic!("VariantArray child_name index {idx} out of bounds"), + fn slot_name(_array: &Self::Array, idx: usize) -> String { + match SLOT_NAMES.get(idx) { + Some(name) => (*name).to_string(), + None => vortex_panic!("VariantArray slot_name index {idx} out of bounds"), } } @@ -140,13 +136,19 @@ impl VTable for Variant { Ok(VariantArray::new(child, dtype.nullability())) } - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()> { + fn with_slots(array: &mut Self::Array, slots: Vec>) -> VortexResult<()> { vortex_ensure!( - children.len() == 1, - "VariantArray expects exactly 1 child, got {}", - children.len() + slots.len() == NUM_SLOTS, + "VariantArray expects exactly {} slot, got {}", + NUM_SLOTS, + slots.len() ); - array.child = children.into_iter().next().vortex_expect("must exist"); + let child = slots + .into_iter() + .next() + .vortex_expect("VariantArray slot vector length was validated") + .ok_or_else(|| vortex_err!("VariantArray child slot must be present"))?; + array.slots = [Some(child)]; Ok(()) } @@ -155,3 +157,22 @@ impl VTable for Variant { Ok(ExecutionStep::done(array.clone().into_array())) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::arrays::PrimitiveArray; + use crate::dtype::Nullability; + + #[test] + fn with_slots_rejects_missing_child() { + let mut array = VariantArray::new( + PrimitiveArray::from_iter([1u8, 2, 3]).into_array(), + Nullability::NonNullable, + ); + + let err = ::with_slots(&mut array, vec![None]).unwrap_err(); + + assert!(err.to_string().contains("child slot must be present")); + } +} diff --git a/vortex-array/src/arrow/executor/struct_.rs b/vortex-array/src/arrow/executor/struct_.rs index 9c989d24ad0..415a7503ee7 100644 --- a/vortex-array/src/arrow/executor/struct_.rs +++ b/vortex-array/src/arrow/executor/struct_.rs @@ -78,7 +78,7 @@ pub(super) fn to_arrow_struct( }; return create_from_fields( target_fields.ok_or_else(|| struct_fields.names().clone()), - array.children(), + &array.children(), None, // Pack is never null, len, ctx, diff --git a/vortex-array/src/builders/struct_.rs b/vortex-array/src/builders/struct_.rs index 825070b327c..fbe6129af6d 100644 --- a/vortex-array/src/builders/struct_.rs +++ b/vortex-array/src/builders/struct_.rs @@ -168,8 +168,7 @@ impl ArrayBuilder for StructBuilder { let array = array.to_struct(); for (a, builder) in array - .unmasked_fields() - .iter() + .iter_unmasked_fields() .zip_eq(self.builders.iter_mut()) { builder.extend_from_array(a); diff --git a/vortex-array/src/display/mod.rs b/vortex-array/src/display/mod.rs index a8d1a9cd6d1..3cc2d6eb1ab 100644 --- a/vortex-array/src/display/mod.rs +++ b/vortex-array/src/display/mod.rs @@ -550,7 +550,7 @@ impl dyn DynArray + '_ { builder.push_record(null_row); } else { let mut row = Vec::new(); - for field_array in struct_.unmasked_fields().iter() { + for field_array in struct_.iter_unmasked_fields() { let value = field_array .scalar_at(row_idx) .map_or_else(|e| format!(""), |s| s.to_string()); diff --git a/vortex-array/src/executor.rs b/vortex-array/src/executor.rs index da05450f8de..367420a4b2f 100644 --- a/vortex-array/src/executor.rs +++ b/vortex-array/src/executor.rs @@ -93,7 +93,7 @@ impl dyn DynArray + '_ { }); let mut current = self.optimize()?; - // Stack frames: (parent, child_idx, done_predicate_for_child) + // Stack frames: (parent, slot_idx, done_predicate_for_slot) let mut stack: Vec<(ArrayRef, usize, DonePredicate)> = Vec::new(); for _ in 0..*MAX_ITERATIONS { @@ -107,8 +107,8 @@ impl dyn DynArray + '_ { ctx.log(format_args!("-> {}", current)); return Ok(current); } - Some((parent, child_idx, _)) => { - current = parent.with_child(child_idx, current)?; + Some((parent, slot_idx, _)) => { + current = parent.with_slot(slot_idx, current)?; current = current.optimize()?; continue; } @@ -123,8 +123,8 @@ impl dyn DynArray + '_ { ctx.log(format_args!("-> canonical (unmatched) {}", current)); return Ok(current); } - Some((parent, child_idx, _)) => { - current = parent.with_child(child_idx, current)?; + Some((parent, slot_idx, _)) => { + current = parent.with_slot(slot_idx, current)?; current = current.optimize()?; continue; } @@ -143,12 +143,12 @@ impl dyn DynArray + '_ { // Execute the array itself match current.vtable().execute(¤t, ctx)? { - ExecutionStep::ExecuteChild(i, done) => { - let child = current - .nth_child(i) - .vortex_expect("ExecuteChild index in bounds"); + ExecutionStep::ExecuteSlot(i, done) => { + let child = current.slots()[i] + .clone() + .vortex_expect("ExecuteSlot index in bounds"); ctx.log(format_args!( - "ExecuteChild({i}): pushing {}, focusing on {}", + "ExecuteSlot({i}): pushing {}, focusing on {}", current, child )); stack.push((current, i, done)); @@ -269,12 +269,14 @@ impl Executable for ArrayRef { } // 2. reduce_parent (child-driven metadata-only rewrites) - for child_idx in 0..array.nchildren() { - let child = array.nth_child(child_idx).vortex_expect("checked length"); - if let Some(reduced_parent) = child.vtable().reduce_parent(&child, &array, child_idx)? { + for (slot_idx, slot) in array.slots().iter().enumerate() { + let Some(child) = slot else { + continue; + }; + if let Some(reduced_parent) = child.vtable().reduce_parent(child, &array, slot_idx)? { ctx.log(format_args!( - "reduce_parent: child[{}]({}) rewrote {} -> {}", - child_idx, + "reduce_parent: slot[{}]({}) rewrote {} -> {}", + slot_idx, child.encoding_id(), array, reduced_parent @@ -285,15 +287,17 @@ impl Executable for ArrayRef { } // 3. execute_parent (child-driven optimized execution) - for child_idx in 0..array.nchildren() { - let child = array.nth_child(child_idx).vortex_expect("checked length"); + for (slot_idx, slot) in array.slots().iter().enumerate() { + let Some(child) = slot else { + continue; + }; if let Some(executed_parent) = child .vtable() - .execute_parent(&child, &array, child_idx, ctx)? + .execute_parent(child, &array, slot_idx, ctx)? { ctx.log(format_args!( - "execute_parent: child[{}]({}) rewrote {} -> {}", - child_idx, + "execute_parent: slot[{}]({}) rewrote {} -> {}", + slot_idx, child.encoding_id(), array, executed_parent @@ -312,27 +316,24 @@ impl Executable for ArrayRef { ctx.log(format_args!("-> {}", result.as_ref())); Ok(result) } - ExecutionStep::ExecuteChild(i, _) => { - // For single-step execution, handle ExecuteChild by executing the child, + ExecutionStep::ExecuteSlot(i, _) => { + // For single-step execution, handle ExecuteSlot by executing the slot, // replacing it, and returning the updated array. - let child = array.nth_child(i).vortex_expect("valid child index"); + let child = array.slots()[i].clone().vortex_expect("valid slot index"); let executed_child = child.execute::(ctx)?; - array.with_child(i, executed_child) + array.with_slot(i, executed_child) } } } } -/// Try execute_parent on each child of the array. +/// Try execute_parent on each occupied slot of the array. fn try_execute_parent(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult> { - for child_idx in 0..array.nchildren() { - let child = array - .nth_child(child_idx) - .vortex_expect("checked nchildren"); - if let Some(result) = child - .vtable() - .execute_parent(&child, array, child_idx, ctx)? - { + for (slot_idx, slot) in array.slots().iter().enumerate() { + let Some(child) = slot else { + continue; + }; + if let Some(result) = child.vtable().execute_parent(child, array, slot_idx, ctx)? { result.statistics().inherit_from(array.statistics()); return Ok(Some(result)); } @@ -349,15 +350,15 @@ pub type DonePredicate = fn(&dyn DynArray) -> bool; /// scheduler what to do next. This enables the scheduler to manage execution iteratively using /// an explicit work stack, run cross-step optimizations, and cache shared sub-expressions. pub enum ExecutionStep { - /// Request that the scheduler execute child at the given index, using the provided - /// [`DonePredicate`] to determine when the child is "done", then replace the child in this + /// Request that the scheduler execute the slot at the given index, using the provided + /// [`DonePredicate`] to determine when the slot is "done", then replace the slot in this /// array and re-enter execution. /// /// Between steps, the scheduler runs reduce/reduce_parent rules to fixpoint, enabling /// cross-step optimization (e.g., pushing scalar functions through newly-decoded children). /// - /// Use [`ExecutionStep::execute_child`] instead of constructing this variant directly. - ExecuteChild(usize, DonePredicate), + /// Use [`ExecutionStep::execute_slot`] instead of constructing this variant directly. + ExecuteSlot(usize, DonePredicate), /// Execution is complete. The result may be in any encoding — not necessarily canonical. /// The scheduler will continue executing the result if it has not yet reached the target form. @@ -365,9 +366,9 @@ pub enum ExecutionStep { } impl ExecutionStep { - /// Request execution of child at `child_idx` until it matches the given [`Matcher`]. - pub fn execute_child(child_idx: usize) -> Self { - ExecutionStep::ExecuteChild(child_idx, M::matches) + /// Request execution of the slot at `slot_idx` until it matches the given [`Matcher`]. + pub fn execute_slot(slot_idx: usize) -> Self { + ExecutionStep::ExecuteSlot(slot_idx, M::matches) } /// Signal that execution is complete with the given result. @@ -379,9 +380,7 @@ impl ExecutionStep { impl fmt::Debug for ExecutionStep { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - ExecutionStep::ExecuteChild(idx, _) => { - f.debug_tuple("ExecuteChild").field(idx).finish() - } + ExecutionStep::ExecuteSlot(idx, _) => f.debug_tuple("ExecuteSlot").field(idx).finish(), ExecutionStep::Done(result) => f.debug_tuple("Done").field(result).finish(), } } diff --git a/vortex-array/src/optimizer/mod.rs b/vortex-array/src/optimizer/mod.rs index 774523a6ca7..71508c67a62 100644 --- a/vortex-array/src/optimizer/mod.rs +++ b/vortex-array/src/optimizer/mod.rs @@ -45,10 +45,15 @@ fn try_optimize(array: &ArrayRef) -> VortexResult> { continue; } - // Apply parent reduction rules to each child in the context of the current array. - // Its important to take all children here, as `current_array` can change inside the loop. - for (idx, child) in current_array.children().iter().enumerate() { - if let Some(new_array) = child.vtable().reduce_parent(child, ¤t_array, idx)? { + // Apply parent reduction rules to each slot in the context of the current array. + // Its important to take all slots here, as `current_array` can change inside the loop. + for (slot_idx, slot) in current_array.slots().iter().enumerate() { + let Some(child) = slot else { continue }; + if let Some(new_array) = + child + .vtable() + .reduce_parent(child, ¤t_array, slot_idx)? + { // If the parent was replaced, then we attempt to reduce it again. current_array = new_array; any_optimizations = true; @@ -78,19 +83,26 @@ fn try_optimize_recursive(array: &ArrayRef) -> VortexResult> { any_optimizations = true; } - let mut new_children = Vec::with_capacity(current_array.nchildren()); - let mut any_child_optimized = false; - for child in current_array.children() { - if let Some(new_child) = try_optimize_recursive(&child)? { - new_children.push(new_child); - any_child_optimized = true; - } else { - new_children.push(child.clone()); + let mut new_slots = Vec::with_capacity(current_array.slots().len()); + let mut any_slot_optimized = false; + for slot in current_array.slots() { + match slot { + Some(child) => { + if let Some(new_child) = try_optimize_recursive(child)? { + new_slots.push(Some(new_child)); + any_slot_optimized = true; + } else { + new_slots.push(Some(child.clone())); + } + } + None => new_slots.push(None), } } - if any_child_optimized { - current_array = current_array.with_children(new_children)?; + if any_slot_optimized { + current_array = current_array + .vtable() + .with_slots(¤t_array, new_slots)?; any_optimizations = true; } diff --git a/vortex-array/src/scalar_fn/fns/between/kernel.rs b/vortex-array/src/scalar_fn/fns/between/kernel.rs index 00d4cb4e85d..552b6863522 100644 --- a/vortex-array/src/scalar_fn/fns/between/kernel.rs +++ b/vortex-array/src/scalar_fn/fns/between/kernel.rs @@ -65,9 +65,8 @@ where let scalar_fn_array = parent .as_opt::() .vortex_expect("ExactScalarFn matcher confirmed ScalarFnArray"); - let children = scalar_fn_array.children(); - let lower = &children[1]; - let upper = &children[2]; + let lower = scalar_fn_array.get_child(1); + let upper = scalar_fn_array.get_child(2); let arr = array.clone().into_array(); if let Some(result) = precondition(&arr, lower, upper)? { return Ok(Some(result)); @@ -100,9 +99,8 @@ where let scalar_fn_array = parent .as_opt::() .vortex_expect("ExactScalarFn matcher confirmed ScalarFnArray"); - let children = scalar_fn_array.children(); - let lower = &children[1]; - let upper = &children[2]; + let lower = scalar_fn_array.get_child(1); + let upper = scalar_fn_array.get_child(2); let arr = array.clone().into_array(); if let Some(result) = precondition(&arr, lower, upper)? { return Ok(Some(result)); diff --git a/vortex-array/src/scalar_fn/fns/binary/compare.rs b/vortex-array/src/scalar_fn/fns/binary/compare.rs index 7c31ee6aab2..6f765964682 100644 --- a/vortex-array/src/scalar_fn/fns/binary/compare.rs +++ b/vortex-array/src/scalar_fn/fns/binary/compare.rs @@ -75,13 +75,11 @@ where let Some(scalar_fn_array) = parent.as_opt::() else { return Ok(None); }; - let children = scalar_fn_array.children(); - // Normalize so `array` is always LHS, swapping the operator if needed // TODO(joe): should be go this here or in the Rule/Kernel let (cmp_op, other) = match child_idx { - 0 => (cmp_op, &children[1]), - 1 => (cmp_op.swap(), &children[0]), + 0 => (cmp_op, scalar_fn_array.get_child(1)), + 1 => (cmp_op.swap(), scalar_fn_array.get_child(0)), _ => return Ok(None), }; diff --git a/vortex-array/src/scalar_fn/fns/fill_null/kernel.rs b/vortex-array/src/scalar_fn/fns/fill_null/kernel.rs index 44f83f5417b..9afd0b11703 100644 --- a/vortex-array/src/scalar_fn/fns/fill_null/kernel.rs +++ b/vortex-array/src/scalar_fn/fns/fill_null/kernel.rs @@ -115,7 +115,8 @@ where let scalar_fn_array = parent .as_opt::() .vortex_expect("ExactScalarFn matcher confirmed ScalarFnArray"); - let fill_value = scalar_fn_array.children()[1] + let fill_value = scalar_fn_array + .get_child(1) .as_constant() .vortex_expect("fill_null fill_value must be constant"); let arr = array.to_array(); @@ -150,7 +151,8 @@ where let scalar_fn_array = parent .as_opt::() .vortex_expect("ExactScalarFn matcher confirmed ScalarFnArray"); - let fill_value = scalar_fn_array.children()[1] + let fill_value = scalar_fn_array + .get_child(1) .as_constant() .vortex_expect("fill_null fill_value must be constant"); let arr = array.to_array(); diff --git a/vortex-array/src/scalar_fn/fns/like/kernel.rs b/vortex-array/src/scalar_fn/fns/like/kernel.rs index 839ffa734e9..b349fe7d5a4 100644 --- a/vortex-array/src/scalar_fn/fns/like/kernel.rs +++ b/vortex-array/src/scalar_fn/fns/like/kernel.rs @@ -69,8 +69,7 @@ where let scalar_fn_array = parent .as_opt::() .vortex_expect("ExactScalarFn matcher confirmed ScalarFnArray"); - let children = scalar_fn_array.children(); - let pattern = &children[1]; + let pattern = scalar_fn_array.get_child(1); let options = *parent.options; ::like(array, pattern, options) } @@ -99,8 +98,7 @@ where let scalar_fn_array = parent .as_opt::() .vortex_expect("ExactScalarFn matcher confirmed ScalarFnArray"); - let children = scalar_fn_array.children(); - let pattern = &children[1]; + let pattern = scalar_fn_array.get_child(1); let options = *parent.options; ::like(array, pattern, options, ctx) } diff --git a/vortex-array/src/scalar_fn/fns/list_contains/kernel.rs b/vortex-array/src/scalar_fn/fns/list_contains/kernel.rs index 13be6bfc26a..71001eb6572 100644 --- a/vortex-array/src/scalar_fn/fns/list_contains/kernel.rs +++ b/vortex-array/src/scalar_fn/fns/list_contains/kernel.rs @@ -64,7 +64,7 @@ where let scalar_fn_array = parent .as_opt::() .vortex_expect("ExactScalarFn matcher confirmed ScalarFnArray"); - let list = &scalar_fn_array.children()[0]; + let list = scalar_fn_array.get_child(0); ::list_contains(list, array) } } @@ -93,7 +93,7 @@ where let scalar_fn_array = parent .as_opt::() .vortex_expect("ExactScalarFn matcher confirmed ScalarFnArray"); - let list = &scalar_fn_array.children()[0]; + let list = scalar_fn_array.get_child(0); ::list_contains(list, array, ctx) } } diff --git a/vortex-array/src/scalar_fn/fns/merge.rs b/vortex-array/src/scalar_fn/fns/merge.rs index 7e4d72fe07c..f0681a418b1 100644 --- a/vortex-array/src/scalar_fn/fns/merge.rs +++ b/vortex-array/src/scalar_fn/fns/merge.rs @@ -159,7 +159,7 @@ impl ScalarFnVTable for Merge { for (field_name, field_array) in array .names() .iter() - .zip_eq(array.unmasked_fields().iter().cloned()) + .zip_eq(array.iter_unmasked_fields().cloned()) { // Update or insert field. if let Some(idx) = field_names.iter().position(|name| name == field_name) { diff --git a/vortex-array/src/scalar_fn/fns/zip/kernel.rs b/vortex-array/src/scalar_fn/fns/zip/kernel.rs index 2ee5b0bf5eb..5c5ee39ac06 100644 --- a/vortex-array/src/scalar_fn/fns/zip/kernel.rs +++ b/vortex-array/src/scalar_fn/fns/zip/kernel.rs @@ -68,9 +68,8 @@ where let scalar_fn_array = parent .as_opt::() .vortex_expect("ExactScalarFn matcher confirmed ScalarFnArray"); - let children = scalar_fn_array.children(); - let if_false = &children[1]; - let mask_array = &children[2]; + let if_false = scalar_fn_array.get_child(1); + let mask_array = scalar_fn_array.get_child(2); ::zip(array, if_false, mask_array) } } @@ -98,9 +97,8 @@ where let scalar_fn_array = parent .as_opt::() .vortex_expect("ExactScalarFn matcher confirmed ScalarFnArray"); - let children = scalar_fn_array.children(); - let if_false = &children[1]; - let mask_array = &children[2]; + let if_false = scalar_fn_array.get_child(1); + let mask_array = scalar_fn_array.get_child(2); ::zip(array, if_false, mask_array, ctx) } } diff --git a/vortex-array/src/vtable/dyn_.rs b/vortex-array/src/vtable/dyn_.rs index 64ca5e99c50..63dae63c6ba 100644 --- a/vortex-array/src/vtable/dyn_.rs +++ b/vortex-array/src/vtable/dyn_.rs @@ -47,7 +47,8 @@ pub trait DynVTable: 'static + private::Sealed + Send + Sync + Debug { children: &dyn ArrayChildren, session: &VortexSession, ) -> VortexResult; - fn with_children(&self, array: &ArrayRef, children: Vec) -> VortexResult; + /// See [`VTable::with_slots`] + fn with_slots(&self, array: &ArrayRef, slots: Vec>) -> VortexResult; /// See [`VTable::reduce`] fn reduce(&self, array: &ArrayRef) -> VortexResult>; @@ -95,9 +96,9 @@ impl DynVTable for ArrayVTableAdapter { Ok(array.into_array()) } - fn with_children(&self, array: &ArrayRef, children: Vec) -> VortexResult { + fn with_slots(&self, array: &ArrayRef, slots: Vec>) -> VortexResult { let mut array = array.as_::().clone(); - V::with_children(&mut array, children)?; + V::with_slots(&mut array, slots)?; Ok(array.into_array()) } diff --git a/vortex-array/src/vtable/mod.rs b/vortex-array/src/vtable/mod.rs index 2b7ab9bd620..aa529674735 100644 --- a/vortex-array/src/vtable/mod.rs +++ b/vortex-array/src/vtable/mod.rs @@ -86,19 +86,41 @@ pub trait VTable: 'static + Sized + Send + Sync + Debug { fn buffer_name(array: &Self::Array, idx: usize) -> Option; /// Returns the number of children in the array. - fn nchildren(array: &Self::Array) -> usize; + /// + /// The default counts non-None slots. + fn nchildren(array: &Self::Array) -> usize { + Self::slots(array).iter().filter(|s| s.is_some()).count() + } /// Returns the child at the given index. /// + /// The default returns the `idx`-th non-None slot. + /// /// # Panics /// Panics if `idx >= nchildren(array)`. - fn child(array: &Self::Array, idx: usize) -> ArrayRef; + fn child(array: &Self::Array, idx: usize) -> ArrayRef { + Self::slots(array) + .iter() + .filter_map(|s| s.clone()) + .nth(idx) + .vortex_expect("child index out of bounds") + } /// Returns the name of the child at the given index. /// + /// The default returns the slot name of the `idx`-th non-None slot. + /// /// # Panics /// Panics if `idx >= nchildren(array)`. - fn child_name(array: &Self::Array, idx: usize) -> String; + fn child_name(array: &Self::Array, idx: usize) -> String { + Self::slots(array) + .iter() + .enumerate() + .filter(|(_, s)| s.is_some()) + .nth(idx) + .map(|(slot_idx, _)| Self::slot_name(array, slot_idx)) + .vortex_expect("child_name index out of bounds") + } /// Exports metadata for an array. /// @@ -177,16 +199,28 @@ pub trait VTable: 'static + Sized + Send + Sync + Debug { children: &dyn ArrayChildren, ) -> VortexResult; - /// Replaces the children in `array` with `children`. The count must be the same and types - /// of children must be expected. - fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()>; + /// Returns the slots of the array as a slice. + /// + /// Slots provide fixed-position storage for child arrays, enabling direct access + /// by slot index without the overhead of dynamic child lookups. Optional children + /// (like validity) are represented as `None` slots rather than shifting dense indices. + fn slots(array: &Self::Array) -> &[Option]; + + /// Returns the name of the slot at the given index. + /// + /// # Panics + /// Panics if `idx >= slots(array).len()`. + fn slot_name(array: &Self::Array, idx: usize) -> String; + + /// Replaces the slots in `array` with `slots`. + fn with_slots(array: &mut Self::Array, slots: Vec>) -> VortexResult<()>; /// Execute this array by returning an [`ExecutionStep`] that tells the scheduler what to /// do next. /// /// Instead of recursively executing children, implementations should return - /// [`ExecutionStep::ExecuteChild`] to request that the scheduler execute a child first, - /// or [`ExecutionStep::Done`] when the + /// [`ExecutionStep::ExecuteSlot(i)`] to request that the scheduler execute a slot first, + /// or [`ExecutionStep::Done(result)`] when the /// encoding can produce a result directly. /// /// Array execution is designed such that repeated execution of an array will eventually diff --git a/vortex-bench/src/datasets/struct_list_of_ints.rs b/vortex-bench/src/datasets/struct_list_of_ints.rs index 34588df5b11..650a04a0b20 100644 --- a/vortex-bench/src/datasets/struct_list_of_ints.rs +++ b/vortex-bench/src/datasets/struct_list_of_ints.rs @@ -116,12 +116,11 @@ impl Dataset for StructListOfInts { // Convert to Arrow RecordBatches and write to parquet let chunked = array.as_::(); - let chunks = chunked.chunks(); let file = File::create(&temp_path)?; let mut writer: Option> = None; - for chunk in chunks.iter() { + for chunk in chunked.iter_chunks() { let converted = recursive_list_from_list_view(chunk.clone())?; let batch = RecordBatch::try_from(converted.as_ref())?; diff --git a/vortex-btrblocks/src/canonical_compressor.rs b/vortex-btrblocks/src/canonical_compressor.rs index 410dda0b599..73556e606f3 100644 --- a/vortex-btrblocks/src/canonical_compressor.rs +++ b/vortex-btrblocks/src/canonical_compressor.rs @@ -228,8 +228,7 @@ impl CanonicalCompressor for BtrBlocksCompressor { Canonical::Decimal(decimal) => compress_decimal(self, &decimal), Canonical::Struct(struct_array) => { let fields = struct_array - .unmasked_fields() - .iter() + .iter_unmasked_fields() .map(|field| self.compress(field)) .collect::, _>>()?; diff --git a/vortex-cuda/gpu-scan-cli/src/main.rs b/vortex-cuda/gpu-scan-cli/src/main.rs index 1c0faddf27e..72b907d896b 100644 --- a/vortex-cuda/gpu-scan-cli/src/main.rs +++ b/vortex-cuda/gpu-scan-cli/src/main.rs @@ -179,8 +179,7 @@ async fn cmd_scan(path: PathBuf, gpu_file: bool, json_output: bool) -> VortexRes let record = next.to_struct(); for (field, field_name) in record - .unmasked_fields() - .iter() + .iter_unmasked_fields() .zip(record.struct_fields().names().iter()) { let field_name = field_name.to_string(); diff --git a/vortex-duckdb/src/convert/vector.rs b/vortex-duckdb/src/convert/vector.rs index d6b008efe64..fb503b57abb 100644 --- a/vortex-duckdb/src/convert/vector.rs +++ b/vortex-duckdb/src/convert/vector.rs @@ -686,7 +686,7 @@ mod tests { let vortex_array = result.to_struct(); assert_eq!(vortex_array.len(), len); - assert_eq!(vortex_array.unmasked_fields().len(), 0); + assert_eq!(vortex_array.struct_fields().nfields(), 0); } #[test] @@ -721,13 +721,13 @@ mod tests { let vortex_array = result.to_struct(); assert_eq!(vortex_array.len(), len); - assert_eq!(vortex_array.unmasked_fields().len(), 2); + assert_eq!(vortex_array.struct_fields().nfields(), 2); assert_arrays_eq!( - &vortex_array.unmasked_fields()[0], + vortex_array.unmasked_field(0), PrimitiveArray::from_option_iter([Some(1i32), Some(2), Some(3), Some(4)]) ); assert_arrays_eq!( - &vortex_array.unmasked_fields()[1], + vortex_array.unmasked_field(1), PrimitiveArray::from_option_iter([Some(5i32), Some(6), Some(7), Some(8)]) ); } diff --git a/vortex-duckdb/src/exporter/mod.rs b/vortex-duckdb/src/exporter/mod.rs index f21d4882a8d..3959c4b4a6c 100644 --- a/vortex-duckdb/src/exporter/mod.rs +++ b/vortex-duckdb/src/exporter/mod.rs @@ -61,8 +61,7 @@ impl ArrayExporter { assert!(validity.all_true()); let fields = array - .unmasked_fields() - .iter() + .iter_unmasked_fields() .map(|field| new_array_exporter(field.clone(), cache, &mut ctx)) .collect::>>()?; diff --git a/vortex-ffi/src/array.rs b/vortex-ffi/src/array.rs index c42ba0e8454..e40a08a2548 100644 --- a/vortex-ffi/src/array.rs +++ b/vortex-ffi/src/array.rs @@ -56,12 +56,12 @@ pub unsafe extern "C-unwind" fn vx_array_get_field( try_or_default(error_out, || { let array = vx_array::as_ref(array); - let field_array = array - .to_struct() - .unmasked_fields() - .get(index as usize) - .ok_or_else(|| vortex_err!("Field index out of bounds"))? - .clone(); + let struct_array = array.to_struct(); + let idx = index as usize; + if idx >= struct_array.struct_fields().nfields() { + return Err(vortex_err!("Field index out of bounds")); + } + let field_array = struct_array.unmasked_field(idx).clone(); Ok(vx_array::new(field_array)) }) diff --git a/vortex-file/src/tests.rs b/vortex-file/src/tests.rs index e477146e5cd..8f92d660460 100644 --- a/vortex-file/src/tests.rs +++ b/vortex-file/src/tests.rs @@ -302,7 +302,7 @@ async fn test_read_projection() { ) ); - let actual = array.to_struct().unmasked_fields()[0].clone(); + let actual = array.to_struct().unmasked_field(0).clone(); let expected = VarBinArray::from(strings_expected.to_vec()).into_array(); assert_arrays_eq!(actual.as_ref(), expected.as_ref()); @@ -324,7 +324,7 @@ async fn test_read_projection() { ) ); - let actual = array.to_struct().unmasked_fields()[0].clone(); + let actual = array.to_struct().unmasked_field(0).clone(); let expected = Buffer::copy_from(numbers_expected).into_array(); assert_arrays_eq!(actual.as_ref(), expected.as_ref()); } @@ -536,13 +536,13 @@ async fn filter_string() { .unwrap(); assert_eq!(result.len(), 1); - let names_actual = result[0].to_struct().unmasked_fields()[0].clone(); + let names_actual = result[0].to_struct().unmasked_field(0).clone(); let names_expected = VarBinArray::from_iter(vec![Some("Joseph")], DType::Utf8(Nullability::Nullable)) .into_array(); assert_arrays_eq!(names_actual.as_ref(), names_expected.as_ref()); - let ages_actual = result[0].to_struct().unmasked_fields()[1].clone(); + let ages_actual = result[0].to_struct().unmasked_field(1).clone(); let ages_expected = PrimitiveArray::from_option_iter([Some(25i32)]).into_array(); assert_arrays_eq!(ages_actual.as_ref(), ages_expected.as_ref()); } @@ -591,7 +591,7 @@ async fn filter_or() { .unwrap(); assert_eq!(result.len(), 1); - let names_actual = result[0].to_struct().unmasked_fields()[0].clone(); + let names_actual = result[0].to_struct().unmasked_field(0).clone(); let names_expected = VarBinArray::from_iter( vec![Some("Joseph"), Some("Angela")], DType::Utf8(Nullability::Nullable), @@ -599,7 +599,7 @@ async fn filter_or() { .into_array(); assert_arrays_eq!(names_actual.as_ref(), names_expected.as_ref()); - let ages_actual = result[0].to_struct().unmasked_fields()[1].clone(); + let ages_actual = result[0].to_struct().unmasked_field(1).clone(); let ages_expected = PrimitiveArray::from_option_iter([Some(25i32), None]).into_array(); assert_arrays_eq!(ages_actual.as_ref(), ages_expected.as_ref()); } @@ -645,7 +645,7 @@ async fn filter_and() { .unwrap(); assert_eq!(result.len(), 1); - let names_actual = result[0].to_struct().unmasked_fields()[0].clone(); + let names_actual = result[0].to_struct().unmasked_field(0).clone(); let names_expected = VarBinArray::from_iter( vec![Some("Joseph"), None], DType::Utf8(Nullability::Nullable), @@ -653,7 +653,7 @@ async fn filter_and() { .into_array(); assert_arrays_eq!(names_actual.as_ref(), names_expected.as_ref()); - let ages_actual = result[0].to_struct().unmasked_fields()[1].clone(); + let ages_actual = result[0].to_struct().unmasked_field(1).clone(); let ages_expected = PrimitiveArray::from_option_iter([Some(25i32), Some(31i32)]).into_array(); assert_arrays_eq!(ages_actual.as_ref(), ages_expected.as_ref()); } @@ -711,7 +711,7 @@ async fn test_with_indices_simple() { .await .unwrap() .to_struct(); - let actual_kept_numbers_array = actual_kept_array.unmasked_fields()[0].to_primitive(); + let actual_kept_numbers_array = actual_kept_array.unmasked_field(0).to_primitive(); let expected_kept_numbers: Vec = kept_indices .iter() @@ -731,7 +731,7 @@ async fn test_with_indices_simple() { .await .unwrap() .to_struct(); - let actual_numbers_array = actual_array.unmasked_fields()[0].clone(); + let actual_numbers_array = actual_array.unmasked_field(0).clone(); let expected_array = Buffer::copy_from(&expected_numbers).into_array(); assert_arrays_eq!(actual_numbers_array.as_ref(), expected_array.as_ref()); } @@ -776,7 +776,7 @@ async fn test_with_indices_on_two_columns() { .to_struct() .to_struct(); - let strings_actual = array.unmasked_fields()[0].clone(); + let strings_actual = array.unmasked_field(0).clone(); let strings_expected_vec: Vec<&str> = kept_indices .iter() .map(|&x| strings_expected[x as usize]) @@ -784,7 +784,7 @@ async fn test_with_indices_on_two_columns() { let strings_expected_array = VarBinArray::from(strings_expected_vec).into_array(); assert_arrays_eq!(strings_actual.as_ref(), strings_expected_array.as_ref()); - let numbers_actual = array.unmasked_fields()[1].clone(); + let numbers_actual = array.unmasked_field(1).clone(); let numbers_expected_vec: Vec = kept_indices .iter() .map(|&x| numbers_expected[x as usize]) @@ -848,7 +848,7 @@ async fn test_with_indices_and_with_row_filter_simple() { .unwrap() .to_struct(); - let actual_kept_numbers_array = actual_kept_array.unmasked_fields()[0].to_primitive(); + let actual_kept_numbers_array = actual_kept_array.unmasked_field(0).to_primitive(); let expected_kept_numbers: Buffer = kept_indices .iter() @@ -871,7 +871,7 @@ async fn test_with_indices_and_with_row_filter_simple() { .unwrap() .to_struct(); - let actual_numbers_array = actual_array.unmasked_fields()[0].clone(); + let actual_numbers_array = actual_array.unmasked_field(0).clone(); let expected_filtered: Buffer = expected_numbers .iter() .filter(|&&x| x > 50) @@ -934,13 +934,13 @@ async fn filter_string_chunked() { .to_struct(); assert_eq!(actual_array.len(), 1); - let names_actual = actual_array.unmasked_fields()[0].clone(); + let names_actual = actual_array.unmasked_field(0).clone(); let names_expected = VarBinArray::from_iter(vec![Some("Joseph")], DType::Utf8(Nullability::Nullable)) .into_array(); assert_arrays_eq!(names_actual.as_ref(), names_expected.as_ref()); - let ages_actual = actual_array.unmasked_fields()[1].clone(); + let ages_actual = actual_array.unmasked_field(1).clone(); let ages_expected = PrimitiveArray::from_option_iter([Some(25i32)]).into_array(); assert_arrays_eq!(ages_actual.as_ref(), ages_expected.as_ref()); } @@ -1025,7 +1025,7 @@ async fn test_pruning_with_or() { .to_struct(); assert_eq!(actual_array.len(), 10); - let letters_actual = actual_array.unmasked_fields()[0].clone(); + let letters_actual = actual_array.unmasked_field(0).clone(); let letters_expected = VarBinViewArray::from_iter_nullable_str([ Some("A".to_owned()), Some("B".to_owned()), @@ -1041,7 +1041,7 @@ async fn test_pruning_with_or() { .into_array(); assert_arrays_eq!(letters_actual.as_ref(), letters_expected.as_ref()); - let numbers_actual = actual_array.unmasked_fields()[1].clone(); + let numbers_actual = actual_array.unmasked_field(1).clone(); let numbers_expected = PrimitiveArray::from_option_iter([ Some(25_i32), Some(31), @@ -1242,7 +1242,7 @@ async fn write_nullable_nested_struct() -> VortexResult<()> { let result = round_trip(&array, Ok).await?.to_struct(); assert_eq!(result.len(), 3); - assert_eq!(result.unmasked_fields().len(), 1); + assert_eq!(result.struct_fields().nfields(), 1); assert!(result.all_valid()?); let nested_struct = result.unmasked_field_by_name("struct")?.to_struct(); diff --git a/vortex-jni/src/array.rs b/vortex-jni/src/array.rs index 3fcdf041cb5..f6670f0bd07 100644 --- a/vortex-jni/src/array.rs +++ b/vortex-jni/src/array.rs @@ -222,13 +222,12 @@ pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getField( let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; try_or_throw(&mut env, |_| { - let field = array_ref - .inner - .to_struct() - .unmasked_fields() - .get(index as usize) - .cloned() - .ok_or_else(|| vortex_err!("Field index out of bounds"))?; + let struct_array = array_ref.inner.to_struct(); + let idx = index as usize; + if idx >= struct_array.struct_fields().nfields() { + return Err(vortex_err!("Field index out of bounds").into()); + } + let field = struct_array.unmasked_field(idx).clone(); Ok(NativeArray::new(field).into_raw()) }) } diff --git a/vortex-layout/src/layouts/file_stats.rs b/vortex-layout/src/layouts/file_stats.rs index 48254684451..8fc2397e959 100644 --- a/vortex-layout/src/layouts/file_stats.rs +++ b/vortex-layout/src/layouts/file_stats.rs @@ -99,7 +99,7 @@ impl FileStatsAccumulator { .accumulators .lock() .iter_mut() - .zip_eq(chunk.unmasked_fields().iter()) + .zip_eq(chunk.iter_unmasked_fields()) { acc.push_chunk(field)?; } diff --git a/vortex-layout/src/layouts/struct_/reader.rs b/vortex-layout/src/layouts/struct_/reader.rs index bb069df9723..1394305ac3e 100644 --- a/vortex-layout/src/layouts/struct_/reader.rs +++ b/vortex-layout/src/layouts/struct_/reader.rs @@ -363,8 +363,7 @@ impl LayoutReader for StructReader { if is_pack_merge { let struct_array = array.to_struct(); let masked_fields: Vec = struct_array - .unmasked_fields() - .iter() + .iter_unmasked_fields() .map(|a| a.clone().mask(validity.clone())) .try_collect()?; diff --git a/vortex-layout/src/layouts/struct_/writer.rs b/vortex-layout/src/layouts/struct_/writer.rs index 06320a10844..6a52f1cb3bc 100644 --- a/vortex-layout/src/layouts/struct_/writer.rs +++ b/vortex-layout/src/layouts/struct_/writer.rs @@ -113,8 +113,7 @@ impl LayoutStrategy for StructStrategy { columns.extend( struct_chunk - .unmasked_fields() - .iter() + .iter_unmasked_fields() .map(|field| (sequence_pointer.advance(), field.to_array())), ); diff --git a/vortex-layout/src/layouts/table.rs b/vortex-layout/src/layouts/table.rs index c8fb8ac6eda..9334bd5b6c4 100644 --- a/vortex-layout/src/layouts/table.rs +++ b/vortex-layout/src/layouts/table.rs @@ -240,8 +240,7 @@ impl LayoutStrategy for TableStrategy { columns.extend( struct_chunk - .unmasked_fields() - .iter() + .iter_unmasked_fields() .map(|field| (sequence_pointer.advance(), field.to_array())), ); diff --git a/vortex-layout/src/layouts/zoned/zone_map.rs b/vortex-layout/src/layouts/zoned/zone_map.rs index 21dac24fdda..9d092b05de2 100644 --- a/vortex-layout/src/layouts/zoned/zone_map.rs +++ b/vortex-layout/src/layouts/zoned/zone_map.rs @@ -335,13 +335,17 @@ mod tests { ] ); assert_eq!( - stats_table.array.unmasked_fields()[1] + stats_table + .array + .unmasked_field(1) .to_bool() .to_bit_buffer(), BitBuffer::from(vec![false, true]) ); assert_eq!( - stats_table.array.unmasked_fields()[3] + stats_table + .array + .unmasked_field(3) .to_bool() .to_bit_buffer(), BitBuffer::from(vec![true, false]) @@ -369,13 +373,17 @@ mod tests { ] ); assert_eq!( - stats_table.array.unmasked_fields()[1] + stats_table + .array + .unmasked_field(1) .to_bool() .to_bit_buffer(), BitBuffer::from(vec![false]) ); assert_eq!( - stats_table.array.unmasked_fields()[3] + stats_table + .array + .unmasked_field(3) .to_bool() .to_bit_buffer(), BitBuffer::from(vec![false]) diff --git a/vortex-python/src/arrays/builtins/chunked.rs b/vortex-python/src/arrays/builtins/chunked.rs index aa376986340..293932c21c2 100644 --- a/vortex-python/src/arrays/builtins/chunked.rs +++ b/vortex-python/src/arrays/builtins/chunked.rs @@ -24,8 +24,7 @@ impl PyChunkedArray { pub fn chunks(self_: PyRef<'_, Self>) -> Vec { self_ .as_array_ref() - .chunks() - .iter() + .iter_chunks() .map(|chunk| PyArrayRef::from(chunk.clone())) .collect() } diff --git a/vortex-python/src/arrays/mod.rs b/vortex-python/src/arrays/mod.rs index 674f45ddc2c..3c1719973a4 100644 --- a/vortex-python/src/arrays/mod.rs +++ b/vortex-python/src/arrays/mod.rs @@ -330,8 +330,7 @@ impl PyArray { let arrow_dtype = chunked_array.dtype().to_arrow_dtype()?; let chunks = chunked_array - .chunks() - .iter() + .iter_chunks() .map(|chunk| -> PyVortexResult<_> { Ok(chunk.clone().into_arrow(&arrow_dtype)?) }) .collect::, _>>()?; diff --git a/vortex-python/src/arrays/py/vtable.rs b/vortex-python/src/arrays/py/vtable.rs index c69961bfa18..320455d78b8 100644 --- a/vortex-python/src/arrays/py/vtable.rs +++ b/vortex-python/src/arrays/py/vtable.rs @@ -87,18 +87,6 @@ impl VTable for Python { None } - fn nchildren(_array: &PythonArray) -> usize { - 0 - } - - fn child(_array: &PythonArray, idx: usize) -> ArrayRef { - vortex_panic!("PythonArray child index {idx} out of bounds") - } - - fn child_name(_array: &PythonArray, idx: usize) -> String { - vortex_panic!("PythonArray child_name index {idx} out of bounds") - } - fn metadata(array: &PythonArray) -> VortexResult { pyo3::Python::attach(|py| { let obj = array.object.bind(py); @@ -146,11 +134,19 @@ impl VTable for Python { todo!() } - fn with_children(_array: &mut Self::Array, children: Vec) -> VortexResult<()> { + fn slots(_array: &PythonArray) -> &[Option] { + &[] + } + + fn slot_name(_array: &PythonArray, idx: usize) -> String { + vortex_panic!("PythonArray has no slots, requested index {idx}") + } + + fn with_slots(_array: &mut PythonArray, slots: Vec>) -> VortexResult<()> { vortex_ensure!( - children.is_empty(), - "PythonArray has no children, got {}", - children.len() + slots.is_empty(), + "PythonArray has no slots, got {}", + slots.len() ); Ok(()) }