diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 64327ae25df..67f1955e17c 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -930,6 +930,8 @@ pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::BoolArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Bool::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Bool::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::bool::BoolArray @@ -1138,6 +1140,8 @@ pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vo pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::ChunkedArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Chunked::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Chunked::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Chunked @@ -1304,6 +1308,8 @@ pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> v pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::ConstantArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Constant::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Constant::with_children(_array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Constant @@ -1510,6 +1516,8 @@ pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::DecimalArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Decimal::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Decimal::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::decimal::DecimalArray @@ -1718,6 +1726,8 @@ pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::dict::Dict::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::dict::Dict::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::dict::Dict @@ -1822,6 +1832,8 @@ pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::dict::Dict::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::dict::Dict::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::dict::Dict @@ -2118,6 +2130,8 @@ pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::ExtensionArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Extension::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Extension::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityChild for vortex_array::arrays::Extension @@ -2254,6 +2268,8 @@ pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::FilterArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Filter::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Filter::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Filter @@ -2470,6 +2486,8 @@ pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::FixedSizeListArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::FixedSizeList::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::FixedSizeList::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::fixed_size_list::FixedSizeListArray @@ -2618,6 +2636,8 @@ pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::ListArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::List::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::List::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::list::ListArray @@ -2786,6 +2806,8 @@ pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vo pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::ListViewArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::ListView::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::ListView::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::listview::ListViewArray @@ -2964,6 +2986,8 @@ pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::MaskedArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Masked::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Masked::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::masked::MaskedArray @@ -3102,6 +3126,8 @@ pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::null::Null::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::null::Null::with_children(_array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::null::Null @@ -3312,6 +3338,8 @@ pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::PrimitiveArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Primitive::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Primitive::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::primitive::PrimitiveArray @@ -3620,6 +3648,8 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Sel pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -3702,6 +3732,8 @@ pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::SharedArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Shared::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Shared::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Shared @@ -3820,6 +3852,8 @@ pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::slice::Slice::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::slice::Slice::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::slice::Slice @@ -4078,6 +4112,8 @@ pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::StructArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Struct::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Struct::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::struct_::StructArray @@ -4294,6 +4330,8 @@ pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vort pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::VarBinArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::VarBin::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::VarBin::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::varbin::VarBinArray @@ -4696,6 +4734,8 @@ pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::VarBinViewArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::VarBinView::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::VarBinView::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::varbinview::VarBinViewArray @@ -4886,6 +4926,8 @@ pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vo pub fn vortex_array::arrays::Variant::stats(array: &Self::Array) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Variant::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Variant::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Variant @@ -5026,6 +5068,8 @@ pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::BoolArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Bool::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Bool::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::BoolArray @@ -5206,6 +5250,8 @@ pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vo pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::ChunkedArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Chunked::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Chunked::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Chunked @@ -5370,6 +5416,8 @@ pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> v pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::ConstantArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Constant::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Constant::with_children(_array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Constant @@ -5512,6 +5560,8 @@ pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::DecimalArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Decimal::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Decimal::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::DecimalArray @@ -5688,6 +5738,8 @@ pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::dict::Dict::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::dict::Dict::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::dict::Dict @@ -5842,6 +5894,8 @@ pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::ExtensionArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Extension::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Extension::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityChild for vortex_array::arrays::Extension @@ -5976,6 +6030,8 @@ pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::FilterArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Filter::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Filter::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Filter @@ -6108,6 +6164,8 @@ pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::FixedSizeListArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::FixedSizeList::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::FixedSizeList::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::FixedSizeListArray @@ -6254,6 +6312,8 @@ pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::ListArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::List::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::List::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::ListArray @@ -6400,6 +6460,8 @@ pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vo pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::ListViewArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::ListView::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::ListView::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::ListViewArray @@ -6558,6 +6620,8 @@ pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::MaskedArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Masked::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Masked::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::MaskedArray @@ -6692,6 +6756,8 @@ pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::null::Null::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::null::Null::with_children(_array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::null::Null @@ -6834,6 +6900,8 @@ pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::PrimitiveArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Primitive::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Primitive::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::PrimitiveArray @@ -7068,6 +7136,8 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Sel pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -7140,6 +7210,8 @@ pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::SharedArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Shared::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Shared::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Shared @@ -7256,6 +7328,8 @@ pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::slice::Slice::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::slice::Slice::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::slice::Slice @@ -7392,6 +7466,8 @@ pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::StructArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Struct::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Struct::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::StructArray @@ -7632,6 +7708,8 @@ pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vort pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::VarBinArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::VarBin::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::VarBin::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::VarBinArray @@ -7848,6 +7926,8 @@ pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::VarBinViewArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::VarBinView::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::VarBinView::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub struct vortex_array::arrays::VarBinViewArray @@ -8026,6 +8106,8 @@ pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vo pub fn vortex_array::arrays::Variant::stats(array: &Self::Array) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Variant::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Variant::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Variant @@ -20402,6 +20484,8 @@ pub fn vortex_array::vtable::DynVTable::reduce(&self, array: &vortex_array::Arra pub fn vortex_array::vtable::DynVTable::reduce_parent(&self, array: &vortex_array::ArrayRef, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> +pub fn vortex_array::vtable::DynVTable::with_buffers(&self, array: &vortex_array::ArrayRef, buffers: alloc::vec::Vec) -> vortex_error::VortexResult + pub fn vortex_array::vtable::DynVTable::with_children(&self, array: &vortex_array::ArrayRef, children: alloc::vec::Vec) -> vortex_error::VortexResult pub trait vortex_array::vtable::OperationsVTable @@ -20544,6 +20628,8 @@ pub fn vortex_array::vtable::VTable::serialize(metadata: Self::Metadata) -> vort pub fn vortex_array::vtable::VTable::stats(array: &Self::Array) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::vtable::VTable::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::vtable::VTable::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Bool @@ -20598,6 +20684,8 @@ pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::BoolArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Bool::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Bool::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Chunked @@ -20652,6 +20740,8 @@ pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vo pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::ChunkedArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Chunked::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Chunked::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Constant @@ -20706,6 +20796,8 @@ pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> v pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::ConstantArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Constant::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Constant::with_children(_array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Decimal @@ -20760,6 +20852,8 @@ pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::DecimalArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Decimal::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Decimal::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Extension @@ -20814,6 +20908,8 @@ pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::ExtensionArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Extension::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Extension::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Filter @@ -20868,6 +20964,8 @@ pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::FilterArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Filter::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Filter::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::FixedSizeList @@ -20922,6 +21020,8 @@ pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::FixedSizeListArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::FixedSizeList::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::FixedSizeList::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::List @@ -20976,6 +21076,8 @@ pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::ListArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::List::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::List::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::ListView @@ -21030,6 +21132,8 @@ pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vo pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::ListViewArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::ListView::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::ListView::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Masked @@ -21084,6 +21188,8 @@ pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::MaskedArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Masked::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Masked::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Primitive @@ -21138,6 +21244,8 @@ pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::PrimitiveArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Primitive::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Primitive::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Shared @@ -21192,6 +21300,8 @@ pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::SharedArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Shared::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Shared::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Struct @@ -21246,6 +21356,8 @@ pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vor pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::StructArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Struct::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Struct::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::VarBin @@ -21300,6 +21412,8 @@ pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vort pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::VarBinArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::VarBin::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::VarBin::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::VarBinView @@ -21354,6 +21468,8 @@ pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::VarBinViewArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::VarBinView::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::VarBinView::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::Variant @@ -21408,6 +21524,8 @@ pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vo pub fn vortex_array::arrays::Variant::stats(array: &Self::Array) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::Variant::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::Variant::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::dict::Dict @@ -21462,6 +21580,8 @@ pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::dict::Dict::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::dict::Dict::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::null::Null @@ -21516,6 +21636,8 @@ pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::null::Null::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::null::Null::with_children(_array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -21570,6 +21692,8 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Sel pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> impl vortex_array::vtable::VTable for vortex_array::arrays::slice::Slice @@ -21624,6 +21748,8 @@ pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceArray) -> vortex_array::stats::StatsSetRef<'_> +pub fn vortex_array::arrays::slice::Slice::with_buffers(array: &mut Self::Array, buffers: alloc::vec::Vec) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::slice::Slice::with_children(array: &mut Self::Array, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> pub trait vortex_array::vtable::ValidityChild @@ -22100,6 +22226,8 @@ pub fn vortex_array::ArrayAdapter::validity_mask(&self) -> vortex_error::Vort pub fn vortex_array::ArrayAdapter::vtable(&self) -> &dyn vortex_array::vtable::DynVTable +pub fn vortex_array::ArrayAdapter::with_buffers(&self, buffers: alloc::vec::Vec) -> vortex_error::VortexResult + pub fn vortex_array::ArrayAdapter::with_children(&self, children: alloc::vec::Vec) -> vortex_error::VortexResult impl vortex_array::scalar_fn::ReduceNode for vortex_array::ArrayAdapter @@ -22484,6 +22612,8 @@ pub fn vortex_array::DynArray::validity_mask(&self) -> vortex_error::VortexResul pub fn vortex_array::DynArray::vtable(&self) -> &dyn vortex_array::vtable::DynVTable +pub fn vortex_array::DynArray::with_buffers(&self, buffers: alloc::vec::Vec) -> vortex_error::VortexResult + pub fn vortex_array::DynArray::with_children(&self, children: alloc::vec::Vec) -> vortex_error::VortexResult impl vortex_array::DynArray for alloc::sync::Arc @@ -22534,6 +22664,8 @@ pub fn alloc::sync::Arc::validity_mask(&self) -> vor pub fn alloc::sync::Arc::vtable(&self) -> &dyn vortex_array::vtable::DynVTable +pub fn alloc::sync::Arc::with_buffers(&self, buffers: alloc::vec::Vec) -> vortex_error::VortexResult + pub fn alloc::sync::Arc::with_children(&self, children: alloc::vec::Vec) -> vortex_error::VortexResult impl vortex_array::DynArray for vortex_array::ArrayAdapter @@ -22584,6 +22716,8 @@ pub fn vortex_array::ArrayAdapter::validity_mask(&self) -> vortex_error::Vort pub fn vortex_array::ArrayAdapter::vtable(&self) -> &dyn vortex_array::vtable::DynVTable +pub fn vortex_array::ArrayAdapter::with_buffers(&self, buffers: alloc::vec::Vec) -> vortex_error::VortexResult + pub fn vortex_array::ArrayAdapter::with_children(&self, children: alloc::vec::Vec) -> vortex_error::VortexResult pub trait vortex_array::DynArrayEq: vortex_array::hash::private::SealedEq diff --git a/vortex-array/src/array/mod.rs b/vortex-array/src/array/mod.rs index 79eb980d317..154a44a4fd2 100644 --- a/vortex-array/src/array/mod.rs +++ b/vortex-array/src/array/mod.rs @@ -165,6 +165,9 @@ pub trait DynArray: /// Replaces the children of the array with the given array references. fn with_children(&self, children: Vec) -> VortexResult; + + /// Replaces the buffers of the array with the given buffer handles. + fn with_buffers(&self, buffers: Vec) -> VortexResult; } impl DynArray for Arc { @@ -278,6 +281,10 @@ impl DynArray for Arc { fn with_children(&self, children: Vec) -> VortexResult { self.as_ref().with_children(children) } + + fn with_buffers(&self, buffers: Vec) -> VortexResult { + self.as_ref().with_buffers(buffers) + } } /// A reference counted pointer to a dynamic [`DynArray`] trait object. @@ -663,6 +670,12 @@ impl DynArray for ArrayAdapter { V::with_children(&mut this, children)?; Ok(this.into_array()) } + + fn with_buffers(&self, buffers: Vec) -> VortexResult { + let mut this = self.0.clone(); + V::with_buffers(&mut this, buffers)?; + Ok(this.into_array()) + } } impl ArrayHash for ArrayAdapter { diff --git a/vortex-array/src/vtable/dyn_.rs b/vortex-array/src/vtable/dyn_.rs index 64ca5e99c50..cdb6e2f3683 100644 --- a/vortex-array/src/vtable/dyn_.rs +++ b/vortex-array/src/vtable/dyn_.rs @@ -48,6 +48,7 @@ pub trait DynVTable: 'static + private::Sealed + Send + Sync + Debug { session: &VortexSession, ) -> VortexResult; fn with_children(&self, array: &ArrayRef, children: Vec) -> VortexResult; + fn with_buffers(&self, array: &ArrayRef, buffers: Vec) -> VortexResult; /// See [`VTable::reduce`] fn reduce(&self, array: &ArrayRef) -> VortexResult>; @@ -101,6 +102,12 @@ impl DynVTable for ArrayVTableAdapter { Ok(array.into_array()) } + fn with_buffers(&self, array: &ArrayRef, buffers: Vec) -> VortexResult { + let mut array = array.as_::().clone(); + V::with_buffers(&mut array, buffers)?; + Ok(array.into_array()) + } + fn reduce(&self, array: &ArrayRef) -> VortexResult> { let Some(reduced) = V::reduce(downcast::(array))? else { return Ok(None); diff --git a/vortex-array/src/vtable/mod.rs b/vortex-array/src/vtable/mod.rs index 2b7ab9bd620..c96ec7a3b9c 100644 --- a/vortex-array/src/vtable/mod.rs +++ b/vortex-array/src/vtable/mod.rs @@ -181,6 +181,23 @@ pub trait VTable: 'static + Sized + Send + Sync + Debug { /// of children must be expected. fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()>; + /// Replaces the buffers in `array` with `buffers`. + /// + /// The default implementation rebuilds the array via [`build()`](VTable::build), which + /// re-runs all validation. This is correct for replacing lazy device buffers with + /// materialized host buffers. + fn with_buffers(array: &mut Self::Array, buffers: Vec) -> VortexResult<()> { + let metadata = Self::metadata(array)?; + let dtype = Self::dtype(array).clone(); + let len = Self::len(array); + let children: Vec = (0..Self::nchildren(array)) + .map(|i| Self::child(array, i)) + .collect(); + let children_slice: &[ArrayRef] = &children; + *array = Self::build(&dtype, len, &metadata, &buffers, &children_slice)?; + Ok(()) + } + /// Execute this array by returning an [`ExecutionStep`] that tells the scheduler what to /// do next. /// diff --git a/vortex-layout/public-api.lock b/vortex-layout/public-api.lock index 4515fc59412..e0bf15ea6e3 100644 --- a/vortex-layout/public-api.lock +++ b/vortex-layout/public-api.lock @@ -12,13 +12,19 @@ pub struct vortex_layout::buffer::LazyBufferHandle impl vortex_layout::buffer::LazyBufferHandle +pub fn vortex_layout::buffer::LazyBufferHandle::alignment(&self) -> vortex_buffer::alignment::Alignment + pub fn vortex_layout::buffer::LazyBufferHandle::byte_ranges(&self) -> core::option::Option<&[core::ops::range::Range]> pub fn vortex_layout::buffer::LazyBufferHandle::filter(&self, ranges: &[core::ops::range::Range]) -> Self +pub fn vortex_layout::buffer::LazyBufferHandle::is_empty(&self) -> bool + +pub fn vortex_layout::buffer::LazyBufferHandle::len(&self) -> usize + pub async fn vortex_layout::buffer::LazyBufferHandle::materialize(&self) -> vortex_error::VortexResult -pub fn vortex_layout::buffer::LazyBufferHandle::new(source: alloc::sync::Arc, segment_id: vortex_layout::segments::SegmentId) -> Self +pub fn vortex_layout::buffer::LazyBufferHandle::new(source: alloc::sync::Arc, segment_id: vortex_layout::segments::SegmentId, alignment: vortex_buffer::alignment::Alignment) -> Self pub fn vortex_layout::buffer::LazyBufferHandle::segment_id(&self) -> vortex_layout::segments::SegmentId @@ -28,10 +34,42 @@ impl core::clone::Clone for vortex_layout::buffer::LazyBufferHandle pub fn vortex_layout::buffer::LazyBufferHandle::clone(&self) -> vortex_layout::buffer::LazyBufferHandle +impl core::cmp::Eq for vortex_layout::buffer::LazyBufferHandle + +impl core::cmp::PartialEq for vortex_layout::buffer::LazyBufferHandle + +pub fn vortex_layout::buffer::LazyBufferHandle::eq(&self, other: &Self) -> bool + impl core::fmt::Debug for vortex_layout::buffer::LazyBufferHandle pub fn vortex_layout::buffer::LazyBufferHandle::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for vortex_layout::buffer::LazyBufferHandle + +pub fn vortex_layout::buffer::LazyBufferHandle::hash(&self, state: &mut H) + +impl vortex_array::buffer::DeviceBuffer for vortex_layout::buffer::LazyBufferHandle + +pub fn vortex_layout::buffer::LazyBufferHandle::aligned(self: alloc::sync::Arc, alignment: vortex_buffer::alignment::Alignment) -> vortex_error::VortexResult> + +pub fn vortex_layout::buffer::LazyBufferHandle::alignment(&self) -> vortex_buffer::alignment::Alignment + +pub fn vortex_layout::buffer::LazyBufferHandle::as_any(&self) -> &dyn core::any::Any + +pub fn vortex_layout::buffer::LazyBufferHandle::copy_to_host(&self, alignment: vortex_buffer::alignment::Alignment) -> vortex_error::VortexResult>> + +pub fn vortex_layout::buffer::LazyBufferHandle::copy_to_host_sync(&self, alignment: vortex_buffer::alignment::Alignment) -> vortex_error::VortexResult + +pub fn vortex_layout::buffer::LazyBufferHandle::filter(&self, ranges: &[core::ops::range::Range]) -> vortex_error::VortexResult> + +pub fn vortex_layout::buffer::LazyBufferHandle::len(&self) -> usize + +pub fn vortex_layout::buffer::LazyBufferHandle::slice(&self, range: core::ops::range::Range) -> alloc::sync::Arc + +pub fn vortex_layout::buffer::create_lazy_array_parts(array_tree: vortex_buffer::ByteBuffer, source: alloc::sync::Arc, segment_id: vortex_layout::segments::SegmentId) -> vortex_error::VortexResult + +pub async fn vortex_layout::buffer::materialize_recursive(array: &vortex_array::array::ArrayRef) -> vortex_error::VortexResult + pub mod vortex_layout::display pub struct vortex_layout::display::DisplayLayoutTree diff --git a/vortex-layout/src/buffer.rs b/vortex-layout/src/buffer.rs index 79235c8d583..eab11c644df 100644 --- a/vortex-layout/src/buffer.rs +++ b/vortex-layout/src/buffer.rs @@ -1,13 +1,26 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::any::Any; use std::fmt::Debug; use std::fmt::Formatter; +use std::hash::Hash; +use std::hash::Hasher; use std::ops::Range; use std::sync::Arc; +use futures::FutureExt; +use futures::future::BoxFuture; +use vortex_array::ArrayRef; +use vortex_array::ArrayVisitor; +use vortex_array::DynArray; use vortex_array::buffer::BufferHandle; +use vortex_array::buffer::DeviceBuffer; +use vortex_array::serde::ArrayParts; +use vortex_buffer::Alignment; +use vortex_buffer::ByteBuffer; use vortex_error::VortexResult; +use vortex_error::vortex_panic; use crate::segments::SegmentId; use crate::segments::SegmentSource; @@ -23,10 +36,11 @@ pub struct LazyBufferHandle { source: Arc, segment_id: SegmentId, selection: Selection, + alignment: Alignment, } /// Byte selection within a segment buffer. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Hash, PartialEq, Eq)] enum Selection { /// The entire segment is selected. All, @@ -36,16 +50,48 @@ enum Selection { Ranges(Arc<[Range]>), } +#[allow(clippy::same_name_method)] impl LazyBufferHandle { /// Create a new lazy handle selecting the entire segment. - pub fn new(source: Arc, segment_id: SegmentId) -> Self { + pub fn new( + source: Arc, + segment_id: SegmentId, + alignment: Alignment, + ) -> Self { Self { source, segment_id, selection: Selection::All, + alignment, } } + /// Returns the length of the selected byte range(s). + /// + /// # Panics + /// + /// Panics if the entire segment is selected ([`Selection::All`]) since the + /// length is not known without performing I/O. + pub fn len(&self) -> usize { + match &self.selection { + Selection::All => { + vortex_panic!("len() is not available for Selection::All; slice first") + } + Selection::Range(r) => r.len(), + Selection::Ranges(rs) => rs.iter().map(|r| r.len()).sum(), + } + } + + /// Returns whether the buffer is empty. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Returns the alignment of the buffer. + pub fn alignment(&self) -> Alignment { + self.alignment + } + /// Returns the segment ID. pub fn segment_id(&self) -> SegmentId { self.segment_id @@ -92,6 +138,7 @@ impl LazyBufferHandle { source: Arc::clone(&self.source), segment_id: self.segment_id, selection, + alignment: self.alignment, } } @@ -142,6 +189,7 @@ impl LazyBufferHandle { source: Arc::clone(&self.source), segment_id: self.segment_id, selection, + alignment: self.alignment, } } @@ -166,10 +214,169 @@ impl Debug for LazyBufferHandle { f.debug_struct("LazyBufferHandle") .field("segment_id", &self.segment_id) .field("selection", &self.selection) + .field("alignment", &self.alignment) .finish() } } +impl PartialEq for LazyBufferHandle { + fn eq(&self, other: &Self) -> bool { + self.segment_id == other.segment_id + && self.selection == other.selection + && self.alignment == other.alignment + } +} + +impl Eq for LazyBufferHandle {} + +impl Hash for LazyBufferHandle { + fn hash(&self, state: &mut H) { + self.segment_id.hash(state); + self.selection.hash(state); + self.alignment.hash(state); + } +} + +impl DeviceBuffer for LazyBufferHandle { + fn as_any(&self) -> &dyn Any { + self + } + + fn len(&self) -> usize { + self.len() + } + + fn alignment(&self) -> Alignment { + self.alignment + } + + fn copy_to_host_sync(&self, alignment: Alignment) -> VortexResult { + futures::executor::block_on(async { + let handle = self.materialize().await?; + Ok(handle.try_into_host_sync()?.aligned(alignment)) + }) + } + + fn copy_to_host( + &self, + alignment: Alignment, + ) -> VortexResult>> { + let this = self.clone(); + Ok(async move { + let handle = this.materialize().await?; + Ok(handle.try_into_host_sync()?.aligned(alignment)) + } + .boxed()) + } + + fn slice(&self, range: Range) -> Arc { + Arc::new(LazyBufferHandle::slice(self, range)) + } + + fn filter(&self, ranges: &[Range]) -> VortexResult> { + Ok(Arc::new(LazyBufferHandle::filter(self, ranges))) + } + + fn aligned(self: Arc, alignment: Alignment) -> VortexResult> { + if self.alignment.is_aligned_to(alignment) { + Ok(self) + } else { + Ok(Arc::new(LazyBufferHandle { + source: Arc::clone(&self.source), + segment_id: self.segment_id, + selection: self.selection.clone(), + alignment, + })) + } + } +} + +/// Build an [`ArrayParts`] with lazy device buffers that defer segment I/O. +/// +/// Each buffer descriptor in the flatbuffer is turned into a [`LazyBufferHandle`] +/// that records the segment source, segment ID, byte range, and alignment but +/// does **not** perform any I/O. The returned [`ArrayParts`] can be decoded into +/// an array tree and manipulated (sliced, filtered, optimized) before the lazy +/// buffers are materialized with [`materialize_recursive`]. +pub fn create_lazy_array_parts( + array_tree: ByteBuffer, + source: Arc, + segment_id: SegmentId, +) -> VortexResult { + use flatbuffers::root; + use vortex_flatbuffers::FlatBuffer; + use vortex_flatbuffers::array as fba; + + let fb_aligned = FlatBuffer::align_from(array_tree.clone()); + let fb_array = root::(fb_aligned.as_ref())?; + + let mut offset: usize = 0; + let buffers: Vec = fb_array + .buffers() + .unwrap_or_default() + .iter() + .map(|fb_buf| { + offset += fb_buf.padding() as usize; + let buffer_len = fb_buf.length() as usize; + let alignment = Alignment::from_exponent(fb_buf.alignment_exponent()); + + let lazy = LazyBufferHandle::new(Arc::clone(&source), segment_id, alignment) + .slice(offset..offset + buffer_len); + + offset += buffer_len; + BufferHandle::new_device(Arc::new(lazy)) + }) + .collect(); + + ArrayParts::from_flatbuffer_with_buffers(array_tree, buffers) +} + +/// Recursively walk the array tree and materialize any [`LazyBufferHandle`] +/// device buffers by performing I/O, returning a new tree with host-resident +/// buffers. +pub async fn materialize_recursive(array: &ArrayRef) -> VortexResult { + // 1. Recursively materialize children. + let children = array.children(); + let mut new_children = Vec::with_capacity(children.len()); + let mut any_child_changed = false; + for child in &children { + let new_child = Box::pin(materialize_recursive(child)).await?; + any_child_changed |= !Arc::ptr_eq(child, &new_child); + new_children.push(new_child); + } + let current = if any_child_changed { + array.with_children(new_children)? + } else { + array.clone() + }; + + // 2. Check for lazy device buffers. + let handles = current.buffer_handles(); + let any_lazy = handles.iter().any(|h| { + h.as_device_opt() + .and_then(|d| d.as_any().downcast_ref::()) + .is_some() + }); + if !any_lazy { + return Ok(current); + } + + // 3. Materialize lazy buffers, ensuring proper alignment. + let mut materialized = Vec::with_capacity(handles.len()); + for handle in &handles { + if let Some(lazy) = handle + .as_device_opt() + .and_then(|d| d.as_any().downcast_ref::()) + { + let buf = lazy.materialize().await?; + materialized.push(buf.ensure_aligned(lazy.alignment())?); + } else { + materialized.push(handle.clone()); + } + } + current.with_buffers(materialized) +} + /// Map a logical byte range into the given set of existing absolute ranges. /// /// The `range` is interpreted as an offset into the concatenated output of @@ -222,6 +429,7 @@ mod tests { use futures::FutureExt; use vortex_array::buffer::BufferHandle; + use vortex_buffer::Alignment; use vortex_buffer::ByteBuffer; use vortex_error::VortexResult; use vortex_io::runtime::single::block_on; @@ -243,7 +451,11 @@ mod tests { fn lazy(data: &[u8]) -> LazyBufferHandle { let buf = BufferHandle::new_host(ByteBuffer::copy_from(data)); - LazyBufferHandle::new(Arc::new(SingleSegment(buf)), SegmentId::from(0u32)) + LazyBufferHandle::new( + Arc::new(SingleSegment(buf)), + SegmentId::from(0u32), + Alignment::none(), + ) } #[test] diff --git a/vortex-layout/src/layouts/flat/reader.rs b/vortex-layout/src/layouts/flat/reader.rs index 727566d3db8..fd7997b75d0 100644 --- a/vortex-layout/src/layouts/flat/reader.rs +++ b/vortex-layout/src/layouts/flat/reader.rs @@ -22,6 +22,8 @@ use vortex_mask::Mask; use vortex_session::VortexSession; use crate::LayoutReader; +use crate::buffer::create_lazy_array_parts; +use crate::buffer::materialize_recursive; use crate::layouts::SharedArrayFuture; use crate::layouts::flat::FlatLayout; use crate::segments::SegmentSource; @@ -60,30 +62,37 @@ impl FlatReader { let row_count = usize::try_from(self.layout.row_count()).vortex_expect("row count must fit in usize"); - // We create the segment_fut here to ensure we give the segment reader visibility into - // how to prioritize this segment, even if the `array` future has already been initialized. - // This is gross... see the function's TODO for a maybe better solution? - let segment_fut = self.segment_source.request(self.layout.segment_id()); - let ctx = self.layout.array_ctx().clone(); let session = self.session.clone(); let dtype = self.layout.dtype().clone(); let array_tree = self.layout.array_tree().cloned(); - async move { - let segment = segment_fut.await?; - let parts = if let Some(array_tree) = array_tree { - // Use the pre-stored flatbuffer from layout metadata combined with segment buffers. - ArrayParts::from_flatbuffer_and_segment(array_tree, segment)? - } else { - // Parse the flatbuffer from the segment itself. - ArrayParts::try_from(segment)? - }; - parts - .decode(&dtype, row_count, &ctx, &session) - .map_err(Arc::new) + + if let Some(array_tree) = array_tree { + // Build lazy — no segment I/O yet. Buffers are LazyBufferHandles wrapped as + // device buffers that will be materialized after slice/filter/optimize. + let source = Arc::clone(&self.segment_source); + let segment_id = self.layout.segment_id(); + async move { + let parts = create_lazy_array_parts(array_tree, source, segment_id)?; + parts + .decode(&dtype, row_count, &ctx, &session) + .map_err(Arc::new) + } + .boxed() + .shared() + } else { + // Legacy path: segment contains both flatbuffer and data buffers. + let segment_fut = self.segment_source.request(self.layout.segment_id()); + async move { + let segment = segment_fut.await?; + let parts = ArrayParts::try_from(segment)?; + parts + .decode(&dtype, row_count, &ctx, &session) + .map_err(Arc::new) + } + .boxed() + .shared() } - .boxed() - .shared() } } @@ -152,6 +161,7 @@ impl LayoutReader for FlatReader { // after this. let array = array.apply(&expr)?; let array = array.filter(mask.clone())?; + let array = materialize_recursive(&array).await?; let mut ctx = session.create_execution_ctx(); let array_mask = array.execute::(&mut ctx)?; @@ -159,6 +169,7 @@ impl LayoutReader for FlatReader { } else { // Run over the full array, with a simpler bitand at the end. let array = array.apply(&expr)?; + let array = materialize_recursive(&array).await?; let mut ctx = session.create_execution_ctx(); let array_mask = array.execute::(&mut ctx)?; @@ -213,6 +224,9 @@ impl LayoutReader for FlatReader { // Evaluate the projection expression. array = array.apply(&expr)?; + // Materialize any remaining lazy device buffers before returning. + array = materialize_recursive(&array).await?; + Ok(array) } .boxed())