update arrow-format

arrow-array/src/array/byte_view_array.rs

@@ -93,7 +93,6 @@ use std::sync::Arc;
 ///                                                                            └───┘
 /// ```
 /// [`GenericByteArray`]: crate::array::GenericByteArray
-
 pub struct GenericByteViewArray<T: ByteViewType + ?Sized> {
     data_type: DataType,
     views: ScalarBuffer<u128>,

arrow-array/src/builder/generic_bytes_view_builder.rs

@@ -20,6 +20,7 @@ use crate::types::{BinaryViewType, ByteViewType, StringViewType};
 use crate::{ArrayRef, GenericByteViewArray};
 use arrow_buffer::{Buffer, BufferBuilder, NullBufferBuilder, ScalarBuffer};
 use arrow_data::ByteView;
+
 use std::any::Any;
 use std::marker::PhantomData;
 use std::sync::Arc;

arrow-ipc/src/convert.rs

@@ -543,7 +543,7 @@ pub(crate) fn get_fb_field_type<'a>(
                 .as_union_value(),
             children: Some(fbb.create_vector(&empty_fields[..])),
         },
-        BinaryView | Utf8View => unimplemented!("BinaryView/Utf8View not implemented"),
+        BinaryView | Utf8View => unimplemented!("unimplemented"),
         Utf8 => FBFieldType {
             type_type: crate::Type::Utf8,
             type_: crate::Utf8Builder::new(fbb).finish().as_union_value(),

arrow-ipc/src/gen/Message.rs

@@ -25,6 +25,8 @@ use flatbuffers::EndianScalar;
 use std::{cmp::Ordering, mem};
 // automatically generated by the FlatBuffers compiler, do not modify
 
+// @generated
+
 #[deprecated(
     since = "2.0.0",
     note = "Use associated constants instead. This will no longer be generated in 2021."
@@ -340,7 +342,7 @@ pub struct MessageHeaderUnionTableOffset {}
 /// Metadata about a field at some level of a nested type tree (but not
 /// its children).
 ///
-/// For example, a `List<Int16>` with values `[[1, 2, 3], null, [4], [5, 6], null]`
+/// For example, a List<Int16> with values `[[1, 2, 3], null, [4], [5, 6], null]`
 /// would have {length: 5, null_count: 2} for its List node, and {length: 6,
 /// null_count: 0} for its Int16 node, as separate FieldNode structs
 // struct FieldNode, aligned to 8
@@ -636,6 +638,7 @@ impl<'a> RecordBatch<'a> {
     pub const VT_NODES: flatbuffers::VOffsetT = 6;
     pub const VT_BUFFERS: flatbuffers::VOffsetT = 8;
     pub const VT_COMPRESSION: flatbuffers::VOffsetT = 10;
+    pub const VT_VARIADICBUFFERCOUNTS: flatbuffers::VOffsetT = 12;
 
     #[inline]
     pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
@@ -648,6 +651,9 @@ impl<'a> RecordBatch<'a> {
     ) -> flatbuffers::WIPOffset<RecordBatch<'bldr>> {
         let mut builder = RecordBatchBuilder::new(_fbb);
         builder.add_length(args.length);
+        if let Some(x) = args.variadicBufferCounts {
+            builder.add_variadicBufferCounts(x);
+        }
         if let Some(x) = args.compression {
             builder.add_compression(x);
         }
@@ -720,6 +726,33 @@ impl<'a> RecordBatch<'a> {
                 )
         }
     }
+    /// Some types such as Utf8View are represented using a variable number of buffers.
+    /// For each such Field in the pre-ordered flattened logical schema, there will be
+    /// an entry in variadicBufferCounts to indicate the number of number of variadic
+    /// buffers which belong to that Field in the current RecordBatch.
+    ///
+    /// For example, the schema
+    ///     col1: Struct<alpha: Int32, beta: BinaryView, gamma: Float64>
+    ///     col2: Utf8View
+    /// contains two Fields with variadic buffers so variadicBufferCounts will have
+    /// two entries, the first counting the variadic buffers of `col1.beta` and the
+    /// second counting `col2`'s.
+    ///
+    /// This field may be omitted if and only if the schema contains no Fields with
+    /// a variable number of buffers, such as BinaryView and Utf8View.
+    #[inline]
+    pub fn variadicBufferCounts(&self) -> Option<flatbuffers::Vector<'a, i64>> {
+        // Safety:
+        // Created from valid Table for this object
+        // which contains a valid value in this slot
+        unsafe {
+            self._tab
+                .get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'a, i64>>>(
+                    RecordBatch::VT_VARIADICBUFFERCOUNTS,
+                    None,
+                )
+        }
+    }
 }
 
 impl flatbuffers::Verifiable for RecordBatch<'_> {
@@ -746,6 +779,11 @@ impl flatbuffers::Verifiable for RecordBatch<'_> {
                 Self::VT_COMPRESSION,
                 false,
             )?
+            .visit_field::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'_, i64>>>(
+                "variadicBufferCounts",
+                Self::VT_VARIADICBUFFERCOUNTS,
+                false,
+            )?
             .finish();
         Ok(())
     }
@@ -755,6 +793,7 @@ pub struct RecordBatchArgs<'a> {
     pub nodes: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, FieldNode>>>,
     pub buffers: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, Buffer>>>,
     pub compression: Option<flatbuffers::WIPOffset<BodyCompression<'a>>>,
+    pub variadicBufferCounts: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, i64>>>,
 }
 impl<'a> Default for RecordBatchArgs<'a> {
     #[inline]
@@ -764,6 +803,7 @@ impl<'a> Default for RecordBatchArgs<'a> {
             nodes: None,
             buffers: None,
             compression: None,
+            variadicBufferCounts: None,
         }
     }
 }
@@ -800,6 +840,16 @@ impl<'a: 'b, 'b> RecordBatchBuilder<'a, 'b> {
             );
     }
     #[inline]
+    pub fn add_variadicBufferCounts(
+        &mut self,
+        variadicBufferCounts: flatbuffers::WIPOffset<flatbuffers::Vector<'b, i64>>,
+    ) {
+        self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(
+            RecordBatch::VT_VARIADICBUFFERCOUNTS,
+            variadicBufferCounts,
+        );
+    }
+    #[inline]
     pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> RecordBatchBuilder<'a, 'b> {
         let start = _fbb.start_table();
         RecordBatchBuilder {
@@ -821,6 +871,7 @@ impl core::fmt::Debug for RecordBatch<'_> {
         ds.field("nodes", &self.nodes());
         ds.field("buffers", &self.buffers());
         ds.field("compression", &self.compression());
+        ds.field("variadicBufferCounts", &self.variadicBufferCounts());
         ds.finish()
     }
 }