feat: Implements ListType and ListVector

src/datatypes2/src/data_type.rs

@@ -23,7 +23,7 @@ use crate::error::{self, Error, Result};
 use crate::type_id::LogicalTypeId;
 use crate::types::{
     BinaryType, BooleanType, DateTimeType, DateType, Float32Type, Float64Type, Int16Type,
-    Int32Type, Int64Type, Int8Type, NullType, StringType, TimestampMicrosecondType,
+    Int32Type, Int64Type, Int8Type, ListType, NullType, StringType, TimestampMicrosecondType,
     TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, TimestampType,
     UInt16Type, UInt32Type, UInt64Type, UInt8Type,
 };
@@ -56,7 +56,9 @@ pub enum ConcreteDataType {
     Date(DateType),
     DateTime(DateTimeType),
     Timestamp(TimestampType),
-    // List(ListType),
+
+    // Compound types:
+    List(ListType),
 }
 
 // TODO(yingwen): Consider moving these methods to the DataType trait.
@@ -145,7 +147,7 @@ impl TryFrom<&ArrowDataType> for ConcreteDataType {
 
     fn try_from(dt: &ArrowDataType) -> Result<ConcreteDataType> {
         let concrete_type = match dt {
-            // ArrowDataType::Null => Self::null_datatype(),
+            ArrowDataType::Null => Self::null_datatype(),
             ArrowDataType::Boolean => Self::boolean_datatype(),
             ArrowDataType::UInt8 => Self::uint8_datatype(),
             ArrowDataType::UInt16 => Self::uint16_datatype(),
@@ -158,13 +160,13 @@ impl TryFrom<&ArrowDataType> for ConcreteDataType {
             ArrowDataType::Float32 => Self::float32_datatype(),
             ArrowDataType::Float64 => Self::float64_datatype(),
             ArrowDataType::Date32 => Self::date_datatype(),
-            // ArrowDataType::Date64 => Self::datetime_datatype(),
+            ArrowDataType::Date64 => Self::datetime_datatype(),
             // ArrowDataType::Timestamp(u, _) => ConcreteDataType::from_arrow_time_unit(u),
             ArrowDataType::Binary | ArrowDataType::LargeBinary => Self::binary_datatype(),
             ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => Self::string_datatype(),
-            // ArrowDataType::List(field) => Self::List(ListType::new(
-            //     ConcreteDataType::from_arrow_type(&field.data_type),
-            // )),
+            ArrowDataType::List(field) => Self::List(ListType::new(
+                ConcreteDataType::from_arrow_type(field.data_type()),
+            )),
             _ => {
                 return error::UnsupportedArrowTypeSnafu {
                     arrow_type: dt.clone(),
@@ -197,10 +199,6 @@ impl_new_concrete_type_functions!(
 );
 
 impl ConcreteDataType {
-    //     pub fn list_datatype(inner_type: ConcreteDataType) -> ConcreteDataType {
-    //         ConcreteDataType::List(ListType::new(inner_type))
-    //     }
-
     pub fn timestamp_second_datatype() -> Self {
         ConcreteDataType::Timestamp(TimestampType::Second(TimestampSecondType::default()))
     }
@@ -229,6 +227,10 @@ impl ConcreteDataType {
             TimeUnit::Nanosecond => Self::timestamp_nanosecond_datatype(),
         }
     }
+
+    pub fn list_datatype(item_type: ConcreteDataType) -> ConcreteDataType {
+        ConcreteDataType::List(ListType::new(item_type))
+    }
 }
 
 /// Data type abstraction.
@@ -269,10 +271,10 @@ mod tests {
 
     #[test]
     fn test_from_arrow_type() {
-        // assert!(matches!(
-        //     ConcreteDataType::from_arrow_type(&ArrowDataType::Null),
-        //     ConcreteDataType::Null(_)
-        // ));
+        assert!(matches!(
+            ConcreteDataType::from_arrow_type(&ArrowDataType::Null),
+            ConcreteDataType::Null(_)
+        ));
         assert!(matches!(
             ConcreteDataType::from_arrow_type(&ArrowDataType::Boolean),
             ConcreteDataType::Boolean(_)

src/datatypes2/src/scalars.rs

@@ -20,9 +20,10 @@ use crate::types::{
     Float32Type, Float64Type, Int16Type, Int32Type, Int64Type, Int8Type, UInt16Type, UInt32Type,
     UInt64Type, UInt8Type,
 };
+use crate::value::{ListValue, ListValueRef, Value};
 use crate::vectors::{
-    BinaryVector, BooleanVector, DateTimeVector, DateVector, MutableVector, PrimitiveVector,
-    StringVector, Vector,
+    BinaryVector, BooleanVector, DateTimeVector, DateVector, ListVector, MutableVector,
+    PrimitiveVector, StringVector, Vector,
 };
 
 fn get_iter_capacity<T, I: Iterator<Item = T>>(iter: &I) -> usize {
@@ -318,42 +319,43 @@ impl<'a> ScalarRef<'a> for DateTime {
 //     }
 // }
 
-// impl Scalar for ListValue {
-//     type VectorType = ListVector;
-//     type RefType<'a> = ListValueRef<'a>;
+impl Scalar for ListValue {
+    type VectorType = ListVector;
+    type RefType<'a> = ListValueRef<'a>;
 
-//     fn as_scalar_ref(&self) -> Self::RefType<'_> {
-//         ListValueRef::Ref { val: self }
-//     }
+    fn as_scalar_ref(&self) -> Self::RefType<'_> {
+        ListValueRef::Ref { val: self }
+    }
 
-//     fn upcast_gat<'short, 'long: 'short>(long: Self::RefType<'long>) -> Self::RefType<'short> {
-//         long
-//     }
-// }
+    fn upcast_gat<'short, 'long: 'short>(long: Self::RefType<'long>) -> Self::RefType<'short> {
+        long
+    }
+}
 
-// impl<'a> ScalarRef<'a> for ListValueRef<'a> {
-//     type ScalarType = ListValue;
+impl<'a> ScalarRef<'a> for ListValueRef<'a> {
+    type ScalarType = ListValue;
 
-//     fn to_owned_scalar(&self) -> Self::ScalarType {
-//         match self {
-//             ListValueRef::Indexed { vector, idx } => match vector.get(*idx) {
-//                 // Normally should not get `Value::Null` if the `ListValueRef` comes
-//                 // from the iterator of the ListVector, but we avoid panic and just
-//                 // returns a default list value in such case since `ListValueRef` may
-//                 // be constructed manually.
-//                 Value::Null => ListValue::default(),
-//                 Value::List(v) => v,
-//                 _ => unreachable!(),
-//             },
-//             ListValueRef::Ref { val } => (*val).clone(),
-//         }
-//     }
-// }
+    fn to_owned_scalar(&self) -> Self::ScalarType {
+        match self {
+            ListValueRef::Indexed { vector, idx } => match vector.get(*idx) {
+                // Normally should not get `Value::Null` if the `ListValueRef` comes
+                // from the iterator of the ListVector, but we avoid panic and just
+                // returns a default list value in such case since `ListValueRef` may
+                // be constructed manually.
+                Value::Null => ListValue::default(),
+                Value::List(v) => v,
+                _ => unreachable!(),
+            },
+            ListValueRef::Ref { val } => (*val).clone(),
+        }
+    }
+}
 
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::vectors::{BinaryVector, Int32Vector};
+    use crate::data_type::ConcreteDataType;
+    use crate::vectors::{BinaryVector, Int32Vector, ListVectorBuilder};
 
     fn build_vector_from_slice<T: ScalarVector>(items: &[Option<T::RefItem<'_>>]) -> T {
         let mut builder = T::Builder::with_capacity(items.len());
@@ -412,40 +414,40 @@ mod tests {
         assert_eq!(date, date.to_owned_scalar());
     }
 
-    // #[test]
-    // fn test_datetime_scalar() {
-    //     let dt = DateTime::new(123);
-    //     assert_eq!(dt, dt.as_scalar_ref());
-    //     assert_eq!(dt, dt.to_owned_scalar());
-    // }
+    #[test]
+    fn test_datetime_scalar() {
+        let dt = DateTime::new(123);
+        assert_eq!(dt, dt.as_scalar_ref());
+        assert_eq!(dt, dt.to_owned_scalar());
+    }
 
-    // #[test]
-    // fn test_list_value_scalar() {
-    //     let list_value = ListValue::new(
-    //         Some(Box::new(vec![Value::Int32(123)])),
-    //         ConcreteDataType::int32_datatype(),
-    //     );
-    //     let list_ref = ListValueRef::Ref { val: &list_value };
-    //     assert_eq!(list_ref, list_value.as_scalar_ref());
-    //     assert_eq!(list_value, list_ref.to_owned_scalar());
-
-    //     let mut builder =
-    //         ListVectorBuilder::with_type_capacity(ConcreteDataType::int32_datatype(), 1);
-    //     builder.push(None);
-    //     builder.push(Some(list_value.as_scalar_ref()));
-    //     let vector = builder.finish();
-
-    //     let ref_on_vec = ListValueRef::Indexed {
-    //         vector: &vector,
-    //         idx: 0,
-    //     };
-    //     assert_eq!(ListValue::default(), ref_on_vec.to_owned_scalar());
-    //     let ref_on_vec = ListValueRef::Indexed {
-    //         vector: &vector,
-    //         idx: 1,
-    //     };
-    //     assert_eq!(list_value, ref_on_vec.to_owned_scalar());
-    // }
+    #[test]
+    fn test_list_value_scalar() {
+        let list_value = ListValue::new(
+            Some(Box::new(vec![Value::Int32(123)])),
+            ConcreteDataType::int32_datatype(),
+        );
+        let list_ref = ListValueRef::Ref { val: &list_value };
+        assert_eq!(list_ref, list_value.as_scalar_ref());
+        assert_eq!(list_value, list_ref.to_owned_scalar());
+
+        let mut builder =
+            ListVectorBuilder::with_type_capacity(ConcreteDataType::int32_datatype(), 1);
+        builder.push(None);
+        builder.push(Some(list_value.as_scalar_ref()));
+        let vector = builder.finish();
+
+        let ref_on_vec = ListValueRef::Indexed {
+            vector: &vector,
+            idx: 0,
+        };
+        assert_eq!(ListValue::default(), ref_on_vec.to_owned_scalar());
+        let ref_on_vec = ListValueRef::Indexed {
+            vector: &vector,
+            idx: 1,
+        };
+        assert_eq!(list_value, ref_on_vec.to_owned_scalar());
+    }
 
     // #[test]
     // fn test_build_timestamp_vector() {

src/datatypes2/src/types.rs

@@ -16,6 +16,7 @@ mod binary_type;
 mod boolean_type;
 mod date_type;
 mod datetime_type;
+mod list_type;
 mod null_type;
 mod primitive_type;
 mod string_type;
@@ -26,6 +27,7 @@ pub use binary_type::BinaryType;
 pub use boolean_type::BooleanType;
 pub use date_type::DateType;
 pub use datetime_type::DateTimeType;
+pub use list_type::ListType;
 pub use null_type::NullType;
 pub use primitive_type::{
     Float32Type, Float64Type, Int16Type, Int32Type, Int64Type, Int8Type, LogicalPrimitiveType,

src/datatypes2/src/types/list_type.rs

@@ -0,0 +1,91 @@
+// Copyright 2022 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use arrow::datatypes::{DataType as ArrowDataType, Field};
+use serde::{Deserialize, Serialize};
+
+use crate::data_type::{ConcreteDataType, DataType};
+use crate::type_id::LogicalTypeId;
+use crate::value::{ListValue, Value};
+use crate::vectors::{ListVectorBuilder, MutableVector};
+
+/// Used to represent the List datatype.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ListType {
+    /// The type of List's item.
+    // Use Box to avoid recursive dependency, as enum ConcreteDataType depends on ListType.
+    item_type: Box<ConcreteDataType>,
+}
+
+impl Default for ListType {
+    fn default() -> Self {
+        ListType::new(ConcreteDataType::null_datatype())
+    }
+}
+
+impl ListType {
+    /// Create a new `ListType` whose item's data type is `item_type`.
+    pub fn new(item_type: ConcreteDataType) -> Self {
+        ListType {
+            item_type: Box::new(item_type),
+        }
+    }
+}
+
+impl DataType for ListType {
+    fn name(&self) -> &str {
+        "List"
+    }
+
+    fn logical_type_id(&self) -> LogicalTypeId {
+        LogicalTypeId::List
+    }
+
+    fn default_value(&self) -> Value {
+        Value::List(ListValue::new(None, *self.item_type.clone()))
+    }
+
+    fn as_arrow_type(&self) -> ArrowDataType {
+        let field = Box::new(Field::new("item", self.item_type.as_arrow_type(), true));
+        ArrowDataType::List(field)
+    }
+
+    fn create_mutable_vector(&self, capacity: usize) -> Box<dyn MutableVector> {
+        Box::new(ListVectorBuilder::with_type_capacity(
+            *self.item_type.clone(),
+            capacity,
+        ))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::value::ListValue;
+
+    #[test]
+    fn test_list_type() {
+        let t = ListType::new(ConcreteDataType::boolean_datatype());
+        assert_eq!("List", t.name());
+        assert_eq!(LogicalTypeId::List, t.logical_type_id());
+        assert_eq!(
+            Value::List(ListValue::new(None, ConcreteDataType::boolean_datatype())),
+            t.default_value()
+        );
+        assert_eq!(
+            ArrowDataType::List(Box::new(Field::new("item", ArrowDataType::Boolean, true))),
+            t.as_arrow_type()
+        );
+    }
+}

src/datatypes2/src/types/primitive_type.rs

@@ -189,8 +189,11 @@ impl WrapperType for DateTime {
 
 macro_rules! define_logical_primitive_type {
     ($Native: ident, $TypeId: ident, $DataType: ident) => {
+        // We need to define it as an empty struct `struct DataType {}` instead of a struct-unit
+        // `struct DataType;` to ensure the serialized JSON string is compatible with previous
+        // implementation.
         #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
-        pub struct $DataType;
+        pub struct $DataType {}
 
         impl LogicalPrimitiveType for $DataType {
             type ArrowPrimitive = arrow::datatypes::$DataType;