[BUG] enable fixed size binary ingest to daft binary (#1612)

* Enables reads of fixed sized binary data * I hit some issues reading UUIDs from parquet files
Eventual-Inc · Nov 16, 2023 · 7149083 · 7149083
1 parent c4b498a
commit 7149083
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 3 deletions.
diff --git a/daft/datatype.py b/daft/datatype.py
@@ -341,7 +341,11 @@ def from_arrow_type(cls, arrow_type: pa.lib.DataType) -> DataType:
             return cls.float64()
         elif pa.types.is_string(arrow_type) or pa.types.is_large_string(arrow_type):
             return cls.string()
-        elif pa.types.is_binary(arrow_type) or pa.types.is_large_binary(arrow_type):
+        elif (
+            pa.types.is_binary(arrow_type)
+            or pa.types.is_large_binary(arrow_type)
+            or pa.types.is_fixed_size_binary(arrow_type)
+        ):
             return cls.binary()
         elif pa.types.is_boolean(arrow_type):
             return cls.bool()

diff --git a/src/daft-core/src/datatypes/dtype.rs b/src/daft-core/src/datatypes/dtype.rs
@@ -400,7 +400,9 @@ impl From<&ArrowType> for DataType {
                 DataType::Time(timeunit.into())
             }
             ArrowType::Duration(timeunit) => DataType::Duration(timeunit.into()),
-            ArrowType::Binary | ArrowType::LargeBinary => DataType::Binary,
+            ArrowType::Binary | ArrowType::LargeBinary | ArrowType::FixedSizeBinary(_) => {
+                DataType::Binary
+            }
             ArrowType::Utf8 | ArrowType::LargeUtf8 => DataType::Utf8,
             ArrowType::Decimal(precision, scale) => DataType::Decimal128(*precision, *scale),
             ArrowType::List(field) | ArrowType::LargeList(field) => {

diff --git a/src/daft-core/src/utils/arrow.rs b/src/daft-core/src/utils/arrow.rs
@@ -15,7 +15,9 @@ fn coerce_to_daft_compatible_type(
 ) -> Option<arrow2::datatypes::DataType> {
     match dtype {
         arrow2::datatypes::DataType::Utf8 => Some(arrow2::datatypes::DataType::LargeUtf8),
-        arrow2::datatypes::DataType::Binary => Some(arrow2::datatypes::DataType::LargeBinary),
+        arrow2::datatypes::DataType::Binary | arrow2::datatypes::DataType::FixedSizeBinary(_) => {
+            Some(arrow2::datatypes::DataType::LargeBinary)
+        }
         arrow2::datatypes::DataType::List(field) => {
             let new_field = match coerce_to_daft_compatible_type(field.data_type()) {
                 Some(new_inner_dtype) => Box::new(