Remove LargeUtf8|Binary, Utf8|BinaryView, and Dictionary from ScalarValue

datafusion-examples/examples/advanced_udf.rs

@@ -96,8 +96,8 @@ impl ScalarUDFImpl for PowUdf {
         // function, but we check again to make sure
         assert_eq!(args.len(), 2);
         let (base, exp) = (&args[0], &args[1]);
-        assert_eq!(base.data_type(), DataType::Float64);
-        assert_eq!(exp.data_type(), DataType::Float64);
+        assert_eq!(base.data_type(), &DataType::Float64);
+        assert_eq!(exp.data_type(), &DataType::Float64);
 
         match (base, exp) {
             // For demonstration purposes we also implement the scalar / scalar
@@ -108,28 +108,31 @@ impl ScalarUDFImpl for PowUdf {
             // the DataFusion expression simplification logic will often invoke
             // this path once during planning, and simply use the result during
             // execution.
-            (
-                ColumnarValue::Scalar(ScalarValue::Float64(base)),
-                ColumnarValue::Scalar(ScalarValue::Float64(exp)),
-            ) => {
-                // compute the output. Note DataFusion treats `None` as NULL.
-                let res = match (base, exp) {
-                    (Some(base), Some(exp)) => Some(base.powf(*exp)),
-                    // one or both arguments were NULL
-                    _ => None,
-                };
-                Ok(ColumnarValue::Scalar(ScalarValue::from(res)))
+            (ColumnarValue::Scalar(base), ColumnarValue::Scalar(exp)) => {
+                match (base.value(), exp.value()) {
+                    (ScalarValue::Float64(base), ScalarValue::Float64(exp)) => {
+                        // compute the output. Note DataFusion treats `None` as NULL.
+                        let res = match (base, exp) {
+                            (Some(base), Some(exp)) => Some(base.powf(*exp)),
+                            // one or both arguments were NULL
+                            _ => None,
+                        };
+                        Ok(ColumnarValue::from(ScalarValue::from(res)))
+                    }
+                    _ => {
+                        internal_err!("Invalid argument types to pow function")
+                    }
+                }
             }
             // special case if the exponent is a constant
-            (
-                ColumnarValue::Array(base_array),
-                ColumnarValue::Scalar(ScalarValue::Float64(exp)),
-            ) => {
-                let result_array = match exp {
+            (ColumnarValue::Array(base_array), ColumnarValue::Scalar(exp)) => {
+                let result_array = match exp.value() {
                     // a ^ null = null
-                    None => new_null_array(base_array.data_type(), base_array.len()),
+                    ScalarValue::Float64(None) => {
+                        new_null_array(base_array.data_type(), base_array.len())
+                    }
                     // a ^ exp
-                    Some(exp) => {
+                    ScalarValue::Float64(Some(exp)) => {
                         // DataFusion has ensured both arguments are Float64:
                         let base_array = base_array.as_primitive::<Float64Type>();
                         // calculate the result for every row. The `unary`
@@ -139,24 +142,25 @@ impl ScalarUDFImpl for PowUdf {
                             compute::unary(base_array, |base| base.powf(*exp));
                         Arc::new(res)
                     }
+                    _ => return internal_err!("Invalid argument types to pow function"),
                 };
                 Ok(ColumnarValue::Array(result_array))
             }
 
             // special case if the base is a constant (note this code is quite
             // similar to the previous case, so we omit comments)
-            (
-                ColumnarValue::Scalar(ScalarValue::Float64(base)),
-                ColumnarValue::Array(exp_array),
-            ) => {
-                let res = match base {
-                    None => new_null_array(exp_array.data_type(), exp_array.len()),
-                    Some(base) => {
+            (ColumnarValue::Scalar(base), ColumnarValue::Array(exp_array)) => {
+                let res = match base.value() {
+                    ScalarValue::Float64(None) => {
+                        new_null_array(exp_array.data_type(), exp_array.len())
+                    }
+                    ScalarValue::Float64(Some(base)) => {
                         let exp_array = exp_array.as_primitive::<Float64Type>();
                         let res: Float64Array =
                             compute::unary(exp_array, |exp| base.powf(exp));
                         Arc::new(res)
                     }
+                    _ => return internal_err!("Invalid argument types to pow function"),
                 };
                 Ok(ColumnarValue::Array(res))
             }
@@ -169,10 +173,6 @@ impl ScalarUDFImpl for PowUdf {
                 )?;
                 Ok(ColumnarValue::Array(Arc::new(res)))
             }
-            // if the types were not float, it is a bug in DataFusion
-            _ => {
-                internal_err!("Invalid argument types to pow function")
-            }
         }
     }
 

datafusion-examples/examples/optimizer_rule.rs

@@ -207,7 +207,7 @@ impl ScalarUDFImpl for MyEq {
     fn invoke(&self, _args: &[ColumnarValue]) -> Result<ColumnarValue> {
         // this example simply returns "true" which is not what a real
         // implementation would do.
-        Ok(ColumnarValue::Scalar(ScalarValue::from(true)))
+        Ok(ColumnarValue::from(ScalarValue::from(true)))
     }
 }
 

datafusion/common/src/lib.rs

@@ -37,6 +37,7 @@ pub mod file_options;
 pub mod format;
 pub mod hash_utils;
 pub mod instant;
+pub mod logical;
 pub mod parsers;
 pub mod rounding;
 pub mod scalar;

datafusion/common/src/logical/equality.rs

@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow_schema::DataType;
+
+pub trait LogicallyEq<Rhs: ?Sized = Self> {
+    #[must_use]
+    fn logically_eq(&self, other: &Rhs) -> bool;
+}
+
+impl LogicallyEq for DataType {
+    fn logically_eq(&self, other: &Self) -> bool {
+        use DataType::*;
+
+        match (self, other) {
+            (Utf8 | LargeUtf8 | Utf8View, Utf8 | LargeUtf8 | Utf8View)
+            | (Binary | LargeBinary | BinaryView, Binary | LargeBinary | BinaryView) => {
+                true
+            }
+            (Dictionary(_, left), Dictionary(_, right)) => left.logically_eq(right),
+            (Dictionary(_, inner), other) | (other, Dictionary(_, inner)) => {
+                other.logically_eq(inner)
+            }
+            (RunEndEncoded(_, left), RunEndEncoded(_, right)) => {
+                left.data_type().logically_eq(right.data_type())
+            }
+            (RunEndEncoded(_, inner), other) | (other, RunEndEncoded(_, inner)) => {
+                other.logically_eq(inner.data_type())
+            }
+            _ => self == other,
+        }
+    }
+}

datafusion/common/src/logical/mod.rs

@@ -0,0 +1,18 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+pub mod equality;