From 562b68ff5af4c83fafa0143f399c8cf795d2b05e Mon Sep 17 00:00:00 2001 From: liukun4515 Date: Tue, 14 Dec 2021 16:47:48 +0800 Subject: [PATCH 1/6] support cast/try_cast for decimal: signed numeric to decimal --- datafusion/Cargo.toml | 6 +- .../src/physical_plan/expressions/cast.rs | 92 ++++++++++++ .../src/physical_plan/expressions/try_cast.rs | 132 +++++++++++++++++- 3 files changed, 227 insertions(+), 3 deletions(-) diff --git a/datafusion/Cargo.toml b/datafusion/Cargo.toml index a8c075ea7a83..2edeb6bdcd9c 100644 --- a/datafusion/Cargo.toml +++ b/datafusion/Cargo.toml @@ -52,8 +52,10 @@ avro = ["avro-rs", "num-traits"] [dependencies] ahash = "0.7" hashbrown = { version = "0.11", features = ["raw"] } -arrow = { version = "6.3.0", features = ["prettyprint"] } -parquet = { version = "6.3.0", features = ["arrow"] } +#arrow = { version = "6.3.0", features = ["prettyprint"] } +#parquet = { version = "6.3.0", features = ["arrow"] } +arrow = { path = "/Users/kliu3/Documents/github/arrow-rs/arrow", features = ["prettyprint"] } +parquet = { path = "/Users/kliu3/Documents/github/arrow-rs/parquet", features = ["arrow"] } sqlparser = "0.13" paste = "^1.0" num_cpus = "1.13.0" diff --git a/datafusion/src/physical_plan/expressions/cast.rs b/datafusion/src/physical_plan/expressions/cast.rs index bba125ebdcc9..1d103330e7ce 100644 --- a/datafusion/src/physical_plan/expressions/cast.rs +++ b/datafusion/src/physical_plan/expressions/cast.rs @@ -129,6 +129,9 @@ pub fn cast_with_options( if expr_type == cast_type { Ok(expr.clone()) } else if can_cast_types(&expr_type, &cast_type) { + // TODO + // support numeric data type to decimal + // support one type decimal to another type decimal Ok(Arc::new(CastExpr::new(expr, cast_type, cast_options))) } else { Err(DataFusionError::Internal(format!( @@ -217,6 +220,95 @@ mod tests { }}; } + #[test] + fn test_cast_numeric_to_decimal() -> Result<()> { + // int32 + generic_test_cast!( + Int32Array, + DataType::Int32, + vec![1, 2, 3, 4, 5], + // TODO + UInt32Array, + DataType::UInt32, + vec![ + Some(1_u32), + Some(2_u32), + Some(3_u32), + Some(4_u32), + Some(5_u32) + ], + DEFAULT_DATAFUSION_CAST_OPTIONS + ); + // int64 + generic_test_cast!( + Int32Array, + DataType::Int64, + vec![1, 2, 3, 4, 5], + // TODO + UInt32Array, + DataType::UInt32, + vec![ + Some(1_u32), + Some(2_u32), + Some(3_u32), + Some(4_u32), + Some(5_u32) + ], + DEFAULT_DATAFUSION_CAST_OPTIONS + ); + // float32 + generic_test_cast!( + Int32Array, + DataType::Float32, + vec![1, 2, 3, 4, 5], + // TODO + UInt32Array, + DataType::UInt32, + vec![ + Some(1_u32), + Some(2_u32), + Some(3_u32), + Some(4_u32), + Some(5_u32) + ], + DEFAULT_DATAFUSION_CAST_OPTIONS + ); + // float64 + generic_test_cast!( + Int32Array, + DataType::Float64, + vec![1, 2, 3, 4, 5], + // TODO + UInt32Array, + DataType::UInt32, + vec![ + Some(1_u32), + Some(2_u32), + Some(3_u32), + Some(4_u32), + Some(5_u32) + ], + DEFAULT_DATAFUSION_CAST_OPTIONS + ); + generic_test_cast!( + Int32Array, + DataType::Decimal(10, 4), + vec![1, 2, 3, 4, 5], + // TODO + UInt32Array, + DataType::UInt32, + vec![ + Some(1_u32), + Some(2_u32), + Some(3_u32), + Some(4_u32), + Some(5_u32) + ], + DEFAULT_DATAFUSION_CAST_OPTIONS + ); + Ok(()) + } + #[test] fn test_cast_i32_u32() -> Result<()> { generic_test_cast!( diff --git a/datafusion/src/physical_plan/expressions/try_cast.rs b/datafusion/src/physical_plan/expressions/try_cast.rs index 1ba4a50260d4..2bc4880520c6 100644 --- a/datafusion/src/physical_plan/expressions/try_cast.rs +++ b/datafusion/src/physical_plan/expressions/try_cast.rs @@ -77,6 +77,14 @@ impl PhysicalExpr for TryCastExpr { fn evaluate(&self, batch: &RecordBatch) -> Result { let value = self.expr.evaluate(batch)?; + // TODO cast signed numeric to decimal + // INT32, INT64, FLOAT, DOUBLE + // private[sql] val IntDecimal = DecimalType(3, 0) + // private[sql] val IntDecimal = DecimalType(5, 0) + // private[sql] val IntDecimal = DecimalType(10, 0) + // private[sql] val LongDecimal = DecimalType(20, 0) + // private[sql] val FloatDecimal = DecimalType(14, 7) + // private[sql] val DoubleDecimal = DecimalType(30, 15) match value { ColumnarValue::Array(array) => Ok(ColumnarValue::Array(kernels::cast::cast( &array, @@ -105,6 +113,11 @@ pub fn try_cast( if expr_type == cast_type { Ok(expr.clone()) } else if can_cast_types(&expr_type, &cast_type) { + // TODO + // support numeric data type to decimal + // support one type decimal to another type decimal + // Now we just support signed numeric type to decimal, + // and will implement decimal to signed numeric data type later. Ok(Arc::new(TryCastExpr::new(expr, cast_type))) } else { Err(DataFusionError::Internal(format!( @@ -121,9 +134,10 @@ mod tests { use crate::physical_plan::expressions::col; use arrow::array::{StringArray, Time64NanosecondArray}; use arrow::{ - array::{Array, Int32Array, Int64Array, TimestampNanosecondArray, UInt32Array}, + array::{Array, Int8Array, Int16Array, Int32Array, Int64Array, DecimalArray, TimestampNanosecondArray, UInt32Array}, datatypes::*, }; + use crate::scalar::ScalarValue::Decimal128; // runs an end-to-end test of physical type cast // 1. construct a record batch with a column "a" of type A @@ -175,6 +189,122 @@ mod tests { }}; } + // #[test] + fn test_try_cast_numeric_to_decimal() -> Result<()> { + // int8 + generic_test_cast!( + Int8Array, + DataType::Int8, + vec![1, 2, 3, 4, 5], + // TODO + DecimalArray, + DataType::Decimal(3,0), + vec![ + Some(Decimal128(Some(1), 3, 0)), + Some(Decimal128(Some(2), 3, 0)), + Some(Decimal128(Some(3), 3, 0)), + Some(Decimal128(Some(4), 3, 0)), + Some(Decimal128(Some(5), 3, 0)) + ] + ); + // int16 + generic_test_cast!( + Int16Array, + DataType::Int16, + vec![1, 2, 3, 4, 5], + // TODO + DecimalArray, + DataType::Decimal(5,0), + vec![ + Some(Decimal128(Some(1), 3, 0)), + Some(Decimal128(Some(2), 3, 0)), + Some(Decimal128(Some(3), 3, 0)), + Some(Decimal128(Some(4), 3, 0)), + Some(Decimal128(Some(5), 3, 0)) + ] + ); + // int32 + generic_test_cast!( + Int32Array, + DataType::Int32, + vec![1, 2, 3, 4, 5], + // TODO + DecimalArray, + DataType::Decimal(10,0), + vec![ + Some(Decimal128(Some(1), 3, 0)), + Some(Decimal128(Some(2), 3, 0)), + Some(Decimal128(Some(3), 3, 0)), + Some(Decimal128(Some(4), 3, 0)), + Some(Decimal128(Some(5), 3, 0)) + ] + ); + // int64 + generic_test_cast!( + Int32Array, + DataType::Int64, + vec![1, 2, 3, 4, 5], + // TODO + UInt32Array, + DataType::UInt32, + vec![ + Some(1_u32), + Some(2_u32), + Some(3_u32), + Some(4_u32), + Some(5_u32) + ] + ); + // float32 + generic_test_cast!( + Int32Array, + DataType::Float32, + vec![1, 2, 3, 4, 5], + // TODO + UInt32Array, + DataType::UInt32, + vec![ + Some(1_u32), + Some(2_u32), + Some(3_u32), + Some(4_u32), + Some(5_u32) + ] + ); + // float64 + generic_test_cast!( + Int32Array, + DataType::Float64, + vec![1, 2, 3, 4, 5], + // TODO + UInt32Array, + DataType::UInt32, + vec![ + Some(1_u32), + Some(2_u32), + Some(3_u32), + Some(4_u32), + Some(5_u32) + ] + ); + generic_test_cast!( + Int32Array, + DataType::Decimal(10, 4), + vec![1, 2, 3, 4, 5], + // TODO + UInt32Array, + DataType::UInt32, + vec![ + Some(1_u32), + Some(2_u32), + Some(3_u32), + Some(4_u32), + Some(5_u32) + ] + ); + Ok(()) + } + #[test] fn test_cast_i32_u32() -> Result<()> { generic_test_cast!( From b9d7427e4d345395e6861b79880b247bc8a9198a Mon Sep 17 00:00:00 2001 From: liukun4515 Date: Thu, 16 Dec 2021 18:56:39 +0800 Subject: [PATCH 2/6] add test case --- .../src/physical_plan/expressions/cast.rs | 138 ++++++++++------- .../src/physical_plan/expressions/try_cast.rs | 142 ++++++++---------- 2 files changed, 150 insertions(+), 130 deletions(-) diff --git a/datafusion/src/physical_plan/expressions/cast.rs b/datafusion/src/physical_plan/expressions/cast.rs index 1d103330e7ce..73f4790d4ed6 100644 --- a/datafusion/src/physical_plan/expressions/cast.rs +++ b/datafusion/src/physical_plan/expressions/cast.rs @@ -129,9 +129,6 @@ pub fn cast_with_options( if expr_type == cast_type { Ok(expr.clone()) } else if can_cast_types(&expr_type, &cast_type) { - // TODO - // support numeric data type to decimal - // support one type decimal to another type decimal Ok(Arc::new(CastExpr::new(expr, cast_type, cast_options))) } else { Err(DataFusionError::Internal(format!( @@ -222,87 +219,120 @@ mod tests { #[test] fn test_cast_numeric_to_decimal() -> Result<()> { + // int8 + generic_test_cast!( + Int8Array, + DataType::Int8, + vec![1, 2, 3, 4, 5], + DecimalArray, + DataType::Decimal(3, 0), + vec![ + Some(Decimal128(Some(1), 3, 0)), + Some(Decimal128(Some(2), 3, 0)), + Some(Decimal128(Some(3), 3, 0)), + Some(Decimal128(Some(4), 3, 0)), + Some(Decimal128(Some(5), 3, 0)) + ], + DEFAULT_DATAFUSION_CAST_OPTIONS + ); + + // int16 + generic_test_cast!( + Int16Array, + DataType::Int16, + vec![1, 2, 3, 4, 5], + DecimalArray, + DataType::Decimal(5, 0), + vec![ + Some(Decimal128(Some(1), 5, 0)), + Some(Decimal128(Some(2), 5, 0)), + Some(Decimal128(Some(3), 5, 0)), + Some(Decimal128(Some(4), 5, 0)), + Some(Decimal128(Some(5), 5, 0)) + ], + DEFAULT_DATAFUSION_CAST_OPTIONS + ); + // int32 generic_test_cast!( Int32Array, DataType::Int32, vec![1, 2, 3, 4, 5], - // TODO - UInt32Array, - DataType::UInt32, + DecimalArray, + DataType::Decimal(10, 0), vec![ - Some(1_u32), - Some(2_u32), - Some(3_u32), - Some(4_u32), - Some(5_u32) + Some(Decimal128(Some(1), 10, 0)), + Some(Decimal128(Some(2), 10, 0)), + Some(Decimal128(Some(3), 10, 0)), + Some(Decimal128(Some(4), 10, 0)), + Some(Decimal128(Some(5), 10, 0)) ], DEFAULT_DATAFUSION_CAST_OPTIONS ); + // int64 generic_test_cast!( - Int32Array, + Int64Array, DataType::Int64, vec![1, 2, 3, 4, 5], - // TODO - UInt32Array, - DataType::UInt32, + DecimalArray, + DataType::Decimal(20, 0), vec![ - Some(1_u32), - Some(2_u32), - Some(3_u32), - Some(4_u32), - Some(5_u32) + Some(Decimal128(Some(1), 20, 0)), + Some(Decimal128(Some(2), 20, 0)), + Some(Decimal128(Some(3), 20, 0)), + Some(Decimal128(Some(4), 20, 0)), + Some(Decimal128(Some(5), 20, 0)) ], DEFAULT_DATAFUSION_CAST_OPTIONS ); - // float32 + generic_test_cast!( - Int32Array, - DataType::Float32, + Int64Array, + DataType::Int64, vec![1, 2, 3, 4, 5], - // TODO - UInt32Array, - DataType::UInt32, + DecimalArray, + DataType::Decimal(20, 2), vec![ - Some(1_u32), - Some(2_u32), - Some(3_u32), - Some(4_u32), - Some(5_u32) + Some(Decimal128(Some(100), 20, 2)), + Some(Decimal128(Some(200), 20, 2)), + Some(Decimal128(Some(300), 20, 2)), + Some(Decimal128(Some(400), 20, 2)), + Some(Decimal128(Some(500), 20, 2)) ], DEFAULT_DATAFUSION_CAST_OPTIONS ); - // float64 + + // float32 generic_test_cast!( - Int32Array, - DataType::Float64, - vec![1, 2, 3, 4, 5], - // TODO - UInt32Array, - DataType::UInt32, + Float32Array, + DataType::Float32, + vec![1.5, 2.5, 3, 1.123_456_8, 5.50], + DecimalArray, + DataType::Decimal(10, 2), vec![ - Some(1_u32), - Some(2_u32), - Some(3_u32), - Some(4_u32), - Some(5_u32) + Some(Decimal128(Some(150), 20, 2)), + Some(Decimal128(Some(250), 20, 2)), + Some(Decimal128(Some(300), 20, 2)), + Some(Decimal128(Some(112), 20, 2)), + Some(Decimal128(Some(550), 20, 2)) ], DEFAULT_DATAFUSION_CAST_OPTIONS ); + + // float64 generic_test_cast!( - Int32Array, - DataType::Decimal(10, 4), - vec![1, 2, 3, 4, 5], - // TODO - UInt32Array, - DataType::UInt32, + Float64Array, + DataType::Float64, + vec![1.5, 2.5, 3, 1.123_456_8, 5.50], + DecimalArray, + DataType::Decimal(20, 4), vec![ - Some(1_u32), - Some(2_u32), - Some(3_u32), - Some(4_u32), - Some(5_u32) + Some(Decimal128(Some(15000), 20, 4)), + Some(Decimal128(Some(25000), 20, 4)), + Some(Decimal128(Some(30000), 20, 4)), + Some(Decimal128(Some(11234), 20, 4)), + Some(Decimal128(Some(55000), 20, 4)) ], DEFAULT_DATAFUSION_CAST_OPTIONS ); diff --git a/datafusion/src/physical_plan/expressions/try_cast.rs b/datafusion/src/physical_plan/expressions/try_cast.rs index 2bc4880520c6..dcd03a15afdd 100644 --- a/datafusion/src/physical_plan/expressions/try_cast.rs +++ b/datafusion/src/physical_plan/expressions/try_cast.rs @@ -77,14 +77,6 @@ impl PhysicalExpr for TryCastExpr { fn evaluate(&self, batch: &RecordBatch) -> Result { let value = self.expr.evaluate(batch)?; - // TODO cast signed numeric to decimal - // INT32, INT64, FLOAT, DOUBLE - // private[sql] val IntDecimal = DecimalType(3, 0) - // private[sql] val IntDecimal = DecimalType(5, 0) - // private[sql] val IntDecimal = DecimalType(10, 0) - // private[sql] val LongDecimal = DecimalType(20, 0) - // private[sql] val FloatDecimal = DecimalType(14, 7) - // private[sql] val DoubleDecimal = DecimalType(30, 15) match value { ColumnarValue::Array(array) => Ok(ColumnarValue::Array(kernels::cast::cast( &array, @@ -113,11 +105,6 @@ pub fn try_cast( if expr_type == cast_type { Ok(expr.clone()) } else if can_cast_types(&expr_type, &cast_type) { - // TODO - // support numeric data type to decimal - // support one type decimal to another type decimal - // Now we just support signed numeric type to decimal, - // and will implement decimal to signed numeric data type later. Ok(Arc::new(TryCastExpr::new(expr, cast_type))) } else { Err(DataFusionError::Internal(format!( @@ -132,12 +119,15 @@ mod tests { use super::*; use crate::error::Result; use crate::physical_plan::expressions::col; - use arrow::array::{StringArray, Time64NanosecondArray}; + use crate::scalar::ScalarValue::Decimal128; + use arrow::array::{Float32Array, Float64Array, StringArray, Time64NanosecondArray}; use arrow::{ - array::{Array, Int8Array, Int16Array, Int32Array, Int64Array, DecimalArray, TimestampNanosecondArray, UInt32Array}, + array::{ + Array, DecimalArray, Int16Array, Int32Array, Int64Array, Int8Array, + TimestampNanosecondArray, UInt32Array, + }, datatypes::*, }; - use crate::scalar::ScalarValue::Decimal128; // runs an end-to-end test of physical type cast // 1. construct a record batch with a column "a" of type A @@ -189,16 +179,15 @@ mod tests { }}; } - // #[test] + #[test] fn test_try_cast_numeric_to_decimal() -> Result<()> { // int8 generic_test_cast!( Int8Array, DataType::Int8, vec![1, 2, 3, 4, 5], - // TODO DecimalArray, - DataType::Decimal(3,0), + DataType::Decimal(3, 0), vec![ Some(Decimal128(Some(1), 3, 0)), Some(Decimal128(Some(2), 3, 0)), @@ -207,99 +196,100 @@ mod tests { Some(Decimal128(Some(5), 3, 0)) ] ); + // int16 generic_test_cast!( Int16Array, DataType::Int16, vec![1, 2, 3, 4, 5], - // TODO DecimalArray, - DataType::Decimal(5,0), + DataType::Decimal(5, 0), vec![ - Some(Decimal128(Some(1), 3, 0)), - Some(Decimal128(Some(2), 3, 0)), - Some(Decimal128(Some(3), 3, 0)), - Some(Decimal128(Some(4), 3, 0)), - Some(Decimal128(Some(5), 3, 0)) + Some(Decimal128(Some(1), 5, 0)), + Some(Decimal128(Some(2), 5, 0)), + Some(Decimal128(Some(3), 5, 0)), + Some(Decimal128(Some(4), 5, 0)), + Some(Decimal128(Some(5), 5, 0)) ] ); + // int32 generic_test_cast!( Int32Array, DataType::Int32, vec![1, 2, 3, 4, 5], - // TODO DecimalArray, - DataType::Decimal(10,0), + DataType::Decimal(10, 0), vec![ - Some(Decimal128(Some(1), 3, 0)), - Some(Decimal128(Some(2), 3, 0)), - Some(Decimal128(Some(3), 3, 0)), - Some(Decimal128(Some(4), 3, 0)), - Some(Decimal128(Some(5), 3, 0)) + Some(Decimal128(Some(1), 10, 0)), + Some(Decimal128(Some(2), 10, 0)), + Some(Decimal128(Some(3), 10, 0)), + Some(Decimal128(Some(4), 10, 0)), + Some(Decimal128(Some(5), 10, 0)) ] ); + // int64 generic_test_cast!( - Int32Array, + Int64Array, DataType::Int64, vec![1, 2, 3, 4, 5], - // TODO - UInt32Array, - DataType::UInt32, + DecimalArray, + DataType::Decimal(20, 0), vec![ - Some(1_u32), - Some(2_u32), - Some(3_u32), - Some(4_u32), - Some(5_u32) + Some(Decimal128(Some(1), 20, 0)), + Some(Decimal128(Some(2), 20, 0)), + Some(Decimal128(Some(3), 20, 0)), + Some(Decimal128(Some(4), 20, 0)), + Some(Decimal128(Some(5), 20, 0)) ] ); - // float32 + + // int64 to different scale generic_test_cast!( - Int32Array, - DataType::Float32, + Int64Array, + DataType::Int64, vec![1, 2, 3, 4, 5], - // TODO - UInt32Array, - DataType::UInt32, + DecimalArray, + DataType::Decimal(20, 2), vec![ - Some(1_u32), - Some(2_u32), - Some(3_u32), - Some(4_u32), - Some(5_u32) + Some(Decimal128(Some(100), 20, 2)), + Some(Decimal128(Some(200), 20, 2)), + Some(Decimal128(Some(300), 20, 2)), + Some(Decimal128(Some(400), 20, 2)), + Some(Decimal128(Some(500), 20, 2)) ] ); - // float64 + + // float32 generic_test_cast!( - Int32Array, - DataType::Float64, - vec![1, 2, 3, 4, 5], - // TODO - UInt32Array, - DataType::UInt32, + Float32Array, + DataType::Float32, + vec![1.5, 2.5, 3, 1.123_456_8, 5.50], + DecimalArray, + DataType::Decimal(10, 2), vec![ - Some(1_u32), - Some(2_u32), - Some(3_u32), - Some(4_u32), - Some(5_u32) + Some(Decimal128(Some(150), 20, 2)), + Some(Decimal128(Some(250), 20, 2)), + Some(Decimal128(Some(300), 20, 2)), + Some(Decimal128(Some(112), 20, 2)), + Some(Decimal128(Some(550), 20, 2)) ] ); + + // float64 generic_test_cast!( - Int32Array, - DataType::Decimal(10, 4), - vec![1, 2, 3, 4, 5], - // TODO - UInt32Array, - DataType::UInt32, + Float64Array, + DataType::Float64, + vec![1.5, 2.5, 3, 1.123_456_8, 5.50], + DecimalArray, + DataType::Decimal(20, 4), vec![ - Some(1_u32), - Some(2_u32), - Some(3_u32), - Some(4_u32), - Some(5_u32) + Some(Decimal128(Some(15000), 20, 4)), + Some(Decimal128(Some(25000), 20, 4)), + Some(Decimal128(Some(30000), 20, 4)), + Some(Decimal128(Some(11234), 20, 4)), + Some(Decimal128(Some(55000), 20, 4)) ] ); Ok(()) From d1e841d318423f8f74a41d5a4e61b5471900a909 Mon Sep 17 00:00:00 2001 From: liukun4515 Date: Thu, 16 Dec 2021 21:51:52 +0800 Subject: [PATCH 3/6] change test case --- .../src/physical_plan/expressions/cast.rs | 96 +++++++++++-------- .../src/physical_plan/expressions/try_cast.rs | 75 +++++++-------- 2 files changed, 95 insertions(+), 76 deletions(-) diff --git a/datafusion/src/physical_plan/expressions/cast.rs b/datafusion/src/physical_plan/expressions/cast.rs index 73f4790d4ed6..4449716f9d30 100644 --- a/datafusion/src/physical_plan/expressions/cast.rs +++ b/datafusion/src/physical_plan/expressions/cast.rs @@ -162,7 +162,10 @@ mod tests { use crate::physical_plan::expressions::col; use arrow::array::{StringArray, Time64NanosecondArray}; use arrow::{ - array::{Array, Int32Array, Int64Array, TimestampNanosecondArray, UInt32Array}, + array::{ + Array, DecimalArray, Float32Array, Float64Array, Int16Array, Int32Array, + Int64Array, Int8Array, TimestampNanosecondArray, UInt32Array, + }, datatypes::*, }; @@ -227,11 +230,11 @@ mod tests { DecimalArray, DataType::Decimal(3, 0), vec![ - Some(Decimal128(Some(1), 3, 0)), - Some(Decimal128(Some(2), 3, 0)), - Some(Decimal128(Some(3), 3, 0)), - Some(Decimal128(Some(4), 3, 0)), - Some(Decimal128(Some(5), 3, 0)) + Some(1_i128), + Some(2_i128), + Some(3_i128), + Some(4_i128), + Some(5_i128), ], DEFAULT_DATAFUSION_CAST_OPTIONS ); @@ -244,11 +247,11 @@ mod tests { DecimalArray, DataType::Decimal(5, 0), vec![ - Some(Decimal128(Some(1), 5, 0)), - Some(Decimal128(Some(2), 5, 0)), - Some(Decimal128(Some(3), 5, 0)), - Some(Decimal128(Some(4), 5, 0)), - Some(Decimal128(Some(5), 5, 0)) + Some(1_i128), + Some(2_i128), + Some(3_i128), + Some(4_i128), + Some(5_i128), ], DEFAULT_DATAFUSION_CAST_OPTIONS ); @@ -261,11 +264,11 @@ mod tests { DecimalArray, DataType::Decimal(10, 0), vec![ - Some(Decimal128(Some(1), 10, 0)), - Some(Decimal128(Some(2), 10, 0)), - Some(Decimal128(Some(3), 10, 0)), - Some(Decimal128(Some(4), 10, 0)), - Some(Decimal128(Some(5), 10, 0)) + Some(1_i128), + Some(2_i128), + Some(3_i128), + Some(4_i128), + Some(5_i128), ], DEFAULT_DATAFUSION_CAST_OPTIONS ); @@ -278,15 +281,16 @@ mod tests { DecimalArray, DataType::Decimal(20, 0), vec![ - Some(Decimal128(Some(1), 20, 0)), - Some(Decimal128(Some(2), 20, 0)), - Some(Decimal128(Some(3), 20, 0)), - Some(Decimal128(Some(4), 20, 0)), - Some(Decimal128(Some(5), 20, 0)) + Some(1_i128), + Some(2_i128), + Some(3_i128), + Some(4_i128), + Some(5_i128), ], DEFAULT_DATAFUSION_CAST_OPTIONS ); + // int64 to different scale generic_test_cast!( Int64Array, DataType::Int64, @@ -294,11 +298,11 @@ mod tests { DecimalArray, DataType::Decimal(20, 2), vec![ - Some(Decimal128(Some(100), 20, 2)), - Some(Decimal128(Some(200), 20, 2)), - Some(Decimal128(Some(300), 20, 2)), - Some(Decimal128(Some(400), 20, 2)), - Some(Decimal128(Some(500), 20, 2)) + Some(100_i128), + Some(200_i128), + Some(300_i128), + Some(400_i128), + Some(500_i128), ], DEFAULT_DATAFUSION_CAST_OPTIONS ); @@ -307,15 +311,15 @@ mod tests { generic_test_cast!( Float32Array, DataType::Float32, - vec![1.5, 2.5, 3, 1.123_456_8, 5.50], + vec![1.5, 2.5, 3.0, 1.123_456_8, 5.50], DecimalArray, DataType::Decimal(10, 2), vec![ - Some(Decimal128(Some(150), 20, 2)), - Some(Decimal128(Some(250), 20, 2)), - Some(Decimal128(Some(300), 20, 2)), - Some(Decimal128(Some(112), 20, 2)), - Some(Decimal128(Some(550), 20, 2)) + Some(150_i128), + Some(250_i128), + Some(300_i128), + Some(112_i128), + Some(550_i128), ], DEFAULT_DATAFUSION_CAST_OPTIONS ); @@ -324,15 +328,15 @@ mod tests { generic_test_cast!( Float64Array, DataType::Float64, - vec![1.5, 2.5, 3, 1.123_456_8, 5.50], + vec![1.5, 2.5, 3.0, 1.123_456_8, 5.50], DecimalArray, DataType::Decimal(20, 4), vec![ - Some(Decimal128(Some(15000), 20, 4)), - Some(Decimal128(Some(25000), 20, 4)), - Some(Decimal128(Some(30000), 20, 4)), - Some(Decimal128(Some(11234), 20, 4)), - Some(Decimal128(Some(55000), 20, 4)) + Some(15000_i128), + Some(25000_i128), + Some(30000_i128), + Some(11234_i128), + Some(55000_i128), ], DEFAULT_DATAFUSION_CAST_OPTIONS ); @@ -341,6 +345,22 @@ mod tests { #[test] fn test_cast_i32_u32() -> Result<()> { + generic_test_cast!( + Int32Array, + DataType::Int32, + vec![1, 2, 3, 4, 5], + UInt32Array, + DataType::UInt32, + vec![ + Some(1_u32), + Some(2_u32), + Some(3_u32), + Some(4_u32), + Some(5_u32) + ], + DEFAULT_DATAFUSION_CAST_OPTIONS + ); + generic_test_cast!( Int32Array, DataType::Int32, diff --git a/datafusion/src/physical_plan/expressions/try_cast.rs b/datafusion/src/physical_plan/expressions/try_cast.rs index dcd03a15afdd..d231ce276a77 100644 --- a/datafusion/src/physical_plan/expressions/try_cast.rs +++ b/datafusion/src/physical_plan/expressions/try_cast.rs @@ -119,7 +119,6 @@ mod tests { use super::*; use crate::error::Result; use crate::physical_plan::expressions::col; - use crate::scalar::ScalarValue::Decimal128; use arrow::array::{Float32Array, Float64Array, StringArray, Time64NanosecondArray}; use arrow::{ array::{ @@ -189,11 +188,11 @@ mod tests { DecimalArray, DataType::Decimal(3, 0), vec![ - Some(Decimal128(Some(1), 3, 0)), - Some(Decimal128(Some(2), 3, 0)), - Some(Decimal128(Some(3), 3, 0)), - Some(Decimal128(Some(4), 3, 0)), - Some(Decimal128(Some(5), 3, 0)) + Some(1_i128), + Some(2_i128), + Some(3_i128), + Some(4_i128), + Some(5_i128), ] ); @@ -205,11 +204,11 @@ mod tests { DecimalArray, DataType::Decimal(5, 0), vec![ - Some(Decimal128(Some(1), 5, 0)), - Some(Decimal128(Some(2), 5, 0)), - Some(Decimal128(Some(3), 5, 0)), - Some(Decimal128(Some(4), 5, 0)), - Some(Decimal128(Some(5), 5, 0)) + Some(1_i128), + Some(2_i128), + Some(3_i128), + Some(4_i128), + Some(5_i128), ] ); @@ -221,11 +220,11 @@ mod tests { DecimalArray, DataType::Decimal(10, 0), vec![ - Some(Decimal128(Some(1), 10, 0)), - Some(Decimal128(Some(2), 10, 0)), - Some(Decimal128(Some(3), 10, 0)), - Some(Decimal128(Some(4), 10, 0)), - Some(Decimal128(Some(5), 10, 0)) + Some(1_i128), + Some(2_i128), + Some(3_i128), + Some(4_i128), + Some(5_i128), ] ); @@ -237,11 +236,11 @@ mod tests { DecimalArray, DataType::Decimal(20, 0), vec![ - Some(Decimal128(Some(1), 20, 0)), - Some(Decimal128(Some(2), 20, 0)), - Some(Decimal128(Some(3), 20, 0)), - Some(Decimal128(Some(4), 20, 0)), - Some(Decimal128(Some(5), 20, 0)) + Some(1_i128), + Some(2_i128), + Some(3_i128), + Some(4_i128), + Some(5_i128), ] ); @@ -253,11 +252,11 @@ mod tests { DecimalArray, DataType::Decimal(20, 2), vec![ - Some(Decimal128(Some(100), 20, 2)), - Some(Decimal128(Some(200), 20, 2)), - Some(Decimal128(Some(300), 20, 2)), - Some(Decimal128(Some(400), 20, 2)), - Some(Decimal128(Some(500), 20, 2)) + Some(100_i128), + Some(200_i128), + Some(300_i128), + Some(400_i128), + Some(500_i128), ] ); @@ -265,15 +264,15 @@ mod tests { generic_test_cast!( Float32Array, DataType::Float32, - vec![1.5, 2.5, 3, 1.123_456_8, 5.50], + vec![1.5, 2.5, 3.0, 1.123_456_8, 5.50], DecimalArray, DataType::Decimal(10, 2), vec![ - Some(Decimal128(Some(150), 20, 2)), - Some(Decimal128(Some(250), 20, 2)), - Some(Decimal128(Some(300), 20, 2)), - Some(Decimal128(Some(112), 20, 2)), - Some(Decimal128(Some(550), 20, 2)) + Some(150_i128), + Some(250_i128), + Some(300_i128), + Some(112_i128), + Some(550_i128), ] ); @@ -281,15 +280,15 @@ mod tests { generic_test_cast!( Float64Array, DataType::Float64, - vec![1.5, 2.5, 3, 1.123_456_8, 5.50], + vec![1.5, 2.5, 3.0, 1.123_456_8, 5.50], DecimalArray, DataType::Decimal(20, 4), vec![ - Some(Decimal128(Some(15000), 20, 4)), - Some(Decimal128(Some(25000), 20, 4)), - Some(Decimal128(Some(30000), 20, 4)), - Some(Decimal128(Some(11234), 20, 4)), - Some(Decimal128(Some(55000), 20, 4)) + Some(15000_i128), + Some(25000_i128), + Some(30000_i128), + Some(11234_i128), + Some(55000_i128), ] ); Ok(()) From 08890b3fff9ab781daebcff114c7d52aaa5368f6 Mon Sep 17 00:00:00 2001 From: liukun4515 Date: Fri, 17 Dec 2021 14:22:29 +0800 Subject: [PATCH 4/6] change dependency --- ballista/rust/core/Cargo.toml | 3 ++- ballista/rust/executor/Cargo.toml | 6 ++++-- datafusion-cli/Cargo.toml | 3 ++- datafusion-examples/Cargo.toml | 3 ++- datafusion/Cargo.toml | 8 ++++---- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/ballista/rust/core/Cargo.toml b/ballista/rust/core/Cargo.toml index 84679d596daf..3fdd7c9fb8ad 100644 --- a/ballista/rust/core/Cargo.toml +++ b/ballista/rust/core/Cargo.toml @@ -43,7 +43,8 @@ tonic = "0.5" uuid = { version = "0.8", features = ["v4"] } chrono = "0.4" -arrow-flight = { version = "6.4.0" } +#arrow-flight = { version = "6.4.0" } +arrow-flight = { path = "/Users/kliu3/Documents/github/arrow-rs/arrow-flight" } datafusion = { path = "../../../datafusion", version = "6.0.0" } diff --git a/ballista/rust/executor/Cargo.toml b/ballista/rust/executor/Cargo.toml index 00f3aab745ff..23cedc97a172 100644 --- a/ballista/rust/executor/Cargo.toml +++ b/ballista/rust/executor/Cargo.toml @@ -29,8 +29,10 @@ edition = "2018" snmalloc = ["snmalloc-rs"] [dependencies] -arrow = { version = "6.4.0" } -arrow-flight = { version = "6.4.0" } +#arrow = { version = "6.4.0" } +#arrow-flight = { version = "6.4.0" } +arrow = { path = "/Users/kliu3/Documents/github/arrow-rs/arrow"} +arrow-flight = { path = "/Users/kliu3/Documents/github/arrow-rs/arrow-flight" } anyhow = "1" async-trait = "0.1.36" ballista-core = { path = "../core", version = "0.6.0" } diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index 394bd1e3a29b..d6fac57151fa 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -31,5 +31,6 @@ clap = "2.33" rustyline = "9.0" tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync"] } datafusion = { path = "../datafusion", version = "6.0.0" } -arrow = { version = "6.4.0" } +#arrow = { version = "6.4.0" } +arrow = { path = "/Users/kliu3/Documents/github/arrow-rs/arrow" } ballista = { path = "../ballista/rust/client", version = "0.6.0" } diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml index f7ef66d99bde..bff4cede444c 100644 --- a/datafusion-examples/Cargo.toml +++ b/datafusion-examples/Cargo.toml @@ -34,7 +34,8 @@ path = "examples/avro_sql.rs" required-features = ["datafusion/avro"] [dev-dependencies] -arrow-flight = { version = "6.4.0" } +#arrow-flight = { version = "6.4.0" } +arrow-flight = { path = "/Users/kliu3/Documents/github/arrow-rs/arrow-flight" } datafusion = { path = "../datafusion" } prost = "0.8" tonic = "0.5" diff --git a/datafusion/Cargo.toml b/datafusion/Cargo.toml index ee809fe37d67..2a03df4a20e1 100644 --- a/datafusion/Cargo.toml +++ b/datafusion/Cargo.toml @@ -54,10 +54,10 @@ ahash = "0.7" hashbrown = { version = "0.11", features = ["raw"] } ##arrow = { version = "6.3.0", features = ["prettyprint"] } ##parquet = { version = "6.3.0", features = ["arrow"] } -#arrow = { path = "/Users/kliu3/Documents/github/arrow-rs/arrow", features = ["prettyprint"] } -#parquet = { path = "/Users/kliu3/Documents/github/arrow-rs/parquet", features = ["arrow"] } -arrow = { version = "6.4.0", features = ["prettyprint"] } -parquet = { version = "6.4.0", features = ["arrow"] } +arrow = { path = "/Users/kliu3/Documents/github/arrow-rs/arrow", features = ["prettyprint"] } +parquet = { path = "/Users/kliu3/Documents/github/arrow-rs/parquet", features = ["arrow"] } +#arrow = { version = "6.4.0", features = ["prettyprint"] } +#parquet = { version = "6.4.0", features = ["arrow"] } sqlparser = "0.13" paste = "^1.0" num_cpus = "1.13.0" From 9f8f3beccc67ede05fabc1561e2503abdb5a732f Mon Sep 17 00:00:00 2001 From: liukun4515 Date: Sat, 18 Dec 2021 20:21:08 +0800 Subject: [PATCH 5/6] remove useless code --- datafusion/src/physical_plan/expressions/cast.rs | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/datafusion/src/physical_plan/expressions/cast.rs b/datafusion/src/physical_plan/expressions/cast.rs index 4449716f9d30..df687b4dd98b 100644 --- a/datafusion/src/physical_plan/expressions/cast.rs +++ b/datafusion/src/physical_plan/expressions/cast.rs @@ -345,22 +345,6 @@ mod tests { #[test] fn test_cast_i32_u32() -> Result<()> { - generic_test_cast!( - Int32Array, - DataType::Int32, - vec![1, 2, 3, 4, 5], - UInt32Array, - DataType::UInt32, - vec![ - Some(1_u32), - Some(2_u32), - Some(3_u32), - Some(4_u32), - Some(5_u32) - ], - DEFAULT_DATAFUSION_CAST_OPTIONS - ); - generic_test_cast!( Int32Array, DataType::Int32, From 6593daf9d0d3fafe0c9fc8cc0855325541dff98e Mon Sep 17 00:00:00 2001 From: liukun4515 Date: Thu, 13 Jan 2022 10:08:25 +0800 Subject: [PATCH 6/6] restore dependency --- ballista/rust/core/Cargo.toml | 3 +-- ballista/rust/executor/Cargo.toml | 6 ++---- datafusion-cli/Cargo.toml | 3 +-- datafusion-examples/Cargo.toml | 3 +-- datafusion/Cargo.toml | 8 ++------ 5 files changed, 7 insertions(+), 16 deletions(-) diff --git a/ballista/rust/core/Cargo.toml b/ballista/rust/core/Cargo.toml index 3fdd7c9fb8ad..84679d596daf 100644 --- a/ballista/rust/core/Cargo.toml +++ b/ballista/rust/core/Cargo.toml @@ -43,8 +43,7 @@ tonic = "0.5" uuid = { version = "0.8", features = ["v4"] } chrono = "0.4" -#arrow-flight = { version = "6.4.0" } -arrow-flight = { path = "/Users/kliu3/Documents/github/arrow-rs/arrow-flight" } +arrow-flight = { version = "6.4.0" } datafusion = { path = "../../../datafusion", version = "6.0.0" } diff --git a/ballista/rust/executor/Cargo.toml b/ballista/rust/executor/Cargo.toml index 23cedc97a172..00f3aab745ff 100644 --- a/ballista/rust/executor/Cargo.toml +++ b/ballista/rust/executor/Cargo.toml @@ -29,10 +29,8 @@ edition = "2018" snmalloc = ["snmalloc-rs"] [dependencies] -#arrow = { version = "6.4.0" } -#arrow-flight = { version = "6.4.0" } -arrow = { path = "/Users/kliu3/Documents/github/arrow-rs/arrow"} -arrow-flight = { path = "/Users/kliu3/Documents/github/arrow-rs/arrow-flight" } +arrow = { version = "6.4.0" } +arrow-flight = { version = "6.4.0" } anyhow = "1" async-trait = "0.1.36" ballista-core = { path = "../core", version = "0.6.0" } diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index d6fac57151fa..394bd1e3a29b 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -31,6 +31,5 @@ clap = "2.33" rustyline = "9.0" tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync"] } datafusion = { path = "../datafusion", version = "6.0.0" } -#arrow = { version = "6.4.0" } -arrow = { path = "/Users/kliu3/Documents/github/arrow-rs/arrow" } +arrow = { version = "6.4.0" } ballista = { path = "../ballista/rust/client", version = "0.6.0" } diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml index bff4cede444c..f7ef66d99bde 100644 --- a/datafusion-examples/Cargo.toml +++ b/datafusion-examples/Cargo.toml @@ -34,8 +34,7 @@ path = "examples/avro_sql.rs" required-features = ["datafusion/avro"] [dev-dependencies] -#arrow-flight = { version = "6.4.0" } -arrow-flight = { path = "/Users/kliu3/Documents/github/arrow-rs/arrow-flight" } +arrow-flight = { version = "6.4.0" } datafusion = { path = "../datafusion" } prost = "0.8" tonic = "0.5" diff --git a/datafusion/Cargo.toml b/datafusion/Cargo.toml index 2a03df4a20e1..fadc9aba9101 100644 --- a/datafusion/Cargo.toml +++ b/datafusion/Cargo.toml @@ -52,12 +52,8 @@ avro = ["avro-rs", "num-traits"] [dependencies] ahash = "0.7" hashbrown = { version = "0.11", features = ["raw"] } -##arrow = { version = "6.3.0", features = ["prettyprint"] } -##parquet = { version = "6.3.0", features = ["arrow"] } -arrow = { path = "/Users/kliu3/Documents/github/arrow-rs/arrow", features = ["prettyprint"] } -parquet = { path = "/Users/kliu3/Documents/github/arrow-rs/parquet", features = ["arrow"] } -#arrow = { version = "6.4.0", features = ["prettyprint"] } -#parquet = { version = "6.4.0", features = ["arrow"] } +arrow = { version = "6.4.0", features = ["prettyprint"] } +parquet = { version = "6.4.0", features = ["arrow"] } sqlparser = "0.13" paste = "^1.0" num_cpus = "1.13.0"