From 6e7855b2d93d70f7896d0cf6d9f72ac27f57364a Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Mon, 6 Dec 2021 21:51:54 +0800 Subject: [PATCH 1/2] dont enable columanr shuffle on unsupported data types Signed-off-by: Yuan Zhou --- .../spark/sql/execution/ColumnarShuffleExchangeExec.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala b/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala index 81a8c1f70..8872f8504 100644 --- a/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala +++ b/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala @@ -97,11 +97,11 @@ case class ColumnarShuffleExchangeExec( // check input datatype for (attr <- child.output) { try { - ConverterUtils.createArrowField(attr) + ConverterUtils.checkIfTypeSupported(attr.dataType) } catch { case e: UnsupportedOperationException => throw new UnsupportedOperationException( - s"${attr.dataType} is not supported in ColumnarShuffleExchange") + s"${attr.dataType} is not supported in ColumnarShuffledHashJoinExec.") } } } From 196407736de37562d42f4c21410c983eecd41a8b Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 7 Dec 2021 13:57:23 +0800 Subject: [PATCH 2/2] Update native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala --- .../spark/sql/execution/ColumnarShuffleExchangeExec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala b/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala index 8872f8504..d36f50530 100644 --- a/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala +++ b/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala @@ -101,7 +101,7 @@ case class ColumnarShuffleExchangeExec( } catch { case e: UnsupportedOperationException => throw new UnsupportedOperationException( - s"${attr.dataType} is not supported in ColumnarShuffledHashJoinExec.") + s"${attr.dataType} is not supported in ColumnarShuffledExchangeExec.") } } }