From af34bbc88d3e1cdc218954acc7f8dfcfaf2f651d Mon Sep 17 00:00:00 2001 From: philo Date: Wed, 9 Mar 2022 21:17:54 +0800 Subject: [PATCH 1/3] Support spark 3.1.2 also --- .../com/intel/oap/sql/shims/spark311/SparkShimProvider.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/shims/spark311/src/main/scala/com/intel/oap/sql/shims/spark311/SparkShimProvider.scala b/shims/spark311/src/main/scala/com/intel/oap/sql/shims/spark311/SparkShimProvider.scala index 725788785..6c96cc0f3 100644 --- a/shims/spark311/src/main/scala/com/intel/oap/sql/shims/spark311/SparkShimProvider.scala +++ b/shims/spark311/src/main/scala/com/intel/oap/sql/shims/spark311/SparkShimProvider.scala @@ -19,8 +19,9 @@ package com.intel.oap.sql.shims.spark311 import com.intel.oap.sql.shims.{SparkShims, SparkShimDescriptor} object SparkShimProvider { - val DESCRIPTOR = SparkShimDescriptor(3, 1, 1) - val DESCRIPTOR_STRINGS = Seq(s"$DESCRIPTOR") + val SPARK311_DESCRIPTOR = SparkShimDescriptor(3, 1, 1) + val SPARK312_DESCRIPTOR = SparkShimDescriptor(3, 1, 2) + val DESCRIPTOR_STRINGS = Seq(s"$SPARK311_DESCRIPTOR", s"$SPARK312_DESCRIPTOR") } class SparkShimProvider extends com.intel.oap.sql.shims.SparkShimProvider { From 7af57be36a84ffbd90232e309c4a832a5ffa273f Mon Sep 17 00:00:00 2001 From: philo Date: Thu, 10 Mar 2022 13:38:37 +0800 Subject: [PATCH 2/3] Upgrade to spark 3.1.3 from 3.1.1 --- native-sql-engine/core/pom.xml | 2 +- pom.xml | 29 +++++++++++++++++-- shims/common/pom.xml | 2 +- .../oap/sql/shims/SparkShimProvider.scala | 2 +- .../com/intel/oap/sql/shims/SparkShims.scala | 6 ++-- shims/pom.xml | 4 +-- .../com.intel.oap.sql.shims.SparkShimProvider | 1 - shims/{spark311 => spark313}/pom.xml | 6 ++-- .../VectorizedParquetRecordReaderChild.java | 0 .../com.intel.oap.sql.shims.SparkShimProvider | 1 + .../oap/execution/ColumnarBatchScanExec.scala | 0 .../sql/shims/spark313/Spark313Shims.scala} | 4 +-- .../shims/spark313}/SparkShimProvider.scala | 9 +++--- .../spark/sql/BasePythonRunnerChild.scala | 0 .../adaptive/AdaptiveSparkPlanExec.scala | 0 .../spark/storage/memory/MemoryStore.scala | 0 .../org/apache/spark/util/ShimUtils.scala | 0 .../shims/spark311/Spark313ShimsSuite.scala} | 4 +-- 18 files changed, 47 insertions(+), 23 deletions(-) delete mode 100644 shims/spark311/src/main/resources/META-INF/services/com.intel.oap.sql.shims.SparkShimProvider rename shims/{spark311 => spark313}/pom.xml (96%) rename shims/{spark311 => spark313}/src/main/java/org/apache/spark/sql/execution/datasources/VectorizedParquetRecordReaderChild.java (100%) create mode 100644 shims/spark313/src/main/resources/META-INF/services/com.intel.oap.sql.shims.SparkShimProvider rename shims/{spark311 => spark313}/src/main/scala/com/intel/oap/execution/ColumnarBatchScanExec.scala (100%) rename shims/{spark311/src/main/scala/com/intel/oap/sql/shims/spark311/Spark311Shims.scala => spark313/src/main/scala/com/intel/oap/sql/shims/spark313/Spark313Shims.scala} (98%) rename shims/{spark311/src/main/scala/com/intel/oap/sql/shims/spark311 => spark313/src/main/scala/com/intel/oap/sql/shims/spark313}/SparkShimProvider.scala (78%) rename shims/{spark311 => spark313}/src/main/scala/org/apache/spark/sql/BasePythonRunnerChild.scala (100%) rename shims/{spark311 => spark313}/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala (100%) rename shims/{spark311 => spark313}/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala (100%) rename shims/{spark311 => spark313}/src/main/scala/org/apache/spark/util/ShimUtils.scala (100%) rename shims/{spark311/src/test/scala/com/intel/oap/sql/shims/spark311/Spark311ShimsSuite.scala => spark313/src/test/scala/com/intel/oap/sql/shims/spark311/Spark313ShimsSuite.scala} (92%) diff --git a/native-sql-engine/core/pom.xml b/native-sql-engine/core/pom.xml index 1cdd279de..0b564b361 100644 --- a/native-sql-engine/core/pom.xml +++ b/native-sql-engine/core/pom.xml @@ -54,7 +54,7 @@ com.intel.oap - spark-sql-columnar-shims-spark311 + spark-sql-columnar-shims-spark313 ${project.version} diff --git a/pom.xml b/pom.xml index 8a92f613c..400756253 100644 --- a/pom.xml +++ b/pom.xml @@ -42,13 +42,13 @@ true - ${spark311.version} + ${spark313.version} spark-3.1 - ${spark311.version} + ${spark313.version} 2.12.10 2.10.0 @@ -123,7 +123,7 @@ 3.1.1 - 3.1.1 + 3.1.1 3.2.1 2.12.10 @@ -317,6 +317,29 @@ + + + snapshots-repo + https://oss.sonatype.org/content/repositories/snapshots + + false + + + true + + + + apache-snapshots-repo + https://repository.apache.org/content/repositories/snapshots/ + + false + + + true + + + + diff --git a/shims/common/pom.xml b/shims/common/pom.xml index 8e8a17831..3e94c8195 100644 --- a/shims/common/pom.xml +++ b/shims/common/pom.xml @@ -78,7 +78,7 @@ org.apache.spark spark-sql_${scala.binary.version} - ${spark311.version} + ${spark313.version} provided diff --git a/shims/common/src/main/scala/com/intel/oap/sql/shims/SparkShimProvider.scala b/shims/common/src/main/scala/com/intel/oap/sql/shims/SparkShimProvider.scala index 9017e6b63..34b0d6be0 100644 --- a/shims/common/src/main/scala/com/intel/oap/sql/shims/SparkShimProvider.scala +++ b/shims/common/src/main/scala/com/intel/oap/sql/shims/SparkShimProvider.scala @@ -20,6 +20,6 @@ package com.intel.oap.sql.shims * Provider interface for matching and retrieving the Shims of a specific Spark version */ trait SparkShimProvider { - def matches(version:String): Boolean + def matches(version: String): Boolean def createShim: SparkShims } diff --git a/shims/common/src/main/scala/com/intel/oap/sql/shims/SparkShims.scala b/shims/common/src/main/scala/com/intel/oap/sql/shims/SparkShims.scala index 62420a6f8..be8f90de0 100644 --- a/shims/common/src/main/scala/com/intel/oap/sql/shims/SparkShims.scala +++ b/shims/common/src/main/scala/com/intel/oap/sql/shims/SparkShims.scala @@ -42,8 +42,10 @@ import org.apache.spark.sql.internal.SQLConf sealed abstract class ShimDescriptor -case class SparkShimDescriptor(major: Int, minor: Int, patch: Int) extends ShimDescriptor { - override def toString(): String = s"$major.$minor.$patch" +// The append arg can either be "" for release version or be "-SNAPSHOT" for snapshot version. +case class SparkShimDescriptor(major: Int, minor: Int, patch: Int, + append: String = "") extends ShimDescriptor { + override def toString(): String = s"$major.$minor.$patch$append" } trait SparkShims { diff --git a/shims/pom.xml b/shims/pom.xml index 9e45e9dd6..740e12e76 100644 --- a/shims/pom.xml +++ b/shims/pom.xml @@ -70,7 +70,7 @@ common - spark311 + spark313 @@ -79,7 +79,7 @@ common - spark311 + spark313 diff --git a/shims/spark311/src/main/resources/META-INF/services/com.intel.oap.sql.shims.SparkShimProvider b/shims/spark311/src/main/resources/META-INF/services/com.intel.oap.sql.shims.SparkShimProvider deleted file mode 100644 index e124437df..000000000 --- a/shims/spark311/src/main/resources/META-INF/services/com.intel.oap.sql.shims.SparkShimProvider +++ /dev/null @@ -1 +0,0 @@ -com.intel.oap.sql.shims.spark311.SparkShimProvider diff --git a/shims/spark311/pom.xml b/shims/spark313/pom.xml similarity index 96% rename from shims/spark311/pom.xml rename to shims/spark313/pom.xml index ef9a8cf40..1266306a1 100644 --- a/shims/spark311/pom.xml +++ b/shims/spark313/pom.xml @@ -24,8 +24,8 @@ ../pom.xml - spark-sql-columnar-shims-spark311 - ${project.name.prefix} Shims for Spark 3.1.1 + spark-sql-columnar-shims-spark313 + ${project.name.prefix} Shims for Spark 3.1.3 jar @@ -89,7 +89,7 @@ org.apache.spark spark-sql_${scala.binary.version} - ${spark311.version} + ${spark313.version} provided diff --git a/shims/spark311/src/main/java/org/apache/spark/sql/execution/datasources/VectorizedParquetRecordReaderChild.java b/shims/spark313/src/main/java/org/apache/spark/sql/execution/datasources/VectorizedParquetRecordReaderChild.java similarity index 100% rename from shims/spark311/src/main/java/org/apache/spark/sql/execution/datasources/VectorizedParquetRecordReaderChild.java rename to shims/spark313/src/main/java/org/apache/spark/sql/execution/datasources/VectorizedParquetRecordReaderChild.java diff --git a/shims/spark313/src/main/resources/META-INF/services/com.intel.oap.sql.shims.SparkShimProvider b/shims/spark313/src/main/resources/META-INF/services/com.intel.oap.sql.shims.SparkShimProvider new file mode 100644 index 000000000..804aeae27 --- /dev/null +++ b/shims/spark313/src/main/resources/META-INF/services/com.intel.oap.sql.shims.SparkShimProvider @@ -0,0 +1 @@ +com.intel.oap.sql.shims.spark313.SparkShimProvider diff --git a/shims/spark311/src/main/scala/com/intel/oap/execution/ColumnarBatchScanExec.scala b/shims/spark313/src/main/scala/com/intel/oap/execution/ColumnarBatchScanExec.scala similarity index 100% rename from shims/spark311/src/main/scala/com/intel/oap/execution/ColumnarBatchScanExec.scala rename to shims/spark313/src/main/scala/com/intel/oap/execution/ColumnarBatchScanExec.scala diff --git a/shims/spark311/src/main/scala/com/intel/oap/sql/shims/spark311/Spark311Shims.scala b/shims/spark313/src/main/scala/com/intel/oap/sql/shims/spark313/Spark313Shims.scala similarity index 98% rename from shims/spark311/src/main/scala/com/intel/oap/sql/shims/spark311/Spark311Shims.scala rename to shims/spark313/src/main/scala/com/intel/oap/sql/shims/spark313/Spark313Shims.scala index e637877e9..7d49a3c50 100644 --- a/shims/spark311/src/main/scala/com/intel/oap/sql/shims/spark311/Spark311Shims.scala +++ b/shims/spark313/src/main/scala/com/intel/oap/sql/shims/spark313/Spark313Shims.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.oap.sql.shims.spark311 +package com.intel.oap.sql.shims.spark313 import com.intel.oap.execution.ColumnarBatchScanExec import com.intel.oap.spark.sql.ArrowWriteQueue @@ -46,7 +46,7 @@ import org.apache.spark.sql.execution.datasources.{DataSourceUtils, OutputWriter import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, REPARTITION, ReusedExchangeExec, ShuffleExchangeExec, ShuffleOrigin} import org.apache.spark.sql.internal.SQLConf -class Spark311Shims extends SparkShims { +class Spark313Shims extends SparkShims { override def getShimDescriptor: ShimDescriptor = SparkShimProvider.DESCRIPTOR diff --git a/shims/spark311/src/main/scala/com/intel/oap/sql/shims/spark311/SparkShimProvider.scala b/shims/spark313/src/main/scala/com/intel/oap/sql/shims/spark313/SparkShimProvider.scala similarity index 78% rename from shims/spark311/src/main/scala/com/intel/oap/sql/shims/spark311/SparkShimProvider.scala rename to shims/spark313/src/main/scala/com/intel/oap/sql/shims/spark313/SparkShimProvider.scala index 6c96cc0f3..99955bafd 100644 --- a/shims/spark311/src/main/scala/com/intel/oap/sql/shims/spark311/SparkShimProvider.scala +++ b/shims/spark313/src/main/scala/com/intel/oap/sql/shims/spark313/SparkShimProvider.scala @@ -14,19 +14,18 @@ * limitations under the License. */ -package com.intel.oap.sql.shims.spark311 +package com.intel.oap.sql.shims.spark313 import com.intel.oap.sql.shims.{SparkShims, SparkShimDescriptor} object SparkShimProvider { - val SPARK311_DESCRIPTOR = SparkShimDescriptor(3, 1, 1) - val SPARK312_DESCRIPTOR = SparkShimDescriptor(3, 1, 2) - val DESCRIPTOR_STRINGS = Seq(s"$SPARK311_DESCRIPTOR", s"$SPARK312_DESCRIPTOR") + val DESCRIPTOR = SparkShimDescriptor(3, 1, 3, "-SNAPSHOT") + val DESCRIPTOR_STRINGS = Seq(s"$DESCRIPTOR") } class SparkShimProvider extends com.intel.oap.sql.shims.SparkShimProvider { def createShim: SparkShims = { - new Spark311Shims() + new Spark313Shims() } def matches(version: String): Boolean = { diff --git a/shims/spark311/src/main/scala/org/apache/spark/sql/BasePythonRunnerChild.scala b/shims/spark313/src/main/scala/org/apache/spark/sql/BasePythonRunnerChild.scala similarity index 100% rename from shims/spark311/src/main/scala/org/apache/spark/sql/BasePythonRunnerChild.scala rename to shims/spark313/src/main/scala/org/apache/spark/sql/BasePythonRunnerChild.scala diff --git a/shims/spark311/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/shims/spark313/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala similarity index 100% rename from shims/spark311/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala rename to shims/spark313/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala diff --git a/shims/spark311/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala b/shims/spark313/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala similarity index 100% rename from shims/spark311/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala rename to shims/spark313/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala diff --git a/shims/spark311/src/main/scala/org/apache/spark/util/ShimUtils.scala b/shims/spark313/src/main/scala/org/apache/spark/util/ShimUtils.scala similarity index 100% rename from shims/spark311/src/main/scala/org/apache/spark/util/ShimUtils.scala rename to shims/spark313/src/main/scala/org/apache/spark/util/ShimUtils.scala diff --git a/shims/spark311/src/test/scala/com/intel/oap/sql/shims/spark311/Spark311ShimsSuite.scala b/shims/spark313/src/test/scala/com/intel/oap/sql/shims/spark311/Spark313ShimsSuite.scala similarity index 92% rename from shims/spark311/src/test/scala/com/intel/oap/sql/shims/spark311/Spark311ShimsSuite.scala rename to shims/spark313/src/test/scala/com/intel/oap/sql/shims/spark311/Spark313ShimsSuite.scala index 6597dace5..01bd86de5 100644 --- a/shims/spark311/src/test/scala/com/intel/oap/sql/shims/spark311/Spark311ShimsSuite.scala +++ b/shims/spark313/src/test/scala/com/intel/oap/sql/shims/spark311/Spark313ShimsSuite.scala @@ -14,13 +14,13 @@ * limitations under the License. */ -package com.intel.oap.sql.shims.spark311 +package com.intel.oap.sql.shims.spark313 import com.intel.oap.sql.shims.{SparkShims, SparkShimLoader, SparkShimDescriptor} import org.scalatest.FunSuite; -class Spark311ShimsSuite extends FunSuite { +class Spark313ShimsSuite extends FunSuite { val descriptor = SparkShimDescriptor(3, 1, 1) test("Spark shims descriptor") { val sparkShims: SparkShims = new SparkShimProvider().createShim From 85bda9a5ac34dc33f4190e74e654e77b1d1c666d Mon Sep 17 00:00:00 2001 From: philo Date: Thu, 10 Mar 2022 13:48:01 +0800 Subject: [PATCH 3/3] Correct the version --- native-sql-engine/core/pom.xml | 2 +- pom.xml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/native-sql-engine/core/pom.xml b/native-sql-engine/core/pom.xml index 0b564b361..10088ac02 100644 --- a/native-sql-engine/core/pom.xml +++ b/native-sql-engine/core/pom.xml @@ -47,7 +47,7 @@ - spark-3.1.1 + spark-3.1.3 true diff --git a/pom.xml b/pom.xml index 400756253..6d072643a 100644 --- a/pom.xml +++ b/pom.xml @@ -122,8 +122,8 @@ - 3.1.1 - 3.1.1 + 3.1.3 + 3.1.3 3.2.1 2.12.10