addressed review comments

Signed-off-by: Raza Jafri <[email protected]>
NVIDIA · Feb 25, 2022 · a2a6b7e · a2a6b7e
1 parent 980eba3
commit a2a6b7e
Show file tree

Hide file tree

Showing 17 changed files with 37 additions and 93 deletions.
diff --git a/jenkins/spark-premerge-build.sh b/jenkins/spark-premerge-build.sh
@@ -95,7 +95,7 @@ rapids_shuffle_smoke_test() {
       PYSP_TEST_spark_cores_max=2 \
       PYSP_TEST_spark_executor_cores=1 \
       SPARK_SUBMIT_FLAGS="--conf spark.executorEnv.UCX_ERROR_SIGNALS=" \
-      PYSP_TEST_spark_shuffle_manager=com.nvidia.spark.rapids.shims.RapidsShuffleManager \
+      PYSP_TEST_spark_shuffle_manager=com.nvidia.spark.rapids.RapidsShuffleManager \
       PYSP_TEST_spark_rapids_memory_gpu_minAllocFraction=0 \
       PYSP_TEST_spark_rapids_memory_gpu_maxAllocFraction=0.1 \
       PYSP_TEST_spark_rapids_memory_gpu_allocFraction=0.1 \

diff --git a/pom.xml b/pom.xml
@@ -116,7 +116,7 @@
                                         <source>${project.basedir}/src/main/301+-nondb/scala</source>
                                         <source>${project.basedir}/src/main/301/scala</source>
                                         <source>${project.basedir}/src/main/301until304/scala</source>
-                                        <source>${project.basedir}/src/main/30X-32X-33X/scala</source>
+                                        <source>${project.basedir}/src/main/30X/scala</source>
                                         <source>${project.basedir}/src/main/301until310-all/scala</source>
                                         <source>${project.basedir}/src/main/301until310-nondb/scala</source>
                                         <source>${project.basedir}/src/main/301until320-all/scala</source>
@@ -167,7 +167,7 @@
                                         <source>${project.basedir}/src/main/301+-nondb/scala</source>
                                         <source>${project.basedir}/src/main/302/scala</source>
                                         <source>${project.basedir}/src/main/301until304/scala</source>
-                                        <source>${project.basedir}/src/main/30X-32X-33X/scala</source>
+                                        <source>${project.basedir}/src/main/30X/scala</source>
                                         <source>${project.basedir}/src/main/301until310-all/scala</source>
                                         <source>${project.basedir}/src/main/301until310-nondb/scala</source>
                                         <source>${project.basedir}/src/main/301until320-all/scala</source>
@@ -227,7 +227,7 @@
                                         <source>${project.basedir}/src/main/301+-nondb/scala</source>
                                         <source>${project.basedir}/src/main/303/scala</source>
                                         <source>${project.basedir}/src/main/301until304/scala</source>
-                                        <source>${project.basedir}/src/main/30X-32X-33X/scala</source>
+                                        <source>${project.basedir}/src/main/30X/scala</source>
                                         <source>${project.basedir}/src/main/301until310-all/scala</source>
                                         <source>${project.basedir}/src/main/301until310-nondb/scala</source>
                                         <source>${project.basedir}/src/main/301until320-all/scala</source>
@@ -281,7 +281,7 @@
                                     <sources>
                                         <source>${project.basedir}/src/main/301+-nondb/scala</source>
                                         <source>${project.basedir}/src/main/304/scala</source>
-                                        <source>${project.basedir}/src/main/30X-32X-33X/scala</source>
+                                        <source>${project.basedir}/src/main/30X/scala</source>
                                         <source>${project.basedir}/src/main/301until310-all/scala</source>
                                         <source>${project.basedir}/src/main/301until310-nondb/scala</source>
                                         <source>${project.basedir}/src/main/301until320-all/scala</source>
@@ -335,6 +335,7 @@
                                     <sources>
                                         <source>${project.basedir}/src/main/301+-nondb/scala</source>
                                         <source>${project.basedir}/src/main/311-nondb/scala</source>
+                                        <source>${project.basedir}/src/main/31X-33X/scala</source>
                                         <source>${project.basedir}/src/main/301until320-all/scala</source>
                                         <source>${project.basedir}/src/main/301until320-noncdh/scala</source>
                                         <source>${project.basedir}/src/main/301until320-nondb/scala</source>
@@ -517,6 +518,7 @@
                                     <sources>
                                         <source>${project.basedir}/src/main/301+-nondb/scala</source>
                                         <source>${project.basedir}/src/main/312-nondb/scala</source>
+                                        <source>${project.basedir}/src/main/31X-33X/scala</source>
                                         <source>${project.basedir}/src/main/301until320-all/scala</source>
                                         <source>${project.basedir}/src/main/301until320-noncdh/scala</source>
                                         <source>${project.basedir}/src/main/301until320-nondb/scala</source>
@@ -575,6 +577,7 @@
                                     <sources>
                                         <source>${project.basedir}/src/main/301+-nondb/scala</source>
                                         <source>${project.basedir}/src/main/313/scala</source>
+                                        <source>${project.basedir}/src/main/31X-33X/scala</source>
                                         <source>${project.basedir}/src/main/301until320-all/scala</source>
                                         <source>${project.basedir}/src/main/301until320-noncdh/scala</source>
                                         <source>${project.basedir}/src/main/301until320-nondb/scala</source>
@@ -632,7 +635,7 @@
                                 <configuration>
                                     <sources>
                                         <source>${project.basedir}/src/main/301+-nondb/scala</source>
-                                        <source>${project.basedir}/src/main/30X-32X-33X/scala</source>
+                                        <source>${project.basedir}/src/main/31X-33X/scala</source>
                                         <source>${project.basedir}/src/main/320/scala</source>
                                         <source>${project.basedir}/src/main/301until330-all/scala</source>
                                         <source>${project.basedir}/src/main/311+-all/scala</source>
@@ -696,7 +699,7 @@
                                 <configuration>
                                     <sources>
                                         <source>${project.basedir}/src/main/301+-nondb/scala</source>
-                                        <source>${project.basedir}/src/main/30X-32X-33X/scala</source>
+                                        <source>${project.basedir}/src/main/31X-33X/scala</source>
                                         <source>${project.basedir}/src/main/321/scala</source>
                                         <source>${project.basedir}/src/main/301until330-all/scala</source>
                                         <source>${project.basedir}/src/main/311+-all/scala</source>
@@ -759,7 +762,7 @@
                                 <configuration>
                                     <sources>
                                         <source>${project.basedir}/src/main/301+-nondb/scala</source>
-                                        <source>${project.basedir}/src/main/30X-32X-33X/scala</source>
+                                        <source>${project.basedir}/src/main/31X-33X/scala</source>
                                         <source>${project.basedir}/src/main/322/scala</source>
                                         <source>${project.basedir}/src/main/301until330-all/scala</source>
                                         <source>${project.basedir}/src/main/311+-all/scala</source>
@@ -824,7 +827,7 @@
                                 <configuration>
                                     <sources>
                                         <source>${project.basedir}/src/main/301+-nondb/scala</source>
-                                        <source>${project.basedir}/src/main/30X-32X-33X/scala</source>
+                                        <source>${project.basedir}/src/main/31X-33X/scala</source>
                                         <source>${project.basedir}/src/main/330/scala</source>
                                         <source>${project.basedir}/src/main/311+-all/scala</source>
                                         <source>${project.basedir}/src/main/311+-nondb/scala</source>
@@ -888,6 +891,7 @@
                                 <configuration>
                                     <sources>
                                         <source>${project.basedir}/src/main/301+-nondb/scala</source>
+                                        <source>${project.basedir}/src/main/31X-33X/scala</source>
                                         <source>${project.basedir}/src/main/311-nondb/scala</source>
                                         <source>${project.basedir}/src/main/311cdh/scala</source>
                                         <source>${project.basedir}/src/main/301until320-all/scala</source>

diff --git a/scripts/spark2diffs/GpuJoinUtils.diff b/scripts/spark2diffs/GpuJoinUtils.diff
@@ -1,7 +1,7 @@
 16,18d15
-< package com.nvidia.spark.rapids.shims
+< package com.nvidia.spark.rapids.shims.v2
 < 
-< import com.nvidia.spark.rapids.shims._
+< import com.nvidia.spark.rapids.shims.v2._
 20,26c17
 < import org.apache.spark.sql.execution.joins.{BuildLeft, BuildRight, BuildSide}
 < 
@@ -11,7 +11,7 @@
 <  */
 < sealed abstract class GpuBuildSide
 ---
-> package com.nvidia.spark.rapids.shims
+> package com.nvidia.spark.rapids.shims.v2
 28c19
 < case object GpuBuildRight extends GpuBuildSide
 ---

diff --git a/scripts/spark2diffs/GpuOverrides.diff b/scripts/spark2diffs/GpuOverrides.diff
@@ -5,9 +5,9 @@
 24a25
 > import ai.rapids.cudf.DType
 26c27
-< import com.nvidia.spark.rapids.shims._
+< import com.nvidia.spark.rapids.shims.v2._
 ---
-> import com.nvidia.spark.rapids.shims.{AQEUtils, GpuHashPartitioning, GpuSpecifiedWindowFrameMeta, GpuWindowExpressionMeta, OffsetWindowFunctionMeta}
+> import com.nvidia.spark.rapids.shims.v2.{AQEUtils, GpuHashPartitioning, GpuSpecifiedWindowFrameMeta, GpuWindowExpressionMeta, OffsetWindowFunctionMeta}
 31a33,35
 > import org.apache.spark.sql.catalyst.expressions.rapids.TimeStamp
 > import org.apache.spark.sql.catalyst.json.rapids.GpuJsonScan
@@ -32,7 +32,7 @@
 > import org.apache.spark.sql.rapids.catalyst.expressions.GpuRand
 50a62,63
 > import org.apache.spark.sql.rapids.execution.python._
-> import org.apache.spark.sql.rapids.shims.GpuTimeAdd
+> import org.apache.spark.sql.rapids.shims.v2.GpuTimeAdd
 63c76
 < abstract class ReplacementRule[INPUT <: BASE, BASE, WRAP_TYPE <: RapidsMeta[INPUT, BASE]](
 ---

diff --git a/scripts/spark2diffs/GpuSortMergeJoinMeta.diff b/scripts/spark2diffs/GpuSortMergeJoinMeta.diff
@@ -3,10 +3,10 @@
 ---
 >  * Copyright (c) 2019-2022, NVIDIA CORPORATION.
 17,20c17
-< package com.nvidia.spark.rapids.shims
+< package com.nvidia.spark.rapids.shims.v2
 < 
 < import com.nvidia.spark.rapids._
-< import com.nvidia.spark.rapids.shims._
+< import com.nvidia.spark.rapids.shims.v2._
 ---
 > package com.nvidia.spark.rapids
 29c26

diff --git a/scripts/spark2diffs/aggregate.diff b/scripts/spark2diffs/aggregate.diff
@@ -12,7 +12,7 @@
 > import ai.rapids.cudf.NvtxColor
 > import com.nvidia.spark.rapids.GpuMetric._
 > import com.nvidia.spark.rapids.RapidsPluginImplicits._
-> import com.nvidia.spark.rapids.shims.ShimUnaryExecNode
+> import com.nvidia.spark.rapids.shims.v2.ShimUnaryExecNode
 > 
 > import org.apache.spark.TaskContext
 > import org.apache.spark.internal.Logging

diff --git a/.../src/main/302/scala/com/nvidia/spark/rapids/shims/spark302/SparkShimServiceProvider.scala b/.../src/main/302/scala/com/nvidia/spark/rapids/shims/spark302/SparkShimServiceProvider.scala
@@ -16,8 +16,7 @@
 
 package com.nvidia.spark.rapids.shims.spark302
 
-import com.nvidia.spark.rapids.{SparkShims, SparkShimVersion}
-import com.nvidia.spark.rapids.shims.SparkShimImpl
+import com.nvidia.spark.rapids.SparkShimVersion
 
 object SparkShimServiceProvider {
   val VERSION = SparkShimVersion(3, 0, 2)
@@ -30,5 +29,4 @@ class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceP
   def matchesVersion(version: String): Boolean = {
     SparkShimServiceProvider.VERSIONNAMES.contains(version)
   }
-
 }
diff --git a/sql-plugin/src/main/304/scala/com/nvidia/spark/rapids/SparkShims.scala b/sql-plugin/src/main/304/scala/com/nvidia/spark/rapids/SparkShims.scala
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020-2022, NVIDIA CORPORATION.
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

diff --git a/.../shims/RapidsShuffleInternalManager.scala → .../shims/RapidsShuffleInternalManager.scala b/.../shims/RapidsShuffleInternalManager.scala → .../shims/RapidsShuffleInternalManager.scala
diff --git a/...ain/311-nondb/scala/com/nvidia/spark/rapids/shims/spark311/SparkShimServiceProvider.scala b/...ain/311-nondb/scala/com/nvidia/spark/rapids/shims/spark311/SparkShimServiceProvider.scala
@@ -25,6 +25,8 @@ object SparkShimServiceProvider {
 
 class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider {
 
+  override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION
+
   def matchesVersion(version: String): Boolean = {
     SparkShimServiceProvider.VERSIONNAMES.contains(version)
   }

diff --git a/...ain/311cdh/scala/com/nvidia/spark/rapids/shims/spark311cdh/SparkShimServiceProvider.scala b/...ain/311cdh/scala/com/nvidia/spark/rapids/shims/spark311cdh/SparkShimServiceProvider.scala
@@ -16,7 +16,7 @@
 
 package com.nvidia.spark.rapids.shims.spark311cdh
 
-import com.nvidia.spark.rapids.{ClouderaShimVersion, SparkShimVersion}
+import com.nvidia.spark.rapids.ClouderaShimVersion
 
 object SparkShimServiceProvider {
   val VERSION = ClouderaShimVersion(3, 1, 1, "3.1.7270")
@@ -26,7 +26,7 @@ object SparkShimServiceProvider {
 
 class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider {
 
-  override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION
+  override def getShimVersion: ClouderaShimVersion = SparkShimServiceProvider.VERSION
 
   def matchesVersion(version: String): Boolean = {
     version.contains(SparkShimServiceProvider.CDH_BASE_VERSION)

diff --git a/...ain/312-nondb/scala/com/nvidia/spark/rapids/shims/spark312/SparkShimServiceProvider.scala b/...ain/312-nondb/scala/com/nvidia/spark/rapids/shims/spark312/SparkShimServiceProvider.scala
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020-2022, NVIDIA CORPORATION.
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

diff --git a/.../shims/RapidsShuffleInternalManager.scala → .../shims/RapidsShuffleInternalManager.scala b/.../shims/RapidsShuffleInternalManager.scala → .../shims/RapidsShuffleInternalManager.scala
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2022, NVIDIA CORPORATION.
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

diff --git a/sql-plugin/src/main/321/scala/com/nvidia/spark/rapids/shims/SparkShims.scala b/sql-plugin/src/main/321/scala/com/nvidia/spark/rapids/shims/SparkShims.scala
@@ -17,7 +17,6 @@
 package com.nvidia.spark.rapids.shims
 
 import com.nvidia.spark.rapids._
-import com.nvidia.spark.rapids.shims.spark321.SparkShimServiceProvider
 
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.SparkSession
@@ -27,7 +26,8 @@ import org.apache.spark.sql.execution.datasources.{FilePartition, FileScanRDD, P
 import org.apache.spark.sql.types.StructType
 
 object SparkShimImpl extends Spark321PlusShims with Spark30Xuntil33XShims {
-  override def getSparkShimVersion: ShimVersion = SparkShimServiceProvider.VERSION
+
+  override def getSparkShimVersion: ShimVersion = ShimLoader.getShimVersion
 
   override def getFileScanRDD(
       sparkSession: SparkSession,

diff --git a/...n/src/main/321/scala/org/apache/spark/sql/rapids/shims/RapidsShuffleInternalManager.scala b/...n/src/main/321/scala/org/apache/spark/sql/rapids/shims/RapidsShuffleInternalManager.scala
diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/ShimLoader.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/ShimLoader.scala
@@ -53,13 +53,13 @@ import org.apache.spark.util.MutableURLClassLoader
 
     E.g., Spark 3.2.0 Shim will use
 
-    jar:file:/home/spark/rapids-4-spark_2.12-22.02.0.jar!/spark3xx-common/
-    jar:file:/home/spark/rapids-4-spark_2.12-22.02.0.jar!/spark320/
+    jar:file:/home/spark/rapids-4-spark_2.12-22.04.0.jar!/spark3xx-common/
+    jar:file:/home/spark/rapids-4-spark_2.12-22.04.0.jar!/spark320/
 
     Spark 3.1.1 will use
 
-    jar:file:/home/spark/rapids-4-spark_2.12-22.02.0.jar!/spark3xx-common/
-    jar:file:/home/spark/rapids-4-spark_2.12-22.02.0.jar!/spark311/
+    jar:file:/home/spark/rapids-4-spark_2.12-22.04.0.jar!/spark3xx-common/
+    jar:file:/home/spark/rapids-4-spark_2.12-22.04.0.jar!/spark311/
 
     Using these Jar URL's allows referencing different bytecode produced from identical sources
     by incompatible Scala / Spark dependencies.

diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/SparkShimServiceProvider.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/SparkShimServiceProvider.scala
@@ -20,6 +20,6 @@ package com.nvidia.spark.rapids
  * A Spark version shim layer interface.
  */
 trait SparkShimServiceProvider {
-  def getShimVersion: SparkShimVersion
+  def getShimVersion: ShimVersion
   def matchesVersion(version:String): Boolean
 }