[NSE-758] Upgrade to spark 3.1.3 from 3.1.1

native-sql-engine/core/pom.xml

@@ -47,14 +47,14 @@
 
   <profiles>
     <profile>
-      <id>spark-3.1.1</id>
+      <id>spark-3.1.3</id>
       <activation>
         <activeByDefault>true</activeByDefault>
       </activation>
       <dependencies>
         <dependency>
           <groupId>com.intel.oap</groupId>
-          <artifactId>spark-sql-columnar-shims-spark311</artifactId>
+          <artifactId>spark-sql-columnar-shims-spark313</artifactId>
           <version>${project.version}</version>
         </dependency>
       </dependencies>

pom.xml

@@ -42,13 +42,13 @@
           <activeByDefault>true</activeByDefault>
       </activation>
       <properties>
-          <spark.version>${spark311.version}</spark.version>
+          <spark.version>${spark313.version}</spark.version>
       </properties>
     </profile>
     <profile>
         <id>spark-3.1</id>
         <properties>
-            <spark.version>${spark311.version}</spark.version>
+            <spark.version>${spark313.version}</spark.version>
             <scala.version>2.12.10</scala.version>
             <jackson.version>2.10.0</jackson.version>
         </properties>
@@ -122,8 +122,8 @@
 
   <properties>
     <!--TODO: need to verify whether the settings will be overridden in profile-->
-    <spark.version>3.1.1</spark.version>
-    <spark311.version>3.1.1</spark311.version>
+    <spark.version>3.1.3</spark.version>
+    <spark313.version>3.1.3</spark313.version>
     <spark321.version>3.2.1</spark321.version>
     <!-- Scala 2.12.10 is the version for default spark 3.1 -->
     <scala.version>2.12.10</scala.version>
@@ -317,6 +317,29 @@
     </dependencies>
   </dependencyManagement>
 
+  <repositories>
+    <repository>
+      <id>snapshots-repo</id>
+      <url>https://oss.sonatype.org/content/repositories/snapshots</url>
+      <releases>
+        <enabled>false</enabled>
+      </releases>
+      <snapshots>
+        <enabled>true</enabled>
+      </snapshots>
+    </repository>
+    <repository>
+      <id>apache-snapshots-repo</id>
+      <url>https://repository.apache.org/content/repositories/snapshots/</url>
+      <releases>
+        <enabled>false</enabled>
+      </releases>
+      <snapshots>
+        <enabled>true</enabled>
+      </snapshots>
+    </repository>
+  </repositories>
+
   <build>
     <plugins>
       <plugin>

shims/common/pom.xml

@@ -78,7 +78,7 @@
       <dependency>
           <groupId>org.apache.spark</groupId>
           <artifactId>spark-sql_${scala.binary.version}</artifactId>
-          <version>${spark311.version}</version>
+          <version>${spark313.version}</version>
           <scope>provided</scope>
       </dependency>
       <dependency>

shims/common/src/main/scala/com/intel/oap/sql/shims/SparkShimProvider.scala

@@ -20,6 +20,6 @@ package com.intel.oap.sql.shims
  * Provider interface for matching and retrieving the Shims of a specific Spark version
  */
 trait SparkShimProvider {
-  def matches(version:String): Boolean
+  def matches(version: String): Boolean
   def createShim: SparkShims
 }

shims/common/src/main/scala/com/intel/oap/sql/shims/SparkShims.scala

@@ -42,8 +42,10 @@ import org.apache.spark.sql.internal.SQLConf
 
 sealed abstract class ShimDescriptor
 
-case class SparkShimDescriptor(major: Int, minor: Int, patch: Int) extends ShimDescriptor {
-  override def toString(): String = s"$major.$minor.$patch"
+// The append arg can either be "" for release version or be "-SNAPSHOT" for snapshot version.
+case class SparkShimDescriptor(major: Int, minor: Int, patch: Int,
+                               append: String = "") extends ShimDescriptor {
+  override def toString(): String = s"$major.$minor.$patch$append"
 }
 
 trait SparkShims {

shims/pom.xml

@@ -70,7 +70,7 @@
         </activation>
         <modules>
           <module>common</module>
-          <module>spark311</module>
+          <module>spark313</module>
         </modules>
     </profile>
     <profile>
@@ -79,7 +79,7 @@
         </properties>
         <modules>
           <module>common</module>
-          <module>spark311</module>
+          <module>spark313</module>
         </modules>
     </profile>
     <profile>

shims/spark311/pom.xml → shims/spark313/pom.xml

@@ -24,8 +24,8 @@
     <relativePath>../pom.xml</relativePath>
   </parent>
 
-  <artifactId>spark-sql-columnar-shims-spark311</artifactId>
-  <name>${project.name.prefix} Shims for Spark 3.1.1</name>
+  <artifactId>spark-sql-columnar-shims-spark313</artifactId>
+  <name>${project.name.prefix} Shims for Spark 3.1.3</name>
   <packaging>jar</packaging>
 
   <build>
@@ -89,7 +89,7 @@
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-sql_${scala.binary.version}</artifactId>
-      <version>${spark311.version}</version>
+      <version>${spark313.version}</version>
     <scope>provided</scope>
     </dependency>
     <dependency>

shims/spark313/src/main/resources/META-INF/services/com.intel.oap.sql.shims.SparkShimProvider

@@ -0,0 +1 @@
+com.intel.oap.sql.shims.spark313.SparkShimProvider

shims/spark311/src/main/scala/com/intel/oap/sql/shims/spark311/Spark311Shims.scala → shims/spark313/src/main/scala/com/intel/oap/sql/shims/spark313/Spark313Shims.scala

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.intel.oap.sql.shims.spark311
+package com.intel.oap.sql.shims.spark313
 
 import com.intel.oap.execution.ColumnarBatchScanExec
 import com.intel.oap.spark.sql.ArrowWriteQueue
@@ -46,7 +46,7 @@ import org.apache.spark.sql.execution.datasources.{DataSourceUtils, OutputWriter
 import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, REPARTITION, ReusedExchangeExec, ShuffleExchangeExec, ShuffleOrigin}
 import org.apache.spark.sql.internal.SQLConf
 
-class Spark311Shims extends SparkShims {
+class Spark313Shims extends SparkShims {
 
   override def getShimDescriptor: ShimDescriptor = SparkShimProvider.DESCRIPTOR
 

shims/spark311/src/main/scala/com/intel/oap/sql/shims/spark311/SparkShimProvider.scala → shims/spark313/src/main/scala/com/intel/oap/sql/shims/spark313/SparkShimProvider.scala

@@ -14,18 +14,18 @@
  * limitations under the License.
  */
 
-package com.intel.oap.sql.shims.spark311
+package com.intel.oap.sql.shims.spark313
 
 import com.intel.oap.sql.shims.{SparkShims, SparkShimDescriptor}
 
 object SparkShimProvider {
-  val DESCRIPTOR = SparkShimDescriptor(3, 1, 1)
+  val DESCRIPTOR = SparkShimDescriptor(3, 1, 3, "-SNAPSHOT")
   val DESCRIPTOR_STRINGS = Seq(s"$DESCRIPTOR")
 }
 
 class SparkShimProvider extends com.intel.oap.sql.shims.SparkShimProvider {
   def createShim: SparkShims = {
-    new Spark311Shims()
+    new Spark313Shims()
   }
 
   def matches(version: String): Boolean = {

shims/spark311/src/test/scala/com/intel/oap/sql/shims/spark311/Spark311ShimsSuite.scala → shims/spark313/src/test/scala/com/intel/oap/sql/shims/spark311/Spark313ShimsSuite.scala

@@ -14,13 +14,13 @@
  * limitations under the License.
  */
 
-package com.intel.oap.sql.shims.spark311
+package com.intel.oap.sql.shims.spark313
 
 import com.intel.oap.sql.shims.{SparkShims, SparkShimLoader, SparkShimDescriptor}
 
 import org.scalatest.FunSuite;
 
-class Spark311ShimsSuite extends FunSuite {
+class Spark313ShimsSuite extends FunSuite {
   val descriptor = SparkShimDescriptor(3, 1, 1)
   test("Spark shims descriptor") {
     val sparkShims: SparkShims = new SparkShimProvider().createShim