diff --git a/native-sql-engine/core/pom.xml b/native-sql-engine/core/pom.xml index 1cdd279de..10088ac02 100644 --- a/native-sql-engine/core/pom.xml +++ b/native-sql-engine/core/pom.xml @@ -47,14 +47,14 @@ - spark-3.1.1 + spark-3.1.3 true com.intel.oap - spark-sql-columnar-shims-spark311 + spark-sql-columnar-shims-spark313 ${project.version} diff --git a/pom.xml b/pom.xml index 8a92f613c..6d072643a 100644 --- a/pom.xml +++ b/pom.xml @@ -42,13 +42,13 @@ true - ${spark311.version} + ${spark313.version} spark-3.1 - ${spark311.version} + ${spark313.version} 2.12.10 2.10.0 @@ -122,8 +122,8 @@ - 3.1.1 - 3.1.1 + 3.1.3 + 3.1.3 3.2.1 2.12.10 @@ -317,6 +317,29 @@ + + + snapshots-repo + https://oss.sonatype.org/content/repositories/snapshots + + false + + + true + + + + apache-snapshots-repo + https://repository.apache.org/content/repositories/snapshots/ + + false + + + true + + + + diff --git a/shims/common/pom.xml b/shims/common/pom.xml index 8e8a17831..3e94c8195 100644 --- a/shims/common/pom.xml +++ b/shims/common/pom.xml @@ -78,7 +78,7 @@ org.apache.spark spark-sql_${scala.binary.version} - ${spark311.version} + ${spark313.version} provided diff --git a/shims/common/src/main/scala/com/intel/oap/sql/shims/SparkShimProvider.scala b/shims/common/src/main/scala/com/intel/oap/sql/shims/SparkShimProvider.scala index 9017e6b63..34b0d6be0 100644 --- a/shims/common/src/main/scala/com/intel/oap/sql/shims/SparkShimProvider.scala +++ b/shims/common/src/main/scala/com/intel/oap/sql/shims/SparkShimProvider.scala @@ -20,6 +20,6 @@ package com.intel.oap.sql.shims * Provider interface for matching and retrieving the Shims of a specific Spark version */ trait SparkShimProvider { - def matches(version:String): Boolean + def matches(version: String): Boolean def createShim: SparkShims } diff --git a/shims/common/src/main/scala/com/intel/oap/sql/shims/SparkShims.scala b/shims/common/src/main/scala/com/intel/oap/sql/shims/SparkShims.scala index 62420a6f8..be8f90de0 100644 --- a/shims/common/src/main/scala/com/intel/oap/sql/shims/SparkShims.scala +++ b/shims/common/src/main/scala/com/intel/oap/sql/shims/SparkShims.scala @@ -42,8 +42,10 @@ import org.apache.spark.sql.internal.SQLConf sealed abstract class ShimDescriptor -case class SparkShimDescriptor(major: Int, minor: Int, patch: Int) extends ShimDescriptor { - override def toString(): String = s"$major.$minor.$patch" +// The append arg can either be "" for release version or be "-SNAPSHOT" for snapshot version. +case class SparkShimDescriptor(major: Int, minor: Int, patch: Int, + append: String = "") extends ShimDescriptor { + override def toString(): String = s"$major.$minor.$patch$append" } trait SparkShims { diff --git a/shims/pom.xml b/shims/pom.xml index 9e45e9dd6..740e12e76 100644 --- a/shims/pom.xml +++ b/shims/pom.xml @@ -70,7 +70,7 @@ common - spark311 + spark313 @@ -79,7 +79,7 @@ common - spark311 + spark313 diff --git a/shims/spark311/src/main/resources/META-INF/services/com.intel.oap.sql.shims.SparkShimProvider b/shims/spark311/src/main/resources/META-INF/services/com.intel.oap.sql.shims.SparkShimProvider deleted file mode 100644 index e124437df..000000000 --- a/shims/spark311/src/main/resources/META-INF/services/com.intel.oap.sql.shims.SparkShimProvider +++ /dev/null @@ -1 +0,0 @@ -com.intel.oap.sql.shims.spark311.SparkShimProvider diff --git a/shims/spark311/pom.xml b/shims/spark313/pom.xml similarity index 96% rename from shims/spark311/pom.xml rename to shims/spark313/pom.xml index ef9a8cf40..1266306a1 100644 --- a/shims/spark311/pom.xml +++ b/shims/spark313/pom.xml @@ -24,8 +24,8 @@ ../pom.xml - spark-sql-columnar-shims-spark311 - ${project.name.prefix} Shims for Spark 3.1.1 + spark-sql-columnar-shims-spark313 + ${project.name.prefix} Shims for Spark 3.1.3 jar @@ -89,7 +89,7 @@ org.apache.spark spark-sql_${scala.binary.version} - ${spark311.version} + ${spark313.version} provided diff --git a/shims/spark311/src/main/java/org/apache/spark/sql/execution/datasources/VectorizedParquetRecordReaderChild.java b/shims/spark313/src/main/java/org/apache/spark/sql/execution/datasources/VectorizedParquetRecordReaderChild.java similarity index 100% rename from shims/spark311/src/main/java/org/apache/spark/sql/execution/datasources/VectorizedParquetRecordReaderChild.java rename to shims/spark313/src/main/java/org/apache/spark/sql/execution/datasources/VectorizedParquetRecordReaderChild.java diff --git a/shims/spark313/src/main/resources/META-INF/services/com.intel.oap.sql.shims.SparkShimProvider b/shims/spark313/src/main/resources/META-INF/services/com.intel.oap.sql.shims.SparkShimProvider new file mode 100644 index 000000000..804aeae27 --- /dev/null +++ b/shims/spark313/src/main/resources/META-INF/services/com.intel.oap.sql.shims.SparkShimProvider @@ -0,0 +1 @@ +com.intel.oap.sql.shims.spark313.SparkShimProvider diff --git a/shims/spark311/src/main/scala/com/intel/oap/execution/ColumnarBatchScanExec.scala b/shims/spark313/src/main/scala/com/intel/oap/execution/ColumnarBatchScanExec.scala similarity index 100% rename from shims/spark311/src/main/scala/com/intel/oap/execution/ColumnarBatchScanExec.scala rename to shims/spark313/src/main/scala/com/intel/oap/execution/ColumnarBatchScanExec.scala diff --git a/shims/spark311/src/main/scala/com/intel/oap/sql/shims/spark311/Spark311Shims.scala b/shims/spark313/src/main/scala/com/intel/oap/sql/shims/spark313/Spark313Shims.scala similarity index 98% rename from shims/spark311/src/main/scala/com/intel/oap/sql/shims/spark311/Spark311Shims.scala rename to shims/spark313/src/main/scala/com/intel/oap/sql/shims/spark313/Spark313Shims.scala index e637877e9..7d49a3c50 100644 --- a/shims/spark311/src/main/scala/com/intel/oap/sql/shims/spark311/Spark311Shims.scala +++ b/shims/spark313/src/main/scala/com/intel/oap/sql/shims/spark313/Spark313Shims.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.intel.oap.sql.shims.spark311 +package com.intel.oap.sql.shims.spark313 import com.intel.oap.execution.ColumnarBatchScanExec import com.intel.oap.spark.sql.ArrowWriteQueue @@ -46,7 +46,7 @@ import org.apache.spark.sql.execution.datasources.{DataSourceUtils, OutputWriter import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, REPARTITION, ReusedExchangeExec, ShuffleExchangeExec, ShuffleOrigin} import org.apache.spark.sql.internal.SQLConf -class Spark311Shims extends SparkShims { +class Spark313Shims extends SparkShims { override def getShimDescriptor: ShimDescriptor = SparkShimProvider.DESCRIPTOR diff --git a/shims/spark311/src/main/scala/com/intel/oap/sql/shims/spark311/SparkShimProvider.scala b/shims/spark313/src/main/scala/com/intel/oap/sql/shims/spark313/SparkShimProvider.scala similarity index 88% rename from shims/spark311/src/main/scala/com/intel/oap/sql/shims/spark311/SparkShimProvider.scala rename to shims/spark313/src/main/scala/com/intel/oap/sql/shims/spark313/SparkShimProvider.scala index 725788785..99955bafd 100644 --- a/shims/spark311/src/main/scala/com/intel/oap/sql/shims/spark311/SparkShimProvider.scala +++ b/shims/spark313/src/main/scala/com/intel/oap/sql/shims/spark313/SparkShimProvider.scala @@ -14,18 +14,18 @@ * limitations under the License. */ -package com.intel.oap.sql.shims.spark311 +package com.intel.oap.sql.shims.spark313 import com.intel.oap.sql.shims.{SparkShims, SparkShimDescriptor} object SparkShimProvider { - val DESCRIPTOR = SparkShimDescriptor(3, 1, 1) + val DESCRIPTOR = SparkShimDescriptor(3, 1, 3, "-SNAPSHOT") val DESCRIPTOR_STRINGS = Seq(s"$DESCRIPTOR") } class SparkShimProvider extends com.intel.oap.sql.shims.SparkShimProvider { def createShim: SparkShims = { - new Spark311Shims() + new Spark313Shims() } def matches(version: String): Boolean = { diff --git a/shims/spark311/src/main/scala/org/apache/spark/sql/BasePythonRunnerChild.scala b/shims/spark313/src/main/scala/org/apache/spark/sql/BasePythonRunnerChild.scala similarity index 100% rename from shims/spark311/src/main/scala/org/apache/spark/sql/BasePythonRunnerChild.scala rename to shims/spark313/src/main/scala/org/apache/spark/sql/BasePythonRunnerChild.scala diff --git a/shims/spark311/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/shims/spark313/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala similarity index 100% rename from shims/spark311/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala rename to shims/spark313/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala diff --git a/shims/spark311/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala b/shims/spark313/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala similarity index 100% rename from shims/spark311/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala rename to shims/spark313/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala diff --git a/shims/spark311/src/main/scala/org/apache/spark/util/ShimUtils.scala b/shims/spark313/src/main/scala/org/apache/spark/util/ShimUtils.scala similarity index 100% rename from shims/spark311/src/main/scala/org/apache/spark/util/ShimUtils.scala rename to shims/spark313/src/main/scala/org/apache/spark/util/ShimUtils.scala diff --git a/shims/spark311/src/test/scala/com/intel/oap/sql/shims/spark311/Spark311ShimsSuite.scala b/shims/spark313/src/test/scala/com/intel/oap/sql/shims/spark311/Spark313ShimsSuite.scala similarity index 92% rename from shims/spark311/src/test/scala/com/intel/oap/sql/shims/spark311/Spark311ShimsSuite.scala rename to shims/spark313/src/test/scala/com/intel/oap/sql/shims/spark311/Spark313ShimsSuite.scala index 6597dace5..01bd86de5 100644 --- a/shims/spark311/src/test/scala/com/intel/oap/sql/shims/spark311/Spark311ShimsSuite.scala +++ b/shims/spark313/src/test/scala/com/intel/oap/sql/shims/spark311/Spark313ShimsSuite.scala @@ -14,13 +14,13 @@ * limitations under the License. */ -package com.intel.oap.sql.shims.spark311 +package com.intel.oap.sql.shims.spark313 import com.intel.oap.sql.shims.{SparkShims, SparkShimLoader, SparkShimDescriptor} import org.scalatest.FunSuite; -class Spark311ShimsSuite extends FunSuite { +class Spark313ShimsSuite extends FunSuite { val descriptor = SparkShimDescriptor(3, 1, 1) test("Spark shims descriptor") { val sparkShims: SparkShims = new SparkShimProvider().createShim