Skip to content
This repository has been archived by the owner on Sep 18, 2023. It is now read-only.

Commit

Permalink
[NSE-1135] Introduce shim layer for supporting spark 3.2.2 (#1146)
Browse files Browse the repository at this point in the history 
* Initial commit

* Fix dependency issues

* Fix compile issue in ColumnarHashAggregateExec

* Cover new profile in assembly

* Remove AdaptiveSparkPlanExec in spark-3.2.x shims module
  • Loading branch information
@PHILO-HE
PHILO-HE authored Oct 31, 2022
1 parent c947bd1 commit 192ecc3
Show file tree
Hide file tree
Showing 19 changed files with 675 additions and 1,035 deletions.
35 changes: 35 additions & 0 deletions gazelle-dist/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,48 @@
<profile>
<id>spark-3.2</id>
<dependencies>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark32</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark321</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</profile>
<profile>
<id>spark-3.2.1</id>
<dependencies>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark32</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark321</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</profile>
<profile>
<id>spark-3.2.2</id>
<dependencies>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark32</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark322</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</profile>
</profiles>

<dependencies>
Expand Down
40 changes: 40 additions & 0 deletions native-sql-engine/core/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,29 @@
<profile>
<id>spark-3.2</id>
<dependencies>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark32</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark321</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>
</profile>
<profile>
<id>spark-3.2.1</id>
<dependencies>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark32</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark321</artifactId>
Expand All @@ -72,6 +95,23 @@
</dependency>
</dependencies>
</profile>
<profile>
<id>spark-3.2.2</id>
<dependencies>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark32</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark322</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>
</profile>
</profiles>

<dependencies>
Expand Down
6 changes: 6 additions & 0 deletions ...ve-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarHashAggregateExec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,12 @@ case class ColumnarHashAggregateExec(
val numaBindingInfo = GazellePluginConfig.getConf.numaBindingInfo
override def supportsColumnar = true

// For compatibility consideration on spark3.2.2.
def isStreaming: Boolean = false

// For compatibility consideration on spark3.2.2.
def numShufflePartitions: Option[Int] = Option.empty[Int]

var resAttributes: Seq[Attribute] = resultExpressions.map(_.toAttribute)

override lazy val allAttributes: AttributeSeq =
Expand Down
23 changes: 22 additions & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,26 @@
<maven.test.skip>true</maven.test.skip>
</properties>
</profile>
<profile>
<id>spark-3.2.1</id>
<properties>
<spark.version>${spark321.version}</spark.version>
<scala.version>2.12.15</scala.version>
<!--Jackson may be directly used in future UT. Align with the version in spark 3.2.-->
<jackson.version>2.12.0</jackson.version>
<maven.test.skip>true</maven.test.skip>
</properties>
</profile>
<profile>
<id>spark-3.2.2</id>
<properties>
<spark.version>${spark322.version}</spark.version>
<scala.version>2.12.15</scala.version>
<!--Jackson may be directly used in future UT. Align with the version in spark 3.2.-->
<jackson.version>2.12.0</jackson.version>
<maven.test.skip>true</maven.test.skip>
</properties>
</profile>
<profile>
<id>hadoop-2.7.4</id>
<properties>
Expand Down Expand Up @@ -122,6 +142,7 @@
<spark.version>3.1.1</spark.version>
<spark311.version>3.1.1</spark311.version>
<spark321.version>3.2.1</spark321.version>
<spark322.version>3.2.2</spark322.version>
<!-- Scala 2.12.10 is the version for default spark 3.1 -->
<scala.version>2.12.10</scala.version>
<java.version>1.8</java.version>
Expand All @@ -145,7 +166,7 @@
<build_jemalloc>ON</build_jemalloc>
<native_avx512>ON</native_avx512>
<project.prefix>spark-sql-columnar</project.prefix>
<project.name.prefix>OAP Project Spark Columnar Plugin</project.name.prefix>
<project.name.prefix>OAP Project Gazelle</project.name.prefix>
</properties>

<dependencyManagement>
Expand Down
3 changes: 2 additions & 1 deletion shims/common/src/main/scala/com/intel/oap/sql/shims/SparkShims.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,8 @@ trait SparkShims {
* Return SparkPlan type since the type name is changed from spark 3.2.
* TODO: need tests.
*/
def newCustomShuffleReaderExec(child: SparkPlan, partitionSpecs : Seq[ShufflePartitionSpec]): SparkPlan
def newCustomShuffleReaderExec(child: SparkPlan, partitionSpecs : Seq[ShufflePartitionSpec]):
SparkPlan

def getChildOfCustomShuffleReaderExec(plan: SparkPlan): SparkPlan

Expand Down
21 changes: 21 additions & 0 deletions shims/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,9 +88,30 @@
</properties>
<modules>
<module>common</module>
<module>spark32</module>
<module>spark321</module>
</modules>
</profile>
<profile>
<id>spark-3.2.1</id>
<properties>
</properties>
<modules>
<module>common</module>
<module>spark32</module>
<module>spark321</module>
</modules>
</profile>
<profile>
<id>spark-3.2.2</id>
<properties>
</properties>
<modules>
<module>common</module>
<module>spark32</module>
<module>spark322</module>
</modules>
</profile>
</profiles>

<build>
Expand Down
114 changes: 114 additions & 0 deletions shims/spark32/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims</artifactId>
<version>1.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

<artifactId>spark-sql-columnar-shims-spark32</artifactId>
<name>${project.name.prefix} Shims Common for Spark 3.2.x</name>
<packaging>jar</packaging>

<build>
<plugins>
<plugin>
<groupId>org.scalastyle</groupId>
<artifactId>scalastyle-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.2.2</version>
<executions>
<execution>
<id>scala-compile-first</id>
<phase>process-resources</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
<execution>
<id>scala-test-compile-first</id>
<phase>process-test-resources</phase>
<goals>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.3</version>
<configuration>
<source>${java.version}</source>
<target>${java.version}</target>
<encoding>UTF-8</encoding>
<maxmem>1024m</maxmem>
<fork>true</fork>
<compilerArgs>
<arg>-Xlint:all,-serial,-path</arg>
</compilerArgs>
</configuration>
</plugin>
</plugins>

<resources>
<resource>
<directory>src/main/resources</directory>
</resource>
</resources>
</build>

<dependencies>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>${project.prefix}-shims-common</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-arrow-datasource-common</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>11.0.2</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>
</project>
0 ...s/VectorizedParquetRecordReaderChild.java → ...s/VectorizedParquetRecordReaderChild.java
View file
File renamed without changes.
0 ...execution/ColumnarBatchScanExecBase.scala → ...execution/ColumnarBatchScanExecBase.scala
View file
File renamed without changes.
Loading

0 comments on commit 192ecc3

Please sign in to comment.