Skip to content
This repository has been archived by the owner on Sep 18, 2023. It is now read-only.

[NSE-1135] Introduce shim layer for supporting spark 3.2.2 #1146

Merged
merged 5 commits into from
Oct 31, 2022
Merged

[NSE-1135] Introduce shim layer for supporting spark 3.2.2 #1146

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
45b5e7c
Initial commit
PHILO-HE Oct 25, 2022
11c904e
Fix dependency issues
PHILO-HE Oct 25, 2022
705b95b
Fix compile issue in ColumnarHashAggregateExec
PHILO-HE Oct 25, 2022
b730535
Cover new profile in assembly
PHILO-HE Oct 25, 2022
c2d0f44
Remove AdaptiveSparkPlanExec in spark-3.2.x shims module
PHILO-HE Oct 26, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions gazelle-dist/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,48 @@
<profile>
<id>spark-3.2</id>
<dependencies>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark32</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark321</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</profile>
<profile>
<id>spark-3.2.1</id>
<dependencies>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark32</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark321</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</profile>
<profile>
<id>spark-3.2.2</id>
<dependencies>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark32</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark322</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</profile>
</profiles>

<dependencies>
Expand Down
40 changes: 40 additions & 0 deletions native-sql-engine/core/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,29 @@
<profile>
<id>spark-3.2</id>
<dependencies>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark32</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark321</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>
</profile>
<profile>
<id>spark-3.2.1</id>
<dependencies>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark32</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark321</artifactId>
Expand All @@ -72,6 +95,23 @@
</dependency>
</dependencies>
</profile>
<profile>
<id>spark-3.2.2</id>
<dependencies>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark32</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark322</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>
</profile>
</profiles>

<dependencies>
Expand Down
6 changes: 6 additions & 0 deletions ...ve-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarHashAggregateExec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,12 @@ case class ColumnarHashAggregateExec(
val numaBindingInfo = GazellePluginConfig.getConf.numaBindingInfo
override def supportsColumnar = true

// For compatibility consideration on spark3.2.2.
def isStreaming: Boolean = false

// For compatibility consideration on spark3.2.2.
def numShufflePartitions: Option[Int] = Option.empty[Int]

var resAttributes: Seq[Attribute] = resultExpressions.map(_.toAttribute)

override lazy val allAttributes: AttributeSeq =
Expand Down
23 changes: 22 additions & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,26 @@
<maven.test.skip>true</maven.test.skip>
</properties>
</profile>
<profile>
<id>spark-3.2.1</id>
<properties>
<spark.version>${spark321.version}</spark.version>
<scala.version>2.12.15</scala.version>
<!--Jackson may be directly used in future UT. Align with the version in spark 3.2.-->
<jackson.version>2.12.0</jackson.version>
<maven.test.skip>true</maven.test.skip>
</properties>
</profile>
<profile>
<id>spark-3.2.2</id>
<properties>
<spark.version>${spark322.version}</spark.version>
<scala.version>2.12.15</scala.version>
<!--Jackson may be directly used in future UT. Align with the version in spark 3.2.-->
<jackson.version>2.12.0</jackson.version>
<maven.test.skip>true</maven.test.skip>
</properties>
</profile>
<profile>
<id>hadoop-2.7.4</id>
<properties>
Expand Down Expand Up @@ -122,6 +142,7 @@
<spark.version>3.1.1</spark.version>
<spark311.version>3.1.1</spark311.version>
<spark321.version>3.2.1</spark321.version>
<spark322.version>3.2.2</spark322.version>
<!-- Scala 2.12.10 is the version for default spark 3.1 -->
<scala.version>2.12.10</scala.version>
<java.version>1.8</java.version>
Expand All @@ -145,7 +166,7 @@
<build_jemalloc>ON</build_jemalloc>
<native_avx512>ON</native_avx512>
<project.prefix>spark-sql-columnar</project.prefix>
<project.name.prefix>OAP Project Spark Columnar Plugin</project.name.prefix>
<project.name.prefix>OAP Project Gazelle</project.name.prefix>
</properties>

<dependencyManagement>
Expand Down
3 changes: 2 additions & 1 deletion shims/common/src/main/scala/com/intel/oap/sql/shims/SparkShims.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,8 @@ trait SparkShims {
* Return SparkPlan type since the type name is changed from spark 3.2.
* TODO: need tests.
*/
def newCustomShuffleReaderExec(child: SparkPlan, partitionSpecs : Seq[ShufflePartitionSpec]): SparkPlan
def newCustomShuffleReaderExec(child: SparkPlan, partitionSpecs : Seq[ShufflePartitionSpec]):
SparkPlan

def getChildOfCustomShuffleReaderExec(plan: SparkPlan): SparkPlan

Expand Down
21 changes: 21 additions & 0 deletions shims/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,9 +88,30 @@
</properties>
<modules>
<module>common</module>
<module>spark32</module>
<module>spark321</module>
</modules>
</profile>
<profile>
<id>spark-3.2.1</id>
<properties>
</properties>
<modules>
<module>common</module>
<module>spark32</module>
<module>spark321</module>
</modules>
</profile>
<profile>
<id>spark-3.2.2</id>
<properties>
</properties>
<modules>
<module>common</module>
<module>spark32</module>
<module>spark322</module>
</modules>
</profile>
</profiles>

<build>
Expand Down
114 changes: 114 additions & 0 deletions shims/spark32/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims</artifactId>
<version>1.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

<artifactId>spark-sql-columnar-shims-spark32</artifactId>
<name>${project.name.prefix} Shims Common for Spark 3.2.x</name>
<packaging>jar</packaging>

<build>
<plugins>
<plugin>
<groupId>org.scalastyle</groupId>
<artifactId>scalastyle-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.2.2</version>
<executions>
<execution>
<id>scala-compile-first</id>
<phase>process-resources</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
<execution>
<id>scala-test-compile-first</id>
<phase>process-test-resources</phase>
<goals>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.3</version>
<configuration>
<source>${java.version}</source>
<target>${java.version}</target>
<encoding>UTF-8</encoding>
<maxmem>1024m</maxmem>
<fork>true</fork>
<compilerArgs>
<arg>-Xlint:all,-serial,-path</arg>
</compilerArgs>
</configuration>
</plugin>
</plugins>

<resources>
<resource>
<directory>src/main/resources</directory>
</resource>
</resources>
</build>

<dependencies>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>${project.prefix}-shims-common</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-arrow-datasource-common</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>11.0.2</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>
</project>
0 ...s/VectorizedParquetRecordReaderChild.java → ...s/VectorizedParquetRecordReaderChild.java
View file
Open in desktop
File renamed without changes.
0 ...execution/ColumnarBatchScanExecBase.scala → ...execution/ColumnarBatchScanExecBase.scala
View file
Open in desktop
File renamed without changes.
Loading