Skip to content

Commit

Permalink
Compatible with spark3 and spark2 (#41)
Browse files Browse the repository at this point in the history
* support spark2.4

* support spark2.2

* support spark3.0

* extract common code

* add profiles for multi scala and spark version

* refactor class name

* update workflow

* update workflow

* add test plugin & remove useless config

* add table config for clickhouse

* refactor common's package name

* update the package name

* update the package name in test
  • Loading branch information
Nicole00 authored Jan 4, 2022
1 parent 657b83b commit d31546b
Show file tree
Hide file tree
Showing 71 changed files with 6,349 additions and 517 deletions.
32 changes: 15 additions & 17 deletions .github/workflows/deploy_release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,32 +25,30 @@ jobs:
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-maven-

- name: download neo4j-contrib & graphframes & pulsar-spark-connector dependency
run: |
wget https://oss-cdn.nebula-graph.com.cn/jar-packages/neo4j-contrib.zip
wget https://oss-cdn.nebula-graph.com.cn/jar-packages/graphframes.zip
wget https://oss-cdn.nebula-graph.com.cn/jar-packages/streamnative.zip
unzip -o -d ~/.m2/repository/ neo4j-contrib.zip
unzip -o -d ~/.m2/repository/ graphframes.zip
rm -rf ~/.m2/repository/io/streamnative
unzip -o -d ~/.m2/repository/io/ streamnative.zip
- name: Install nebula-graph
run: |
mkdir tmp
pushd tmp
git clone https://github.com/vesoft-inc/nebula-docker-compose.git
pushd nebula-docker-compose/
cp ../../nebula-exchange/src/test/resources/docker-compose.yaml .
cp ../../exchange-common/src/test/resources/docker-compose.yaml .
docker-compose up -d
sleep 10
popd
popd
- name: Deploy release to Maven
uses: samuelmeuli/action-maven-publish@v1
- name: Build with Maven
run: |
mvn clean package -pl nebula-exchange_spark_2.2 -am -Pscala-2.11 -Pspark-2.2
mvn clean package -pl nebula-exchange_spark_2.4 -am -Pscala-2.11 -Pspark-2.4
mvn clean package -pl nebula-exchange_spark_3.0 -am -Pscala-2.12 -Pspark-3.0
- name: upload to release assets
uses: softprops/action-gh-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
gpg_passphrase: ${{ secrets.GPG_PASSPHRASE }}
nexus_username: ${{ secrets.OSSRH_USERNAME }}
nexus_password: ${{ secrets.OSSRH_TOKEN }}
files: |
nebula-exchange_spark_2.2/target/nebula-exchange_spark_2.2-*-SNAPSHOT.jar
nebula-exchange_spark_2.4/target/nebula-exchange_spark_2.4-*-SNAPSHOT.jar
nebula-exchange_spark_3.0/target/nebula-exchange_spark_3.0-*-SNAPSHOT.jar
32 changes: 15 additions & 17 deletions .github/workflows/deploy_snapshot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,32 +27,30 @@ jobs:
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-maven-

- name: download neo4j-contrib & graphframes & pulsar-spark-connector dependency
run: |
wget https://oss-cdn.nebula-graph.com.cn/jar-packages/neo4j-contrib.zip
wget https://oss-cdn.nebula-graph.com.cn/jar-packages/graphframes.zip
wget https://oss-cdn.nebula-graph.com.cn/jar-packages/streamnative.zip
unzip -o -d ~/.m2/repository/ neo4j-contrib.zip
unzip -o -d ~/.m2/repository/ graphframes.zip
rm -rf ~/.m2/repository/io/streamnative
unzip -o -d ~/.m2/repository/io/ streamnative.zip
- name: Install nebula-graph
run: |
mkdir tmp
pushd tmp
git clone https://github.com/vesoft-inc/nebula-docker-compose.git
pushd nebula-docker-compose/
cp ../../nebula-exchange/src/test/resources/docker-compose.yaml .
cp ../../exchange-common/src/test/resources/docker-compose.yaml .
docker-compose up -d
sleep 10
popd
popd
- name: Deploy SNAPSHOT to Sonatype
uses: samuelmeuli/action-maven-publish@v1
- name: Build with Maven
run: |
mvn clean package -pl nebula-exchange_spark_2.2 -am -Pscala-2.11 -Pspark-2.2
mvn clean package -pl nebula-exchange_spark_2.4 -am -Pscala-2.11 -Pspark-2.4
mvn clean package -pl nebula-exchange_spark_3.0 -am -Pscala-2.12 -Pspark-3.0
- name: upload to snapshot assets
uses: softprops/action-gh-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
gpg_passphrase: ${{ secrets.GPG_PASSPHRASE }}
nexus_username: ${{ secrets.OSSRH_USERNAME }}
nexus_password: ${{ secrets.OSSRH_TOKEN }}
files: |
nebula-exchange_spark_2.2/target/nebula-exchange_spark_2.2-*-SNAPSHOT.jar
nebula-exchange_spark_2.4/target/nebula-exchange_spark_2.4-*-SNAPSHOT.jar
nebula-exchange_spark_3.0/target/nebula-exchange_spark_3.0-*-SNAPSHOT.jar
17 changes: 5 additions & 12 deletions .github/workflows/maven.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,29 +30,22 @@ jobs:
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-maven-

- name: download neo4j-contrib & graphframes & pulsar-spark-connector dependency
run: |
wget https://oss-cdn.nebula-graph.com.cn/jar-packages/neo4j-contrib.zip
wget https://oss-cdn.nebula-graph.com.cn/jar-packages/graphframes.zip
wget https://oss-cdn.nebula-graph.com.cn/jar-packages/streamnative.zip
unzip -o -d ~/.m2/repository/ neo4j-contrib.zip
unzip -o -d ~/.m2/repository/ graphframes.zip
rm -rf ~/.m2/repository/io/streamnative
unzip -o -d ~/.m2/repository/io/ streamnative.zip
- name: Install nebula-graph
run: |
mkdir tmp
pushd tmp
git clone https://github.com/vesoft-inc/nebula-docker-compose.git
pushd nebula-docker-compose/
cp ../../nebula-exchange/src/test/resources/docker-compose.yaml .
cp ../../exchange-common/src/test/resources/docker-compose.yaml .
docker-compose up -d
sleep 10
popd
popd
- name: Build with Maven
run: mvn -B package
run: |
mvn clean package -pl nebula-exchange_spark_2.2 -am -Pscala-2.11 -Pspark-2.2
mvn clean package -pl nebula-exchange_spark_2.4 -am -Pscala-2.11 -Pspark-2.4
mvn clean package -pl nebula-exchange_spark_3.0 -am -Pscala-2.12 -Pspark-3.0
- uses: codecov/codecov-action@v2
252 changes: 252 additions & 0 deletions exchange-common/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,252 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>exchange</artifactId>
<groupId>com.vesoft</groupId>
<version>2.5-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

<artifactId>exchange-common</artifactId>

<properties>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<nebula.version>2.0.0-SNAPSHOT</nebula.version>
<scalatest.version>3.2.0</scalatest.version>
<scala-logging.version>3.9.2</scala-logging.version>
</properties>

<dependencies>
<dependency>
<groupId>com.vesoft</groupId>
<artifactId>client</artifactId>
<version>${nebula.version}</version>
</dependency>
<dependency>
<groupId>org.locationtech.jts</groupId>
<artifactId>jts-core</artifactId>
<version>1.16.1</version>
</dependency>

<!-- scala dependency -->
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.binary.version}</artifactId>
<version>${scalatest.version}</version>
<scope>test</scope>
</dependency>

<!-- spark dependency -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>

<build>
<plugins>
<!-- deploy plugin -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.2</version>
<executions>
<execution>
<id>default-deploy</id>
<phase>deploy</phase>
<configuration>
<skip>true</skip>
</configuration>
</execution>
</executions>
</plugin>
<!-- skip nexus staging deploy -->
<plugin>
<groupId>org.sonatype.plugins</groupId>
<artifactId>nexus-staging-maven-plugin</artifactId>
<executions>
<execution>
<id>default-deploy</id>
<phase>deploy</phase>
<goals>
<goal>deploy</goal>
</goals>
<configuration>
<serverId>ossrh</serverId>
<skipNexusStagingDeployMojo>true</skipNexusStagingDeployMojo>
</configuration>
</execution>
</executions>
</plugin>

<!-- Source plugin -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>3.2.0</version>
<executions>
<execution>
<id>attach-sources</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Javadoc plugin -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>3.2.0</version>
<executions>
<execution>
<id>attach-javadocs</id>
<phase>package</phase>
<goals>
<goal>jar</goal>
</goals>
<configuration>
<encoding>UTF-8</encoding>
<charset>UTF-8</charset>
<additionalOptions>
<additionalparam>-source 8</additionalparam>
<additionalOption>-Xdoclint:none</additionalOption>
</additionalOptions>
</configuration>
</execution>
</executions>
</plugin>
<!-- test plugin -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.12.4</version>
<configuration>
<includes>
<include>**/*Test.*</include>
<include>**/*Suite.*</include>
</includes>
</configuration>
</plugin>
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<version>2.0.0</version>
<executions>
<execution>
<id>test</id>
<goals>
<goal>test</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<version>2.15.2</version>
<configuration>
<scalaVersion>${scala.version}</scalaVersion>
<args>
<arg>-target:jvm-1.8</arg>
</args>
<jvmArgs>
<jvmArg>-Xss4096K</jvmArg>
</jvmArgs>
</configuration>
<executions>
<execution>
<id>scala-compile</id>
<goals>
<goal>compile</goal>
</goals>
<configuration>
<excludes>
<exclude>com/vesoft/tools/**</exclude>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</configuration>
</execution>
<execution>
<id>scala-test-compile</id>
<goals>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>4.4.0</version>
<executions>
<execution>
<id>Scaladoc</id>
<goals>
<goal>doc</goal>
</goals>
<phase>prepare-package</phase>
<configuration>
<args>
<arg>-nobootcp</arg>
<arg>-no-link-warnings</arg>
</args>
</configuration>
</execution>
<execution>
<id>attach-javadocs</id>
<goals>
<goal>doc-jar</goal>
</goals>
<configuration>
<args>
<arg>-nobootcp</arg>
<arg>-no-link-warnings</arg>
</args>
</configuration>
</execution>
<execution>
<id>scala-compile-first</id>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- jacoco plugin -->
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.7</version>
<executions>
<execution>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
Loading

0 comments on commit d31546b

Please sign in to comment.