Skip to content

Commit

Permalink
Use async client in Spark writer (#1405)
Browse files Browse the repository at this point in the history
  • Loading branch information
@dutor
yaphet authored and dutor committed Jan 2, 2020
1 parent 0b2f4f4 commit c87f1b1
Show file tree
Hide file tree
Showing 4 changed files with 445 additions and 72 deletions.
1 change: 1 addition & 0 deletions src/tools/spark-sstfile-generator/.gitignore
View file
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
dependency-reduced-pom.xml
*.iml
242 changes: 225 additions & 17 deletions src/tools/spark-sstfile-generator/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

<groupId>com.vesoft</groupId>
<artifactId>sst.generator</artifactId>
<version>1.0.0-beta</version>
<version>1.0.0-rc2</version>

<properties>
<java.version>1.8</java.version>
Expand All @@ -21,8 +21,7 @@
<config.version>1.4.0</config.version>
<scala-logging.version>3.9.2</scala-logging.version>
<scopt.version>3.7.1</scopt.version>
<config.version>1.4.0</config.version>
<nebula.version>1.0.0-beta</nebula.version>
<nebula.version>1.0.0-rc2</nebula.version>
<s2.version>1.0.0</s2.version>
</properties>

Expand All @@ -49,11 +48,11 @@
</goals>
<configuration>
<excludes>
com/vesoft/tools/**
<exclude>com/vesoft/tools/**</exclude>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</configuration>
</execution>
<execution>
Expand All @@ -80,6 +79,20 @@
<goal>shade</goal>
</goals>
<configuration>
<artifactSet>
<excludes>
<exclude>org.apache.spark:*</exclude>
<exclude>org.apache.hadoop:*</exclude>
<exclude>org.apache.hive:*</exclude>
<exclude>log4j:log4j</exclude>
<exclude>org.apache.orc:*</exclude>
<exclude>xml-apis:xml-apis</exclude>
<exclude>javax.inject:javax.inject</exclude>
<exclude>org.spark-project.hive:hive-exec</exclude>
<exclude>stax:stax-api</exclude>
<exclude>org.glassfish.hk2.external:aopalliance-repackaged</exclude>
</excludes>
</artifactSet>
<filters>
<filter>
<artifact>*:*</artifact>
Expand All @@ -104,26 +117,212 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>${spark.version}</version>
<exclusions>
<exclusion>
<artifactId>snappy-java</artifactId>
<groupId>org.xerial.snappy</groupId>
</exclusion>
<exclusion>
<artifactId>paranamer</artifactId>
<groupId>com.thoughtworks.paranamer</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>commons-codec</artifactId>
<groupId>commons-codec</groupId>
</exclusion>
<exclusion>
<artifactId>avro</artifactId>
<groupId>org.apache.avro</groupId>
</exclusion>
<exclusion>
<artifactId>commons-lang</artifactId>
<groupId>commons-lang</groupId>
</exclusion>
<exclusion>
<artifactId>commons-collections</artifactId>
<groupId>commons-collections</groupId>
</exclusion>
<exclusion>
<artifactId>commons-compress</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>commons-math3</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>guava</artifactId>
<groupId>com.google.guava</groupId>
</exclusion>
<exclusion>
<artifactId>httpclient</artifactId>
<groupId>org.apache.httpcomponents</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>netty</artifactId>
<groupId>io.netty</groupId>
</exclusion>
<exclusion>
<artifactId>jackson-annotations</artifactId>
<groupId>com.fasterxml.jackson.core</groupId>
</exclusion>
<exclusion>
<artifactId>scala-reflect</artifactId>
<groupId>org.scala-lang</groupId>
</exclusion>
<exclusion>
<artifactId>scala-library</artifactId>
<groupId>org.scala-lang</groupId>
</exclusion>
<exclusion>
<artifactId>jackson-databind</artifactId>
<groupId>com.fasterxml.jackson.core</groupId>
</exclusion>
<exclusion>
<artifactId>scala-xml_2.11</artifactId>
<groupId>org.scala-lang.modules</groupId>
</exclusion>
<exclusion>
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<version>${spark.version}</version>
<exclusions>
<exclusion>
<artifactId>snappy-java</artifactId>
<groupId>org.xerial.snappy</groupId>
</exclusion>
<exclusion>
<artifactId>jsr305</artifactId>
<groupId>com.google.code.findbugs</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>jackson-core</artifactId>
<groupId>com.fasterxml.jackson.core</groupId>
</exclusion>
<exclusion>
<artifactId>joda-time</artifactId>
<groupId>joda-time</groupId>
</exclusion>
<exclusion>
<artifactId>commons-codec</artifactId>
<groupId>commons-codec</groupId>
</exclusion>
<exclusion>
<artifactId>snappy-java</artifactId>
<groupId>org.xerial.snappy</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_2.11</artifactId>
<version>${spark.version}</version>
<exclusions>
<exclusion>
<artifactId>commons-codec</artifactId>
<groupId>commons-codec</groupId>
</exclusion>
<exclusion>
<artifactId>commons-logging</artifactId>
<groupId>commons-logging</groupId>
</exclusion>
<exclusion>
<artifactId>avro</artifactId>
<groupId>org.apache.avro</groupId>
</exclusion>
<exclusion>
<artifactId>commons-compress</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>commons-lang3</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>jackson-mapper-asl</artifactId>
<groupId>org.codehaus.jackson</groupId>
</exclusion>
<exclusion>
<artifactId>antlr-runtime</artifactId>
<groupId>org.antlr</groupId>
</exclusion>
<exclusion>
<artifactId>jackson-core-asl</artifactId>
<groupId>org.codehaus.jackson</groupId>
</exclusion>
<exclusion>
<artifactId>derby</artifactId>
<groupId>org.apache.derby</groupId>
</exclusion>
<exclusion>
<artifactId>httpclient</artifactId>
<groupId>org.apache.httpcomponents</groupId>
</exclusion>
<exclusion>
<artifactId>httpcore</artifactId>
<groupId>org.apache.httpcomponents</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-yarn_2.11</artifactId>
<version>${spark.version}</version>
<exclusions>
<exclusion>
<artifactId>guava</artifactId>
<groupId>com.google.guava</groupId>
</exclusion>
<exclusion>
<artifactId>commons-codec</artifactId>
<groupId>commons-codec</groupId>
</exclusion>
<exclusion>
<artifactId>commons-compress</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>activation</artifactId>
<groupId>javax.activation</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.databricks</groupId>
<artifactId>spark-csv_2.11</artifactId>
<version>1.5.0</version>
<exclusions>
<exclusion>
<artifactId>scala-library</artifactId>
<groupId>org.scala-lang</groupId>
</exclusion>
<exclusion>
<artifactId>univocity-parsers</artifactId>
<groupId>com.univocity</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
Expand All @@ -145,22 +344,26 @@
<groupId>com.typesafe.scala-logging</groupId>
<artifactId>scala-logging_2.11</artifactId>
<version>${scala-logging.version}</version>
<exclusions>
<exclusion>
<artifactId>scala-library</artifactId>
<groupId>org.scala-lang</groupId>
</exclusion>
<exclusion>
<artifactId>scala-reflect</artifactId>
<groupId>org.scala-lang</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.github.scopt</groupId>
<artifactId>scopt_2.11</artifactId>
<version>${scopt.version}</version>
</dependency>
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
<version>${config.version}</version>
</dependency>
<dependency>
<groupId>com.vesoft</groupId>
<artifactId>client</artifactId>
<version>${nebula.version}</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
Expand All @@ -171,5 +374,10 @@
<artifactId>s2-geometry-library-java</artifactId>
<version>${s2.version}</version>
</dependency>
<dependency>
<groupId>com.vesoft</groupId>
<artifactId>client</artifactId>
<version>${nebula.version}</version>
</dependency>
</dependencies>
</project>
14 changes: 12 additions & 2 deletions src/tools/spark-sstfile-generator/src/main/resources/application.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,10 @@
hive-field-1: nebula-field-1,
hive-field-2: nebula-field-2
}
vertex: {
field: hive-field-0
policy: "hash"
}
vertex: hive-field-0
partition: 32
}
Expand All @@ -72,8 +76,14 @@
hive-field-1: nebula-field-1,
hive-field-2: nebula-field-2
}
source: hive-field-0
target: hive-field-1
source: {
field: hive-field-0
policy: "hash"
}
target: {
field:hive-field-1
policy: "uuid"
}
ranking: hive-field-2
partition: 32
}
Expand Down
Loading

0 comments on commit c87f1b1

Please sign in to comment.