Skip to content

Commit

Permalink
[MSPARK-45] Move Spark-OJAI connector code to Spark github repo (apac…
Browse files Browse the repository at this point in the history 
…he#132)

* SPARK-45 Move Spark-OJAI connector code to Spark github repo

* Fixing pom versions for maprdb spark connector.

* Changes made to the connector code to be compatible with 5.2.* and 6.0 clients.

(cherry picked from commit 3d44f64)
  • Loading branch information
@HanumathRao
HanumathRao authored and Mikhail Gorbov committed Jan 2, 2018
1 parent 1bc3f26 commit 3bf7630
Show file tree
Hide file tree
Showing 79 changed files with 15,350 additions and 0 deletions.
65 changes: 65 additions & 0 deletions external/maprdb/build.gradle
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
import PomUtil

apply from: "${topDir}/javatest.gradle"
apply plugin: "scala"
apply plugin: "maven-publish"

defaultTasks "publishToMavenLocal"

group = "com.mapr.db"
version = maprVersion

dependencies {
compile project(path: ":src/java-utils", configuration: "main")
compile project(path: ":src/fs/proto-java", configuration: "main")
compile project(path: ":src/fs/client/fileclient/java/maprfs", configuration: "main")
compile project(path: ":src/fs/client/fileclient/java/maprfs-jni", configuration: "main")
compile project(path: ":src/fs/client/dbclient-parent/client", configuration: "main")
compile project(path: ":src/fs/client/dbclient-parent/map-reduce", configuration: "main")
compile(group: "org.ojai", name: "ojai", version: ojaiVersion) {
exclude group: "org.apache.hadoop"
}
compile(group: "org.ojai", name: "ojai-scala", version: ojaiVersion) {
exclude group: "org.apache.hadoop"
}
compile group: "org.apache.spark", name: "spark-core_${scalaCompatVersion}", version: sparkCoreVersion
compile group: "org.apache.spark", name: "spark-streaming_${scalaCompatVersion}", version: sparkCoreVersion
compile group: "org.apache.spark", name: "spark-sql_${scalaCompatVersion}", version: sparkCoreVersion
compile group: "org.scala-lang", name: "scala-library", version: scalaVersion
compile group: "com.fasterxml.jackson.module", name: "jackson-module-scala_${scalaCompatVersion}", version: jacksonVersion
compile group: "org.specs2", name: "specs2-junit_${scalaCompatVersion}", version: specs2Version
compile group: "org.scalatest", name: "scalatest_${scalaCompatVersion}", version: scalatestVersion

testCompile project(path: ":src/fs/client/dbclient-parent/client", configuration: "testjar")
}

tasks.withType(Jar) {
baseName "maprdb-spark"
}

File originalPom = file("pom.xml")
List<Node> originalNodes = null

task eatPom() {
doFirst {
originalNodes = PomUtil.parsePom(originalPom, maprVersion)
}
}

publishing {
publications {
mvnArtifact {
artifactId "maprdb-spark"
pom.withXml {
originalNodes.each { asNode().append(it) }
}
}
}
}

model {
tasks.generatePomFileForMvnArtifactPublication {
dependsOn "eatPom"
}
}

145 changes: 145 additions & 0 deletions external/maprdb/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
<?xml version="1.0"?>
<!-- Copyright (c) 2015 MapR, Inc. -->
<project
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<version>2.1.0-mapr-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>

<groupId>com.mapr.db</groupId>
<artifactId>maprdb-spark</artifactId>
<packaging>jar</packaging>

<name>MapR-DB Spark Library</name>
<url>http://www.mapr.com/</url>

<dependencies>
<dependency>
<groupId>com.mapr.db</groupId>
<artifactId>maprdb</artifactId>
<version>6.0.0-mapr-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
<version>${fasterxml.jackson.version}</version>
</dependency>
<dependency>
<groupId>org.specs2</groupId>
<artifactId>specs2-core_${scala.binary.version}</artifactId>
<version>2.4.17</version>
</dependency>
<dependency>
<groupId>org.specs2</groupId>
<artifactId>specs2-junit_${scala.binary.version}</artifactId>
<version>2.4.17</version>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.binary.version}</artifactId>
</dependency>
<dependency>
<groupId>com.mapr.db</groupId>
<artifactId>maprdb-mapreduce</artifactId>
<version>6.0.0-mapr-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.ojai</groupId>
<artifactId>ojai</artifactId>
<version>2.0-mapr-1707-beta</version>
</dependency>
<dependency>
<groupId>org.ojai</groupId>
<artifactId>ojai-scala</artifactId>
<version>2.0-mapr-1707-beta</version>
</dependency>
<dependency>
<groupId>com.mapr</groupId>
<artifactId>mapr-java-utils</artifactId>
<version>6.0.0-mapr-SNAPSHOT</version>
<classifier>tests</classifier>
<scope>test</scope>
</dependency>
</dependencies>

<build>
<sourceDirectory>src/main/scala</sourceDirectory>
<testSourceDirectory>src/test/scala</testSourceDirectory>
<plugins>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<configuration>
<recompileMode>incremental</recompileMode>
<args>
<arg>-explaintypes</arg>
</args>
</configuration>
<executions>
<execution>
<id>scala-compile-first</id>
<phase>process-resources</phase>
<goals>
<goal>add-source</goal>
<goal>compile</goal>
</goals>
</execution>
<execution>
<id>scala-test-compile</id>
<phase>process-test-resources</phase>
<goals>
<goal>add-source</goal>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<includes>
<include>**/*Tests*.*</include>
</includes>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
</plugins>
</build>
</project>
145 changes: 145 additions & 0 deletions external/maprdb/pom.xml.5.0
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
<?xml version="1.0"?>
<!-- Copyright (c) 2015 MapR, Inc. -->
<project
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<version>2.1.0-mapr-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>

<groupId>com.mapr.db</groupId>
<artifactId>maprdb-spark</artifactId>
<packaging>jar</packaging>

<name>MapR-DB Spark Library</name>
<url>http://www.mapr.com/</url>

<dependencies>
<dependency>
<groupId>com.mapr.db</groupId>
<artifactId>maprdb</artifactId>
<version>5.2.2-mapr</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
<version>${fasterxml.jackson.version}</version>
</dependency>
<dependency>
<groupId>org.specs2</groupId>
<artifactId>specs2-core_${scala.binary.version}</artifactId>
<version>2.4.17</version>
</dependency>
<dependency>
<groupId>org.specs2</groupId>
<artifactId>specs2-junit_${scala.binary.version}</artifactId>
<version>2.4.17</version>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.binary.version}</artifactId>
</dependency>
<dependency>
<groupId>com.mapr.db</groupId>
<artifactId>maprdb-mapreduce</artifactId>
<version>5.2.2-mapr</version>
</dependency>
<dependency>
<groupId>org.ojai</groupId>
<artifactId>ojai</artifactId>
<version>1.1-mapr-1703</version>
</dependency>
<dependency>
<groupId>org.ojai</groupId>
<artifactId>ojai-scala</artifactId>
<version>1.1-mapr-1703</version>
</dependency>
<dependency>
<groupId>com.mapr</groupId>
<artifactId>mapr-java-utils</artifactId>
<version>5.2.2-mapr</version>
<classifier>tests</classifier>
<scope>test</scope>
</dependency>
</dependencies>

<build>
<sourceDirectory>src/main/scala</sourceDirectory>
<testSourceDirectory>src/test/scala</testSourceDirectory>
<plugins>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<configuration>
<recompileMode>incremental</recompileMode>
<args>
<arg>-explaintypes</arg>
</args>
</configuration>
<executions>
<execution>
<id>scala-compile-first</id>
<phase>process-resources</phase>
<goals>
<goal>add-source</goal>
<goal>compile</goal>
</goals>
</execution>
<execution>
<id>scala-test-compile</id>
<phase>process-test-resources</phase>
<goals>
<goal>add-source</goal>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<includes>
<include>**/*Tests*.*</include>
</includes>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
</plugins>
</build>
</project>
Loading

0 comments on commit 3bf7630

Please sign in to comment.