Acxiom#206 Fixed issue causing the Metadata Extractor to pull in olde…

…r version of libraries which caused a conflict. Acxiom#207 Removed support for Spark 2.3/Scala 2.11 and Spark 2.4/Scala 2.12
dafreels · Jan 21, 2021 · 1692ee7 · 1692ee7
1 parent 2a19daa
commit 1692ee7
Show file tree

Hide file tree

Showing 30 changed files with 315 additions and 247 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -10,24 +10,6 @@ jobs:
   allow_failures:
     - os: windows
   include:
-    - stage: Build
-      name: Spark 2.3 Scala 2.11 Linux Build
-      os: linux
-      dist: trusty
-      scala:
-        - 2.11.12
-      jdk:
-        - oraclejdk8
-      install: true
-      script: mvn -B -P spark_2.3 clean install
-    -
-      name: Spark 2.3 Scala 2.11 OSX Build
-      os: osx
-      osx_image: xcode9.3
-      scala:
-        - 2.11.12
-      install: true
-      script: mvn -B -P spark_2.3 clean install
     -
       name: Spark 2.4 Scala 2.11 Linux Build
       os: linux
@@ -44,24 +26,6 @@ jobs:
         - 2.11.12
       install: true
       script: mvn -B -P spark_2.4 clean install
-    -
-      name: Spark 2.4 Scala 2.12 Linux Build
-      os: linux
-      dist: trusty
-      scala:
-        - 2.12.10
-      jdk:
-        - oraclejdk8
-      install: true
-      script: mvn -B -P spark_2.4,scala_2.12 clean install
-    -
-      name: Spark 2.4 Scala 2.12 OSX Build
-      os: osx
-      osx_image: xcode9.3
-      scala:
-        - 2.12.10
-      install: true
-      script: mvn -B -P spark_2.4,scala_2.12 clean install
     -
       name: Spark 3.0 Scala 2.12 Linux Build
       os: linux
@@ -103,36 +67,6 @@ jobs:
       script: mvn -B -Dsonar.organization=$SONAR_ORGANIZATION -Dsonar.login=$SONAR_TOKEN -Dsonar.projectKey=Acxiom_spark-pipeline-driver-develop -Dsonar.host.url=https://sonarcloud.io scoverage:report scalastyle:check org.jacoco:jacoco-maven-plugin:prepare-agent sonar:sonar
 
     - stage: Release
-      if: branch = master AND type != pull_request AND fork = false AND repo = Acxiom/metalus
-      name: Spark 2.3 Scala 2.11 Build
-      os: linux
-      dist: trusty
-      scala:
-        - 2.11.12
-      jdk:
-        - oraclejdk8
-      before_install:
-        - openssl aes-256-cbc -K $encrypted_60fb54548dfb_key -iv $encrypted_60fb54548dfb_iv -in deployment/acxsigningkey.asc.enc -out deployment/acxsigningkey.asc -d
-        - mvn -P spark_2.3 -B versions:set -DremoveSnapshot
-      install:
-        - mvn -B -DskipTests=true -P spark_2.3 clean install
-        - mvn -B -P spark_2.3 -DrepoToken=$coverallsToken -Dsonar.organization=$SONAR_ORGANIZATION -Dsonar.login=$SONAR_TOKEN -Dsonar.projectKey=Acxiom_spark-pipeline-driver-release -Dsonar.host.url=https://sonarcloud.io scoverage:report scala:doc scalastyle:check coveralls:report org.jacoco:jacoco-maven-plugin:prepare-agent sonar:sonar
-      before_script:
-        - gpg --keyring=$TRAVIS_BUILD_DIR/pubring.gpg --no-default-keyring --import deployment/acxsigningkey.asc
-        - gpg --secret-keyring=$TRAVIS_BUILD_DIR/secring.gpg --no-default-keyring --import deployment/acxsigningkey.asc
-      script: mvn --settings deployment/release-settings.xml -pl .,metalus-core,metalus-aws,metalus-common,metalus-gcp,metalus-kafka,metalus-mongo,metalus-utils -B -P release,spark_2.3 -DskipTests=true -Dgpg.executable=gpg -Dgpg.keyname=F40721C83ED4BD317EA73B225B996D862829DE42 -Dgpg.passphrase=$PASSPHRASE -Dgpg.publicKeyring=$TRAVIS_BUILD_DIR/pubring.gpg -Dgpg.secretKeyring=$TRAVIS_BUILD_DIR/secring.gpg deploy
-      before_deploy:
-        - export project_version=$(mvn -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive exec:exec)
-      deploy:
-        provider: releases
-        api_key: $GITHUB_OAUTH_TOKEN
-        name: PipelineDriver-Spark_2.3-Scala_2.11-${project_version}
-        skip_cleanup: true
-        file_glob: true
-        file:
-          - metalus-utils/target/metalus-utils_2.11-spark_2.3-${project_version}.tar.gz
-          - metalus-application/target/metalus-application_2.11-spark_2.3-${project_version}.jar
-    -
       if: branch = master AND type != pull_request AND fork = false AND repo = Acxiom/metalus
       name: Spark 2.4 Scala 2.11 Build
       os: linux
@@ -161,34 +95,6 @@ jobs:
         file:
           - metalus-utils/target/metalus-utils_2.11-spark_2.4-${project_version}.tar.gz
           - metalus-application/target/metalus-application_2.11-spark_2.4-${project_version}.jar
-    - if: branch = master AND type != pull_request AND fork = false AND repo = Acxiom/metalus
-      name: Spark 2.4 Scala 2.12 Build
-      os: linux
-      dist: trusty
-      scala:
-        - 2.12.10
-      jdk:
-        - oraclejdk8
-      before_install:
-        - openssl aes-256-cbc -K $encrypted_60fb54548dfb_key -iv $encrypted_60fb54548dfb_iv -in deployment/acxsigningkey.asc.enc -out deployment/acxsigningkey.asc -d
-        - mvn -B versions:set -DremoveSnapshot
-      install:
-          - mvn -B -DskipTests=true -P spark_2.4,scala_2.12 clean install
-      before_script:
-        - gpg --keyring=$TRAVIS_BUILD_DIR/pubring.gpg --no-default-keyring --import deployment/acxsigningkey.asc
-        - gpg --secret-keyring=$TRAVIS_BUILD_DIR/secring.gpg --no-default-keyring --import deployment/acxsigningkey.asc
-      script: mvn --settings deployment/release-settings.xml -pl metalus-core,metalus-aws,metalus-common,metalus-gcp,metalus-kafka,metalus-mongo,metalus-utils -B -P release,spark_2.4,scala_2.12 -DskipTests=true -Dgpg.executable=gpg -Dgpg.keyname=F40721C83ED4BD317EA73B225B996D862829DE42 -Dgpg.passphrase=$PASSPHRASE -Dgpg.publicKeyring=$TRAVIS_BUILD_DIR/pubring.gpg -Dgpg.secretKeyring=$TRAVIS_BUILD_DIR/secring.gpg deploy
-      before_deploy:
-        - export project_version=$(mvn -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive exec:exec)
-      deploy:
-        provider: releases
-        api_key: $GITHUB_OAUTH_TOKEN
-        name: PipelineDriver-Spark_2.4-Scala_2.12-${project_version}
-        skip_cleanup: true
-        file_glob: true
-        file:
-          - metalus-utils/target/metalus-utils_2.12-spark_2.4-${project_version}.tar.gz
-          - metalus-application/target/metalus-application_2.12-spark_2.4-${project_version}.jar
     - if: branch = master AND type != pull_request AND fork = false AND repo = Acxiom/metalus
       name: Spark 3.0 Scala 2.12 Build
       os: linux

diff --git a/docs/application-example.md b/docs/application-example.md
@@ -1062,21 +1062,6 @@ The application commands below provide the proper templates to run the example:
 * _<jar_path>_ - The fully qualified path to the built jars
 * _<data_location>_ - The fully qualified path to the example data
 
-### Spark 2.3/Scala 2.11
-```bash
-spark-submit --class com.acxiom.pipeline.drivers.DefaultPipelineDriver \
---master spark://localhost:7077 \
---deploy-mode client \
---jars metalus-common_2.11-spark_2.3-<VERSION>.jar,metalus-examples_2.11-spark_2.3-<VERSION>.jar,metalus-mongo_2.11-spark_2.3-<VERSION>.jar,mongo-spark-connector_2.11-2.3.2.jar,mongo-java-driver-3.11.2.jar  \
-<jar_path>/metalus-application_2.11-spark_2.3-<VERSION>.jar \
---driverSetupClass com.acxiom.pipeline.applications.DefaultApplicationDriverSetup \
---applicationConfigPath <data_location>/application-example.json \
---input_url <data_location>/orders.csv \
---input_format csv \
---input_separator , \
---mongoURI mongodb://localhost:27017/application_examples \
---logLevel DEBUG
-```
 ### Spark 2.4/Scala 2.11
 ```bash
 spark-submit --class com.acxiom.pipeline.drivers.DefaultPipelineDriver \
@@ -1092,27 +1077,12 @@ spark-submit --class com.acxiom.pipeline.drivers.DefaultPipelineDriver \
 --mongoURI mongodb://localhost:27017/application_examples \
 --logLevel DEBUG
 ```
-### Spark 2.4/Scala 2.12
-```bash
-spark-submit --class com.acxiom.pipeline.drivers.DefaultPipelineDriver \
---master spark://localhost:7077 \
---deploy-mode client \
---jars metalus-common_2.12-spark_2.4-<VERSION>.jar,metalus-examples_2.12-spark_2.4-<VERSION>.jar,metalus-mongo_2.12-spark_2.4-<VERSION>.jar,mongo-spark-connector_2.12-2.4.1.jar,mongo-java-driver-3.11.2.jar  \
-<jar_path>/metalus-application_2.12-spark_2.4-<VERSION>.jar \
---driverSetupClass com.acxiom.pipeline.applications.DefaultApplicationDriverSetup \
---applicationConfigPath <data_location>/application-example.json \
---input_url <data_location>/orders.csv \
---input_format csv \
---input_separator , \
---mongoURI mongodb://localhost:27017/application_examples \
---logLevel DEBUG
-```
 ### Spark 3.0/Scala 2.12
 ```bash
 spark-submit --class com.acxiom.pipeline.drivers.DefaultPipelineDriver \
 --master spark://localhost:7077 \
 --deploy-mode client \
---jars metalus-common_2.12-spark_3.0-<VERSION>.jar,metalus-examples_2.12-spark_3.0-<VERSION>.jar,metalus-mongo_2.12-spark_3.0-<VERSION>.jar,mongo-spark-connector_2.12-2.4.1.jar,mongo-java-driver-3.11.2.jar  \
+--jars metalus-common_2.12-spark_3.0-<VERSION>.jar,metalus-examples_2.12-spark_3.0-<VERSION>.jar,metalus-mongo_2.12-spark_3.0-<VERSION>.jar,mongo-spark-connector_2.12-3.0.0.jar,mongodb-driver-sync-4.0.5.jar,mongodb-driver-core-4.0.5.jar,bson-4.0.5.jar  \
 <jar_path>/metalus-application_2.12-spark_3.0-<VERSION>.jar \
 --driverSetupClass com.acxiom.pipeline.applications.DefaultApplicationDriverSetup \
 --applicationConfigPath <data_location>/application-example.json \

diff --git a/docs/contributions.md b/docs/contributions.md
@@ -24,20 +24,23 @@ get familiar with the coding style and learn if the functionality already exists
 ## Building
 The project is built using [Apache Maven](http://maven.apache.org/).
 
-To build the project using Scala 2.11 and Spark 2.3 run:
-
-	mvn
-
 To build the project using Scala 2.11 and Spark 2.4 run:
 
-	mvn -P spark_2.4
+	mvn
 
-To build the project using Scala 2.12 and Spark 2.4 run:
+To build the project using Scala 2.12 and Spark 3.0 run:
 
-	mvn -P spark_2.4,scala_2.12
+	mvn -P spark_3.0
 
 
 (This will clean, build, test and package the jars and generate documentation)
 
 ## Running tests
-Tests are part of the main build.
+In addition to the unit tests that are part of the main build, a set of manual tests are available
+that will build the project for each version, verify the Metadata Extractor and run the example 
+application against the appropriate Spark version. Mongo will need to be installed and on the path.
+
+Command to execute manual test:
+```shell
+manual_tests/manual-tests.sh
+```
diff --git a/docs/dependency-manager.md b/docs/dependency-manager.md
@@ -22,12 +22,12 @@ Here is a working example:
       {
         "groupId": "org.mongodb.spark",
         "artifactId": "mongo-spark-connector_2.11",
-        "version": "2.3.2"
+        "version": "2.4.2"
       },
       {
         "groupId": "org.mongodb",
         "artifactId": "mongo-java-driver",
-        "version": "3.11.2",
+        "version": "3.12.2",
         "scope": "extraction"
       }
     ]
@@ -81,7 +81,7 @@ bin/dependency-resolver.sh --jar-files /tmp/steps.jar,/tmp/common-steps.jar --ou
 Example Output with a _path-prefix_ of _hdfs://acxiom/jars/udl_:
 
 ```shell script
-hdfs://acxiom/jars/udl/metalus-mongo_2.11-spark_2.3-1.6.0-SNAPSHOT.jar:hdfs://acxiom/jars/udl/mongo-spark-connector_2.11-2.3.2.jar:hdfs://acxiom/jars/udl/mongo-java-driver-3.11.2.jar
+hdfs://acxiom/jars/udl/metalus-mongo_2.11-spark_2.4-1.6.0-SNAPSHOT.jar:hdfs://acxiom/jars/udl/mongo-spark-connector_2.11-2.4.2.jar:hdfs://acxiom/jars/udl/mongo-java-driver-3.12.7.jar
 ```
 
 ## Maven Dependency Resolver
@@ -96,12 +96,12 @@ Developers may choose to override the repo within the _dependencies.json_ file b
       {
         "groupId": "org.mongodb.spark",
         "artifactId": "mongo-spark-connector_2.11",
-        "version": "2.3.2"
+        "version": "2.4.2"
       },
       {
         "groupId": "org.mongodb",
         "artifactId": "mongo-java-driver",
-        "version": "3.11.2",
+        "version": "3.12.7",
         "scope": "extraction"
       }
     ]

diff --git a/docs/getting-started.md b/docs/getting-started.md
@@ -13,9 +13,7 @@ Below are the basic build commands required to build the Metalus libraries:
 
 |Spark Version|Scala Version|Command|
 |-------------|-------------|-------|
-|2.3          |2.11         |mvn -P spark_2.3 clean install|
 |2.4          |2.11         |mvn clean install|
-|2.4          |2.12         |mvn -P spark_2.4,scala_2.12 clean install|
 |3.0          |2.12         |mvn -P spark_3.0 clean install|
 
 
@@ -25,7 +23,7 @@ This example requires a local Mongo instance to be running. A free version can b
 
 ## Spark Setup
 Download the desired version of [Spark](http://spark.apache.org/downloads.html) and unpack. Metalus supports versions 
-2.3, 2.4 or 3.0.
+2.4 or 3.0.
 
 Start the spark server from the unpacked directory with the following commands:
 

diff --git a/docs/introduction.md b/docs/introduction.md
@@ -40,24 +40,21 @@ Metalus core provides the base library required to run metalus applications and
 
 |Spark Version|Scala Version|Library|
 |-------------|-------------|-------|
-|2.3          |2.11         |[Maven 2.11 Spark 2.3 library](https://search.maven.org/search?q=a:metalus-core_2.11-spark_2.3)|
 |2.4          |2.11         |[Maven 2.11 Spark 2.4 library](https://search.maven.org/search?q=a:metalus-core_2.11-spark_2.4)|
-|2.4          |2.12         |[Maven 2.12 Spark 2.4 library](https://search.maven.org/search?q=a:metalus-core_2.12-spark_2.4)|
+|3.0          |2.12         |[Maven 2.12 Spark 3.0 library](https://search.maven.org/search?q=a:metalus-core_2.12-spark_3.0)|
 
 ## Metalus Common
 Metalus common provides a step library for building basic applications.
 
 |Spark Version|Scala Version|Library|
 |-------------|-------------|-------|
-|2.3          |2.11         |[Maven 2.11 Spark 2.3 library](https://search.maven.org/search?q=a:metalus-common_2.11-spark_2.3)|
 |2.4          |2.11         |[Maven 2.11 Spark 2.4 library](https://search.maven.org/search?q=a:metalus-common_2.11-spark_2.4)|
-|2.4          |2.12         |[Maven 2.12 Spark 2.4 library](https://search.maven.org/search?q=a:metalus-common_2.12-spark_2.4)|
+|3.0          |2.12         |[Maven 2.12 Spark 3.0 library](https://search.maven.org/search?q=a:metalus-common_2.12-spark_3.0)|
 
 ## Metalus AWS
 Metalus AWS provides a step library for working with AWS technologies.
 
 |Spark Version|Scala Version|Library|
 |-------------|-------------|-------|
-|2.3          |2.11         |[Maven 2.11 Spark 2.3 library](https://search.maven.org/search?q=a:metalus-aws_2.11-spark_2.3)|
 |2.4          |2.11         |[Maven 2.11 Spark 2.4 library](https://search.maven.org/search?q=a:metalus-aws_2.11-spark_2.4)|
-|2.4          |2.12         |[Maven 2.12 Spark 2.4 library](https://search.maven.org/search?q=a:metalus-aws_2.12-spark_2.4)|
+|3.0          |2.12         |[Maven 2.12 Spark 3.0 library](https://search.maven.org/search?q=a:metalus-aws_2.12-spark_3.0)|
diff --git a/docs/json-pipelines.md b/docs/json-pipelines.md
@@ -75,7 +75,6 @@ Below is an example of how a basic two step pipeline may look once complete:
                 "pkg": "com.acxiom.pipeline.steps"
             },
             "tags": [
-              "metalus-common_2.11-spark_2.3-1.5.0-SNAPSHOT.jar",
               "metalus-common_2.11-spark_2.4-1.5.0-SNAPSHOT.jar"
             ],
             "stepId": "87db259d-606e-46eb-b723-82923349640f"