[CELEBORN-999] MR deps check

.github/workflows/deps.yml

@@ -50,6 +50,7 @@ jobs:
           - 'flink-1.14'
           - 'flink-1.15'
           - 'flink-1.17'
+          - 'mr'
     steps:
     - uses: actions/checkout@v2
     - name: Setup JDK 8
@@ -79,6 +80,7 @@ jobs:
           - 'flink-1.14'
           - 'flink-1.15'
           - 'flink-1.17'
+          - 'mr'
     steps:
     - uses: actions/checkout@v2
     - name: Setup JDK 8

dev/dependencies.sh

@@ -186,6 +186,10 @@ case "$MODULE" in
     MVN_MODULES="client-flink/flink-1.17"
     SBT_PROJECT="celeborn-client-flink-1_17"
     ;;
+  "mr")
+    MVN_MODULES="client-mr/mr"
+    SBT_PROJECT="celeborn-client-mr"
+    ;;
   *)
     MODULE="server"
     MVN_MODULES="worker,master"

dev/deps/dependencies-client-mr

@@ -0,0 +1,186 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+HikariCP-java7/2.4.12//HikariCP-java7-2.4.12.jar
+RoaringBitmap/0.9.32//RoaringBitmap-0.9.32.jar
+accessors-smart/2.4.7//accessors-smart-2.4.7.jar
+aopalliance/1.0//aopalliance-1.0.jar
+asm/9.1//asm-9.1.jar
+audience-annotations/0.5.0//audience-annotations-0.5.0.jar
+avro/1.7.7//avro-1.7.7.jar
+commons-beanutils/1.9.4//commons-beanutils-1.9.4.jar
+commons-cli/1.2//commons-cli-1.2.jar
+commons-codec/1.11//commons-codec-1.11.jar
+commons-collections/3.2.2//commons-collections-3.2.2.jar
+commons-compress/1.4.1//commons-compress-1.4.1.jar
+commons-configuration2/2.1.1//commons-configuration2-2.1.1.jar
+commons-crypto/1.0.0//commons-crypto-1.0.0.jar
+commons-daemon/1.0.13//commons-daemon-1.0.13.jar
+commons-io/2.13.0//commons-io-2.13.0.jar
+commons-lang3/3.12.0//commons-lang3-3.12.0.jar
+commons-logging/1.1.3//commons-logging-1.1.3.jar
+commons-math3/3.1.1//commons-math3-3.1.1.jar
+commons-net/3.6//commons-net-3.6.jar
+commons-text/1.4//commons-text-1.4.jar
+curator-client/2.13.0//curator-client-2.13.0.jar
+curator-framework/2.13.0//curator-framework-2.13.0.jar
+curator-recipes/2.13.0//curator-recipes-2.13.0.jar
+dnsjava/2.1.7//dnsjava-2.1.7.jar
+ehcache/3.3.1//ehcache-3.3.1.jar
+geronimo-jcache_1.0_spec/1.0-alpha-1//geronimo-jcache_1.0_spec-1.0-alpha-1.jar
+gson/2.9.0//gson-2.9.0.jar
+guava/14.0.1//guava-14.0.1.jar
+guice-servlet/4.0//guice-servlet-4.0.jar
+guice/4.0//guice-4.0.jar
+hadoop-annotations/3.2.4//hadoop-annotations-3.2.4.jar
+hadoop-auth/3.2.4//hadoop-auth-3.2.4.jar
+hadoop-client-api/3.2.4//hadoop-client-api-3.2.4.jar
+hadoop-client-runtime/3.2.4//hadoop-client-runtime-3.2.4.jar
+hadoop-common/3.2.4//hadoop-common-3.2.4.jar
+hadoop-hdfs-client/3.2.4//hadoop-hdfs-client-3.2.4.jar
+hadoop-mapreduce-client-app/3.2.4//hadoop-mapreduce-client-app-3.2.4.jar
+hadoop-mapreduce-client-common/3.2.4//hadoop-mapreduce-client-common-3.2.4.jar
+hadoop-mapreduce-client-core/3.2.4//hadoop-mapreduce-client-core-3.2.4.jar
+hadoop-mapreduce-client-shuffle/3.2.4//hadoop-mapreduce-client-shuffle-3.2.4.jar
+hadoop-yarn-api/3.2.4//hadoop-yarn-api-3.2.4.jar
+hadoop-yarn-client/3.2.4//hadoop-yarn-client-3.2.4.jar
+hadoop-yarn-common/3.2.4//hadoop-yarn-common-3.2.4.jar
+hadoop-yarn-registry/3.2.4//hadoop-yarn-registry-3.2.4.jar
+hadoop-yarn-server-common/3.2.4//hadoop-yarn-server-common-3.2.4.jar
+hadoop-yarn-server-nodemanager/3.2.4//hadoop-yarn-server-nodemanager-3.2.4.jar
+hadoop-yarn-server-web-proxy/3.2.4//hadoop-yarn-server-web-proxy-3.2.4.jar
+htrace-core4/4.1.0-incubating//htrace-core4-4.1.0-incubating.jar
+httpclient/4.5.13//httpclient-4.5.13.jar
+httpcore/4.4.13//httpcore-4.4.13.jar
+jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
+jackson-core/2.10.5//jackson-core-2.10.5.jar
+jackson-jaxrs-base/2.10.5//jackson-jaxrs-base-2.10.5.jar
+jackson-jaxrs-json-provider/2.10.5//jackson-jaxrs-json-provider-2.10.5.jar
+jackson-jaxrs/1.9.2//jackson-jaxrs-1.9.2.jar
+jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
+jackson-module-jaxb-annotations/2.10.5//jackson-module-jaxb-annotations-2.10.5.jar
+jackson-xc/1.9.2//jackson-xc-1.9.2.jar
+jakarta.activation-api/1.2.1//jakarta.activation-api-1.2.1.jar
+jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
+javax.activation-api/1.2.0//javax.activation-api-1.2.0.jar
+javax.inject/1//javax.inject-1.jar
+javax.servlet-api/3.1.0//javax.servlet-api-3.1.0.jar
+jaxb-api/2.2.11//jaxb-api-2.2.11.jar
+jaxb-impl/2.2.3-1//jaxb-impl-2.2.3-1.jar
+jcip-annotations/1.0-1//jcip-annotations-1.0-1.jar
+jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
+jersey-client/1.19//jersey-client-1.19.jar
+jersey-core/1.19//jersey-core-1.19.jar
+jersey-guice/1.19//jersey-guice-1.19.jar
+jersey-json/1.19//jersey-json-1.19.jar
+jersey-server/1.19//jersey-server-1.19.jar
+jersey-servlet/1.19//jersey-servlet-1.19.jar
+jettison/1.1//jettison-1.1.jar
+jetty-http/9.4.43.v20210629//jetty-http-9.4.43.v20210629.jar
+jetty-io/9.4.43.v20210629//jetty-io-9.4.43.v20210629.jar
+jetty-security/9.4.43.v20210629//jetty-security-9.4.43.v20210629.jar
+jetty-server/9.4.43.v20210629//jetty-server-9.4.43.v20210629.jar
+jetty-servlet/9.4.43.v20210629//jetty-servlet-9.4.43.v20210629.jar
+jetty-util-ajax/9.4.43.v20210629//jetty-util-ajax-9.4.43.v20210629.jar
+jetty-util/9.4.43.v20210629//jetty-util-9.4.43.v20210629.jar
+jetty-webapp/9.4.43.v20210629//jetty-webapp-9.4.43.v20210629.jar
+jetty-xml/9.4.43.v20210629//jetty-xml-9.4.43.v20210629.jar
+jsch/0.1.55//jsch-0.1.55.jar
+json-smart/2.4.7//json-smart-2.4.7.jar
+jsp-api/2.1//jsp-api-2.1.jar
+jsr305/1.3.9//jsr305-1.3.9.jar
+jsr311-api/1.1.1//jsr311-api-1.1.1.jar
+jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
+kerb-admin/1.0.1//kerb-admin-1.0.1.jar
+kerb-client/1.0.1//kerb-client-1.0.1.jar
+kerb-common/1.0.1//kerb-common-1.0.1.jar
+kerb-core/1.0.1//kerb-core-1.0.1.jar
+kerb-crypto/1.0.1//kerb-crypto-1.0.1.jar
+kerb-identity/1.0.1//kerb-identity-1.0.1.jar
+kerb-server/1.0.1//kerb-server-1.0.1.jar
+kerb-simplekdc/1.0.1//kerb-simplekdc-1.0.1.jar
+kerb-util/1.0.1//kerb-util-1.0.1.jar
+kerby-asn1/1.0.1//kerby-asn1-1.0.1.jar
+kerby-config/1.0.1//kerby-config-1.0.1.jar
+kerby-pkix/1.0.1//kerby-pkix-1.0.1.jar
+kerby-util/1.0.1//kerby-util-1.0.1.jar
+kerby-xdr/1.0.1//kerby-xdr-1.0.1.jar
+leveldbjni-all/1.8//leveldbjni-all-1.8.jar
+lz4-java/1.8.0//lz4-java-1.8.0.jar
+metrics-core/3.2.6//metrics-core-3.2.6.jar
+metrics-graphite/3.2.6//metrics-graphite-3.2.6.jar
+metrics-jvm/3.2.6//metrics-jvm-3.2.6.jar
+mssql-jdbc/6.2.1.jre7//mssql-jdbc-6.2.1.jre7.jar
+netty-all/4.1.93.Final//netty-all-4.1.93.Final.jar
+netty-buffer/4.1.93.Final//netty-buffer-4.1.93.Final.jar
+netty-codec-dns/4.1.93.Final//netty-codec-dns-4.1.93.Final.jar
+netty-codec-haproxy/4.1.93.Final//netty-codec-haproxy-4.1.93.Final.jar
+netty-codec-http/4.1.93.Final//netty-codec-http-4.1.93.Final.jar
+netty-codec-http2/4.1.93.Final//netty-codec-http2-4.1.93.Final.jar
+netty-codec-memcache/4.1.93.Final//netty-codec-memcache-4.1.93.Final.jar
+netty-codec-mqtt/4.1.93.Final//netty-codec-mqtt-4.1.93.Final.jar
+netty-codec-redis/4.1.93.Final//netty-codec-redis-4.1.93.Final.jar
+netty-codec-smtp/4.1.93.Final//netty-codec-smtp-4.1.93.Final.jar
+netty-codec-socks/4.1.93.Final//netty-codec-socks-4.1.93.Final.jar
+netty-codec-stomp/4.1.93.Final//netty-codec-stomp-4.1.93.Final.jar
+netty-codec-xml/4.1.93.Final//netty-codec-xml-4.1.93.Final.jar
+netty-codec/4.1.93.Final//netty-codec-4.1.93.Final.jar
+netty-common/4.1.93.Final//netty-common-4.1.93.Final.jar
+netty-handler-proxy/4.1.93.Final//netty-handler-proxy-4.1.93.Final.jar
+netty-handler/4.1.93.Final//netty-handler-4.1.93.Final.jar
+netty-resolver-dns-classes-macos/4.1.93.Final//netty-resolver-dns-classes-macos-4.1.93.Final.jar
+netty-resolver-dns-native-macos/4.1.93.Final/osx-aarch_64/netty-resolver-dns-native-macos-4.1.93.Final-osx-aarch_64.jar
+netty-resolver-dns-native-macos/4.1.93.Final/osx-x86_64/netty-resolver-dns-native-macos-4.1.93.Final-osx-x86_64.jar
+netty-resolver-dns/4.1.93.Final//netty-resolver-dns-4.1.93.Final.jar
+netty-resolver/4.1.93.Final//netty-resolver-4.1.93.Final.jar
+netty-transport-classes-epoll/4.1.93.Final//netty-transport-classes-epoll-4.1.93.Final.jar
+netty-transport-classes-kqueue/4.1.93.Final//netty-transport-classes-kqueue-4.1.93.Final.jar
+netty-transport-native-epoll/4.1.93.Final/linux-aarch_64/netty-transport-native-epoll-4.1.93.Final-linux-aarch_64.jar
+netty-transport-native-epoll/4.1.93.Final/linux-x86_64/netty-transport-native-epoll-4.1.93.Final-linux-x86_64.jar
+netty-transport-native-kqueue/4.1.93.Final/osx-aarch_64/netty-transport-native-kqueue-4.1.93.Final-osx-aarch_64.jar
+netty-transport-native-kqueue/4.1.93.Final/osx-x86_64/netty-transport-native-kqueue-4.1.93.Final-osx-x86_64.jar
+netty-transport-native-unix-common/4.1.93.Final//netty-transport-native-unix-common-4.1.93.Final.jar
+netty-transport-rxtx/4.1.93.Final//netty-transport-rxtx-4.1.93.Final.jar
+netty-transport-sctp/4.1.93.Final//netty-transport-sctp-4.1.93.Final.jar
+netty-transport-udt/4.1.93.Final//netty-transport-udt-4.1.93.Final.jar
+netty-transport/4.1.93.Final//netty-transport-4.1.93.Final.jar
+netty/3.10.6.Final//netty-3.10.6.Final.jar
+nimbus-jose-jwt/9.8.1//nimbus-jose-jwt-9.8.1.jar
+okhttp/2.7.5//okhttp-2.7.5.jar
+okio/1.6.0//okio-1.6.0.jar
+paranamer/2.3//paranamer-2.3.jar
+protobuf-java/3.19.2//protobuf-java-3.19.2.jar
+ratis-client/2.5.1//ratis-client-2.5.1.jar
+ratis-common/2.5.1//ratis-common-2.5.1.jar
+ratis-proto/2.5.1//ratis-proto-2.5.1.jar
+ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar
+re2j/1.1//re2j-1.1.jar
+reload4j/1.2.18.3//reload4j-1.2.18.3.jar
+scala-library/2.12.15//scala-library-2.12.15.jar
+scala-reflect/2.12.15//scala-reflect-2.12.15.jar
+shims/0.9.32//shims-0.9.32.jar
+slf4j-api/1.7.36//slf4j-api-1.7.36.jar
+slf4j-reload4j/1.7.35//slf4j-reload4j-1.7.35.jar
+snakeyaml/1.33//snakeyaml-1.33.jar
+snappy-java/1.0.5//snappy-java-1.0.5.jar
+spotbugs-annotations/3.1.9//spotbugs-annotations-3.1.9.jar
+stax2-api/4.2.1//stax2-api-4.2.1.jar
+token-provider/1.0.1//token-provider-1.0.1.jar
+woodstox-core/5.3.0//woodstox-core-5.3.0.jar
+xz/1.0//xz-1.0.jar
+zookeeper/3.4.14//zookeeper-3.4.14.jar
+zstd-jni/1.5.2-1//zstd-jni-1.5.2-1.jar

project/CelebornBuild.scala

@@ -36,8 +36,10 @@ object Dependencies {
   val lz4JavaVersion = sparkClientProjects.map(_.lz4JavaVersion).getOrElse("1.8.0")
 
   // Dependent library versions
+  val commonsCompressVersion = "1.4.1"
   val commonsCryptoVersion = "1.0.0"
   val commonsIoVersion = "2.13.0"
+  val commonsLoggingVersion = "1.1.3"
   val commonsLang3Version = "3.12.0"
   val findbugsVersion = "1.3.9"
   val guavaVersion = "14.0.1"
@@ -63,16 +65,23 @@ object Dependencies {
   val protocVersion = "3.19.2"
   val protoVersion = "3.19.2"
 
+  val commonsCompress = "org.apache.commons" % "commons-compress" % commonsCompressVersion
   val commonsCrypto = "org.apache.commons" % "commons-crypto" % commonsCryptoVersion excludeAll(
     ExclusionRule("net.java.dev.jna", "jna"))
   val commonsIo = "commons-io" % "commons-io" % commonsIoVersion
   val commonsLang3 = "org.apache.commons" % "commons-lang3" % commonsLang3Version
+  val commonsLogging = "commons-logging" % "commons-logging" % commonsLoggingVersion
   val findbugsJsr305 = "com.google.code.findbugs" % "jsr305" % findbugsVersion
   val guava = "com.google.guava" % "guava" % guavaVersion
   val hadoopClientApi = "org.apache.hadoop" % "hadoop-client-api" % hadoopVersion
   val hadoopClientRuntime = "org.apache.hadoop" % "hadoop-client-runtime" % hadoopVersion
-  val hadoopMapreduceClientApp = "org.apache.hadoop" % "hadoop-mapreduce-client-app" % hadoopVersion excludeAll (
-    ExclusionRule("com.google.guava", "guava"))
+  val hadoopMapreduceClientApp = "org.apache.hadoop" % "hadoop-mapreduce-client-app" % hadoopVersion excludeAll(
+    ExclusionRule("com.google.guava", "guava"),
+    ExclusionRule("com.fasterxml.jackson.core", "jackson-annotations"),
+    ExclusionRule("com.fasterxml.jackson.core", "jackson-databind"),
+    ExclusionRule("jline", "jline"),
+    ExclusionRule("log4j", "log4j"),
+    ExclusionRule("org.slf4j", "slf4j-log4j12"))
   val ioDropwizardMetricsCore = "io.dropwizard.metrics" % "metrics-core" % metricsVersion
   val ioDropwizardMetricsGraphite = "io.dropwizard.metrics" % "metrics-graphite" % metricsVersion
   val ioDropwizardMetricsJvm = "io.dropwizard.metrics" % "metrics-jvm" % metricsVersion
@@ -144,6 +153,8 @@ object CelebornCommonSettings {
     Compile / compile / javacOptions ++= Seq("-target", "1.8"),
 
     dependencyOverrides := Seq(
+      Dependencies.commonsCompress,
+      Dependencies.commonsLogging,
       Dependencies.findbugsJsr305,
       Dependencies.slf4jApi),