From 31cf3efbc5ca0f13f2aa970e539e134d6bb764cd Mon Sep 17 00:00:00 2001 From: sychen Date: Wed, 20 Sep 2023 14:17:36 +0800 Subject: [PATCH 1/7] deps --- .github/workflows/deps.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/deps.yml b/.github/workflows/deps.yml index 1567ab4016b..32d551ce188 100644 --- a/.github/workflows/deps.yml +++ b/.github/workflows/deps.yml @@ -50,6 +50,7 @@ jobs: - 'flink-1.14' - 'flink-1.15' - 'flink-1.17' + - 'mr' steps: - uses: actions/checkout@v2 - name: Setup JDK 8 @@ -79,6 +80,7 @@ jobs: - 'flink-1.14' - 'flink-1.15' - 'flink-1.17' + - 'mr' steps: - uses: actions/checkout@v2 - name: Setup JDK 8 From 7dc79233a1694d1421eb9b804aa60e26a61c50e1 Mon Sep 17 00:00:00 2001 From: sychen Date: Wed, 20 Sep 2023 14:30:53 +0800 Subject: [PATCH 2/7] add mr deps --- dev/dependencies.sh | 4 + dev/deps/dependencies-client-mr | 186 ++++++++++++++++++++++++++++++++ 2 files changed, 190 insertions(+) create mode 100644 dev/deps/dependencies-client-mr diff --git a/dev/dependencies.sh b/dev/dependencies.sh index 6dc486a451e..5824c0ba561 100755 --- a/dev/dependencies.sh +++ b/dev/dependencies.sh @@ -186,6 +186,10 @@ case "$MODULE" in MVN_MODULES="client-flink/flink-1.17" SBT_PROJECT="celeborn-client-flink-1_17" ;; + "mr") + MVN_MODULES="client-mr/mr" + SBT_PROJECT="celeborn-client-mr" + ;; *) MODULE="server" MVN_MODULES="worker,master" diff --git a/dev/deps/dependencies-client-mr b/dev/deps/dependencies-client-mr new file mode 100644 index 00000000000..8f6a629affa --- /dev/null +++ b/dev/deps/dependencies-client-mr @@ -0,0 +1,186 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +HikariCP-java7/2.4.12//HikariCP-java7-2.4.12.jar +RoaringBitmap/0.9.32//RoaringBitmap-0.9.32.jar +accessors-smart/2.4.7//accessors-smart-2.4.7.jar +aopalliance/1.0//aopalliance-1.0.jar +asm/9.1//asm-9.1.jar +audience-annotations/0.5.0//audience-annotations-0.5.0.jar +avro/1.7.7//avro-1.7.7.jar +commons-beanutils/1.9.4//commons-beanutils-1.9.4.jar +commons-cli/1.2//commons-cli-1.2.jar +commons-codec/1.11//commons-codec-1.11.jar +commons-collections/3.2.2//commons-collections-3.2.2.jar +commons-compress/1.4.1//commons-compress-1.4.1.jar +commons-configuration2/2.1.1//commons-configuration2-2.1.1.jar +commons-crypto/1.0.0//commons-crypto-1.0.0.jar +commons-daemon/1.0.13//commons-daemon-1.0.13.jar +commons-io/2.13.0//commons-io-2.13.0.jar +commons-lang3/3.12.0//commons-lang3-3.12.0.jar +commons-logging/1.1.3//commons-logging-1.1.3.jar +commons-math3/3.1.1//commons-math3-3.1.1.jar +commons-net/3.6//commons-net-3.6.jar +commons-text/1.4//commons-text-1.4.jar +curator-client/2.13.0//curator-client-2.13.0.jar +curator-framework/2.13.0//curator-framework-2.13.0.jar +curator-recipes/2.13.0//curator-recipes-2.13.0.jar +dnsjava/2.1.7//dnsjava-2.1.7.jar +ehcache/3.3.1//ehcache-3.3.1.jar +geronimo-jcache_1.0_spec/1.0-alpha-1//geronimo-jcache_1.0_spec-1.0-alpha-1.jar +gson/2.9.0//gson-2.9.0.jar +guava/14.0.1//guava-14.0.1.jar +guice-servlet/4.0//guice-servlet-4.0.jar +guice/4.0//guice-4.0.jar +hadoop-annotations/3.2.4//hadoop-annotations-3.2.4.jar +hadoop-auth/3.2.4//hadoop-auth-3.2.4.jar +hadoop-client-api/3.2.4//hadoop-client-api-3.2.4.jar +hadoop-client-runtime/3.2.4//hadoop-client-runtime-3.2.4.jar +hadoop-common/3.2.4//hadoop-common-3.2.4.jar +hadoop-hdfs-client/3.2.4//hadoop-hdfs-client-3.2.4.jar +hadoop-mapreduce-client-app/3.2.4//hadoop-mapreduce-client-app-3.2.4.jar +hadoop-mapreduce-client-common/3.2.4//hadoop-mapreduce-client-common-3.2.4.jar +hadoop-mapreduce-client-core/3.2.4//hadoop-mapreduce-client-core-3.2.4.jar +hadoop-mapreduce-client-shuffle/3.2.4//hadoop-mapreduce-client-shuffle-3.2.4.jar +hadoop-yarn-api/3.2.4//hadoop-yarn-api-3.2.4.jar +hadoop-yarn-client/3.2.4//hadoop-yarn-client-3.2.4.jar +hadoop-yarn-common/3.2.4//hadoop-yarn-common-3.2.4.jar +hadoop-yarn-registry/3.2.4//hadoop-yarn-registry-3.2.4.jar +hadoop-yarn-server-common/3.2.4//hadoop-yarn-server-common-3.2.4.jar +hadoop-yarn-server-nodemanager/3.2.4//hadoop-yarn-server-nodemanager-3.2.4.jar +hadoop-yarn-server-web-proxy/3.2.4//hadoop-yarn-server-web-proxy-3.2.4.jar +htrace-core4/4.1.0-incubating//htrace-core4-4.1.0-incubating.jar +httpclient/4.5.13//httpclient-4.5.13.jar +httpcore/4.4.13//httpcore-4.4.13.jar +jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar +jackson-core/2.10.5//jackson-core-2.10.5.jar +jackson-jaxrs-base/2.10.5//jackson-jaxrs-base-2.10.5.jar +jackson-jaxrs-json-provider/2.10.5//jackson-jaxrs-json-provider-2.10.5.jar +jackson-jaxrs/1.9.2//jackson-jaxrs-1.9.2.jar +jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar +jackson-module-jaxb-annotations/2.10.5//jackson-module-jaxb-annotations-2.10.5.jar +jackson-xc/1.9.2//jackson-xc-1.9.2.jar +jakarta.activation-api/1.2.1//jakarta.activation-api-1.2.1.jar +jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar +javax.activation-api/1.2.0//javax.activation-api-1.2.0.jar +javax.inject/1//javax.inject-1.jar +javax.servlet-api/3.1.0//javax.servlet-api-3.1.0.jar +jaxb-api/2.2.11//jaxb-api-2.2.11.jar +jaxb-impl/2.2.3-1//jaxb-impl-2.2.3-1.jar +jcip-annotations/1.0-1//jcip-annotations-1.0-1.jar +jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar +jersey-client/1.19//jersey-client-1.19.jar +jersey-core/1.19//jersey-core-1.19.jar +jersey-guice/1.19//jersey-guice-1.19.jar +jersey-json/1.19//jersey-json-1.19.jar +jersey-server/1.19//jersey-server-1.19.jar +jersey-servlet/1.19//jersey-servlet-1.19.jar +jettison/1.1//jettison-1.1.jar +jetty-http/9.4.43.v20210629//jetty-http-9.4.43.v20210629.jar +jetty-io/9.4.43.v20210629//jetty-io-9.4.43.v20210629.jar +jetty-security/9.4.43.v20210629//jetty-security-9.4.43.v20210629.jar +jetty-server/9.4.43.v20210629//jetty-server-9.4.43.v20210629.jar +jetty-servlet/9.4.43.v20210629//jetty-servlet-9.4.43.v20210629.jar +jetty-util-ajax/9.4.43.v20210629//jetty-util-ajax-9.4.43.v20210629.jar +jetty-util/9.4.43.v20210629//jetty-util-9.4.43.v20210629.jar +jetty-webapp/9.4.43.v20210629//jetty-webapp-9.4.43.v20210629.jar +jetty-xml/9.4.43.v20210629//jetty-xml-9.4.43.v20210629.jar +jsch/0.1.55//jsch-0.1.55.jar +json-smart/2.4.7//json-smart-2.4.7.jar +jsp-api/2.1//jsp-api-2.1.jar +jsr305/1.3.9//jsr305-1.3.9.jar +jsr311-api/1.1.1//jsr311-api-1.1.1.jar +jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar +kerb-admin/1.0.1//kerb-admin-1.0.1.jar +kerb-client/1.0.1//kerb-client-1.0.1.jar +kerb-common/1.0.1//kerb-common-1.0.1.jar +kerb-core/1.0.1//kerb-core-1.0.1.jar +kerb-crypto/1.0.1//kerb-crypto-1.0.1.jar +kerb-identity/1.0.1//kerb-identity-1.0.1.jar +kerb-server/1.0.1//kerb-server-1.0.1.jar +kerb-simplekdc/1.0.1//kerb-simplekdc-1.0.1.jar +kerb-util/1.0.1//kerb-util-1.0.1.jar +kerby-asn1/1.0.1//kerby-asn1-1.0.1.jar +kerby-config/1.0.1//kerby-config-1.0.1.jar +kerby-pkix/1.0.1//kerby-pkix-1.0.1.jar +kerby-util/1.0.1//kerby-util-1.0.1.jar +kerby-xdr/1.0.1//kerby-xdr-1.0.1.jar +leveldbjni-all/1.8//leveldbjni-all-1.8.jar +lz4-java/1.8.0//lz4-java-1.8.0.jar +metrics-core/3.2.6//metrics-core-3.2.6.jar +metrics-graphite/3.2.6//metrics-graphite-3.2.6.jar +metrics-jvm/3.2.6//metrics-jvm-3.2.6.jar +mssql-jdbc/6.2.1.jre7//mssql-jdbc-6.2.1.jre7.jar +netty-all/4.1.93.Final//netty-all-4.1.93.Final.jar +netty-buffer/4.1.93.Final//netty-buffer-4.1.93.Final.jar +netty-codec-dns/4.1.93.Final//netty-codec-dns-4.1.93.Final.jar +netty-codec-haproxy/4.1.93.Final//netty-codec-haproxy-4.1.93.Final.jar +netty-codec-http/4.1.93.Final//netty-codec-http-4.1.93.Final.jar +netty-codec-http2/4.1.93.Final//netty-codec-http2-4.1.93.Final.jar +netty-codec-memcache/4.1.93.Final//netty-codec-memcache-4.1.93.Final.jar +netty-codec-mqtt/4.1.93.Final//netty-codec-mqtt-4.1.93.Final.jar +netty-codec-redis/4.1.93.Final//netty-codec-redis-4.1.93.Final.jar +netty-codec-smtp/4.1.93.Final//netty-codec-smtp-4.1.93.Final.jar +netty-codec-socks/4.1.93.Final//netty-codec-socks-4.1.93.Final.jar +netty-codec-stomp/4.1.93.Final//netty-codec-stomp-4.1.93.Final.jar +netty-codec-xml/4.1.93.Final//netty-codec-xml-4.1.93.Final.jar +netty-codec/4.1.93.Final//netty-codec-4.1.93.Final.jar +netty-common/4.1.93.Final//netty-common-4.1.93.Final.jar +netty-handler-proxy/4.1.93.Final//netty-handler-proxy-4.1.93.Final.jar +netty-handler/4.1.93.Final//netty-handler-4.1.93.Final.jar +netty-resolver-dns-classes-macos/4.1.93.Final//netty-resolver-dns-classes-macos-4.1.93.Final.jar +netty-resolver-dns-native-macos/4.1.93.Final/osx-aarch_64/netty-resolver-dns-native-macos-4.1.93.Final-osx-aarch_64.jar +netty-resolver-dns-native-macos/4.1.93.Final/osx-x86_64/netty-resolver-dns-native-macos-4.1.93.Final-osx-x86_64.jar +netty-resolver-dns/4.1.93.Final//netty-resolver-dns-4.1.93.Final.jar +netty-resolver/4.1.93.Final//netty-resolver-4.1.93.Final.jar +netty-transport-classes-epoll/4.1.93.Final//netty-transport-classes-epoll-4.1.93.Final.jar +netty-transport-classes-kqueue/4.1.93.Final//netty-transport-classes-kqueue-4.1.93.Final.jar +netty-transport-native-epoll/4.1.93.Final/linux-aarch_64/netty-transport-native-epoll-4.1.93.Final-linux-aarch_64.jar +netty-transport-native-epoll/4.1.93.Final/linux-x86_64/netty-transport-native-epoll-4.1.93.Final-linux-x86_64.jar +netty-transport-native-kqueue/4.1.93.Final/osx-aarch_64/netty-transport-native-kqueue-4.1.93.Final-osx-aarch_64.jar +netty-transport-native-kqueue/4.1.93.Final/osx-x86_64/netty-transport-native-kqueue-4.1.93.Final-osx-x86_64.jar +netty-transport-native-unix-common/4.1.93.Final//netty-transport-native-unix-common-4.1.93.Final.jar +netty-transport-rxtx/4.1.93.Final//netty-transport-rxtx-4.1.93.Final.jar +netty-transport-sctp/4.1.93.Final//netty-transport-sctp-4.1.93.Final.jar +netty-transport-udt/4.1.93.Final//netty-transport-udt-4.1.93.Final.jar +netty-transport/4.1.93.Final//netty-transport-4.1.93.Final.jar +netty/3.10.6.Final//netty-3.10.6.Final.jar +nimbus-jose-jwt/9.8.1//nimbus-jose-jwt-9.8.1.jar +okhttp/2.7.5//okhttp-2.7.5.jar +okio/1.6.0//okio-1.6.0.jar +paranamer/2.3//paranamer-2.3.jar +protobuf-java/3.19.2//protobuf-java-3.19.2.jar +ratis-client/2.5.1//ratis-client-2.5.1.jar +ratis-common/2.5.1//ratis-common-2.5.1.jar +ratis-proto/2.5.1//ratis-proto-2.5.1.jar +ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar +re2j/1.1//re2j-1.1.jar +reload4j/1.2.18.3//reload4j-1.2.18.3.jar +scala-library/2.12.15//scala-library-2.12.15.jar +scala-reflect/2.12.15//scala-reflect-2.12.15.jar +shims/0.9.32//shims-0.9.32.jar +slf4j-api/1.7.36//slf4j-api-1.7.36.jar +slf4j-reload4j/1.7.35//slf4j-reload4j-1.7.35.jar +snakeyaml/1.33//snakeyaml-1.33.jar +snappy-java/1.0.5//snappy-java-1.0.5.jar +spotbugs-annotations/3.1.9//spotbugs-annotations-3.1.9.jar +stax2-api/4.2.1//stax2-api-4.2.1.jar +token-provider/1.0.1//token-provider-1.0.1.jar +woodstox-core/5.3.0//woodstox-core-5.3.0.jar +xz/1.0//xz-1.0.jar +zookeeper/3.4.14//zookeeper-3.4.14.jar +zstd-jni/1.5.2-1//zstd-jni-1.5.2-1.jar From 26c147eac78d66940838e0528438fd3bb4a49a23 Mon Sep 17 00:00:00 2001 From: sychen Date: Wed, 20 Sep 2023 22:24:13 +0800 Subject: [PATCH 3/7] skip sbt mr --- .github/workflows/deps.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/deps.yml b/.github/workflows/deps.yml index 32d551ce188..17426c0ee2c 100644 --- a/.github/workflows/deps.yml +++ b/.github/workflows/deps.yml @@ -50,7 +50,6 @@ jobs: - 'flink-1.14' - 'flink-1.15' - 'flink-1.17' - - 'mr' steps: - uses: actions/checkout@v2 - name: Setup JDK 8 From 5b2a5146a4bdd5393b6a3ef96603784c3ce08d8d Mon Sep 17 00:00:00 2001 From: sychen Date: Wed, 20 Sep 2023 22:47:48 +0800 Subject: [PATCH 4/7] trigger test From b4ed2c3219f4c613e39dcd74ac00672e7e84d3a4 Mon Sep 17 00:00:00 2001 From: sychen Date: Mon, 9 Oct 2023 12:53:01 +0800 Subject: [PATCH 5/7] Revert "skip sbt mr" This reverts commit 26c147eac78d66940838e0528438fd3bb4a49a23. --- .github/workflows/deps.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deps.yml b/.github/workflows/deps.yml index 17426c0ee2c..32d551ce188 100644 --- a/.github/workflows/deps.yml +++ b/.github/workflows/deps.yml @@ -50,6 +50,7 @@ jobs: - 'flink-1.14' - 'flink-1.15' - 'flink-1.17' + - 'mr' steps: - uses: actions/checkout@v2 - name: Setup JDK 8 From b1406fd571c5b5322f39391ddf6ec18490803e4d Mon Sep 17 00:00:00 2001 From: sychen Date: Mon, 9 Oct 2023 12:53:19 +0800 Subject: [PATCH 6/7] update sbt deps --- project/CelebornBuild.scala | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala index e7600c2edc5..0b146c50221 100644 --- a/project/CelebornBuild.scala +++ b/project/CelebornBuild.scala @@ -36,8 +36,10 @@ object Dependencies { val lz4JavaVersion = sparkClientProjects.map(_.lz4JavaVersion).getOrElse("1.8.0") // Dependent library versions + val commonsCompressVersion = "1.4.1" val commonsCryptoVersion = "1.0.0" val commonsIoVersion = "2.13.0" + val commonsLoggingVersion = "1.1.3" val commonsLang3Version = "3.12.0" val findbugsVersion = "1.3.9" val guavaVersion = "14.0.1" @@ -63,16 +65,25 @@ object Dependencies { val protocVersion = "3.19.2" val protoVersion = "3.19.2" + val commonsCompress = "org.apache.commons" % "commons-compress" % commonsCompressVersion val commonsCrypto = "org.apache.commons" % "commons-crypto" % commonsCryptoVersion excludeAll( ExclusionRule("net.java.dev.jna", "jna")) val commonsIo = "commons-io" % "commons-io" % commonsIoVersion val commonsLang3 = "org.apache.commons" % "commons-lang3" % commonsLang3Version + val commonsLogging = "commons-logging" % "commons-logging" % commonsLoggingVersion val findbugsJsr305 = "com.google.code.findbugs" % "jsr305" % findbugsVersion val guava = "com.google.guava" % "guava" % guavaVersion val hadoopClientApi = "org.apache.hadoop" % "hadoop-client-api" % hadoopVersion val hadoopClientRuntime = "org.apache.hadoop" % "hadoop-client-runtime" % hadoopVersion - val hadoopMapreduceClientApp = "org.apache.hadoop" % "hadoop-mapreduce-client-app" % hadoopVersion excludeAll ( - ExclusionRule("com.google.guava", "guava")) + val hadoopMapreduceClientApp = "org.apache.hadoop" % "hadoop-mapreduce-client-app" % hadoopVersion excludeAll( + ExclusionRule("org.apache.commons", "commons-compress"), + ExclusionRule("commons-logging", "commons-logging"), + ExclusionRule("com.google.guava", "guava"), + ExclusionRule("com.fasterxml.jackson.core", "jackson-annotations"), + ExclusionRule("com.fasterxml.jackson.core", "jackson-databind"), + ExclusionRule("jline", "jline"), + ExclusionRule("log4j", "log4j"), + ExclusionRule("org.slf4j", "slf4j-log4j12")) val ioDropwizardMetricsCore = "io.dropwizard.metrics" % "metrics-core" % metricsVersion val ioDropwizardMetricsGraphite = "io.dropwizard.metrics" % "metrics-graphite" % metricsVersion val ioDropwizardMetricsJvm = "io.dropwizard.metrics" % "metrics-jvm" % metricsVersion @@ -895,6 +906,8 @@ object MRClientProjects { .settings( commonSettings, libraryDependencies ++= Seq( + Dependencies.commonsCompress, + Dependencies.commonsLogging, Dependencies.hadoopClientApi, Dependencies.hadoopClientRuntime, Dependencies.hadoopMapreduceClientApp From 938459b5b64092d6b97adced6d430629a8ab77e8 Mon Sep 17 00:00:00 2001 From: sychen Date: Tue, 10 Oct 2023 12:19:12 +0800 Subject: [PATCH 7/7] use dependencyOverrides --- project/CelebornBuild.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala index 0b146c50221..f27001ce473 100644 --- a/project/CelebornBuild.scala +++ b/project/CelebornBuild.scala @@ -76,8 +76,6 @@ object Dependencies { val hadoopClientApi = "org.apache.hadoop" % "hadoop-client-api" % hadoopVersion val hadoopClientRuntime = "org.apache.hadoop" % "hadoop-client-runtime" % hadoopVersion val hadoopMapreduceClientApp = "org.apache.hadoop" % "hadoop-mapreduce-client-app" % hadoopVersion excludeAll( - ExclusionRule("org.apache.commons", "commons-compress"), - ExclusionRule("commons-logging", "commons-logging"), ExclusionRule("com.google.guava", "guava"), ExclusionRule("com.fasterxml.jackson.core", "jackson-annotations"), ExclusionRule("com.fasterxml.jackson.core", "jackson-databind"), @@ -155,6 +153,8 @@ object CelebornCommonSettings { Compile / compile / javacOptions ++= Seq("-target", "1.8"), dependencyOverrides := Seq( + Dependencies.commonsCompress, + Dependencies.commonsLogging, Dependencies.findbugsJsr305, Dependencies.slf4jApi), @@ -906,8 +906,6 @@ object MRClientProjects { .settings( commonSettings, libraryDependencies ++= Seq( - Dependencies.commonsCompress, - Dependencies.commonsLogging, Dependencies.hadoopClientApi, Dependencies.hadoopClientRuntime, Dependencies.hadoopMapreduceClientApp