From a70687a6fae60164e7ac4fd8bcdc963312c8da75 Mon Sep 17 00:00:00 2001 From: Vacha Date: Tue, 30 Nov 2021 14:17:21 -0800 Subject: [PATCH] Upgrading commons-codec in hdfs-fixture and cleaning up dependencies in repository-hdfs (#1603) (#1621) Signed-off-by: Vacha --- plugins/repository-hdfs/build.gradle | 55 ++++++++++++++++--- .../hadoop-annotations-3.3.1.jar.sha1 | 1 - .../licenses/hadoop-auth-3.3.1.jar.sha1 | 1 - .../licenses/hadoop-client-3.3.1.jar.sha1 | 1 - .../licenses/hadoop-client-api-3.3.1.jar.sha1 | 1 + .../hadoop-client-runtime-3.3.1.jar.sha1 | 1 + .../licenses/hadoop-common-3.3.1.jar.sha1 | 1 - .../hadoop-hdfs-client-3.3.1.jar.sha1 | 1 - .../hadoop-shaded-guava-1.1.1.jar.sha1 | 1 - .../hadoop-shaded-protobuf_3_7-1.1.1.jar.sha1 | 1 - test/fixtures/hdfs-fixture/build.gradle | 1 + 11 files changed, 51 insertions(+), 14 deletions(-) delete mode 100644 plugins/repository-hdfs/licenses/hadoop-annotations-3.3.1.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-auth-3.3.1.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-client-3.3.1.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/hadoop-client-api-3.3.1.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.1.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-common-3.3.1.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-hdfs-client-3.3.1.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-shaded-guava-1.1.1.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-shaded-protobuf_3_7-1.1.1.jar.sha1 diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 44a0b324b72ec..f5f619cc075fb 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -58,15 +58,10 @@ configurations { } dependencies { - api "org.apache.hadoop:hadoop-client:${versions.hadoop3}" - api "org.apache.hadoop:hadoop-common:${versions.hadoop3}" - api "org.apache.hadoop:hadoop-annotations:${versions.hadoop3}" - api "org.apache.hadoop:hadoop-auth:${versions.hadoop3}" + api "org.apache.hadoop:hadoop-client-api:${versions.hadoop3}" + runtimeOnly "org.apache.hadoop:hadoop-client-runtime:${versions.hadoop3}" api "org.apache.hadoop:hadoop-hdfs:${versions.hadoop3}" - api "org.apache.hadoop:hadoop-hdfs-client:${versions.hadoop3}" api 'org.apache.htrace:htrace-core4:4.1.0-incubating' - api "org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.1.1" - api "org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7:1.1.1" api "org.apache.logging.log4j:log4j-core:${versions.log4j}" api 'org.apache.avro:avro:1.10.2' api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}" @@ -378,6 +373,52 @@ thirdPartyAudit { 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$JvmMemoryAccessor', 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$MemoryAccessor', + 'org.apache.hadoop.shaded.com.google.common.cache.Striped64', + 'org.apache.hadoop.shaded.com.google.common.cache.Striped64$1', + 'org.apache.hadoop.shaded.com.google.common.cache.Striped64$Cell', + 'org.apache.hadoop.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray', + 'org.apache.hadoop.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1', + 'org.apache.hadoop.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2', + 'org.apache.hadoop.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3', + 'org.apache.hadoop.shaded.com.google.common.hash.Striped64', + 'org.apache.hadoop.shaded.com.google.common.hash.Striped64$1', + 'org.apache.hadoop.shaded.com.google.common.hash.Striped64$Cell', + 'org.apache.hadoop.shaded.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator', + 'org.apache.hadoop.shaded.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1', + 'org.apache.hadoop.shaded.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper', + 'org.apache.hadoop.shaded.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1', + 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe', + 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeBooleanField', + 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeByteField', + 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeCachedField', + 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeCharField', + 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeCustomEncodedField', + 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeDoubleField', + 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeFloatField', + 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeIntField', + 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeLongField', + 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeObjectField', + 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeShortField', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.cache.Striped64', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.cache.Striped64$1', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.cache.Striped64$Cell', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.Striped64', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.Striped64$1', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.Striped64$Cell', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1', + 'org.apache.hadoop.shaded.org.xbill.DNS.spi.DNSJavaNameServiceDescriptor', + 'org.apache.hadoop.shaded.org.xerial.snappy.pure.PureJavaSnappy', + 'org.apache.hadoop.shaded.org.xerial.snappy.pure.SnappyRawCompressor', + 'org.apache.hadoop.shaded.org.xerial.snappy.pure.SnappyRawDecompressor', + 'org.apache.hadoop.shaded.org.xerial.snappy.pure.UnsafeUtil', + 'org.apache.avro.reflect.FieldAccessUnsafe', 'org.apache.avro.reflect.FieldAccessUnsafe$UnsafeBooleanField', 'org.apache.avro.reflect.FieldAccessUnsafe$UnsafeByteField', diff --git a/plugins/repository-hdfs/licenses/hadoop-annotations-3.3.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-annotations-3.3.1.jar.sha1 deleted file mode 100644 index eb4fa5cf79397..0000000000000 --- a/plugins/repository-hdfs/licenses/hadoop-annotations-3.3.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6d70d2a8682f650f0dd6e66945448a2e2222a44a \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-auth-3.3.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-auth-3.3.1.jar.sha1 deleted file mode 100644 index 945e4b9b34452..0000000000000 --- a/plugins/repository-hdfs/licenses/hadoop-auth-3.3.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -862a3da7b7c75cca2e0ffe48f1f449d0897245de \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-client-3.3.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-3.3.1.jar.sha1 deleted file mode 100644 index 10a68ca91b034..0000000000000 --- a/plugins/repository-hdfs/licenses/hadoop-client-3.3.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6cb172bd4287acb088d85c345c931a3d953467e2 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.1.jar.sha1 new file mode 100644 index 0000000000000..dc2f20e310d30 --- /dev/null +++ b/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.1.jar.sha1 @@ -0,0 +1 @@ +4b9c9cdd9967495838fb521001699c4c9dddf183 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.1.jar.sha1 new file mode 100644 index 0000000000000..feb37ecc90255 --- /dev/null +++ b/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.1.jar.sha1 @@ -0,0 +1 @@ +f3a55d882328ee87a1054f99d62ba987fa9029a4 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-common-3.3.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-common-3.3.1.jar.sha1 deleted file mode 100644 index b678b52ef3d93..0000000000000 --- a/plugins/repository-hdfs/licenses/hadoop-common-3.3.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -227027e98079d3f0f24c56f323fe27a129658073 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-hdfs-client-3.3.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-hdfs-client-3.3.1.jar.sha1 deleted file mode 100644 index d23e66191ca6b..0000000000000 --- a/plugins/repository-hdfs/licenses/hadoop-hdfs-client-3.3.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5ad71520a3632a9b5b2c65f9f53d1c9d80544ee0 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-shaded-guava-1.1.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-shaded-guava-1.1.1.jar.sha1 deleted file mode 100644 index 1709abd759b01..0000000000000 --- a/plugins/repository-hdfs/licenses/hadoop-shaded-guava-1.1.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -02419d851c01139edf9e19b81056382163d9bfab \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-shaded-protobuf_3_7-1.1.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-shaded-protobuf_3_7-1.1.1.jar.sha1 deleted file mode 100644 index cd274b0e29fa2..0000000000000 --- a/plugins/repository-hdfs/licenses/hadoop-shaded-protobuf_3_7-1.1.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -57b7ba0ca94313c342b03bd31830fe4a8f34bc1a \ No newline at end of file diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index 70d6d11d7348b..815dd2c4afba3 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -33,6 +33,7 @@ apply plugin: 'opensearch.java' dependencies { api "org.apache.hadoop:hadoop-minicluster:3.3.1" api "org.apache.commons:commons-compress:1.21" + api "commons-codec:commons-codec:${versions.commonscodec}" api "org.apache.logging.log4j:log4j-core:${versions.log4j}" api "io.netty:netty-all:${versions.netty}" api 'com.google.code.gson:gson:2.8.9'