From e9a6bbe930b0eb59dafe8eb9fdef0e11283534a5 Mon Sep 17 00:00:00 2001 From: Tianli Feng Date: Tue, 25 Jul 2023 11:32:32 -0700 Subject: [PATCH] Bump version of Hadoop dependencies to 3.3.5 (#6995) Signed-off-by: Tianli Feng Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + distribution/src/config/jvm.options | 3 ++ .../SecuredForkJoinWorkerThreadFactory.java | 46 +++++++++++++++++++ plugins/repository-hdfs/build.gradle | 6 ++- .../licenses/hadoop-client-api-3.3.4.jar.sha1 | 1 - .../licenses/hadoop-client-api-3.3.6.jar.sha1 | 1 + .../hadoop-client-runtime-3.3.4.jar.sha1 | 1 - .../hadoop-client-runtime-3.3.6.jar.sha1 | 1 + .../licenses/hadoop-hdfs-3.3.4.jar.sha1 | 1 - .../licenses/hadoop-hdfs-3.3.6.jar.sha1 | 1 + .../hdfs/HdfsClientThreadLeakFilter.java | 5 +- 11 files changed, 62 insertions(+), 5 deletions(-) create mode 100644 libs/secure-sm/src/main/java/org/opensearch/secure_sm/SecuredForkJoinWorkerThreadFactory.java delete mode 100644 plugins/repository-hdfs/licenses/hadoop-client-api-3.3.4.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/hadoop-client-api-3.3.6.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.4.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.6.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.4.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.6.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index b2bd4bcd25785..9d241eac356c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.netflix.nebula.ospackage-base` from 11.3.0 to 11.4.0 ([#8838](https://github.com/opensearch-project/OpenSearch/pull/8838)) - Bump `com.google.http-client:google-http-client-gson` from 1.43.2 to 1.43.3 ([#8840](https://github.com/opensearch-project/OpenSearch/pull/8840)) - OpenJDK Update (July 2023 Patch releases) ([#8868](https://github.com/opensearch-project/OpenSearch/pull/8868) +- Bump `hadoop` libraries from 3.3.4 to 3.3.6 ([#6995](https://github.com/opensearch-project/OpenSearch/pull/6995)) ### Changed - Perform aggregation postCollection in ContextIndexSearcher after searching leaves ([#8303](https://github.com/opensearch-project/OpenSearch/pull/8303)) diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options index e15afc0f677c3..952110c6c0289 100644 --- a/distribution/src/config/jvm.options +++ b/distribution/src/config/jvm.options @@ -82,3 +82,6 @@ ${error.file} # JDK 20+ Incubating Vector Module for SIMD optimizations; # disabling may reduce performance on vector optimized lucene 20:--add-modules=jdk.incubator.vector + +# HDFS ForkJoinPool.common() support by SecurityManager +-Djava.util.concurrent.ForkJoinPool.common.threadFactory=org.opensearch.secure_sm.SecuredForkJoinWorkerThreadFactory diff --git a/libs/secure-sm/src/main/java/org/opensearch/secure_sm/SecuredForkJoinWorkerThreadFactory.java b/libs/secure-sm/src/main/java/org/opensearch/secure_sm/SecuredForkJoinWorkerThreadFactory.java new file mode 100644 index 0000000000000..fe239fea8129e --- /dev/null +++ b/libs/secure-sm/src/main/java/org/opensearch/secure_sm/SecuredForkJoinWorkerThreadFactory.java @@ -0,0 +1,46 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.secure_sm; + +import java.security.AccessControlContext; +import java.security.AccessController; +import java.security.Permission; +import java.security.Permissions; +import java.security.PrivilegedAction; +import java.security.ProtectionDomain; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.ForkJoinPool.ForkJoinWorkerThreadFactory; +import java.util.concurrent.ForkJoinWorkerThread; + +public class SecuredForkJoinWorkerThreadFactory implements ForkJoinWorkerThreadFactory { + static AccessControlContext contextWithPermissions(Permission... perms) { + Permissions permissions = new Permissions(); + for (Permission perm : perms) + permissions.add(perm); + return new AccessControlContext(new ProtectionDomain[] { new ProtectionDomain(null, permissions) }); + } + + // ACC for access to the factory + private static final AccessControlContext ACC = contextWithPermissions( + new RuntimePermission("getClassLoader"), + new RuntimePermission("setContextClassLoader"), + new RuntimePermission("modifyThreadGroup"), + new RuntimePermission("modifyThread") + ); + + public final ForkJoinWorkerThread newThread(ForkJoinPool pool) { + return AccessController.doPrivileged(new PrivilegedAction<>() { + public ForkJoinWorkerThread run() { + return new ForkJoinWorkerThread(pool) { + + }; + } + }, ACC); + } +} diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 0b13e18ba3f4d..6626bfccc6662 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -48,7 +48,7 @@ opensearchplugin { } versions << [ - 'hadoop3': '3.3.4' + 'hadoop3': '3.3.6' ] testFixtures.useFixture ":test:fixtures:krb5kdc-fixture", "hdfs" @@ -440,3 +440,7 @@ thirdPartyAudit { 'org.apache.avro.reflect.FieldAccessUnsafe$UnsafeShortField', ) } + +tasks.withType(JavaForkOptions) { + systemProperty "java.util.concurrent.ForkJoinPool.common.threadFactory", "org.opensearch.secure_sm.SecuredForkJoinWorkerThreadFactory" +} diff --git a/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.4.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.4.jar.sha1 deleted file mode 100644 index dd79b8a10cebc..0000000000000 --- a/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6339a8f7279310c8b1f7ef314b592d8c71ca72ef \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.6.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.6.jar.sha1 new file mode 100644 index 0000000000000..d99793bc56522 --- /dev/null +++ b/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.6.jar.sha1 @@ -0,0 +1 @@ +12ac6f103a0ff29fce17a078c7c64d25320b6165 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.4.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.4.jar.sha1 deleted file mode 100644 index 32d58d1dc501a..0000000000000 --- a/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -21f7a9a2da446f1e5b3e5af16ebf956d3ee43ee0 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.6.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.6.jar.sha1 new file mode 100644 index 0000000000000..ea22d763b7bfa --- /dev/null +++ b/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.6.jar.sha1 @@ -0,0 +1 @@ +81065531e63fccbe85fb04a3274709593fb00d3c \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.4.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.4.jar.sha1 deleted file mode 100644 index 532d25a44531f..0000000000000 --- a/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -036ef2f86dc44410d2bb5d54ce40435d2484d9a5 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.6.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.6.jar.sha1 new file mode 100644 index 0000000000000..fe60968056eb7 --- /dev/null +++ b/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.6.jar.sha1 @@ -0,0 +1 @@ +ba40aca60f39599d5b1f1d32b35295bfde1f3c8b \ No newline at end of file diff --git a/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsClientThreadLeakFilter.java b/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsClientThreadLeakFilter.java index b9b0e9e87dd0c..2758bd020e979 100644 --- a/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsClientThreadLeakFilter.java +++ b/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsClientThreadLeakFilter.java @@ -43,6 +43,9 @@ * to ignore the offending thread until a version of Hadoop is released that addresses the incorrect * interrupt handling. * + * In Hadoop 3.3.6, the org.apache.hadoop.fs.statistics.impl.EvaluatingStatisticsMap uses ForkJoinPool + * to perform statistics calculation, leaving dangling workers. + * * @see https://issues.apache.org/jira/browse/HADOOP-12829 * @see "org.apache.hadoop.fs.FileSystem.Statistics.StatisticsDataReferenceCleaner" * @see "org.apache.hadoop.fs.FileSystem.Statistics" @@ -53,6 +56,6 @@ public final class HdfsClientThreadLeakFilter implements ThreadFilter { @Override public boolean reject(Thread t) { - return t.getName().equals(OFFENDING_THREAD_NAME); + return t.getName().equals(OFFENDING_THREAD_NAME) || t.getName().startsWith("ForkJoinPool.commonPool-"); } }