Bump version of Hadoop dependencies to 3.3.5 (#6995)

Signed-off-by: Tianli Feng <[email protected]>
Signed-off-by: Andriy Redko <[email protected]>

CHANGELOG.md

@@ -87,6 +87,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.netflix.nebula.ospackage-base` from 11.3.0 to 11.4.0 ([#8838](https://github.com/opensearch-project/OpenSearch/pull/8838))
 - Bump `com.google.http-client:google-http-client-gson` from 1.43.2 to 1.43.3 ([#8840](https://github.com/opensearch-project/OpenSearch/pull/8840))
 - OpenJDK Update (July 2023 Patch releases) ([#8868](https://github.com/opensearch-project/OpenSearch/pull/8868)
+- Bump `hadoop` libraries from 3.3.4 to 3.3.6 ([#6995](https://github.com/opensearch-project/OpenSearch/pull/6995))
 
 ### Changed
 - Perform aggregation postCollection in ContextIndexSearcher after searching leaves ([#8303](https://github.com/opensearch-project/OpenSearch/pull/8303))

distribution/src/config/jvm.options

@@ -82,3 +82,6 @@ ${error.file}
 # JDK 20+ Incubating Vector Module for SIMD optimizations;
 # disabling may reduce performance on vector optimized lucene
 20:--add-modules=jdk.incubator.vector
+
+# HDFS ForkJoinPool.common() support by SecurityManager
+-Djava.util.concurrent.ForkJoinPool.common.threadFactory=org.opensearch.secure_sm.SecuredForkJoinWorkerThreadFactory

libs/secure-sm/src/main/java/org/opensearch/secure_sm/SecuredForkJoinWorkerThreadFactory.java

@@ -0,0 +1,46 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.secure_sm;
+
+import java.security.AccessControlContext;
+import java.security.AccessController;
+import java.security.Permission;
+import java.security.Permissions;
+import java.security.PrivilegedAction;
+import java.security.ProtectionDomain;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.ForkJoinPool.ForkJoinWorkerThreadFactory;
+import java.util.concurrent.ForkJoinWorkerThread;
+
+public class SecuredForkJoinWorkerThreadFactory implements ForkJoinWorkerThreadFactory {
+    static AccessControlContext contextWithPermissions(Permission... perms) {
+        Permissions permissions = new Permissions();
+        for (Permission perm : perms)
+            permissions.add(perm);
+        return new AccessControlContext(new ProtectionDomain[] { new ProtectionDomain(null, permissions) });
+    }
+
+    // ACC for access to the factory
+    private static final AccessControlContext ACC = contextWithPermissions(
+        new RuntimePermission("getClassLoader"),
+        new RuntimePermission("setContextClassLoader"),
+        new RuntimePermission("modifyThreadGroup"),
+        new RuntimePermission("modifyThread")
+    );
+
+    public final ForkJoinWorkerThread newThread(ForkJoinPool pool) {
+        return AccessController.doPrivileged(new PrivilegedAction<>() {
+            public ForkJoinWorkerThread run() {
+                return new ForkJoinWorkerThread(pool) {
+
+                };
+            }
+        }, ACC);
+    }
+}

plugins/repository-hdfs/build.gradle

@@ -48,7 +48,7 @@ opensearchplugin {
 }
 
 versions << [
-  'hadoop3': '3.3.4'
+  'hadoop3': '3.3.6'
 ]
 
 testFixtures.useFixture ":test:fixtures:krb5kdc-fixture", "hdfs"
@@ -440,3 +440,7 @@ thirdPartyAudit {
     'org.apache.avro.reflect.FieldAccessUnsafe$UnsafeShortField',
   )
 }
+
+tasks.withType(JavaForkOptions) {
+  systemProperty "java.util.concurrent.ForkJoinPool.common.threadFactory", "org.opensearch.secure_sm.SecuredForkJoinWorkerThreadFactory"
+}

plugins/repository-hdfs/licenses/hadoop-client-api-3.3.6.jar.sha1

@@ -0,0 +1 @@
+12ac6f103a0ff29fce17a078c7c64d25320b6165

plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.6.jar.sha1

@@ -0,0 +1 @@
+81065531e63fccbe85fb04a3274709593fb00d3c

plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.6.jar.sha1

@@ -0,0 +1 @@
+ba40aca60f39599d5b1f1d32b35295bfde1f3c8b

plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsClientThreadLeakFilter.java

@@ -43,6 +43,9 @@
  * to ignore the offending thread until a version of Hadoop is released that addresses the incorrect
  * interrupt handling.
  *
+ * In Hadoop 3.3.6, the org.apache.hadoop.fs.statistics.impl.EvaluatingStatisticsMap uses ForkJoinPool
+ * to perform statistics calculation, leaving dangling workers.
+ *
  * @see <a href="https://issues.apache.org/jira/browse/HADOOP-12829">https://issues.apache.org/jira/browse/HADOOP-12829</a>
  * @see "org.apache.hadoop.fs.FileSystem.Statistics.StatisticsDataReferenceCleaner"
  * @see "org.apache.hadoop.fs.FileSystem.Statistics"
@@ -53,6 +56,6 @@ public final class HdfsClientThreadLeakFilter implements ThreadFilter {
 
     @Override
     public boolean reject(Thread t) {
-        return t.getName().equals(OFFENDING_THREAD_NAME);
+        return t.getName().equals(OFFENDING_THREAD_NAME) || t.getName().startsWith("ForkJoinPool.commonPool-");
     }
 }