Upgrade to Netty 4.1.25.Final (#31232)

This commit upgrades us to Netty 4.1.25. This upgrade is more
challenging than past upgrades, all because of a new object cleaner
thread that they have added. This thread requires an additional security
permission (set context class loader, needed to avoid leaks in certain
scenarios). Additionally, there is not a clean way to shutdown this
thread which means that the thread can fail thread leak control during
tests. As such, we have to filter this thread from thread leak control.

modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWithAuthTests.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.reindex;
 
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.ElasticsearchStatusException;
@@ -41,6 +42,7 @@
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.index.reindex.test.ObjectCleanerThreadThreadFilter;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestStatus;
@@ -64,6 +66,7 @@
 import static org.elasticsearch.index.reindex.ReindexTestCase.matcher;
 import static org.hamcrest.Matchers.containsString;
 
+@ThreadLeakFilters(filters = {ObjectCleanerThreadThreadFilter.class})
 public class ReindexFromRemoteWithAuthTests extends ESSingleNodeTestCase {
     private TransportAddress address;
 

modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.reindex;
 
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
@@ -32,6 +33,7 @@
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.reindex.test.ObjectCleanerThreadThreadFilter;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -55,6 +57,7 @@
  * Integration test for retry behavior. Useful because retrying relies on the way that the
  * rest of Elasticsearch throws exceptions and unit tests won't verify that.
  */
+@ThreadLeakFilters(filters = {ObjectCleanerThreadThreadFilter.class})
 public class RetryTests extends ESIntegTestCase {
 
     private static final int DOC_COUNT = 20;

modules/reindex/src/test/java/org/elasticsearch/index/reindex/test/ObjectCleanerThreadThreadFilter.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex.test;
+
+import com.carrotsearch.randomizedtesting.ThreadFilter;
+
+public class ObjectCleanerThreadThreadFilter implements ThreadFilter {
+
+    @Override
+    public boolean reject(final Thread t) {
+        // TODO: replace with constant from Netty when https://github.com/netty/netty/pull/8014 is integrated
+        return "ObjectCleanerThread".equals(t.getName());
+    }
+
+}

modules/transport-netty4/build.gradle

@@ -34,13 +34,13 @@ compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-tr
 
 dependencies {
     // network stack
-    compile "io.netty:netty-buffer:4.1.16.Final"
-    compile "io.netty:netty-codec:4.1.16.Final"
-    compile "io.netty:netty-codec-http:4.1.16.Final"
-    compile "io.netty:netty-common:4.1.16.Final"
-    compile "io.netty:netty-handler:4.1.16.Final"
-    compile "io.netty:netty-resolver:4.1.16.Final"
-    compile "io.netty:netty-transport:4.1.16.Final"
+    compile "io.netty:netty-buffer:4.1.25.Final"
+    compile "io.netty:netty-codec:4.1.25.Final"
+    compile "io.netty:netty-codec-http:4.1.25.Final"
+    compile "io.netty:netty-common:4.1.25.Final"
+    compile "io.netty:netty-handler:4.1.25.Final"
+    compile "io.netty:netty-resolver:4.1.25.Final"
+    compile "io.netty:netty-transport:4.1.25.Final"
 }
 
 dependencyLicenses {
@@ -161,6 +161,6 @@ thirdPartyAudit.excludes = [
 
         'org.conscrypt.AllocatedBuffer',
         'org.conscrypt.BufferAllocator',
-        'org.conscrypt.Conscrypt$Engines',
+        'org.conscrypt.Conscrypt',
         'org.conscrypt.HandshakeListener'
 ]

modules/transport-netty4/licenses/netty-buffer-4.1.25.Final.jar.sha1

@@ -0,0 +1 @@
+f366d0cc87b158ca064d27507127e3cc4eb2f089

modules/transport-netty4/licenses/netty-codec-4.1.25.Final.jar.sha1

@@ -0,0 +1 @@
+3e465c75bead40d06b5b9c0612b37cf77c548887

modules/transport-netty4/licenses/netty-codec-http-4.1.25.Final.jar.sha1

@@ -0,0 +1 @@
+70888d3f2a829541378f68503ddd52c3193df35a

modules/transport-netty4/licenses/netty-common-4.1.25.Final.jar.sha1

@@ -0,0 +1 @@
+e17d5c05c101fe14536ce3fb34b36c54e04791f6

modules/transport-netty4/licenses/netty-handler-4.1.25.Final.jar.sha1

@@ -0,0 +1 @@
+ecdfb8fe93a8b75db3ea8746d3437eed845c24bd

modules/transport-netty4/licenses/netty-resolver-4.1.25.Final.jar.sha1

@@ -0,0 +1 @@
+dc0965d00746b782b33f419b005cbc130973030d

modules/transport-netty4/licenses/netty-transport-4.1.25.Final.jar.sha1

@@ -0,0 +1 @@
+19a6f1f649894b6705aa9d8cbcced188dff133b0

modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy

@@ -21,6 +21,8 @@ grant codeBase "${codebase.netty-common}" {
    // for reading the system-wide configuration for the backlog of established sockets
    permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read";
 
+   permission java.lang.RuntimePermission "setContextClassLoader";
+
    // netty makes and accepts socket connections
    permission java.net.SocketPermission "*", "accept,connect";
 };

modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4BadRequestTests.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.http.netty4;
 
 import io.netty.handler.codec.http.FullHttpResponse;
+import org.elasticsearch.test.Netty4TestCase;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
@@ -33,7 +34,6 @@
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.After;
@@ -48,7 +48,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 
-public class Netty4BadRequestTests extends ESTestCase {
+public class Netty4BadRequestTests extends Netty4TestCase {
 
     private NetworkService networkService;
     private MockBigArrays bigArrays;

modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpChannelTests.java

@@ -41,6 +41,7 @@
 import io.netty.handler.codec.http.HttpVersion;
 import io.netty.util.Attribute;
 import io.netty.util.AttributeKey;
+import org.elasticsearch.test.Netty4TestCase;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.bytes.ReleasablePagedBytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -64,7 +65,6 @@
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.netty4.Netty4Utils;
@@ -90,7 +90,7 @@
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
-public class Netty4HttpChannelTests extends ESTestCase {
+public class Netty4HttpChannelTests extends Netty4TestCase {
 
     private NetworkService networkService;
     private ThreadPool threadPool;

modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandlerTests.java

@@ -36,9 +36,9 @@
 import io.netty.handler.codec.http.HttpVersion;
 import io.netty.handler.codec.http.LastHttpContent;
 import io.netty.handler.codec.http.QueryStringDecoder;
+import org.elasticsearch.test.Netty4TestCase;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.http.HttpPipelinedRequest;
-import org.elasticsearch.test.ESTestCase;
 import org.junit.After;
 
 import java.nio.channels.ClosedChannelException;
@@ -61,7 +61,7 @@
 import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
 import static org.hamcrest.core.Is.is;
 
-public class Netty4HttpPipeliningHandlerTests extends ESTestCase {
+public class Netty4HttpPipeliningHandlerTests extends Netty4TestCase {
 
     private final ExecutorService handlerService = Executors.newFixedThreadPool(randomIntBetween(4, 8));
     private final ExecutorService eventLoopService = Executors.newFixedThreadPool(1);

modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java

@@ -20,7 +20,7 @@
 package org.elasticsearch.http.netty4;
 
 import io.netty.handler.codec.http.FullHttpResponse;
-import org.elasticsearch.ESNetty4IntegTestCase;
+import org.elasticsearch.test.Netty4IntegTestCase;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
@@ -45,7 +45,7 @@
  * a single node "cluster". We also force test infrastructure to use the node client instead of the transport client for the same reason.
  */
 @ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numClientNodes = 0, numDataNodes = 1, transportClientRatio = 0)
-public class Netty4HttpRequestSizeLimitIT extends ESNetty4IntegTestCase {
+public class Netty4HttpRequestSizeLimitIT extends Netty4IntegTestCase {
 
     private static final ByteSizeValue LIMIT = new ByteSizeValue(2, ByteSizeUnit.KB);
 

modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java

@@ -32,6 +32,7 @@
 import io.netty.handler.codec.http.HttpHeaderNames;
 import io.netty.handler.codec.http.HttpResponseStatus;
 import io.netty.handler.codec.http.HttpVersion;
+import org.elasticsearch.test.Netty4TestCase;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
@@ -42,7 +43,6 @@
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.http.NullDispatcher;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
-import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.After;
@@ -61,7 +61,7 @@
 /**
  * This test just tests, if he pipelining works in general with out any connection the Elasticsearch handler
  */
-public class Netty4HttpServerPipeliningTests extends ESTestCase {
+public class Netty4HttpServerPipeliningTests extends Netty4TestCase {
     private NetworkService networkService;
     private ThreadPool threadPool;
     private MockBigArrays bigArrays;

modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java

@@ -38,6 +38,7 @@
 import io.netty.handler.codec.http.HttpResponseStatus;
 import io.netty.handler.codec.http.HttpUtil;
 import io.netty.handler.codec.http.HttpVersion;
+import org.elasticsearch.test.Netty4TestCase;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -59,7 +60,6 @@
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.After;
@@ -94,7 +94,7 @@
 /**
  * Tests for the {@link Netty4HttpServerTransport} class.
  */
-public class Netty4HttpServerTransportTests extends ESTestCase {
+public class Netty4HttpServerTransportTests extends Netty4TestCase {
 
     private NetworkService networkService;
     private ThreadPool threadPool;

modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java

@@ -20,7 +20,7 @@
 package org.elasticsearch.http.netty4;
 
 import io.netty.handler.codec.http.FullHttpResponse;
-import org.elasticsearch.ESNetty4IntegTestCase;
+import org.elasticsearch.test.Netty4IntegTestCase;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
@@ -33,7 +33,7 @@
 import static org.hamcrest.Matchers.is;
 
 @ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1)
-public class Netty4PipeliningIT extends ESNetty4IntegTestCase {
+public class Netty4PipeliningIT extends Netty4IntegTestCase {
 
     @Override
     protected boolean addMockHttpTransport() {

modules/transport-netty4/src/test/java/org/elasticsearch/ESNetty4IntegTestCase.java → modules/transport-netty4/src/test/java/org/elasticsearch/test/Netty4IntegTestCase.java

@@ -16,19 +16,21 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch;
 
+package org.elasticsearch.test;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.transport.Netty4Plugin;
 import org.elasticsearch.transport.netty4.Netty4Transport;
 
 import java.util.Arrays;
 import java.util.Collection;
 
-public abstract class ESNetty4IntegTestCase extends ESIntegTestCase {
+@ThreadLeakFilters(filters = {ObjectCleanerThreadThreadFilter.class})
+public abstract class Netty4IntegTestCase extends ESIntegTestCase {
 
     @Override
     protected boolean ignoreExternalCluster() {

modules/transport-netty4/src/test/java/org/elasticsearch/test/Netty4TestCase.java

@@ -0,0 +1,26 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+
+@ThreadLeakFilters(filters = {ObjectCleanerThreadThreadFilter.class})
+public abstract class Netty4TestCase extends ESTestCase {
+}