Remove connect SocketPermissions from core (#22797)

This is related to #22116. Core no longer needs `SocketPermission`
`connect`.

This permission is relegated to these modules/plugins:
- transport-netty4 module
- reindex module
- repository-url module
- discovery-azure-classic plugin
- discovery-ec2 plugin
- discovery-gce plugin
- repository-azure plugin
- repository-gcs plugin
- repository-hdfs plugin
- repository-s3 plugin

And for tests:
- mocksocket jar
- rest client
- httpcore-nio jar
- httpasyncclient jar

buildSrc/version.properties

@@ -1,3 +1,4 @@
+# When updating elasticsearch, please update 'rest' version in core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
 elasticsearch     = 6.0.0-alpha1
 lucene            = 6.4.0
 
@@ -15,11 +16,16 @@ jna               = 4.2.2
 randomizedrunner  = 2.4.0
 junit             = 4.11
 httpclient        = 4.5.2
+# When updating httpcore, please also update core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
 httpcore          = 4.4.5
+# When updating httpasyncclient, please also update core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
+httpasyncclient   = 4.1.2
 commonslogging    = 1.1.3
 commonscodec      = 1.10
 hamcrest          = 1.3
 securemock        = 1.2
+# When updating mocksocket, please also update core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
 mocksocket        = 1.1
+
 # benchmark dependencies
 jmh               = 1.17.3

client/rest/build.gradle

@@ -33,7 +33,7 @@ group = 'org.elasticsearch.client'
 dependencies {
   compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
-  compile "org.apache.httpcomponents:httpasyncclient:4.1.2"
+  compile "org.apache.httpcomponents:httpasyncclient:${versions.httpasyncclient}"
   compile "org.apache.httpcomponents:httpcore-nio:${versions.httpcore}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
   compile "commons-logging:commons-logging:${versions.commonslogging}"

core/src/main/resources/org/elasticsearch/bootstrap/security.policy

@@ -55,8 +55,8 @@ grant {
   // third party code, to safeguard these against unprivileged code like scripts.
   permission org.elasticsearch.SpecialPermission;
 
-  // Allow connecting to the internet anywhere
-  permission java.net.SocketPermission "*", "connect,resolve";
+  // Allow host/ip name service lookups
+  permission java.net.SocketPermission "*", "resolve";
 
   // Allow read access to all system properties
   permission java.util.PropertyPermission "*", "read";

core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy

@@ -64,6 +64,22 @@ grant codeBase "${codebase.junit-4.11.jar}" {
 };
 
 grant codeBase "${codebase.mocksocket-1.1.jar}" {
-  // mocksocket accepts socket connections
-  permission java.net.SocketPermission "*", "accept";
+  // mocksocket makes and accepts socket connections
+  permission java.net.SocketPermission "*", "accept,connect";
+};
+
+
+grant codeBase "${codebase.rest-6.0.0-alpha1-SNAPSHOT.jar}" {
+  // rest makes socket connections for rest tests
+  permission java.net.SocketPermission "*", "connect";
+};
+
+grant codeBase "${codebase.httpcore-nio-4.4.5.jar}" {
+  // httpcore makes socket connections for rest tests
+  permission java.net.SocketPermission "*", "connect";
+};
+
+grant codeBase "${codebase.httpasyncclient-4.1.2.jar}" {
+  // httpasyncclient makes socket connections for rest tests
+  permission java.net.SocketPermission "*", "connect";
 };

modules/reindex/src/main/plugin-metadata/plugin-security.policy

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+grant {
+   // reindex opens socket connections using the rest client
+   permission java.net.SocketPermission "*", "connect";
+};

modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java

@@ -29,6 +29,9 @@
 import java.io.InputStream;
 import java.net.URL;
 import java.nio.file.NoSuchFileException;
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
 import java.util.Map;
 
 /**
@@ -102,7 +105,7 @@ public boolean blobExists(String blobName) {
     @Override
     public InputStream readBlob(String name) throws IOException {
         try {
-            return new BufferedInputStream(new URL(path, name).openStream(), blobStore.bufferSizeInBytes());
+            return new BufferedInputStream(getInputStream(new URL(path, name)), blobStore.bufferSizeInBytes());
         } catch (FileNotFoundException fnfe) {
             throw new NoSuchFileException("[" + name + "] blob not found");
         }
@@ -113,4 +116,12 @@ public void writeBlob(String blobName, InputStream inputStream, long blobSize) t
         throw new UnsupportedOperationException("URL repository doesn't support this operation");
     }
 
+    private static InputStream getInputStream(URL url) throws IOException {
+        try {
+            return AccessController.doPrivileged((PrivilegedExceptionAction<InputStream>) url::openStream);
+        } catch (PrivilegedActionException e) {
+            throw (IOException) e.getCause();
+        }
+    }
+
 }

modules/repository-url/src/main/plugin-metadata/plugin-security.policy

@@ -0,0 +1,22 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+grant {
+  permission java.net.SocketPermission "*", "connect";
+};

modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy

@@ -18,8 +18,8 @@
  */
 
 grant {
-   // netty accepts socket connections
-   permission java.net.SocketPermission "*", "accept";
+   // netty makes and accepts socket connections
+   permission java.net.SocketPermission "*", "accept,connect";
 };
 
 grant codeBase "${codebase.netty-common-4.1.7.Final.jar}" {

plugins/discovery-azure-classic/src/main/plugin-metadata/plugin-security.policy

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+grant {
+  // azure client opens socket connections for discovery
+  permission java.net.SocketPermission "*", "connect";
+};

plugins/discovery-ec2/src/main/plugin-metadata/plugin-security.policy

@@ -24,4 +24,7 @@ grant {
   // NOTE: no tests fail without this, but we know the problem
   // exists in AWS sdk, and tests here are not thorough
   permission java.lang.RuntimePermission "getClassLoader";
+
+  // ec2 client opens socket connections for discovery
+  permission java.net.SocketPermission "*", "connect";
 };

plugins/discovery-gce/src/main/plugin-metadata/plugin-security.policy

@@ -22,4 +22,7 @@ grant {
   permission java.lang.RuntimePermission "accessDeclaredMembers";
   permission java.lang.RuntimePermission "setFactory";
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+
+  // gce client opens socket connections for discovery
+  permission java.net.SocketPermission "*", "connect";
 };

plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+grant {
+  // azure client opens socket connections for to access repository
+  permission java.net.SocketPermission "*", "connect";
+};

plugins/repository-gcs/src/main/plugin-metadata/plugin-security.policy

@@ -23,4 +23,7 @@ grant {
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
   permission java.net.URLPermission "http://www.googleapis.com/*", "*";
   permission java.net.URLPermission "https://www.googleapis.com/*", "*";
+
+  // gcs client opens socket connections for to access repository
+  permission java.net.SocketPermission "*", "connect";
 };

plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java

@@ -19,20 +19,17 @@
 package org.elasticsearch.repositories.hdfs;
 
 import org.apache.hadoop.fs.CreateFlag;
-import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Options.CreateOpts;
 import org.apache.hadoop.fs.Path;
-import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.support.AbstractBlobContainer;
 import org.elasticsearch.common.blobstore.support.PlainBlobMetaData;
 import org.elasticsearch.repositories.hdfs.HdfsBlobStore.Operation;
 
-import java.io.BufferedInputStream;
 import java.io.FilterInputStream;
 import java.io.IOException;
 import java.io.InputStream;

plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy

@@ -35,4 +35,7 @@ grant {
   permission javax.security.auth.AuthPermission "getSubject";
   permission javax.security.auth.AuthPermission "doAs";
   permission javax.security.auth.AuthPermission "modifyPrivateCredentials";
+
+  // hdfs client opens socket connections for to access repository
+  permission java.net.SocketPermission "*", "connect";
 };

plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy

@@ -34,4 +34,7 @@ grant {
   // TODO: get these fixed in aws sdk
   // See https://github.com/aws/aws-sdk-java/issues/766
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+
+  // s3 client opens socket connections for to access repository
+  permission java.net.SocketPermission "*", "connect";
 };