Add proxy settings for GCS repository (#2096)

Added proxy settings for GCS repository. Security settings: - gcs.client.*.proxy.username - Proxy user name - gcs.client.*.proxy.password - Proxy user password Common settings: - gcs.client.*.proxy.type - java Proxy.Type names: HTTP, SOCKS. default is DIRECT - gcs.client.*.proxy.host - Proxy host name - gcs.client.*.proxy.port - Proxy port Signed-off-by: Andrey Pleskach <[email protected]>
opensearch-project · Feb 17, 2022 · b9ff91d · b9ff91d
1 parent 9ea25c4
commit b9ff91d
Show file tree

Hide file tree

Showing 7 changed files with 340 additions and 16 deletions.
diff --git a/...y-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageClientSettings.java b/...y-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageClientSettings.java
@@ -36,17 +36,23 @@
 
 import org.opensearch.common.Strings;
 import org.opensearch.common.settings.SecureSetting;
+import org.opensearch.common.settings.SecureString;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.settings.SettingsException;
 import org.opensearch.common.unit.TimeValue;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UncheckedIOException;
+import java.net.InetAddress;
+import java.net.Proxy;
 import java.net.URI;
+import java.net.UnknownHostException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Locale;
 import java.util.Map;
 import java.util.function.Function;
 
@@ -114,6 +120,54 @@ public class GoogleCloudStorageClientSettings {
         key -> new Setting<>(key, "repository-gcs", Function.identity(), Setting.Property.NodeScope, Setting.Property.Deprecated)
     );
 
+    /** Proxy type */
+    static final Setting.AffixSetting<Proxy.Type> PROXY_TYPE_SETTING = Setting.affixKeySetting(
+        PREFIX,
+        "proxy.type",
+        (key) -> new Setting<Proxy.Type>(
+            key,
+            Proxy.Type.DIRECT.name(),
+            s -> Proxy.Type.valueOf(s.toUpperCase(Locale.ROOT)),
+            Setting.Property.NodeScope
+        )
+    );
+
+    /** The host of a proxy to connect */
+    static final Setting.AffixSetting<String> PROXY_HOST_SETTING = Setting.affixKeySetting(
+        PREFIX,
+        "proxy.host",
+        key -> Setting.simpleString(key, Setting.Property.NodeScope),
+        () -> PROXY_TYPE_SETTING
+    );
+
+    /** The port of a proxy to connect */
+    static final Setting.AffixSetting<Integer> PROXY_PORT_SETTING = Setting.affixKeySetting(
+        PREFIX,
+        "proxy.port",
+        key -> Setting.intSetting(key, 0, 0, (1 << 16) - 1, Setting.Property.NodeScope),
+        () -> PROXY_TYPE_SETTING,
+        () -> PROXY_HOST_SETTING
+    );
+
+    /** The username of a proxy to connect */
+    static final Setting.AffixSetting<SecureString> PROXY_USERNAME_SETTING = Setting.affixKeySetting(
+        PREFIX,
+        "proxy.username",
+        key -> SecureSetting.secureString(key, null),
+        () -> PROXY_TYPE_SETTING,
+        () -> PROXY_HOST_SETTING
+    );
+
+    /** The password of a proxy to connect */
+    static final Setting.AffixSetting<SecureString> PROXY_PASSWORD_SETTING = Setting.affixKeySetting(
+        PREFIX,
+        "proxy.password",
+        key -> SecureSetting.secureString(key, null),
+        () -> PROXY_TYPE_SETTING,
+        () -> PROXY_HOST_SETTING,
+        () -> PROXY_USERNAME_SETTING
+    );
+
     /** The credentials used by the client to connect to the Storage endpoint. */
     private final ServiceAccountCredentials credential;
 
@@ -135,14 +189,18 @@ public class GoogleCloudStorageClientSettings {
     /** The token server URI. This leases access tokens in the oauth flow. */
     private final URI tokenUri;
 
+    /** The GCS SDK Proxy settings. */
+    private final ProxySettings proxySettings;
+
     GoogleCloudStorageClientSettings(
         final ServiceAccountCredentials credential,
         final String endpoint,
         final String projectId,
         final TimeValue connectTimeout,
         final TimeValue readTimeout,
         final String applicationName,
-        final URI tokenUri
+        final URI tokenUri,
+        final ProxySettings proxySettings
     ) {
         this.credential = credential;
         this.endpoint = endpoint;
@@ -151,6 +209,7 @@ public class GoogleCloudStorageClientSettings {
         this.readTimeout = readTimeout;
         this.applicationName = applicationName;
         this.tokenUri = tokenUri;
+        this.proxySettings = proxySettings;
     }
 
     public ServiceAccountCredentials getCredential() {
@@ -181,6 +240,10 @@ public URI getTokenUri() {
         return tokenUri;
     }
 
+    public ProxySettings getProxySettings() {
+        return proxySettings;
+    }
+
     public static Map<String, GoogleCloudStorageClientSettings> load(final Settings settings) {
         final Map<String, GoogleCloudStorageClientSettings> clients = new HashMap<>();
         for (final String clientName : settings.getGroups(PREFIX).keySet()) {
@@ -202,10 +265,39 @@ static GoogleCloudStorageClientSettings getClientSettings(final Settings setting
             getConfigValue(settings, clientName, CONNECT_TIMEOUT_SETTING),
             getConfigValue(settings, clientName, READ_TIMEOUT_SETTING),
             getConfigValue(settings, clientName, APPLICATION_NAME_SETTING),
-            getConfigValue(settings, clientName, TOKEN_URI_SETTING)
+            getConfigValue(settings, clientName, TOKEN_URI_SETTING),
+            validateAndCreateProxySettings(settings, clientName)
         );
     }
 
+    static ProxySettings validateAndCreateProxySettings(final Settings settings, final String clientName) {
+        final Proxy.Type proxyType = getConfigValue(settings, clientName, PROXY_TYPE_SETTING);
+        final String proxyHost = getConfigValue(settings, clientName, PROXY_HOST_SETTING);
+        final int proxyPort = getConfigValue(settings, clientName, PROXY_PORT_SETTING);
+        final SecureString proxyUserName = getConfigValue(settings, clientName, PROXY_USERNAME_SETTING);
+        final SecureString proxyPassword = getConfigValue(settings, clientName, PROXY_PASSWORD_SETTING);
+        // Validate proxy settings
+        if (proxyType == Proxy.Type.DIRECT
+            && (proxyPort != 0 || Strings.hasText(proxyHost) || Strings.hasText(proxyUserName) || Strings.hasText(proxyPassword))) {
+            throw new SettingsException(
+                "Google Cloud Storage proxy port or host or username or password have been set but proxy type is not defined."
+            );
+        }
+        if (proxyType != Proxy.Type.DIRECT && (proxyPort == 0 || Strings.isEmpty(proxyHost))) {
+            throw new SettingsException("Google Cloud Storage proxy type has been set but proxy host or port is not defined.");
+        }
+        if (proxyType == Proxy.Type.DIRECT) {
+            return ProxySettings.NO_PROXY_SETTINGS;
+        }
+
+        try {
+            final InetAddress proxyHostAddress = InetAddress.getByName(proxyHost);
+            return new ProxySettings(proxyType, proxyHostAddress, proxyPort, proxyUserName.toString(), proxyPassword.toString());
+        } catch (final UnknownHostException e) {
+            throw new SettingsException("Google Cloud Storage proxy host is unknown.", e);
+        }
+    }
+
     /**
      * Loads the service account file corresponding to a given client name. If no
      * file is defined for the client, a {@code null} credential is returned.

diff --git a/...epository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStoragePlugin.java b/...epository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStoragePlugin.java
@@ -92,7 +92,12 @@ public List<Setting<?>> getSettings() {
             GoogleCloudStorageClientSettings.CONNECT_TIMEOUT_SETTING,
             GoogleCloudStorageClientSettings.READ_TIMEOUT_SETTING,
             GoogleCloudStorageClientSettings.APPLICATION_NAME_SETTING,
-            GoogleCloudStorageClientSettings.TOKEN_URI_SETTING
+            GoogleCloudStorageClientSettings.TOKEN_URI_SETTING,
+            GoogleCloudStorageClientSettings.PROXY_TYPE_SETTING,
+            GoogleCloudStorageClientSettings.PROXY_HOST_SETTING,
+            GoogleCloudStorageClientSettings.PROXY_PORT_SETTING,
+            GoogleCloudStorageClientSettings.PROXY_USERNAME_SETTING,
+            GoogleCloudStorageClientSettings.PROXY_PASSWORD_SETTING
         );
     }
 

diff --git a/...pository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageService.java b/...pository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageService.java
@@ -50,6 +50,9 @@
 import org.opensearch.common.unit.TimeValue;
 
 import java.io.IOException;
+import java.net.Authenticator;
+import java.net.PasswordAuthentication;
+import java.net.Proxy;
 import java.net.URI;
 import java.util.Map;
 
@@ -142,13 +145,7 @@ synchronized void closeRepositoryClient(String repositoryName) {
      */
     private Storage createClient(GoogleCloudStorageClientSettings clientSettings, GoogleCloudStorageOperationsStats stats)
         throws IOException {
-        final HttpTransport httpTransport = SocketAccess.doPrivilegedIOException(() -> {
-            final NetHttpTransport.Builder builder = new NetHttpTransport.Builder();
-            // requires java.lang.RuntimePermission "setFactory"
-            // Pin the TLS trust certificates.
-            builder.trustCertificates(GoogleUtils.getCertificateTrustStore());
-            return builder.build();
-        });
+        final HttpTransport httpTransport = createHttpTransport(clientSettings);
 
         final GoogleCloudStorageHttpStatsCollector httpStatsCollector = new GoogleCloudStorageHttpStatsCollector(stats);
 
@@ -175,6 +172,28 @@ public HttpRequestInitializer getHttpRequestInitializer(ServiceOptions<?, ?> ser
         return storageOptions.getService();
     }
 
+    private HttpTransport createHttpTransport(final GoogleCloudStorageClientSettings clientSettings) throws IOException {
+        return SocketAccess.doPrivilegedIOException(() -> {
+            final NetHttpTransport.Builder builder = new NetHttpTransport.Builder();
+            // requires java.lang.RuntimePermission "setFactory"
+            // Pin the TLS trust certificates.
+            builder.trustCertificates(GoogleUtils.getCertificateTrustStore());
+            final ProxySettings proxySettings = clientSettings.getProxySettings();
+            if (proxySettings != ProxySettings.NO_PROXY_SETTINGS) {
+                if (proxySettings.isAuthenticated()) {
+                    Authenticator.setDefault(new Authenticator() {
+                        @Override
+                        protected PasswordAuthentication getPasswordAuthentication() {
+                            return new PasswordAuthentication(proxySettings.getUsername(), proxySettings.getPassword().toCharArray());
+                        }
+                    });
+                }
+                builder.setProxy(new Proxy(proxySettings.getType(), proxySettings.getAddress()));
+            }
+            return builder.build();
+        });
+    }
+
     StorageOptions createStorageOptions(
         final GoogleCloudStorageClientSettings clientSettings,
         final HttpTransportOptions httpTransportOptions

diff --git a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/ProxySettings.java b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/ProxySettings.java
@@ -0,0 +1,80 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.repositories.gcs;
+
+import org.opensearch.common.Strings;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.Proxy;
+import java.util.Objects;
+
+public class ProxySettings {
+
+    public static final ProxySettings NO_PROXY_SETTINGS = new ProxySettings(Proxy.Type.DIRECT, null, -1, null, null);
+
+    private final Proxy.Type type;
+
+    private final InetAddress host;
+
+    private final String username;
+
+    private final String password;
+
+    private final int port;
+
+    public ProxySettings(final Proxy.Type type, final InetAddress host, final int port, final String username, final String password) {
+        this.type = type;
+        this.host = host;
+        this.port = port;
+        this.username = username;
+        this.password = password;
+    }
+
+    public Proxy.Type getType() {
+        return this.type;
+    }
+
+    public InetSocketAddress getAddress() {
+        return new InetSocketAddress(host, port);
+    }
+
+    public String getUsername() {
+        return this.username;
+    }
+
+    public String getPassword() {
+        return this.password;
+    }
+
+    public boolean isAuthenticated() {
+        return Strings.isNullOrEmpty(username) == false && Strings.isNullOrEmpty(password) == false;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        final ProxySettings that = (ProxySettings) o;
+        return port == that.port
+            && type == that.type
+            && Objects.equals(host, that.host)
+            && Objects.equals(username, that.username)
+            && Objects.equals(password, that.password);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(type, host, username, password, port);
+    }
+}
diff --git a/plugins/repository-gcs/src/main/plugin-metadata/plugin-security.policy b/plugins/repository-gcs/src/main/plugin-metadata/plugin-security.policy
@@ -40,4 +40,7 @@ grant {
 
   // gcs client opens socket connections for to access repository
   permission java.net.SocketPermission "*", "connect";
+
+  // gcs client set Authenticator for proxy username/password
+  permission java.net.NetPermission "setDefaultAuthenticator";
 };