Add http proxy support for OIDC realm (#57039)

This change introduces support for using an http proxy for egress
communication of the OpenID Connect realm.

docs/reference/settings/security-settings.asciidoc

@@ -1347,6 +1347,24 @@ id tokens with regards to their creation and expiration times.
 Specifies whether to populate the {es} user's metadata with the values that are
 provided by the OpenID Connect claims. Defaults to `true`.
 
+`http.proxy.host`::
+Specifies the address of the proxy server that will be used by the internal
+http client for all back-channel communication to the OpenID Connect Provider
+endpoints. This includes requests to the Token Endpoint, the Userinfo Endpoint
+and requests to fetch the JSON Web Key Set from the OP if `op.jwkset_path` is
+set as a URL. 
+
+`http.proxy.scheme`::
+Specifies the protocol to use to connect to the proxy server that will be 
+used by the http client for all back-channel communication to the OpenID 
+Connect Provider endpoints. Defaults to `http`. Allowed values are 
+`http` or `https`.
+
+`http.proxy.port`::
+Specifies the port of the proxy server that will be used by the http
+client for all backchannel communication to the OpenID Connect Provider
+endpoints. Defaults to `80`.
+
 `http.connect_timeout`::
 Controls the behavior of the http client used for back-channel communication to
 the OpenID Connect Provider endpoints. Specifies the timeout until a connection

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/oidc/OpenIdConnectRealmSettings.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.core.security.authc.oidc;
 
+import org.apache.http.HttpHost;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.unit.TimeValue;
@@ -19,7 +20,9 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.function.Function;
 
@@ -139,6 +142,48 @@ private OpenIdConnectRealmSettings() {
     public static final Setting.AffixSetting<Integer> HTTP_MAX_ENDPOINT_CONNECTIONS
         = Setting.affixKeySetting(RealmSettings.realmSettingPrefix(TYPE), "http.max_endpoint_connections",
         key -> Setting.intSetting(key, 200, Setting.Property.NodeScope));
+    public static final Setting.AffixSetting<String> HTTP_PROXY_HOST
+        = Setting.affixKeySetting(RealmSettings.realmSettingPrefix(TYPE), "http.proxy.host",
+        key -> Setting.simpleString(key, new Setting.Validator<String>() {
+            @Override
+            public void validate(String value) {
+                // There is no point in validating the hostname in itself without the scheme and port
+            }
+
+            @Override
+            public void validate(String value, Map<Setting<?>, Object> settings) {
+                final String namespace = HTTP_PROXY_HOST.getNamespace(HTTP_PROXY_HOST.getConcreteSetting(key));
+                final Setting<Integer> portSetting = HTTP_PROXY_PORT.getConcreteSettingForNamespace(namespace);
+                final Integer port = (Integer) settings.get(portSetting);
+                final Setting<String> schemeSetting = HTTP_PROXY_SCHEME.getConcreteSettingForNamespace(namespace);
+                final String scheme = (String) settings.get(schemeSetting);
+                try {
+                    new HttpHost(value, port, scheme);
+                } catch (Exception e) {
+                    throw new IllegalArgumentException("HTTP host for hostname [" + value + "] (from [" + key + "])," +
+                        " port [" + port + "] (from [" + portSetting.getKey() + "]) and " +
+                        "scheme [" + scheme + "] (from ([" + schemeSetting.getKey() + "]) is invalid");
+                }
+            }
+
+            @Override
+            public Iterator<Setting<?>> settings() {
+                final String namespace = HTTP_PROXY_HOST.getNamespace(HTTP_PROXY_HOST.getConcreteSetting(key));
+                final List<Setting<?>> settings = List.of(HTTP_PROXY_PORT.getConcreteSettingForNamespace(namespace),
+                    HTTP_PROXY_SCHEME.getConcreteSettingForNamespace(namespace));
+                return settings.iterator();
+            }
+        }, Setting.Property.NodeScope));
+    public static final Setting.AffixSetting<Integer> HTTP_PROXY_PORT
+        = Setting.affixKeySetting(RealmSettings.realmSettingPrefix(TYPE), "http.proxy.port",
+        key -> Setting.intSetting(key, 80, 1, 65535, Setting.Property.NodeScope), () -> HTTP_PROXY_HOST);
+    public static final Setting.AffixSetting<String> HTTP_PROXY_SCHEME
+        = Setting.affixKeySetting(RealmSettings.realmSettingPrefix(TYPE), "http.proxy.scheme",
+        key -> Setting.simpleString(key, "http", value -> {
+            if (value.equals("http") == false && value.equals("https") == false) {
+                throw new IllegalArgumentException("Invalid value [" + value + "] for [" + key + "]. Only `http` or `https` are allowed.");
+            }
+        }, Setting.Property.NodeScope));
 
     public static final ClaimSetting PRINCIPAL_CLAIM = new ClaimSetting("principal");
     public static final ClaimSetting GROUPS_CLAIM = new ClaimSetting("groups");
@@ -151,7 +196,8 @@ public static Set<Setting.AffixSetting<?>> getSettings() {
             RP_CLIENT_ID, RP_REDIRECT_URI, RP_RESPONSE_TYPE, RP_REQUESTED_SCOPES, RP_CLIENT_SECRET, RP_SIGNATURE_ALGORITHM,
             RP_POST_LOGOUT_REDIRECT_URI, OP_AUTHORIZATION_ENDPOINT, OP_TOKEN_ENDPOINT, OP_USERINFO_ENDPOINT,
             OP_ENDSESSION_ENDPOINT, OP_ISSUER, OP_JWKSET_PATH, POPULATE_USER_METADATA, HTTP_CONNECT_TIMEOUT, HTTP_CONNECTION_READ_TIMEOUT,
-            HTTP_SOCKET_TIMEOUT, HTTP_MAX_CONNECTIONS, HTTP_MAX_ENDPOINT_CONNECTIONS, ALLOWED_CLOCK_SKEW);
+            HTTP_SOCKET_TIMEOUT, HTTP_MAX_CONNECTIONS, HTTP_MAX_ENDPOINT_CONNECTIONS, HTTP_PROXY_HOST, HTTP_PROXY_PORT,
+            HTTP_PROXY_SCHEME, ALLOWED_CLOCK_SKEW);
         set.addAll(DelegatedAuthorizationSettings.getSettings(TYPE));
         set.addAll(RealmSettings.getStandardSettings(TYPE));
         set.addAll(SSLConfigurationSettings.getRealmSettings(TYPE));

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthenticator.java

@@ -42,6 +42,7 @@
 import org.apache.commons.codec.Charsets;
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
+import org.apache.http.HttpHost;
 import org.apache.http.HttpResponse;
 import org.apache.http.NameValuePair;
 import org.apache.http.auth.AuthenticationException;
@@ -56,6 +57,7 @@
 import org.apache.http.entity.ContentType;
 import org.apache.http.impl.auth.BasicScheme;
 import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
+import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
 import org.apache.http.impl.nio.client.HttpAsyncClients;
 import org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager;
 import org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor;
@@ -112,6 +114,9 @@
 import static org.elasticsearch.xpack.core.security.authc.oidc.OpenIdConnectRealmSettings.HTTP_CONNECT_TIMEOUT;
 import static org.elasticsearch.xpack.core.security.authc.oidc.OpenIdConnectRealmSettings.HTTP_MAX_CONNECTIONS;
 import static org.elasticsearch.xpack.core.security.authc.oidc.OpenIdConnectRealmSettings.HTTP_MAX_ENDPOINT_CONNECTIONS;
+import static org.elasticsearch.xpack.core.security.authc.oidc.OpenIdConnectRealmSettings.HTTP_PROXY_HOST;
+import static org.elasticsearch.xpack.core.security.authc.oidc.OpenIdConnectRealmSettings.HTTP_PROXY_PORT;
+import static org.elasticsearch.xpack.core.security.authc.oidc.OpenIdConnectRealmSettings.HTTP_PROXY_SCHEME;
 import static org.elasticsearch.xpack.core.security.authc.oidc.OpenIdConnectRealmSettings.HTTP_SOCKET_TIMEOUT;
 
 /**
@@ -579,10 +584,14 @@ private CloseableHttpAsyncClient createHttpClient() {
                         .setConnectTimeout(Math.toIntExact(realmConfig.getSetting(HTTP_CONNECT_TIMEOUT).getMillis()))
                         .setConnectionRequestTimeout(Math.toIntExact(realmConfig.getSetting(HTTP_CONNECTION_READ_TIMEOUT).getSeconds()))
                         .setSocketTimeout(Math.toIntExact(realmConfig.getSetting(HTTP_SOCKET_TIMEOUT).getMillis())).build();
-                    CloseableHttpAsyncClient httpAsyncClient = HttpAsyncClients.custom()
+                    HttpAsyncClientBuilder httpAsyncClientBuilder = HttpAsyncClients.custom()
                         .setConnectionManager(connectionManager)
-                        .setDefaultRequestConfig(requestConfig)
-                        .build();
+                        .setDefaultRequestConfig(requestConfig);
+                    if (realmConfig.hasSetting(HTTP_PROXY_HOST)) {
+                        httpAsyncClientBuilder.setProxy(new HttpHost(realmConfig.getSetting(HTTP_PROXY_HOST),
+                            realmConfig.getSetting(HTTP_PROXY_PORT), realmConfig.getSetting(HTTP_PROXY_SCHEME)));
+                    }
+                    CloseableHttpAsyncClient httpAsyncClient = httpAsyncClientBuilder.build();
                     httpAsyncClient.start();
                     return httpAsyncClient;
                 });

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectRealm.java

@@ -257,9 +257,9 @@ private RelyingPartyConfiguration buildRelyingPartyConfiguration(RealmConfig con
         }
         final ResponseType responseType;
         try {
-            // This should never happen as it's already validated in the settings
             responseType = ResponseType.parse(require(config, RP_RESPONSE_TYPE));
         } catch (ParseException e) {
+            // This should never happen as it's already validated in the settings
             throw new SettingsException("Invalid value for " + RP_RESPONSE_TYPE.getKey(), e);
         }
 

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectRealmSettingsTests.java

@@ -264,6 +264,69 @@ public void testMissingClientSecretThrowsError() {
             Matchers.containsString(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.RP_CLIENT_SECRET)));
     }
 
+    public void testInvalidProxySchemeThrowsError() {
+        final Settings.Builder settingsBuilder = Settings.builder()
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.OP_AUTHORIZATION_ENDPOINT), "https://op.example.com/login")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.OP_TOKEN_ENDPOINT), "https://op.example.com/token")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.OP_ISSUER), "https://op.example.com")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.OP_JWKSET_PATH), "https://op.example.com/jwks.json")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.PRINCIPAL_CLAIM.getClaim()), "sub")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.RP_REDIRECT_URI), "https://rp.my.com")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.RP_CLIENT_ID), "rp-my")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.RP_RESPONSE_TYPE), "code")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.HTTP_PROXY_HOST), "proxyhostname.org")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.HTTP_PROXY_SCHEME), "invalid");
+        settingsBuilder.setSecureSettings(getSecureSettings());
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> {
+            buildConfig(settingsBuilder.build()).getSetting(OpenIdConnectRealmSettings.HTTP_PROXY_SCHEME);
+        });
+        assertThat(exception.getMessage(),
+            Matchers.containsString(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.HTTP_PROXY_SCHEME)));
+    }
+
+    public void testInvalidProxyPortThrowsError() {
+        final Settings.Builder settingsBuilder = Settings.builder()
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.OP_AUTHORIZATION_ENDPOINT), "https://op.example.com/login")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.OP_TOKEN_ENDPOINT), "https://op.example.com/token")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.OP_ISSUER), "https://op.example.com")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.OP_JWKSET_PATH), "https://op.example.com/jwks.json")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.PRINCIPAL_CLAIM.getClaim()), "sub")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.RP_REDIRECT_URI), "https://rp.my.com")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.RP_CLIENT_ID), "rp-my")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.RP_RESPONSE_TYPE), "code")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.HTTP_PROXY_HOST), "proxyhostname.org")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.HTTP_PROXY_PORT), 123456);
+        settingsBuilder.setSecureSettings(getSecureSettings());
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> {
+            buildConfig(settingsBuilder.build()).getSetting(OpenIdConnectRealmSettings.HTTP_PROXY_PORT);
+        });
+        assertThat(exception.getMessage(),
+            Matchers.containsString(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.HTTP_PROXY_PORT)));
+    }
+
+    public void testInvalidProxyHostThrowsError() {
+        final Settings.Builder settingsBuilder = Settings.builder()
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.OP_AUTHORIZATION_ENDPOINT), "https://op.example.com/login")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.OP_TOKEN_ENDPOINT), "https://op.example.com/token")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.OP_ISSUER), "https://op.example.com")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.OP_JWKSET_PATH), "https://op.example.com/jwks.json")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.PRINCIPAL_CLAIM.getClaim()), "sub")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.RP_REDIRECT_URI), "https://rp.my.com")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.RP_CLIENT_ID), "rp-my")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.RP_RESPONSE_TYPE), "code")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.HTTP_PROXY_HOST), "proxy hostname.org")
+            .put(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.HTTP_PROXY_PORT), 8080);
+        settingsBuilder.setSecureSettings(getSecureSettings());
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> {
+            buildConfig(settingsBuilder.build()).getSetting(OpenIdConnectRealmSettings.HTTP_PROXY_HOST);
+        });
+        assertThat(exception.getMessage(), Matchers.allOf(
+            Matchers.containsString(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.HTTP_PROXY_HOST)),
+            Matchers.containsString(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.HTTP_PROXY_PORT)),
+            Matchers.containsString(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.HTTP_PROXY_SCHEME))
+        ));
+    }
+
     private MockSecureSettings getSecureSettings() {
         MockSecureSettings secureSettings = new MockSecureSettings();
         secureSettings.setString(getFullSettingKey(REALM_NAME, OpenIdConnectRealmSettings.RP_CLIENT_SECRET),