Security: remove SSL settings fallback (#36846)

This commit removes the fallback for SSL settings. While this may be
seen as a non user friendly change, the intention behind this change
is to simplify the reasoning needed to understand what is actually
being used for a given SSL configuration. Each configuration now needs
to be explicitly specified as there is no global configuration or
fallback to some other configuration.

Closes #29797

client/rest-high-level/build.gradle

@@ -105,7 +105,7 @@ integTestCluster {
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.security.authc.token.enabled', 'true'
   // Truststore settings are not used since TLS is not enabled. Included for testing the get certificates API
-  setting 'xpack.ssl.certificate_authorities', 'testnode.crt'
+  setting 'xpack.security.http.ssl.certificate_authorities', 'testnode.crt'
   setting 'xpack.security.transport.ssl.truststore.path', 'testnode.jks'
   keystoreSetting 'xpack.security.transport.ssl.truststore.secure_password', 'testnode'
   setupCommand 'setupDummyUser',

docs/reference/migration/migrate_7_0/settings.asciidoc

@@ -121,3 +121,13 @@ xpack.security.authc.realms:
 Any realm specific secure settings that have been stored in the elasticsearch
 keystore (such as ldap bind passwords, or passwords for ssl keys) must be updated
 in a similar way.
+
+[float]
+[[tls-setting-fallback]]
+==== TLS/SSL settings
+
+The default TLS/SSL settings, which were prefixed by `xpack.ssl`, have been removed.
+The removal of these default settings also removes the ability for a component to
+fallback to a default configuration when using TLS. Each component (realm, transport, http,
+http client, etc) must now be configured with their own settings for TLS if it is being
+used.

docs/reference/security/securing-communications/configuring-tls-docker.asciidoc

@@ -114,11 +114,14 @@ services:
       - xpack.license.self_generated.type=trial <2>
       - xpack.security.enabled=true
       - xpack.security.http.ssl.enabled=true
+      - xpack.security.http.ssl.key=$CERTS_DIR/es01/es01.key
+      - xpack.security.http.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
+      - xpack.security.http.ssl.certificate=$CERTS_DIR/es01/es01.crt
       - xpack.security.transport.ssl.enabled=true
       - xpack.security.transport.ssl.verification_mode=certificate <3>
-      - xpack.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
-      - xpack.ssl.certificate=$CERTS_DIR/es01/es01.crt
-      - xpack.ssl.key=$CERTS_DIR/es01/es01.key
+      - xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
+      - xpack.security.transport.ssl.certificate=$CERTS_DIR/es01/es01.crt
+      - xpack.security.transport.ssl.key=$CERTS_DIR/es01/es01.key
     volumes: ['esdata_01:/usr/share/elasticsearch/data', './certs:$CERTS_DIR']
     ports:
       - 9200:9200
@@ -140,11 +143,14 @@ services:
       - xpack.license.self_generated.type=trial
       - xpack.security.enabled=true
       - xpack.security.http.ssl.enabled=true
+      - xpack.security.http.ssl.key=$CERTS_DIR/es02/es02.key
+      - xpack.security.http.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
+      - xpack.security.http.ssl.certificate=$CERTS_DIR/es02/es02.crt
       - xpack.security.transport.ssl.enabled=true
-      - xpack.security.transport.ssl.verification_mode=certificate
-      - xpack.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
-      - xpack.ssl.certificate=$CERTS_DIR/es02/es02.crt
-      - xpack.ssl.key=$CERTS_DIR/es02/es02.key
+      - xpack.security.transport.ssl.verification_mode=certificate <3>
+      - xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
+      - xpack.security.transport.ssl.certificate=$CERTS_DIR/es02/es02.crt
+      - xpack.security.transport.ssl.key=$CERTS_DIR/es02/es02.key
     volumes: ['esdata_02:/usr/share/elasticsearch/data', './certs:$CERTS_DIR']
 
   wait_until_ready:
@@ -199,9 +205,9 @@ WARNING: Windows users not running PowerShell will need to remove `\` and join l
 ----
 docker exec es01 /bin/bash -c "bin/elasticsearch-setup-passwords \
 auto --batch \
--Expack.ssl.certificate=certificates/es01/es01.crt \
--Expack.ssl.certificate_authorities=certificates/ca/ca.crt \
--Expack.ssl.key=certificates/es01/es01.key \
+-Expack.security.http.ssl.certificate=certificates/es01/es01.crt \
+-Expack.security.http.ssl.certificate_authorities=certificates/ca/ca.crt \
+-Expack.security.http.ssl.key=certificates/es01/es01.key \
 --url https://localhost:9200"
 ----
 --

docs/reference/security/securing-communications/tls-transport.asciidoc

@@ -25,7 +25,7 @@ xpack.security.transport.ssl.truststore.path: certs/elastic-certificates.p12 <3>
 <1> If you used the `--dns` or `--ip` options with the `elasticsearch-certutil cert` command
 and you want to enable strict hostname checking, set the verification mode to
 `full`.
-See <<ssl-tls-settings, `xpack.ssl.verification_mode`>> for a description of these values.
+See <<ssl-tls-settings, `xpack.security.transport.ssl.verification_mode`>> for a description of these values.
 
 <2> If you created a separate certificate for each node, then you might need to
 customize this path on each node. If the filename matches the node name, you can
@@ -54,7 +54,7 @@ xpack.security.transport.ssl.certificate_authorities: [ "/home/es/config/ca.crt"
 <1> If you used the `--dns` or `--ip` options with the `elasticsearch-certutil cert` command
 and you want to enable strict hostname checking, set the verification mode to
 `full`.
-See <<ssl-tls-settings, `xpack.ssl.verification_mode`>> for a description of these values.
+See <<ssl-tls-settings, `xpack.security.transport.ssl.verification_mode`>> for a description of these values.
 <2> The full path to the node key file. This must be a location within the
     {es} configuration directory.
 <3> The full path to the node certificate. This must be a location within the

docs/reference/settings/security-settings.asciidoc

@@ -475,20 +475,18 @@ The default is `jks`.
 `ssl.verification_mode`::
 Indicates the type of verification when using `ldaps` to protect against man
 in the middle attacks and certificate forgery. Values are `none`, `certificate`,
-and `full`. Defaults to the value of `xpack.ssl.verification_mode`.
+and `full`. Defaults to `full`.
 +
-See <<ssl-tls-settings,`xpack.ssl.verification_mode`>> for an explanation of
-these values.
+See <<ssl-tls-settings,`ssl.verification_mode`>> for an explanation of these values.
 
 `ssl.supported_protocols`::
-Supported protocols for TLS/SSL (with versions). Defaults to the value of
-`xpack.ssl.supported_protocols`.
+Supported protocols for TLS/SSL (with versions). Defaults to `TLSv1.2,TLSv1.1,TLSv1`.
 
 `ssl.cipher_suites`:: Specifies the cipher suites that should be supported when 
 communicating with the LDAP server. 
 Supported cipher suites can be found in Oracle's http://docs.oracle.com/javase/8/docs/technotes/guides/security/SunProviders.html[
-Java Cryptography Architecture documentation]. Defaults to the value of
-`xpack.ssl.cipher_suites`.
+Java Cryptography Architecture documentation]. See <<ssl-tls-settings,`ssl.cipher_suites`>>
+for the default value.
 
 `cache.ttl`::
 Specifies the time-to-live for cached user entries. A user and a hash of its 
@@ -721,20 +719,18 @@ The default is `jks`.
 `ssl.verification_mode`::
 Indicates the type of verification when using `ldaps` to protect against man
 in the middle attacks and certificate forgery. Values are `none`, `certificate`,
-and `full`. Defaults to the value of `xpack.ssl.verification_mode`.
+and `full`. Defaults to `full`.
 +
-See <<ssl-tls-settings,`xpack.ssl.verification_mode`>> for an explanation of
-these values.
+See <<ssl-tls-settings,`ssl.verification_mode`>> for an explanation of these values.
 
 `ssl.supported_protocols`::
-Supported protocols for TLS/SSL (with versions). Defaults to the value of
-`xpack.ssl.supported_protocols`.
+Supported protocols for TLS/SSL (with versions). Defaults to `TLSv1.2, TLSv1.1, TLSv1`.
 
 `ssl.cipher_suites`:: Specifies the cipher suites that should be supported when 
 communicating with the Active Directory server. 
 Supported cipher suites can be found in Oracle's http://docs.oracle.com/javase/8/docs/technotes/guides/security/SunProviders.html[
-Java Cryptography Architecture documentation]. Defaults to the value of
-`xpack.ssl.cipher_suites`.
+Java Cryptography Architecture documentation]. See <<ssl-tls-settings,`ssl.cipher_suites`>> for
+the default values.
 
 `cache.ttl`::
 Specifies the time-to-live for cached user entries. A user and a hash of its 
@@ -1133,8 +1129,7 @@ One of `full`
 certificate path, but not the hostname) or `none` (perform no verification).
 Defaults to `full`.
 +
-See <<ssl-tls-settings,`xpack.ssl.verification_mode`>> for a more detailed
-explanation of these values.
+See <<ssl-tls-settings,`ssl.verification_mode`>> for a more detailed explanation of these values.
 
 `ssl.supported_protocols`::
 Specifies the supported protocols for TLS/SSL.
@@ -1204,14 +1199,12 @@ through the list of URLs will continue until a successful connection is made.
 
 [float]
 [[ssl-tls-settings]]
-==== Default TLS/SSL settings
-You can configure the following TLS/SSL settings in
-`elasticsearch.yml`. For more information, see
-{stack-ov}/encrypting-communications.html[Encrypting communications]. These
-settings are used unless they have been overridden by more specific
-settings such as those for HTTP or Transport.
-
-`xpack.ssl.supported_protocols`::
+==== Default values for TLS/SSL settings
+In general, the values below represent the default values for the various TLS
+settings. For more information, see
+{stack-ov}/encrypting-communications.html[Encrypting communications].
+
+`ssl.supported_protocols`::
 Supported protocols with versions. Valid protocols: `SSLv2Hello`,
 `SSLv3`, `TLSv1`, `TLSv1.1`, `TLSv1.2`. Defaults to `TLSv1.2`, `TLSv1.1`,
 `TLSv1`. 
@@ -1221,15 +1214,15 @@ NOTE: If `xpack.security.fips_mode.enabled` is `true`, you cannot use `SSLv2Hell
 or `SSLv3`. See <<fips-140-compliance>>.
 --
 
-`xpack.ssl.client_authentication`::
+`ssl.client_authentication`::
 Controls the server's behavior in regard to requesting a certificate
 from client connections. Valid values are `required`, `optional`, and `none`.
 `required` forces a client to present a certificate, while `optional`
 requests a client certificate but the client is not required to present one.
-Defaults to `required`. This global setting is not applicable for HTTP, see
+Defaults to `required`, except for HTTP, which defaults to `none`. See
  <<http-tls-ssl-settings>>.
 
-`xpack.ssl.verification_mode`::
+`ssl.verification_mode`::
 Controls the verification of certificates. Valid values are:
 - `full`, which verifies that the provided certificate is signed by a trusted
        authority (CA) and also verifies that the server's hostname (or IP
@@ -1244,7 +1237,7 @@ Controls the verification of certificates. Valid values are:
 +
 The default value is `full`.
 
-`xpack.ssl.cipher_suites`::
+`ssl.cipher_suites`::
 Supported cipher suites can be found in Oracle's http://docs.oracle.com/javase/8/docs/technotes/guides/security/SunProviders.html[
 Java Cryptography Architecture documentation]. Defaults to `TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256`,
 `TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256`, `TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA`, `TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA`,
@@ -1255,112 +1248,15 @@ Jurisdiction Policy Files_ has been installed, the default value also includes `
 
 [float]
 [[tls-ssl-key-settings]]
-===== Default TLS/SSL key and trusted certificate settings
+===== TLS/SSL key and trusted certificate settings
 
 The following settings are used to specify a private key, certificate, and the
-trusted certificates that should be used when communicating over an SSL/TLS connection.
-If none of the settings below are specified, the 
-<<ssl-tls-settings,default settings>> are used. If no trusted certificates are configured, the default certificates that are trusted by the JVM will be
-trusted along with the certificate(s) from the <<tls-ssl-key-settings, key settings>>. The key and certificate must be in place
-for connections that require client authentication or when acting as a SSL enabled server.
-
-[float]
-===== PEM encoded files
-
-When using PEM encoded files, use the following settings:
-
-`xpack.ssl.key`::
-Path to the PEM encoded file containing the private key.
-
-`xpack.ssl.key_passphrase`::
-The passphrase that is used to decrypt the private key. This value is
-optional as the key might not be encrypted.
-
-`xpack.ssl.secure_key_passphrase` (<<secure-settings,Secure>>)::
-The passphrase that is used to decrypt the private key. This value is
-optional as the key might not be encrypted.
-
-`xpack.ssl.certificate`::
-Path to a PEM encoded file containing the certificate (or certificate chain)
-that will be presented to clients when they connect.
-
-`xpack.ssl.certificate_authorities`::
-List of paths to the PEM encoded certificate files that should be trusted.
-
-[float]
-===== Java keystore files
-
-When using Java keystore files (JKS), which contain the private key, certificate
-and certificates that should be trusted, use the following settings:
-
-`xpack.ssl.keystore.path`::
-Path to the keystore that holds the private key and certificate.
-
-`xpack.ssl.keystore.password`::
-Password to the keystore.
-
-`xpack.ssl.keystore.secure_password` (<<secure-settings,Secure>>)::
-Password to the keystore.
-
-`xpack.ssl.keystore.key_password`::
-Password for the private key in the keystore. Defaults to the
-same value as `xpack.ssl.keystore.password`.
-
-`xpack.ssl.keystore.secure_key_password` (<<secure-settings,Secure>>)::
-Password for the private key in the keystore.
-
-`xpack.ssl.truststore.path`::
-Path to the truststore file.
-
-`xpack.ssl.truststore.password`::
-Password to the truststore.
-
-`xpack.ssl.truststore.secure_password` (<<secure-settings,Secure>>)::
-Password to the truststore.
-
-WARNING: If `xpack.security.fips_mode.enabled` is `true`, you cannot use Java 
-keystore files. See <<fips-140-compliance>>. 
-
-[float]
-===== PKCS#12 files
-
-When using PKCS#12 container files (`.p12` or `.pfx`), which contain the
-private key, certificate, and certificates that should be trusted, use
-the following settings:
-
-`xpack.ssl.keystore.path`::
-Path to the PKCS#12 file that holds the private key and certificate.
-
-`xpack.ssl.keystore.type`::
-Set this to `PKCS12`.
-
-`xpack.ssl.keystore.password`::
-Password to the PKCS#12 file.
-
-`xpack.ssl.keystore.secure_password` (<<secure-settings,Secure>>)::
-Password to the PKCS#12 file.
-
-`xpack.ssl.keystore.key_password`::
-Password for the private key in the PKCS12 file.
-Defaults to the same value as `xpack.ssl.keystore.password`.
-
-`xpack.ssl.keystore.secure_key_password` (<<secure-settings,Secure>>)::
-Password for the private key in the PKCS12 file.
-
-`xpack.ssl.truststore.path`::
-Path to the truststore file.
-
-`xpack.ssl.truststore.type`::
-Set this to `PKCS12`.
-
-`xpack.ssl.truststore.password`::
-Password to the truststore.
-
-`xpack.ssl.truststore.secure_password` (<<secure-settings,Secure>>)::
-Password to the truststore.
-
-WARNING: If `xpack.security.fips_mode.enabled` is `true`, you cannot use PKCS#12 
-keystore files. See <<fips-140-compliance>>. 
+trusted certificates that should be used when communicating over an SSL/TLS
+connection. If no trusted certificates are configured, the default certificates
+that are trusted by the JVM will be trusted along with the certificate(s)
+associated with a key in the same context. The key and certificate must be in
+place for connections that require client authentication or when acting as a
+SSL enabled server.
 
 [[pkcs12-truststore-note]]
 [NOTE]
@@ -1375,33 +1271,6 @@ a PKCS#12 container includes trusted certificate ("anchor") entries look for
 `openssl pkcs12 -info` output, or `trustedCertEntry` in the
 `keytool -list` output.
 
-[float]
-===== PKCS#11 tokens
-
-When using a PKCS#11 cryptographic token, which contains the
-private key, certificate, and certificates that should be trusted, use
-the following settings:
-
-`xpack.ssl.keystore.type`::
-Set this to `PKCS11`.
-
-`xpack.ssl.truststore.type`::
-Set this to `PKCS11`.
-
-
-[[pkcs11-truststore-note]]
-[NOTE]
-When configuring the PKCS#11 token that your JVM is configured to use as
-a keystore or a truststore for Elasticsearch, the PIN for the token can be
-configured by setting the appropriate value to `xpack.ssl.truststore.password`
-or `xpack.ssl.truststore.secure_password`. In the absence of the above, {es} will
-fallback to use he appropriate JVM setting (`-Djavax.net.ssl.trustStorePassword`)
-if that is set.
-Since there can only be one PKCS#11 token configured, only one keystore and
-truststore will be usable for configuration in {es}. This in turn means
-that only one certificate can be used for TLS both in the transport and the
-http layer.
-
 [[http-tls-ssl-settings]]
 :ssl-prefix:             xpack.security.http
 :component:              HTTP
@@ -1435,7 +1304,7 @@ append the portion of the setting after `xpack.security.transport.`. For the key
 setting, this would be `transport.profiles.$PROFILE.xpack.security.ssl.key`.
 
 [[auditing-tls-ssl-settings]]
-:ssl-prefix:             xpack.security.audit.index.client.xpack
+:ssl-prefix:             xpack.security.audit.index.client.xpack.security.transport
 :component:              Auditing
 :client-auth-default!:
 :server!: