From 33cecb086a11af9ae064e257fe2a9569efff3db0 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Tue, 18 Jan 2022 18:12:03 +0200 Subject: [PATCH] Defer security auto-configuration on initial node startup (#82574) (#82733) The new security autoconfiguration in version 8 prints information to the terminal. This information is required in order to access the node (eg the generated elastic user password). But this contends for screen lines with the normal log output of a starting node, to the extent that the security autoconfiguration can pass by unnoticed by the user. This PR addresses this concern by: * making sure that auto-configuration is triggered only after the node is started (ClusterPlugin#onNodeStarted). Auto-configuration requires the http bind address, which is guaranteed to be available only after this point in time. * Additionally deferring the auto-configuration by a fixed time amount (9 sec), even after all the depended resources have finished initializing. --- .../InitialNodeSecurityAutoConfiguration.java | 170 ++++++++++-------- .../xpack/security/Security.java | 13 +- 2 files changed, 111 insertions(+), 72 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/InitialNodeSecurityAutoConfiguration.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/InitialNodeSecurityAutoConfiguration.java index 6b34999abca3f..b03d0d3c24481 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/InitialNodeSecurityAutoConfiguration.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/InitialNodeSecurityAutoConfiguration.java @@ -15,8 +15,10 @@ import org.elasticsearch.bootstrap.BootstrapInfo; import org.elasticsearch.client.Client; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.env.Environment; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ssl.SSLService; import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore; import org.elasticsearch.xpack.security.enrollment.BaseEnrollmentTokenGenerator; @@ -51,7 +53,9 @@ public static void maybeGenerateEnrollmentTokensAndElasticCredentialsOnNodeStart SecurityIndexManager securityIndexManager, SSLService sslService, Client client, - Environment environment + Environment environment, + OnNodeStartedListener onNodeStartedListener, + ThreadPool threadPool ) { // Assume the following auto-configuration must NOT run if enrollment is disabled when the node starts, // so no credentials or HTTPS CA fingerprint will be displayed in this case (in addition to no enrollment @@ -88,81 +92,101 @@ public static void maybeGenerateEnrollmentTokensAndElasticCredentialsOnNodeStart // is now a system index), it's not a catastrophic position to be in either, because it only entails // that new tokens and possibly credentials are generated anew // TODO maybe we can improve the check that this is indeed the initial node - String fingerprint; - try { - fingerprint = enrollmentTokenGenerator.getHttpsCaFingerprint(); - LOGGER.info( - "HTTPS has been configured with automatically generated certificates, " - + "and the CA's hex-encoded SHA-256 fingerprint is [" - + fingerprint - + "]" - ); - } catch (Exception e) { - fingerprint = null; - LOGGER.error("Failed to compute the HTTPS CA fingerprint, probably the certs are not auto-generated", e); - } - final String httpsCaFingerprint = fingerprint; - GroupedActionListener> groupedActionListener = new GroupedActionListener<>( - ActionListener.wrap(results -> { - final Map allResultsMap = new HashMap<>(); - for (Map result : results) { - allResultsMap.putAll(result); - } - final String elasticPassword = allResultsMap.get("generated_elastic_user_password"); - final String kibanaEnrollmentToken = allResultsMap.get("kibana_enrollment_token"); - final String nodeEnrollmentToken = allResultsMap.get("node_enrollment_token"); - outputInformationToConsole(elasticPassword, kibanaEnrollmentToken, nodeEnrollmentToken, httpsCaFingerprint, out); - }, e -> { LOGGER.error("Unexpected exception during security auto-configuration", e); }), - 3 - ); - // we only generate the elastic user password if the node has been auto-configured in a specific way, such that the first - // time a node starts it will form a cluster by itself and can hold the .security index (which we assume it is when - // {@code ENROLLMENT_ENABLED} is true), that the node process's output is a terminal and that the password is not - // specified already via the two secure settings - if (false == BOOTSTRAP_ELASTIC_PASSWORD.exists(environment.settings()) - && false == AUTOCONFIG_ELASTIC_PASSWORD_HASH.exists(environment.settings())) { - final char[] elasticPassword = generatePassword(20); - nativeUsersStore.createElasticUser(elasticPassword, ActionListener.wrap(aVoid -> { - LOGGER.debug("elastic credentials generated successfully"); - groupedActionListener.onResponse(Map.of("generated_elastic_user_password", new String(elasticPassword))); - }, e -> { - LOGGER.error("Failed to generate credentials for the elastic built-in superuser", e); - // null password in case of error - groupedActionListener.onResponse(Map.of()); - })); - } else { - if (false == BOOTSTRAP_ELASTIC_PASSWORD.exists(environment.settings())) { - LOGGER.info( - "Auto-configuration will not generate a password for the elastic built-in superuser, " - + "you should use the password specified in the node's secure setting [" - + BOOTSTRAP_ELASTIC_PASSWORD.getKey() - + "] in order to authenticate as elastic" - ); + // a lot of stuff runs when a node just started, and the autoconfiguration is not time-critical + // and nothing else depends on it; be a good sport and wait a couple + onNodeStartedListener.run(() -> threadPool.schedule(new AbstractRunnable() { + + @Override + public void onFailure(Exception e) { + LOGGER.error("Unexpected exception when auto configuring the initial node for Security", e); } - // empty password in case password generation is skipped - groupedActionListener.onResponse(Map.of("generated_elastic_user_password", "")); - } - final Iterator backoff = BACKOFF_POLICY.iterator(); - enrollmentTokenGenerator.createKibanaEnrollmentToken(kibanaToken -> { - if (kibanaToken != null) { + + @Override + protected void doRun() { + // the HTTP address is guaranteed to be bound only after the node started + String fingerprint; try { - LOGGER.debug("Successfully generated the kibana enrollment token"); - groupedActionListener.onResponse(Map.of("kibana_enrollment_token", kibanaToken.getEncoded())); + fingerprint = enrollmentTokenGenerator.getHttpsCaFingerprint(); + LOGGER.info( + "HTTPS has been configured with automatically generated certificates, " + + "and the CA's hex-encoded SHA-256 fingerprint is [" + + fingerprint + + "]" + ); } catch (Exception e) { - LOGGER.error("Failed to encode kibana enrollment token", e); - groupedActionListener.onResponse(Map.of()); + fingerprint = null; + LOGGER.error("Failed to compute the HTTPS CA fingerprint, probably the certs are not auto-generated", e); } - } else { - groupedActionListener.onResponse(Map.of()); - } - }, backoff); - enrollmentTokenGenerator.maybeCreateNodeEnrollmentToken(encodedNodeToken -> { - if (encodedNodeToken != null) { - groupedActionListener.onResponse(Map.of("node_enrollment_token", encodedNodeToken)); - } else { - groupedActionListener.onResponse(Map.of()); + final String httpsCaFingerprint = fingerprint; + GroupedActionListener> groupedActionListener = new GroupedActionListener<>( + ActionListener.wrap(results -> { + final Map allResultsMap = new HashMap<>(); + for (Map result : results) { + allResultsMap.putAll(result); + } + final String elasticPassword = allResultsMap.get("generated_elastic_user_password"); + final String kibanaEnrollmentToken = allResultsMap.get("kibana_enrollment_token"); + final String nodeEnrollmentToken = allResultsMap.get("node_enrollment_token"); + outputInformationToConsole( + elasticPassword, + kibanaEnrollmentToken, + nodeEnrollmentToken, + httpsCaFingerprint, + out + ); + }, e -> LOGGER.error("Unexpected exception during security auto-configuration", e)), + 3 + ); + // we only generate the elastic user password if the node has been auto-configured in a specific way, such that the + // first time a node starts it will form a cluster by itself and can hold the .security index (which we assume + // it is when {@code ENROLLMENT_ENABLED} is true), that the node process's output is a terminal and that the + // password is not specified already via the two secure settings + if (false == BOOTSTRAP_ELASTIC_PASSWORD.exists(environment.settings()) + && false == AUTOCONFIG_ELASTIC_PASSWORD_HASH.exists(environment.settings())) { + final char[] elasticPassword = generatePassword(20); + nativeUsersStore.createElasticUser(elasticPassword, ActionListener.wrap(aVoid -> { + LOGGER.debug("elastic credentials generated successfully"); + groupedActionListener.onResponse(Map.of("generated_elastic_user_password", new String(elasticPassword))); + }, e -> { + LOGGER.error("Failed to generate credentials for the elastic built-in superuser", e); + // null password in case of error + groupedActionListener.onResponse(Map.of()); + })); + } else { + if (false == BOOTSTRAP_ELASTIC_PASSWORD.exists(environment.settings())) { + LOGGER.info( + "Auto-configuration will not generate a password for the elastic built-in superuser, " + + "you should use the password specified in the node's secure setting [" + + BOOTSTRAP_ELASTIC_PASSWORD.getKey() + + "] in order to authenticate as elastic" + ); + } + // empty password in case password generation is skipped + groupedActionListener.onResponse(Map.of("generated_elastic_user_password", "")); + } + final Iterator backoff = BACKOFF_POLICY.iterator(); + enrollmentTokenGenerator.createKibanaEnrollmentToken(kibanaToken -> { + if (kibanaToken != null) { + try { + LOGGER.debug("Successfully generated the kibana enrollment token"); + groupedActionListener.onResponse(Map.of("kibana_enrollment_token", kibanaToken.getEncoded())); + } catch (Exception e) { + LOGGER.error("Failed to encode kibana enrollment token", e); + groupedActionListener.onResponse(Map.of()); + } + } else { + groupedActionListener.onResponse(Map.of()); + } + }, backoff); + enrollmentTokenGenerator.maybeCreateNodeEnrollmentToken(encodedNodeToken -> { + if (encodedNodeToken != null) { + groupedActionListener.onResponse(Map.of("node_enrollment_token", encodedNodeToken)); + } else { + groupedActionListener.onResponse(Map.of()); + } + }, backoff); } - }, backoff); + }, TimeValue.timeValueSeconds(9), ThreadPool.Names.GENERIC)); } }); } @@ -253,4 +277,8 @@ private static void outputInformationToConsole( builder.append(System.lineSeparator()); out.println(builder); } + + interface OnNodeStartedListener { + void run(Runnable runnable); + } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index f4858acc9cf46..12ab04fe2877b 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -41,6 +41,7 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.util.concurrent.ListenableFuture; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.Nullable; @@ -450,6 +451,8 @@ public class Security extends Plugin private final Settings settings; private final boolean enabled; + private final ListenableFuture nodeStartedListenable; + /* what a PITA that we need an extra indirection to initialize this. Yet, once we got rid of guice we can thing about how * to fix this or make it simpler. Today we need several service that are created in createComponents but we need to register * an instance of TransportInterceptor way earlier before createComponents is called. */ @@ -480,6 +483,7 @@ public Security(Settings settings, final Path configPath) { this.settings = settings; // TODO this is wrong, we should only use the environment that is provided to createComponents this.enabled = XPackSettings.SECURITY_ENABLED.get(settings); + this.nodeStartedListenable = new ListenableFuture<>(); if (enabled) { runStartupChecks(settings); Automatons.updateConfiguration(settings); @@ -756,7 +760,9 @@ Collection createComponents( securityIndex.get(), getSslService(), client, - environment + environment, + (runnable -> nodeStartedListenable.addListener(ActionListener.wrap(runnable))), + threadPool ); // to keep things simple, just invalidate all cached entries on license change. this happens so rarely that the impact should be @@ -1274,6 +1280,11 @@ public Map getProcessors(Processor.Parameters paramet ); } + @Override + public void onNodeStarted() { + this.nodeStartedListenable.onResponse(null); + } + /** * Realm settings were changed in 7.0. This method validates that the settings in use on this node match the new style of setting. * In 6.x a realm config would be