From 05ddae809c8e98714353bf76ed38695293c4a09e Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Tue, 18 Jan 2022 12:45:43 +0200 Subject: [PATCH] Defer security auto-configuration on initial node startup (#82574) The new security autoconfiguration in version 8 prints information to the terminal. This information is required in order to access the node (eg the generated elastic user password). But this contends for screen lines with the normal log output of a starting node, to the extent that the security autoconfiguration can pass by unnoticed by the user. This PR addresses this concern by: * making sure that auto-configuration is triggered only after the node is started (ClusterPlugin#onNodeStarted). Auto-configuration requires the http bind address, which is guaranteed to be available only after this point in time. * Additionally deferring the auto-configuration by a fixed time amount (9 sec), even after all the depended resources have finished initializing. --- .../InitialNodeSecurityAutoConfiguration.java | 170 ++++++++++-------- .../xpack/security/Security.java | 12 +- 2 files changed, 110 insertions(+), 72 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/InitialNodeSecurityAutoConfiguration.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/InitialNodeSecurityAutoConfiguration.java index 903ab07aa13b4..5a2284ed21830 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/InitialNodeSecurityAutoConfiguration.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/InitialNodeSecurityAutoConfiguration.java @@ -15,8 +15,10 @@ import org.elasticsearch.bootstrap.BootstrapInfo; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.env.Environment; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ssl.SSLService; import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore; import org.elasticsearch.xpack.security.enrollment.BaseEnrollmentTokenGenerator; @@ -51,7 +53,9 @@ public static void maybeGenerateEnrollmentTokensAndElasticCredentialsOnNodeStart SecurityIndexManager securityIndexManager, SSLService sslService, Client client, - Environment environment + Environment environment, + OnNodeStartedListener onNodeStartedListener, + ThreadPool threadPool ) { // Assume the following auto-configuration must NOT run if enrollment is disabled when the node starts, // so no credentials or HTTPS CA fingerprint will be displayed in this case (in addition to no enrollment @@ -88,81 +92,101 @@ public static void maybeGenerateEnrollmentTokensAndElasticCredentialsOnNodeStart // is now a system index), it's not a catastrophic position to be in either, because it only entails // that new tokens and possibly credentials are generated anew // TODO maybe we can improve the check that this is indeed the initial node - String fingerprint; - try { - fingerprint = enrollmentTokenGenerator.getHttpsCaFingerprint(); - LOGGER.info( - "HTTPS has been configured with automatically generated certificates, " - + "and the CA's hex-encoded SHA-256 fingerprint is [" - + fingerprint - + "]" - ); - } catch (Exception e) { - fingerprint = null; - LOGGER.error("Failed to compute the HTTPS CA fingerprint, probably the certs are not auto-generated", e); - } - final String httpsCaFingerprint = fingerprint; - GroupedActionListener> groupedActionListener = new GroupedActionListener<>( - ActionListener.wrap(results -> { - final Map allResultsMap = new HashMap<>(); - for (Map result : results) { - allResultsMap.putAll(result); - } - final String elasticPassword = allResultsMap.get("generated_elastic_user_password"); - final String kibanaEnrollmentToken = allResultsMap.get("kibana_enrollment_token"); - final String nodeEnrollmentToken = allResultsMap.get("node_enrollment_token"); - outputInformationToConsole(elasticPassword, kibanaEnrollmentToken, nodeEnrollmentToken, httpsCaFingerprint, out); - }, e -> { LOGGER.error("Unexpected exception during security auto-configuration", e); }), - 3 - ); - // we only generate the elastic user password if the node has been auto-configured in a specific way, such that the first - // time a node starts it will form a cluster by itself and can hold the .security index (which we assume it is when - // {@code ENROLLMENT_ENABLED} is true), that the node process's output is a terminal and that the password is not - // specified already via the two secure settings - if (false == BOOTSTRAP_ELASTIC_PASSWORD.exists(environment.settings()) - && false == AUTOCONFIG_ELASTIC_PASSWORD_HASH.exists(environment.settings())) { - final char[] elasticPassword = generatePassword(20); - nativeUsersStore.createElasticUser(elasticPassword, ActionListener.wrap(aVoid -> { - LOGGER.debug("elastic credentials generated successfully"); - groupedActionListener.onResponse(Map.of("generated_elastic_user_password", new String(elasticPassword))); - }, e -> { - LOGGER.error("Failed to generate credentials for the elastic built-in superuser", e); - // null password in case of error - groupedActionListener.onResponse(Map.of()); - })); - } else { - if (false == BOOTSTRAP_ELASTIC_PASSWORD.exists(environment.settings())) { - LOGGER.info( - "Auto-configuration will not generate a password for the elastic built-in superuser, " - + "you should use the password specified in the node's secure setting [" - + BOOTSTRAP_ELASTIC_PASSWORD.getKey() - + "] in order to authenticate as elastic" - ); + // a lot of stuff runs when a node just started, and the autoconfiguration is not time-critical + // and nothing else depends on it; be a good sport and wait a couple + onNodeStartedListener.run(() -> threadPool.schedule(new AbstractRunnable() { + + @Override + public void onFailure(Exception e) { + LOGGER.error("Unexpected exception when auto configuring the initial node for Security", e); } - // empty password in case password generation is skipped - groupedActionListener.onResponse(Map.of("generated_elastic_user_password", "")); - } - final Iterator backoff = BACKOFF_POLICY.iterator(); - enrollmentTokenGenerator.createKibanaEnrollmentToken(kibanaToken -> { - if (kibanaToken != null) { + + @Override + protected void doRun() { + // the HTTP address is guaranteed to be bound only after the node started + String fingerprint; try { - LOGGER.debug("Successfully generated the kibana enrollment token"); - groupedActionListener.onResponse(Map.of("kibana_enrollment_token", kibanaToken.getEncoded())); + fingerprint = enrollmentTokenGenerator.getHttpsCaFingerprint(); + LOGGER.info( + "HTTPS has been configured with automatically generated certificates, " + + "and the CA's hex-encoded SHA-256 fingerprint is [" + + fingerprint + + "]" + ); } catch (Exception e) { - LOGGER.error("Failed to encode kibana enrollment token", e); - groupedActionListener.onResponse(Map.of()); + fingerprint = null; + LOGGER.error("Failed to compute the HTTPS CA fingerprint, probably the certs are not auto-generated", e); } - } else { - groupedActionListener.onResponse(Map.of()); - } - }, backoff); - enrollmentTokenGenerator.maybeCreateNodeEnrollmentToken(encodedNodeToken -> { - if (encodedNodeToken != null) { - groupedActionListener.onResponse(Map.of("node_enrollment_token", encodedNodeToken)); - } else { - groupedActionListener.onResponse(Map.of()); + final String httpsCaFingerprint = fingerprint; + GroupedActionListener> groupedActionListener = new GroupedActionListener<>( + ActionListener.wrap(results -> { + final Map allResultsMap = new HashMap<>(); + for (Map result : results) { + allResultsMap.putAll(result); + } + final String elasticPassword = allResultsMap.get("generated_elastic_user_password"); + final String kibanaEnrollmentToken = allResultsMap.get("kibana_enrollment_token"); + final String nodeEnrollmentToken = allResultsMap.get("node_enrollment_token"); + outputInformationToConsole( + elasticPassword, + kibanaEnrollmentToken, + nodeEnrollmentToken, + httpsCaFingerprint, + out + ); + }, e -> LOGGER.error("Unexpected exception during security auto-configuration", e)), + 3 + ); + // we only generate the elastic user password if the node has been auto-configured in a specific way, such that the + // first time a node starts it will form a cluster by itself and can hold the .security index (which we assume + // it is when {@code ENROLLMENT_ENABLED} is true), that the node process's output is a terminal and that the + // password is not specified already via the two secure settings + if (false == BOOTSTRAP_ELASTIC_PASSWORD.exists(environment.settings()) + && false == AUTOCONFIG_ELASTIC_PASSWORD_HASH.exists(environment.settings())) { + final char[] elasticPassword = generatePassword(20); + nativeUsersStore.createElasticUser(elasticPassword, ActionListener.wrap(aVoid -> { + LOGGER.debug("elastic credentials generated successfully"); + groupedActionListener.onResponse(Map.of("generated_elastic_user_password", new String(elasticPassword))); + }, e -> { + LOGGER.error("Failed to generate credentials for the elastic built-in superuser", e); + // null password in case of error + groupedActionListener.onResponse(Map.of()); + })); + } else { + if (false == BOOTSTRAP_ELASTIC_PASSWORD.exists(environment.settings())) { + LOGGER.info( + "Auto-configuration will not generate a password for the elastic built-in superuser, " + + "you should use the password specified in the node's secure setting [" + + BOOTSTRAP_ELASTIC_PASSWORD.getKey() + + "] in order to authenticate as elastic" + ); + } + // empty password in case password generation is skipped + groupedActionListener.onResponse(Map.of("generated_elastic_user_password", "")); + } + final Iterator backoff = BACKOFF_POLICY.iterator(); + enrollmentTokenGenerator.createKibanaEnrollmentToken(kibanaToken -> { + if (kibanaToken != null) { + try { + LOGGER.debug("Successfully generated the kibana enrollment token"); + groupedActionListener.onResponse(Map.of("kibana_enrollment_token", kibanaToken.getEncoded())); + } catch (Exception e) { + LOGGER.error("Failed to encode kibana enrollment token", e); + groupedActionListener.onResponse(Map.of()); + } + } else { + groupedActionListener.onResponse(Map.of()); + } + }, backoff); + enrollmentTokenGenerator.maybeCreateNodeEnrollmentToken(encodedNodeToken -> { + if (encodedNodeToken != null) { + groupedActionListener.onResponse(Map.of("node_enrollment_token", encodedNodeToken)); + } else { + groupedActionListener.onResponse(Map.of()); + } + }, backoff); } - }, backoff); + }, TimeValue.timeValueSeconds(9), ThreadPool.Names.GENERIC)); } }); } @@ -253,4 +277,8 @@ private static void outputInformationToConsole( builder.append(System.lineSeparator()); out.println(builder); } + + interface OnNodeStartedListener { + void run(Runnable runnable); + } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index f947c668f1d24..bce044fa01d84 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -40,6 +40,7 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.util.concurrent.ListenableFuture; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.Nullable; @@ -443,6 +444,7 @@ public class Security extends Plugin private final Settings settings; private final boolean enabled; private final SecuritySystemIndices systemIndices; + private final ListenableFuture nodeStartedListenable; /* what a PITA that we need an extra indirection to initialize this. Yet, once we got rid of guice we can thing about how * to fix this or make it simpler. Today we need several service that are created in createComponents but we need to register @@ -475,6 +477,7 @@ public Security(Settings settings, final Path configPath) { // TODO this is wrong, we should only use the environment that is provided to createComponents this.enabled = XPackSettings.SECURITY_ENABLED.get(settings); this.systemIndices = new SecuritySystemIndices(); + this.nodeStartedListenable = new ListenableFuture<>(); if (enabled) { runStartupChecks(settings); Automatons.updateConfiguration(settings); @@ -765,7 +768,9 @@ Collection createComponents( systemIndices.getMainIndexManager(), getSslService(), client, - environment + environment, + (runnable -> nodeStartedListenable.addListener(ActionListener.wrap(runnable))), + threadPool ); // to keep things simple, just invalidate all cached entries on license change. this happens so rarely that the impact should be @@ -1287,6 +1292,11 @@ public Map getProcessors(Processor.Parameters paramet ); } + @Override + public void onNodeStarted() { + this.nodeStartedListenable.onResponse(null); + } + /** * Realm settings were changed in 7.0. This method validates that the settings in use on this node match the new style of setting. * In 6.x a realm config would be