Bring along updates from elastic#74890

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/enrollment/tool/CreateEnrollmentTokenTool.java

@@ -32,18 +32,23 @@ public class CreateEnrollmentTokenTool extends BaseRunAsSuperuserCommand {
     static final List<String> ALLOWED_SCOPES = List.of("node", "kibana");
 
     CreateEnrollmentTokenTool() {
-        this(environment -> new CommandLineHttpClient(environment),
+        this(
+            environment -> new CommandLineHttpClient(environment),
             environment -> KeyStoreWrapper.load(environment.configFile()),
-            environment -> new CreateEnrollmentToken(environment));
+            environment -> new CreateEnrollmentToken(environment)
+        );
     }
 
-    CreateEnrollmentTokenTool(Function<Environment, CommandLineHttpClient> clientFunction,
+    CreateEnrollmentTokenTool(
+        Function<Environment, CommandLineHttpClient> clientFunction,
         CheckedFunction<Environment, KeyStoreWrapper, Exception> keyStoreFunction,
-        CheckedFunction<Environment, CreateEnrollmentToken, Exception> createEnrollmentTokenFunction) {
-        super(clientFunction, keyStoreFunction);
+        CheckedFunction<Environment, CreateEnrollmentToken, Exception> createEnrollmentTokenFunction
+    ) {
+        super(clientFunction, keyStoreFunction, "Creates enrollment tokens for elasticsearch nodes and kibana instances");
         this.createEnrollmentTokenFunction = createEnrollmentTokenFunction;
-        scope = parser.acceptsAll(List.of("scope", "s"),  "The scope of this enrollment token, can be either \"node\" or \"kibana\"")
-            .withRequiredArg().required();
+        scope = parser.acceptsAll(List.of("scope", "s"), "The scope of this enrollment token, can be either \"node\" or \"kibana\"")
+            .withRequiredArg()
+            .required();
     }
 
     public static void main(String[] args) throws Exception {
@@ -52,13 +57,15 @@ public static void main(String[] args) throws Exception {
 
     @Override
     protected void validate(Terminal terminal, OptionSet options, Environment env) throws Exception {
-        if (XPackSettings.ENROLLMENT_ENABLED.get(env.settings()) == false){
-            throw new UserException(ExitCodes.CONFIG,
-                "[xpack.security.enrollment.enabled] must be set to `true` to create an enrollment token");
+        if (XPackSettings.ENROLLMENT_ENABLED.get(env.settings()) == false) {
+            throw new UserException(
+                ExitCodes.CONFIG,
+                "[xpack.security.enrollment.enabled] must be set to `true` to create an enrollment token"
+            );
         }
         final String tokenScope = scope.value(options);
         if (ALLOWED_SCOPES.contains(tokenScope) == false) {
-            terminal.errorPrintln("The scope of this enrollment token, can only be one of "+ ALLOWED_SCOPES);
+            terminal.errorPrintln("The scope of this enrollment token, can only be one of " + ALLOWED_SCOPES);
             throw new UserException(ExitCodes.USAGE, "Invalid scope");
         }
     }
@@ -75,7 +82,7 @@ protected void executeCommand(Terminal terminal, OptionSet options, Environment
                 terminal.println(createEnrollmentTokenService.createKibanaEnrollmentToken(username, password));
             }
         } catch (Exception e) {
-            terminal.errorPrintln("Unable to create enrollment token for scope [" + tokenScope +"]");
+            terminal.errorPrintln("Unable to create enrollment token for scope [" + tokenScope + "]");
             throw new UserException(ExitCodes.CANT_CREATE, e.getMessage(), e.getCause());
         }
     }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/tool/BaseRunAsSuperuserCommand.java

@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.security.tool;
 
 import joptsimple.OptionSet;
+import joptsimple.OptionSpecBuilder;
 
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.KeyStoreAwareCommand;
@@ -20,6 +21,9 @@
 import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.security.authc.RealmConfig;
+import org.elasticsearch.xpack.core.security.authc.RealmSettings;
+import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.security.authc.file.FileUserPasswdStore;
 import org.elasticsearch.xpack.security.authc.file.FileUserRolesStore;
@@ -35,9 +39,11 @@
 import java.security.SecureRandom;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
 /**
  * A {@link KeyStoreAwareCommand} that can be extended fpr any CLI tool that needs to allow a local user with
@@ -49,6 +55,7 @@ public abstract class BaseRunAsSuperuserCommand extends KeyStoreAwareCommand {
     private static final String[] ROLES = new String[] { "superuser" };
     private static final int PASSWORD_LENGTH = 14;
 
+    private final OptionSpecBuilder force;
     private final Function<Environment, CommandLineHttpClient> clientFunction;
     private final CheckedFunction<Environment, KeyStoreWrapper, Exception> keyStoreFunction;
     private SecureString password;
@@ -57,11 +64,14 @@ public abstract class BaseRunAsSuperuserCommand extends KeyStoreAwareCommand {
 
     public BaseRunAsSuperuserCommand(
         Function<Environment, CommandLineHttpClient> clientFunction,
-        CheckedFunction<Environment, KeyStoreWrapper, Exception> keyStoreFunction
+        CheckedFunction<Environment, KeyStoreWrapper, Exception> keyStoreFunction,
+        String description
     ) {
-        super("description");
+        super(description);
         this.clientFunction = clientFunction;
         this.keyStoreFunction = keyStoreFunction;
+        force = parser.acceptsAll(List.of("f", "force"),
+            "Use this option to force execution of the command against a cluster that is currently unhealthy.");
     }
 
     @Override
@@ -118,7 +128,8 @@ protected final void execute(Terminal terminal, OptionSet options, Environment e
             FileUserRolesStore.writeFile(userRoles, rolesFile);
 
             attributesChecker.check(terminal);
-            checkClusterHealthWithRetries(newEnv, terminal, 5);
+            final boolean forceExecution = options.has(force);
+            checkClusterHealthWithRetries(newEnv, terminal, 5, forceExecution);
             executeCommand(terminal, options, newEnv);
         } catch (Exception e) {
             int exitCode;
@@ -136,7 +147,7 @@ protected final void execute(Terminal terminal, OptionSet options, Environment e
     /**
      * Removes temporary file realm user from users and roles file
      */
-    protected void cleanup(Terminal terminal, Environment env) throws Exception {
+    private void cleanup(Terminal terminal, Environment env) throws Exception {
         final Path passwordFile = FileUserPasswdStore.resolveFile(env);
         final Path rolesFile = FileUserRolesStore.resolveFile(env);
         FileAttributesChecker attributesChecker = new FileAttributesChecker(passwordFile, rolesFile);
@@ -175,17 +186,22 @@ protected String getUsername() {
     }
 
     private void ensureFileRealmEnabled(Settings settings) throws Exception {
-        Map<String, Settings> fileRealmSettings = settings.getGroups("xpack.security.authc.realms.file");
+        final Map<RealmConfig.RealmIdentifier, Settings> realms = RealmSettings.getRealmSettings(settings);
+        Map<RealmConfig.RealmIdentifier, Settings> fileRealmSettings = realms.entrySet().stream()
+            .filter(e -> e.getKey().getType().equals(FileRealmSettings.TYPE))
+            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
         if (fileRealmSettings.size() > 1) {
             throw new UserException(
                 ExitCodes.CONFIG,
                 "Multiple file realms are configured. "
                     + "[file] is an internal realm type and therefore there can only be one such realm configured"
             );
-        } else if (fileRealmSettings.size() == 1
-            && fileRealmSettings.entrySet().stream().anyMatch(s -> s.getValue().get("enabled").equals("false"))) {
-                throw new UserException(ExitCodes.CONFIG, "File realm must be enabled");
-            }
+        } else if (fileRealmSettings.size() == 1) {
+            final String fileRealmName = fileRealmSettings.entrySet().iterator().next().getKey().getName();
+            if (RealmSettings.ENABLED_SETTING.apply(FileRealmSettings.TYPE)
+                .getConcreteSettingForNamespace(fileRealmName)
+                .get(settings) == false) throw new UserException(ExitCodes.CONFIG, "File realm must be enabled");
+        }
         // Else it's either explicitly enabled, or not defined in the settings so it is implicitly enabled.
     }
 
@@ -194,7 +210,7 @@ private void ensureFileRealmEnabled(Settings settings) throws Exception {
      * retries as the file realm might not have reloaded the users file yet in order to authenticate our
      * newly created file realm user.
      */
-    private void checkClusterHealthWithRetries(Environment env, Terminal terminal, int retries) throws Exception {
+    private void checkClusterHealthWithRetries(Environment env, Terminal terminal, int retries, boolean force) throws Exception {
         CommandLineHttpClient client = clientFunction.apply(env);
         final URL clusterHealthUrl = createURL(new URL(client.getDefaultURL()), "_cluster/health", "?pretty");
         final HttpResponse response;
@@ -214,7 +230,7 @@ private void checkClusterHealthWithRetries(Environment env, Terminal terminal, i
                 );
                 Thread.sleep(1000);
                 retries -= 1;
-                checkClusterHealthWithRetries(env, terminal, retries);
+                checkClusterHealthWithRetries(env, terminal, retries, force);
             } else {
                 throw new UserException(
                     ExitCodes.DATA_ERROR,
@@ -226,10 +242,21 @@ private void checkClusterHealthWithRetries(Environment env, Terminal terminal, i
             if (clusterStatus.isEmpty()) {
                 throw new UserException(
                     ExitCodes.DATA_ERROR,
-                    "Failed to determine the health of the cluster. Cluster health API did not return a status value"
+                    "Failed to determine the health of the cluster. Cluster health API did not return a status value."
                 );
-            } else if ("red".equalsIgnoreCase(clusterStatus)) {
-                throw new UserException(ExitCodes.UNAVAILABLE, "Cluster health is currently RED.");
+            } else if ("red".equalsIgnoreCase(clusterStatus) && force == false) {
+                terminal.errorPrintln("Failed to determine the health of the cluster. Cluster health is currently RED.");
+                terminal.errorPrintln("This means that some cluster data is unavailable and your cluster is not fully functional.");
+                terminal.errorPrintln("The cluster logs (https://www.elastic.co/guide/en/elasticsearch/reference/master/logging.html)" +
+                    "might contain information/indications for the underlying cause");
+                terminal.errorPrintln(
+                    "It is recommended that you resolve the issues with your cluster before continuing");
+                terminal.errorPrintln("It is very likely that the command will fail when run against an unhealthy cluster.");
+                terminal.errorPrintln("");
+                terminal.errorPrintln("If you still want to attempt to execute this command against an unhealthy cluster," +
+                    " you can pass the `-f` parameter.");
+                throw new UserException(ExitCodes.UNAVAILABLE,
+                    "Failed to determine the health of the cluster. Cluster health is currently RED.");
             }
             // else it is yellow or green so we can continue
         }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/tool/ResetElasticPasswordTool.java

@@ -46,7 +46,7 @@ public static void main(String[] args) throws Exception {
     protected ResetElasticPasswordTool(
         Function<Environment, CommandLineHttpClient> clientFunction,
         CheckedFunction<Environment, KeyStoreWrapper, Exception> keyStoreFunction) {
-        super(clientFunction, keyStoreFunction);
+        super(clientFunction, keyStoreFunction, "Resets the password or the elastic built-in user");
         interactive = parser.acceptsAll(List.of("i", "interactive"));
         auto = parser.acceptsAll(List.of("a", "auto")); // default
         batch = parser.acceptsAll(List.of("b", "batch"));

x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/security/enrollment/tool/BaseRunAsSuperuserCommandTests.java

@@ -56,6 +56,7 @@
 import static org.hamcrest.Matchers.emptyString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.stringContainsInOrder;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyString;
 import static org.mockito.Matchers.eq;
@@ -84,7 +85,7 @@ protected Environment createEnv(Map<String, String> settings) throws UserExcepti
     }
 
     @BeforeClass
-    public static void setupJimfs() throws IOException {
+    public static void setupJimfs() {
         String view = randomFrom("basic", "posix");
         Configuration conf = Configuration.unix().toBuilder().setAttributeViews(view).build();
         jimfs = Jimfs.newFileSystem(conf);
@@ -135,6 +136,10 @@ public static void closeJimfs() throws IOException {
 
     public void testSuccessfulCommand() throws Exception {
         execute();
+        assertThat(terminal.getOutput(), is(emptyString()));
+        assertThat(terminal.getErrorOutput(), is(emptyString()));
+        assertNoUsers();
+        assertNoUsersRoles();
     }
 
     public void testFailureWhenFileRealmIsDisabled() throws Exception {
@@ -176,6 +181,24 @@ public void testUnhealthyCluster() throws Exception {
         assertThat(e.exitCode, equalTo(ExitCodes.UNAVAILABLE));
         assertThat(e.getMessage(), containsString("RED"));
         assertThat(terminal.getOutput(), is(emptyString()));
+        String error = terminal.getErrorOutput();
+        assertThat(error, stringContainsInOrder("Failed to determine the health of the cluster. Cluster health is currently RED.",
+            "This means that some cluster data is unavailable and your cluster is not fully functional.",
+            "The cluster logs (https://www.elastic.co/guide/en/elasticsearch/reference/master/logging.html)" +
+                "might contain information/indications for the underlying cause"));
+        assertNoUsers();
+        assertNoUsersRoles();
+    }
+
+    public void testUnhealthyClusterWithForce() throws Exception {
+        URL url = new URL(client.getDefaultURL());
+        HttpResponse healthResponse =
+            new HttpResponse(HttpURLConnection.HTTP_OK, Map.of("status", randomFrom("red")));
+        when(client.execute(anyString(), eq(clusterHealthUrl(url)), anyString(), any(SecureString.class), any(CheckedSupplier.class),
+            any(CheckedFunction.class))).thenReturn(healthResponse);
+        execute("-f");
+        assertThat(terminal.getOutput(), is(emptyString()));
+        assertThat(terminal.getErrorOutput(), is(emptyString()));
         assertNoUsers();
         assertNoUsersRoles();
     }
@@ -249,7 +272,7 @@ static class DummyRunAsSuperuserCommand extends BaseRunAsSuperuserCommand {
             Function<Environment, CommandLineHttpClient> clientFunction,
             CheckedFunction<Environment, KeyStoreWrapper, Exception> keyStoreFunction
         ) {
-            super(clientFunction, keyStoreFunction);
+            super(clientFunction, keyStoreFunction, "dummy command");
         }
 
         @Override

x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/security/enrollment/tool/CreateEnrollmentTokenToolTests.java

@@ -161,6 +161,16 @@ public void testUnhealthyCluster() throws Exception {
         assertThat(e.getMessage(), containsString("RED"));
     }
 
+    public void testUnhealthyClusterWithForce() throws Exception {
+        String scope = randomBoolean() ? "node" : "kibana";
+        String output = execute("--scope", scope);
+        if (scope.equals("kibana")) {
+            assertThat(output, containsString("WU5OlRHaHF5UU9UVENhUEJpOVZQak1iOWcifQ=="));
+        } else {
+            assertThat(output, containsString("25FOnRkdUgzTmNTVHNTOGN0c3AwaWNUeEEifQ=="));
+        }
+    }
+
     public void testEnrollmentDisabled() throws Exception {
         settings = Settings.builder()
             .put(settings)

x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/security/enrollment/tool/ResetElasticPasswordToolTests.java

@@ -191,6 +191,21 @@ public void testFailureUnableToChangePassword() throws Exception {
         assertThat(output, containsString("The password will be printed in the console."));
     }
 
+    public void testFailureClusterUnhealthyWithForce() throws Exception {
+        terminal.addTextInput("y");
+        final URL url = new URL(client.getDefaultURL());
+        HttpResponse healthResponse =
+            new HttpResponse(HttpURLConnection.HTTP_OK, Map.of("status", randomFrom("red")));
+        when(client.execute(anyString(), eq(clusterHealthUrl(url)), anyString(), any(SecureString.class), any(CheckedSupplier.class),
+            any(CheckedFunction.class))).thenReturn(healthResponse);
+        execute("-a", randomFrom("-f", "--force"));
+        String output = terminal.getOutput();
+        assertThat(output, containsString("This tool will reset the password of the [elastic] user to an autogenerated value."));
+        assertThat(output, containsString("The password will be printed in the console."));
+        assertThat(output, containsString("Password for the elastic user successfully reset."));
+        assertThat(output, containsString("New value:"));
+    }
+
     public void testAutoInteractiveModesMutuallyExclusive() throws Exception {
         UserException e = expectThrows(UserException.class, () -> execute(randomFrom("-i", "--interactive"), randomFrom("-a", "--auto")));
         assertThat(e.exitCode, equalTo(ExitCodes.USAGE));