Warn users if security is implicitly disabled (#70114)

* Warn users if security is implicitly disabled

Elasticsearch has security features implicitly disabled by default for
Basic and Trial licenses, unless explicitly set in the configuration
file.
This may be good for onboarding, but it also lead to unintended insecure
 clusters.
 This change introduces clear warnings when security features are
 implicitly disabled.
 - a warning header in each REST response if security is implicitly
 disabled;
 - a log message during cluster boot.

distribution/docker/src/test/java/org/elasticsearch/docker/test/DockerYmlTestSuiteIT.java

@@ -10,7 +10,6 @@
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.client.Request;
-import org.elasticsearch.common.CharArrays;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
@@ -24,11 +23,8 @@
 
 import java.io.IOException;
 import java.net.URISyntaxException;
-import java.nio.CharBuffer;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.Arrays;
-import java.util.Base64;
 
 public class DockerYmlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -130,22 +126,4 @@ protected String getProtocol() {
         }
         return "https";
     }
-
-    private static String basicAuthHeaderValue(String username, SecureString passwd) {
-        CharBuffer chars = CharBuffer.allocate(username.length() + passwd.length() + 1);
-        byte[] charBytes = null;
-        try {
-            chars.put(username).put(':').put(passwd.getChars());
-            charBytes = CharArrays.toUtf8Bytes(chars.array());
-
-            //TODO we still have passwords in Strings in headers. Maybe we can look into using a CharSequence?
-            String basicToken = Base64.getEncoder().encodeToString(charBytes);
-            return "Basic " + basicToken;
-        } finally {
-            Arrays.fill(chars.array(), (char) 0);
-            if (charBytes != null) {
-                Arrays.fill(charBytes, (byte) 0);
-            }
-        }
-    }
 }

docs/build.gradle

@@ -68,6 +68,7 @@ testClusters.matching { it.name == "integTest"}.configureEach {
 
   // enable regexes in painless so our tests don't complain about example snippets that use them
   setting 'script.painless.regex.enabled', 'true'
+  setting 'xpack.security.enabled', 'false'
   setting 'path.repo', "${buildDir}/cluster/shared/repo"
   Closure configFile = {
     extraConfigFile it, file("src/test/cluster/config/$it")

docs/reference/rest-api/usage.asciidoc

@@ -48,15 +48,7 @@ GET /_xpack/usage
 {
   "security" : {
     "available" : true,
-    "enabled" : false,
-    "ssl" : {
-      "http" : {
-        "enabled" : false
-      },
-      "transport" : {
-        "enabled" : false
-      }
-    }
+    "enabled" : false
   },
   "monitoring" : {
     "available" : true,

modules/ingest-geoip/qa/file-based-update/build.gradle

@@ -12,6 +12,8 @@ apply plugin: 'elasticsearch.rest-test'
 testClusters.all {
   testDistribution = 'DEFAULT'
   setting 'resource.reload.interval.high', '100ms'
+  setting 'xpack.security.enabled', 'true'
+  user username: 'admin', password: 'admin-password', role: 'superuser'
 }
 
 tasks.named("integTest").configure {

modules/ingest-geoip/qa/file-based-update/src/test/java/org/elasticsearch/ingest/geoip/UpdateDatabasesIT.java

@@ -11,6 +11,9 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
@@ -53,4 +56,12 @@ private static Map<String, Object> toMap(Response response) throws IOException {
         return XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false);
     }
 
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue("admin", new SecureString("admin-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+
 }

plugins/examples/painless-whitelist/build.gradle

@@ -23,6 +23,7 @@ dependencies {
 
 testClusters.all {
   testDistribution = 'DEFAULT'
+  setting 'xpack.security.enabled', 'false'
 }
 
 tasks.named("test").configure { enabled = false }

qa/ccs-rolling-upgrade-remote-cluster/build.gradle

@@ -35,12 +35,14 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
       numberOfNodes = 2
       versions = [bwcVersionStr, project.version]
       setting 'cluster.remote.node.attr', 'gateway'
+      setting 'xpack.security.enabled', 'false'
     }
     "${baseName}-remote" {
       numberOfNodes = 3
       versions = [bwcVersionStr, project.version]
       firstNode.setting 'node.attr.gateway', 'true'
       lastNode.setting 'node.attr.gateway', 'true'
+      setting 'xpack.security.enabled', 'false'
     }
   }
 

qa/ccs-unavailable-clusters/build.gradle

@@ -10,6 +10,11 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 apply plugin: 'elasticsearch.test-with-dependencies'
 
+testClusters.matching { it.name == "integTest" }.configureEach {
+  setting 'xpack.security.enabled', 'true'
+  user username: 'admin', password: 'admin-password', role: 'superuser'
+}
+
 dependencies {
   testImplementation project(":client:rest-high-level")
 }

qa/ccs-unavailable-clusters/src/test/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java

@@ -38,7 +38,9 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.search.aggregations.InternalAggregations;
@@ -327,4 +329,12 @@ private HighLevelClient(RestClient restClient) {
             super(restClient, (client) -> {}, Collections.emptyList());
         }
     }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue("admin", new SecureString("admin-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
 }

qa/die-with-dignity/build.gradle

@@ -21,6 +21,8 @@ tasks.named("javaRestTest").configure {
 
 testClusters.matching { it.name == "javaRestTest" }.configureEach {
   systemProperty "die.with.dignity.test", "whatever"
+  setting 'xpack.security.enabled', 'true'
+  user username: 'admin', password: 'admin-password', role: 'superuser'
 }
 
 tasks.named("test").configure {

qa/die-with-dignity/src/javaRestTest/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java

@@ -10,7 +10,9 @@
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.rest.ESRestTestCase;
 
 import java.io.BufferedReader;
@@ -99,13 +101,14 @@ protected boolean preserveClusterUponCompletion() {
 
     @Override
     protected final Settings restClientSettings() {
+        String token = basicAuthHeaderValue("admin", new SecureString("admin-password".toCharArray()));
         return Settings.builder()
             .put(super.restClientSettings())
+            .put(ThreadContext.PREFIX + ".Authorization", token)
             // increase the timeout here to 90 seconds to handle long waits for a green
             // cluster health. the waits for green need to be longer than a minute to
             // account for delayed shards
             .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "1s")
             .build();
     }
-
 }

qa/full-cluster-restart/build.gradle

@@ -26,6 +26,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) {
       // some tests rely on the translog not being flushed
       setting 'indices.memory.shard_inactive_time', '60m'
       setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
+      setting 'xpack.security.enabled', 'false'
     }
   }
 
@@ -59,4 +60,4 @@ for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) {
   tasks.register(bwcTaskName(bwcVersion)) {
     dependsOn tasks.named("${baseName}#upgradedClusterTest")
   }
-}
+}

qa/logging-config/build.gradle

@@ -11,6 +11,10 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 apply plugin: 'elasticsearch.standalone-test'
 
+testClusters.all {
+  setting 'xpack.security.enabled', 'false'
+}
+
 testClusters.matching { it.name == "integTest" }.configureEach {
   /**
    * Provide a custom log4j configuration where layout is an old style pattern and confirm that Elasticsearch

qa/mixed-cluster/build.gradle

@@ -38,6 +38,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
       numberOfNodes = 4
 
       setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
+      setting 'xpack.security.enabled', 'false'
     }
   }
 

qa/multi-cluster-search/build.gradle

@@ -25,6 +25,7 @@ testClusters {
     'remote-cluster' {
         numberOfNodes = 2
         setting 'node.roles', '[data,ingest,master]'
+        setting 'xpack.security.enabled', 'false'
     }
 }
 
@@ -38,6 +39,7 @@ testClusters.matching { it.name == "mixedClusterTest"}.configureEach {
   setting 'cluster.remote.my_remote_cluster.seeds',
     { "\"${testClusters.'remote-cluster'.getAllTransportPortURI().get(0)}\"" }
   setting 'cluster.remote.connections_per_cluster', '1'
+  setting 'xpack.security.enabled', 'false'
 }
 
 tasks.register("integTest") {

qa/remote-clusters/src/test/java/org/elasticsearch/cluster/remote/test/AbstractMultiClusterRemoteTestCase.java

@@ -13,7 +13,6 @@
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestHighLevelClient;
-import org.elasticsearch.common.CharArrays;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
@@ -26,11 +25,8 @@
 
 import java.io.IOException;
 import java.net.URISyntaxException;
-import java.nio.CharBuffer;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.Arrays;
-import java.util.Base64;
 import java.util.Collections;
 
 public abstract class AbstractMultiClusterRemoteTestCase extends ESRestTestCase {
@@ -150,24 +146,6 @@ protected String getProtocol() {
         return "https";
     }
 
-    private static String basicAuthHeaderValue(String username, SecureString passwd) {
-        CharBuffer chars = CharBuffer.allocate(username.length() + passwd.length() + 1);
-        byte[] charBytes = null;
-        try {
-            chars.put(username).put(':').put(passwd.getChars());
-            charBytes = CharArrays.toUtf8Bytes(chars.array());
-
-            //TODO we still have passwords in Strings in headers. Maybe we can look into using a CharSequence?
-            String basicToken = Base64.getEncoder().encodeToString(charBytes);
-            return "Basic " + basicToken;
-        } finally {
-            Arrays.fill(chars.array(), (char) 0);
-            if (charBytes != null) {
-                Arrays.fill(charBytes, (byte) 0);
-            }
-        }
-    }
-
     private String getProperty(String key) {
         String value = System.getProperty(key);
         if (value == null) {

qa/repository-multi-version/build.gradle

@@ -30,6 +30,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) {
       version = v
       numberOfNodes = 2
       setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
+      setting 'xpack.security.enabled', 'false'
     }
   }
 
@@ -76,4 +77,4 @@ for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) {
   tasks.register(bwcTaskName(bwcVersion)) {
     dependsOn tasks.named("${baseName}#Step4NewClusterTest")
   }
-}
+}

qa/rolling-upgrade/build.gradle

@@ -38,6 +38,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
 
       setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
       setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
+      setting 'xpack.security.enabled', 'false'
     }
   }
 

qa/smoke-test-http/build.gradle

@@ -17,6 +17,10 @@ dependencies {
   testImplementation project(':plugins:transport-nio') // for http
 }
 
+testClusters.all {
+  setting 'xpack.security.enabled', 'false'
+}
+
 tasks.named("integTest").configure {
   /*
    * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each

qa/smoke-test-ingest-disabled/build.gradle

@@ -15,6 +15,10 @@ dependencies {
   testImplementation project(':modules:ingest-common')
 }
 
+testClusters.all {
+  setting 'xpack.security.enabled', 'false'
+}
+
 testClusters.matching { it.name == "integTest" }.configureEach {
   setting 'node.roles', '[data,master,remote_cluster_client]'
 }

qa/smoke-test-ingest-with-all-dependencies/build.gradle

@@ -19,6 +19,10 @@ dependencies {
   testImplementation project(':modules:reindex')
 }
 
+testClusters.all {
+  setting 'xpack.security.enabled', 'false'
+}
+
 tasks.named("testingConventions").configure {
   naming {
     IT {

qa/smoke-test-multinode/build.gradle

@@ -23,6 +23,10 @@ testClusters.matching { it.name == "integTest" }.configureEach {
   setting 'path.repo', repo.absolutePath
 }
 
+testClusters.all {
+  setting 'xpack.security.enabled', 'false'
+}
+
 tasks.named("integTest").configure {
   doFirst {
     project.delete(repo)

qa/smoke-test-plugins/build.gradle

@@ -27,6 +27,7 @@ testClusters.matching { it.name == "integTest" }.configureEach {
   pluginPaths.each { pluginPath ->
     plugin pluginPath
   }
+  setting 'xpack.security.enabled', 'false'
 }
 
 ext.expansions = [

qa/unconfigured-node-name/build.gradle

@@ -1,5 +1,3 @@
-import org.elasticsearch.gradle.OS
-
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
@@ -12,6 +10,10 @@ apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
+testClusters.all {
+  setting 'xpack.security.enabled', 'false'
+}
+
 testClusters.matching { it.name == "integTest" }.configureEach {
   nameCustomization = { null }
 }

qa/verify-version-constants/build.gradle

@@ -21,6 +21,8 @@ for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) {
   testClusters {
     "${baseName}" {
       version = bwcVersion.toString()
+      setting 'xpack.security.enabled', 'true'
+      user username: 'admin', password: 'admin-password', role: 'superuser'
     }
   }