Deprecate the behaviour of implicitly disabling file/native realm

docs/reference/migration/index.asciidoc

@@ -28,6 +28,7 @@ For more information about {minor-version},
 see the <<release-highlights>> and <<es-release-notes>>.
 For information about how to upgrade your cluster, see <<setup-upgrade>>.
 
+* <<breaking-changes-7.13,Migrating to 7.13>>
 * <<breaking-changes-7.12,Migrating to 7.12>>
 * <<breaking-changes-7.11,Migrating to 7.11>>
 * <<breaking-changes-7.10,Migrating to 7.10>>
@@ -44,6 +45,7 @@ For information about how to upgrade your cluster, see <<setup-upgrade>>.
 
 --
 
+include::migrate_7_13.asciidoc[]
 include::migrate_7_12.asciidoc[]
 include::migrate_7_11.asciidoc[]
 include::migrate_7_10.asciidoc[]

docs/reference/migration/migrate_7_13.asciidoc

@@ -0,0 +1,60 @@
+[[migrating-7.13]]
+== Migrating to 7.13
+++++
+<titleabbrev>7.13</titleabbrev>
+++++
+
+This section discusses the changes that you need to be aware of when migrating
+your application to {es} 7.13.
+
+See also <<release-highlights>> and <<es-release-notes>>.
+
+// * <<breaking_713_blah_changes>>
+// * <<breaking_713_blah_changes>>
+
+//NOTE: The notable-breaking-changes tagged regions are re-used in the
+//Installation and Upgrade Guide
+
+//tag::notable-breaking-changes[]
+
+[discrete]
+[[breaking-changes-7.13]]
+=== Breaking changes
+
+The following changes in {es} 7.13 might affect your applications
+and prevent them from operating normally.
+Before upgrading to 7.13, review these changes and take the described steps
+to mitigate the impact.
+
+NOTE: Breaking changes introduced in minor versions are
+normally limited to security and bug fixes.
+Significant changes in behavior are deprecated in a minor release and
+the old behavior is supported until the next major release.
+To find out if you are using any deprecated functionality,
+enable <<deprecation-logging, deprecation logging>>.
+
+
+[discrete]
+[[deprecated-7.13]]
+=== Deprecations
+
+The following functionality has been deprecated in {es} 7.13
+and will be removed in 8.0
+While this won't have an immediate impact on your applications,
+we strongly encourage you take the described steps to update your code
+after upgrading to 7.13.
+
+NOTE: Significant changes in behavior are deprecated in a minor release and
+the old behavior is supported until the next major release.
+To find out if you are using any deprecated functionality,
+enable <<deprecation-logging, deprecation logging>>.
+
+[discrete]
+[[breaking_713_security_changes]]
+==== Security deprecations
+
+[[implicitly-disabled-native-realms]]
+Currently, if native and file realms are not configured, they are implicitly disabled if there
+are other explicitly configured realms. This behaviour is deprecated.
+In version 8.0.0, the native and file realms will always be enabled unless explicitly
+disabled.

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RealmSettings.java

@@ -30,9 +30,10 @@
 public class RealmSettings {
 
     public static final String PREFIX = "xpack.security.authc.realms.";
+    public static final String ENABLED_SETTING_KEY = "enabled";
     public static final String ORDER_SETTING_KEY = "order";
 
-    public static final Function<String, Setting.AffixSetting<Boolean>> ENABLED_SETTING = affixSetting("enabled",
+    public static final Function<String, Setting.AffixSetting<Boolean>> ENABLED_SETTING = affixSetting(ENABLED_SETTING_KEY,
             key -> Setting.boolSetting(key, true, Setting.Property.NodeScope));
     public static final Function<String, Setting.AffixSetting<Integer>> ORDER_SETTING = affixSetting(ORDER_SETTING_KEY,
             key -> Setting.intSetting(key, Integer.MAX_VALUE, Setting.Property.NodeScope));

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java

@@ -47,6 +47,7 @@ private DeprecationChecks() {
             NodeDeprecationChecks::checkProcessors,
             NodeDeprecationChecks::checkMissingRealmOrders,
             NodeDeprecationChecks::checkUniqueRealmOrders,
+            NodeDeprecationChecks::checkImplicitlyDisabledNativeRealms,
             (settings, pluginsAndModules) -> NodeDeprecationChecks.checkThreadPoolListenerQueueSize(settings),
             (settings, pluginsAndModules) -> NodeDeprecationChecks.checkThreadPoolListenerSize(settings),
             NodeDeprecationChecks::checkClusterRemoteConnectSetting,

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java

@@ -9,6 +9,7 @@
 
 import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules;
 import org.elasticsearch.bootstrap.JavaVersion;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -19,8 +20,12 @@
 import org.elasticsearch.threadpool.FixedExecutorBuilder;
 import org.elasticsearch.transport.RemoteClusterService;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
+import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.RealmSettings;
+import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings;
+import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -52,6 +57,7 @@ static DeprecationIssue checkMissingRealmOrders(final Settings settings, final P
         final Set<String> orderNotConfiguredRealms = RealmSettings.getRealmSettings(settings).entrySet()
                 .stream()
                 .filter(e -> false == e.getValue().hasValue(RealmSettings.ORDER_SETTING_KEY))
+                .filter(e -> e.getValue().getAsBoolean(RealmSettings.ENABLED_SETTING_KEY, true))
                 .map(e -> RealmSettings.realmSettingPrefix(e.getKey()) + RealmSettings.ORDER_SETTING_KEY)
                 .collect(Collectors.toSet());
 
@@ -104,6 +110,40 @@ static DeprecationIssue checkUniqueRealmOrders(final Settings settings, final Pl
         );
     }
 
+    static DeprecationIssue checkImplicitlyDisabledNativeRealms(final Settings settings, final PluginsAndModules pluginsAndModules) {
+        final Map<RealmConfig.RealmIdentifier, Settings> realmSettings = RealmSettings.getRealmSettings(settings);
+        if (realmSettings.isEmpty()) {
+            return null;
+        }
+        // If all configured realms are disabled, this equals to no realm is configured. The implicitly behaviour in this case
+        // is to add file and native realms. So we are good here.
+        if (false == realmSettings.entrySet().stream().anyMatch(
+            e -> e.getValue().getAsBoolean(RealmSettings.ENABLED_SETTING_KEY, true))) {
+            return null;
+        }
+        final List<String> implicitlyDisabledNativeRealmTypes =
+            new ArrayList<>(org.elasticsearch.common.collect.List.of(FileRealmSettings.TYPE, NativeRealmSettings.TYPE));
+        realmSettings.keySet().forEach(ri -> implicitlyDisabledNativeRealmTypes.remove(ri.getType()));
+        if (implicitlyDisabledNativeRealmTypes.isEmpty()) {
+            return null;
+        }
+
+        final String details = String.format(
+            Locale.ROOT,
+            "Found implicitly disabled native %s: [%s]. %s disabled because there are other explicitly configured realms." +
+                "In next major release, native realms will always be enabled unless explicitly disabled.",
+            implicitlyDisabledNativeRealmTypes.size() == 1 ? "realm" : "realms",
+            Strings.collectionToDelimitedString(implicitlyDisabledNativeRealmTypes, ","),
+            implicitlyDisabledNativeRealmTypes.size() == 1 ? "It is" : "They are");
+
+        return new DeprecationIssue(
+            DeprecationIssue.Level.CRITICAL,
+            "File and/or native realms cannot be implicitly disabled in next major release.",
+            "https://www.elastic.co/guide/en/elasticsearch/reference/7.13/deprecated-7.13.html#implicitly-disabled-native-realms",
+            details
+        );
+    }
+
     static DeprecationIssue checkThreadPoolListenerQueueSize(final Settings settings) {
         return checkThreadPoolListenerSetting("thread_pool.listener.queue_size", settings);
     }

x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java

@@ -98,11 +98,13 @@ public void testCheckProcessors() {
 
     public void testCheckMissingRealmOrders() {
         final RealmConfig.RealmIdentifier invalidRealm =
-            new RealmConfig.RealmIdentifier(randomAlphaOfLengthBetween(4, 12), randomAlphaOfLengthBetween(4, 12));
+            new RealmConfig.RealmIdentifier(randomRealmTypeOtherThanFileOrNative(), randomAlphaOfLengthBetween(4, 12));
         final RealmConfig.RealmIdentifier validRealm =
-            new RealmConfig.RealmIdentifier(randomAlphaOfLengthBetween(4, 12), randomAlphaOfLengthBetween(4, 12));
+            new RealmConfig.RealmIdentifier(randomRealmTypeOtherThanFileOrNative(), randomAlphaOfLengthBetween(4, 12));
         final Settings settings =
             Settings.builder()
+                .put("xpack.security.authc.realms.file.default_file.enabled", false)
+                .put("xpack.security.authc.realms.native.default_native.enabled", false)
                 .put("xpack.security.authc.realms." + invalidRealm.getType() + "." + invalidRealm.getName() + ".enabled", "true")
                 .put("xpack.security.authc.realms." + validRealm.getType() + "." + validRealm.getName() + ".order", randomInt())
                 .build();
@@ -123,16 +125,30 @@ public void testCheckMissingRealmOrders() {
         ), deprecationIssues.get(0));
     }
 
+    public void testRealmOrderIsNotRequiredIfRealmIsDisabled() {
+        final RealmConfig.RealmIdentifier realmIdentifier =
+            new RealmConfig.RealmIdentifier(randomAlphaOfLengthBetween(4, 12), randomAlphaOfLengthBetween(4, 12));
+        final Settings settings =
+            Settings.builder()
+                .put("xpack.security.authc.realms." + realmIdentifier.getType() + "." + realmIdentifier.getName() + ".enabled", "false")
+            .build();
+        final PluginsAndModules pluginsAndModules = new PluginsAndModules(Collections.emptyList(), Collections.emptyList());
+        final List<DeprecationIssue> deprecationIssues = getDeprecationIssues(settings, pluginsAndModules);
+        assertTrue(deprecationIssues.isEmpty());
+    }
+
     public void testCheckUniqueRealmOrders() {
         final int order = randomInt(9999);
 
         final RealmConfig.RealmIdentifier invalidRealm1 =
-            new RealmConfig.RealmIdentifier(randomAlphaOfLengthBetween(4, 12), randomAlphaOfLengthBetween(4, 12));
+            new RealmConfig.RealmIdentifier(randomRealmTypeOtherThanFileOrNative(), randomAlphaOfLengthBetween(4, 12));
         final RealmConfig.RealmIdentifier invalidRealm2 =
-            new RealmConfig.RealmIdentifier(randomAlphaOfLengthBetween(4, 12), randomAlphaOfLengthBetween(4, 12));
+            new RealmConfig.RealmIdentifier(randomRealmTypeOtherThanFileOrNative(), randomAlphaOfLengthBetween(4, 12));
         final RealmConfig.RealmIdentifier validRealm =
-            new RealmConfig.RealmIdentifier(randomAlphaOfLengthBetween(4, 12), randomAlphaOfLengthBetween(4, 12));
+            new RealmConfig.RealmIdentifier(randomRealmTypeOtherThanFileOrNative(), randomAlphaOfLengthBetween(4, 12));
         final Settings settings = Settings.builder()
+            .put("xpack.security.authc.realms.file.default_file.enabled", false)
+            .put("xpack.security.authc.realms.native.default_native.enabled", false)
             .put("xpack.security.authc.realms."
                 + invalidRealm1.getType() + "." + invalidRealm1.getName() + ".order", order)
             .put("xpack.security.authc.realms."
@@ -159,16 +175,107 @@ public void testCheckUniqueRealmOrders() {
     public void testCorrectRealmOrders() {
         final int order = randomInt(9999);
         final Settings settings = Settings.builder()
+            .put("xpack.security.authc.realms.file.default_file.enabled", false)
+            .put("xpack.security.authc.realms.native.default_native.enabled", false)
             .put("xpack.security.authc.realms."
-                + randomAlphaOfLengthBetween(4, 12) + "." + randomAlphaOfLengthBetween(4, 12) + ".order", order)
+                + randomRealmTypeOtherThanFileOrNative() + "." + randomAlphaOfLengthBetween(4, 12) + ".order", order)
             .put("xpack.security.authc.realms."
-                + randomAlphaOfLengthBetween(4, 12) + "." + randomAlphaOfLengthBetween(4, 12) + ".order", order + 1)
+                + randomRealmTypeOtherThanFileOrNative() + "." + randomAlphaOfLengthBetween(4, 12) + ".order", order + 1)
             .build();
 
         final PluginsAndModules pluginsAndModules = new PluginsAndModules(Collections.emptyList(), Collections.emptyList());
         final List<DeprecationIssue> deprecationIssues = getDeprecationIssues(settings, pluginsAndModules);
 
-        assertEquals(0, deprecationIssues.size());
+        assertTrue(deprecationIssues.isEmpty());
+    }
+
+    public void testCheckImplicitlyDisabledNativeRealms() {
+        final Settings.Builder builder = Settings.builder();
+
+        final boolean otherRealmConfigured = randomBoolean();
+        final boolean otherRealmEnabled = randomBoolean();
+        if (otherRealmConfigured) {
+            final int otherRealmId = randomIntBetween(0, 9);
+            final String otherRealmName = randomAlphaOfLengthBetween(4, 12);
+            if (otherRealmEnabled) {
+                builder.put("xpack.security.authc.realms.type_" + otherRealmId + ".realm_" + otherRealmName + ".order", 1);
+            } else {
+                builder.put("xpack.security.authc.realms.type_" + otherRealmId + ".realm_" + otherRealmName + ".enabled", false);
+            }
+        }
+        final boolean fileRealmConfigured = randomBoolean();
+        final boolean fileRealmEnabled = randomBoolean();
+        if (fileRealmConfigured) {
+            final String fileRealmName = randomAlphaOfLengthBetween(4, 12);
+            // Configure file realm or explicitly disable it
+            if (fileRealmEnabled) {
+                builder.put("xpack.security.authc.realms.file." + fileRealmName + ".order", 10);
+            } else {
+                builder.put("xpack.security.authc.realms.file." + fileRealmName + ".enabled", false);
+            }
+        }
+        final boolean nativeRealmConfigured = randomBoolean();
+        final boolean nativeRealmEnabled = randomBoolean();
+        if (nativeRealmConfigured) {
+            final String nativeRealmName = randomAlphaOfLengthBetween(4, 12);
+            // Configure native realm or explicitly disable it
+            if (nativeRealmEnabled) {
+                builder.put("xpack.security.authc.realms.native." + nativeRealmName + ".order", 20);
+            } else {
+                builder.put("xpack.security.authc.realms.native." + nativeRealmName + ".enabled", false);
+            }
+        }
+        final Settings settings = builder.build();
+        final PluginsAndModules pluginsAndModules = new PluginsAndModules(Collections.emptyList(), Collections.emptyList());
+        final List<DeprecationIssue> deprecationIssues = getDeprecationIssues(settings, pluginsAndModules);
+
+        if (otherRealmConfigured && otherRealmEnabled) {
+            if (false == fileRealmConfigured && false == nativeRealmConfigured) {
+                assertCommonImplicitDisabledRealms(deprecationIssues);
+                assertEquals("Found implicitly disabled native realms: [file,native]. " +
+                        "They are disabled because there are other explicitly configured realms." +
+                        "In next major release, native realms will always be enabled unless explicitly disabled.",
+                    deprecationIssues.get(0).getDetails());
+            } else if (false == fileRealmConfigured) {
+                assertCommonImplicitDisabledRealms(deprecationIssues);
+                assertEquals("Found implicitly disabled native realm: [file]. " +
+                        "It is disabled because there are other explicitly configured realms." +
+                        "In next major release, native realms will always be enabled unless explicitly disabled.",
+                    deprecationIssues.get(0).getDetails());
+            } else if (false == nativeRealmConfigured) {
+                assertCommonImplicitDisabledRealms(deprecationIssues);
+                assertEquals("Found implicitly disabled native realm: [native]. " +
+                        "It is disabled because there are other explicitly configured realms." +
+                        "In next major release, native realms will always be enabled unless explicitly disabled.",
+                    deprecationIssues.get(0).getDetails());
+            } else {
+                assertTrue(deprecationIssues.isEmpty());
+            }
+        } else {
+            if (false == fileRealmConfigured && false == nativeRealmConfigured) {
+                assertTrue(deprecationIssues.isEmpty());
+            } else if (false == fileRealmConfigured) {
+                if (nativeRealmEnabled) {
+                    assertCommonImplicitDisabledRealms(deprecationIssues);
+                    assertEquals("Found implicitly disabled native realm: [file]. " +
+                            "It is disabled because there are other explicitly configured realms." +
+                            "In next major release, native realms will always be enabled unless explicitly disabled.",
+                        deprecationIssues.get(0).getDetails());
+                } else {
+                    assertTrue(deprecationIssues.isEmpty());
+                }
+            } else if (false == nativeRealmConfigured) {
+                if (fileRealmEnabled) {
+                    assertCommonImplicitDisabledRealms(deprecationIssues);
+                    assertEquals("Found implicitly disabled native realm: [native]. " +
+                            "It is disabled because there are other explicitly configured realms." +
+                            "In next major release, native realms will always be enabled unless explicitly disabled.",
+                        deprecationIssues.get(0).getDetails());
+                } else {
+                    assertTrue(deprecationIssues.isEmpty());
+                }
+            }
+        }
     }
 
     public void testThreadPoolListenerQueueSize() {
@@ -349,4 +456,18 @@ private List<DeprecationIssue> getDeprecationIssues(Settings settings, PluginsAn
 
         return issues;
     }
+
+    private void assertCommonImplicitDisabledRealms(List<DeprecationIssue> deprecationIssues) {
+        assertEquals(1, deprecationIssues.size());
+        assertEquals("File and/or native realms cannot be implicitly disabled in next major release.",
+            deprecationIssues.get(0).getMessage());
+        assertEquals("https://www.elastic.co/guide/en/elasticsearch/reference" +
+                "/7.13/deprecated-7.13.html#implicitly-disabled-native-realms",
+            deprecationIssues.get(0).getUrl());
+    }
+
+    private String randomRealmTypeOtherThanFileOrNative() {
+        return randomValueOtherThanMany(t -> org.elasticsearch.common.collect.Set.of("file", "native").contains(t),
+            () -> randomAlphaOfLengthBetween(4, 12));
+    }
 }