Use a licensed feature per realm-type (+custom) (elastic#79121)

This commit changes the licensed feature usage tracking for realms to
record each realm type as its own separate feature.
Custom realms continue to fall under a single catch-all feature.

Backport of: elastic#78810

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -351,13 +351,25 @@ public class Security extends Plugin implements SystemIndexPlugin, IngestPlugin,
     public static final LicensedFeature.Momentary AUDITING_FEATURE =
         LicensedFeature.momentaryLenient(null, "security_auditing", License.OperationMode.GOLD);
 
+    private static final String REALMS_FEATURE_FAMILY = "security-realms";
     // Builtin realms (file/native) realms are Basic licensed, so don't need to be checked or tracked
-    // Standard realms (LDAP, AD, PKI, etc) are Gold+
+    // Some realms (LDAP, AD, PKI) are Gold+
+    public static final LicensedFeature.Persistent LDAP_REALM_FEATURE =
+        LicensedFeature.persistentLenient(REALMS_FEATURE_FAMILY, "ldap", License.OperationMode.GOLD);
+    public static final LicensedFeature.Persistent AD_REALM_FEATURE =
+        LicensedFeature.persistentLenient(REALMS_FEATURE_FAMILY, "active-directory", License.OperationMode.GOLD);
+    public static final LicensedFeature.Persistent PKI_REALM_FEATURE =
+        LicensedFeature.persistentLenient(REALMS_FEATURE_FAMILY, "pki", License.OperationMode.GOLD);
     // SSO realms are Platinum+
-    public static final LicensedFeature.Persistent STANDARD_REALMS_FEATURE =
-        LicensedFeature.persistentLenient(null, "security_standard_realms", License.OperationMode.GOLD);
-    public static final LicensedFeature.Persistent ALL_REALMS_FEATURE =
-        LicensedFeature.persistentLenient(null, "security_all_realms", License.OperationMode.PLATINUM);
+    public static final LicensedFeature.Persistent SAML_REALM_FEATURE =
+        LicensedFeature.persistentLenient(REALMS_FEATURE_FAMILY, "saml", License.OperationMode.PLATINUM);
+    public static final LicensedFeature.Persistent OIDC_REALM_FEATURE =
+        LicensedFeature.persistentLenient(REALMS_FEATURE_FAMILY, "oidc", License.OperationMode.PLATINUM);
+    public static final LicensedFeature.Persistent KERBEROS_REALM_FEATURE =
+        LicensedFeature.persistentLenient(REALMS_FEATURE_FAMILY, "kerberos", License.OperationMode.PLATINUM);
+    // Custom realms are Platinum+
+    public static final LicensedFeature.Persistent CUSTOM_REALMS_FEATURE =
+        LicensedFeature.persistentLenient(REALMS_FEATURE_FAMILY, "custom", License.OperationMode.PLATINUM);
 
     private static final Logger logger = LogManager.getLogger(Security.class);
 

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/InternalRealms.java

@@ -7,9 +7,12 @@
 package org.elasticsearch.xpack.security.authc;
 
 import org.elasticsearch.bootstrap.BootstrapCheck;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.license.LicensedFeature;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xpack.core.security.authc.Realm;
@@ -23,6 +26,7 @@
 import org.elasticsearch.xpack.core.security.authc.pki.PkiRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.saml.SamlRealmSettings;
 import org.elasticsearch.xpack.core.ssl.SSLService;
+import org.elasticsearch.xpack.security.Security;
 import org.elasticsearch.xpack.security.authc.esnative.NativeRealm;
 import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore;
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
@@ -53,36 +57,64 @@
  */
 public final class InternalRealms {
 
+    static final String RESERVED_TYPE = ReservedRealm.TYPE;
+    static final String NATIVE_TYPE = NativeRealmSettings.TYPE;
+    static final String FILE_TYPE = FileRealmSettings.TYPE;
+    static final String LDAP_TYPE = LdapRealmSettings.LDAP_TYPE;
+    static final String AD_TYPE = LdapRealmSettings.AD_TYPE;
+    static final String PKI_TYPE = PkiRealmSettings.TYPE;
+    static final String SAML_TYPE = SamlRealmSettings.TYPE;
+    static final String OIDC_TYPE = OpenIdConnectRealmSettings.TYPE;
+    static final String KERBEROS_TYPE = KerberosRealmSettings.TYPE;
+
+    private static final Set<String> BUILTIN_TYPES = Sets.newHashSet(NATIVE_TYPE, FILE_TYPE);
+
     /**
-     * The list of all <em>internal</em> realm types, excluding {@link ReservedRealm#TYPE}.
+     * The map of all <em>licensed</em> internal realm types to their licensed feature
      */
-    private static final Set<String> XPACK_TYPES = Collections
-        .unmodifiableSet(Sets.newHashSet(NativeRealmSettings.TYPE, FileRealmSettings.TYPE, LdapRealmSettings.AD_TYPE,
-            LdapRealmSettings.LDAP_TYPE, PkiRealmSettings.TYPE, SamlRealmSettings.TYPE, KerberosRealmSettings.TYPE,
-            OpenIdConnectRealmSettings.TYPE));
+    private static final Map<String, LicensedFeature.Persistent> LICENSED_REALMS = org.elasticsearch.core.Map.ofEntries(
+        org.elasticsearch.core.Map.entry(AD_TYPE, Security.AD_REALM_FEATURE),
+        org.elasticsearch.core.Map.entry(LDAP_TYPE, Security.LDAP_REALM_FEATURE),
+        org.elasticsearch.core.Map.entry(PKI_TYPE, Security.PKI_REALM_FEATURE),
+        org.elasticsearch.core.Map.entry(SAML_TYPE, Security.SAML_REALM_FEATURE),
+        org.elasticsearch.core.Map.entry(KERBEROS_TYPE, Security.KERBEROS_REALM_FEATURE),
+        org.elasticsearch.core.Map.entry(OIDC_TYPE, Security.OIDC_REALM_FEATURE)
+    );
 
     /**
-     * The list of all standard realm types, which are those provided by x-pack and do not have extensive
-     * interaction with third party sources
+     * The set of all <em>internal</em> realm types, excluding {@link ReservedRealm#TYPE}
+     * @deprecated Use of this method (other than in tests) is discouraged.
      */
-    private static final Set<String> STANDARD_TYPES = Collections.unmodifiableSet(Sets.newHashSet(NativeRealmSettings.TYPE,
-        FileRealmSettings.TYPE, LdapRealmSettings.AD_TYPE, LdapRealmSettings.LDAP_TYPE, PkiRealmSettings.TYPE));
-
+    @Deprecated
     public static Collection<String> getConfigurableRealmsTypes() {
-        return Collections.unmodifiableSet(XPACK_TYPES);
+        return org.elasticsearch.core.Set.copyOf(Sets.union(BUILTIN_TYPES, LICENSED_REALMS.keySet()));
     }
 
-    /**
-     * Determines whether <code>type</code> is an internal realm-type that is provided by x-pack,
-     * excluding the {@link ReservedRealm} and realms that have extensive interaction with
-     * third party sources
-     */
-    static boolean isStandardRealm(String type) {
-        return STANDARD_TYPES.contains(type);
+    static boolean isInternalRealm(String type) {
+        return RESERVED_TYPE.equals(type) || BUILTIN_TYPES.contains(type) || LICENSED_REALMS.containsKey(type);
     }
 
     static boolean isBuiltinRealm(String type) {
-        return FileRealmSettings.TYPE.equals(type) || NativeRealmSettings.TYPE.equals(type);
+        return BUILTIN_TYPES.contains(type);
+    }
+
+    /**
+     * @return The licensed feature for the given realm type, or {@code null} if the realm does not require a specific license type
+     * @throws IllegalArgumentException if the provided type is not an {@link #isInternalRealm(String) internal realm}
+     */
+    @Nullable
+    static LicensedFeature.Persistent getLicensedFeature(String type) {
+        if (Strings.isNullOrEmpty(type)) {
+            throw new IllegalArgumentException("Empty realm type [" + type + "]");
+        }
+        if (type.equals(RESERVED_TYPE) || isBuiltinRealm(type)) {
+            return null;
+        }
+        final LicensedFeature.Persistent feature = LICENSED_REALMS.get(type);
+        if (feature == null) {
+            throw new IllegalArgumentException("Unsupported realm type [" + type + "]");
+        }
+        return feature;
     }
 
     /**

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java

@@ -17,7 +17,9 @@
 import org.elasticsearch.common.util.concurrent.CountDown;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.license.LicensedFeature;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.security.authc.Realm;
@@ -127,18 +129,25 @@ protected void recomputeActiveRealms() {
             Strings.collectionToCommaDelimitedString(licensedRealms)
         );
 
+        stopTrackingInactiveRealms(licenseStateSnapshot, licensedRealms);
+
+        activeRealms = licensedRealms;
+    }
+
+    // Can be overridden in testing
+    protected void stopTrackingInactiveRealms(XPackLicenseState licenseStateSnapshot, List<Realm> licensedRealms) {
         // Stop license-tracking for any previously-active realms that are no longer allowed
         if (activeRealms != null) {
             activeRealms.stream().filter(r -> licensedRealms.contains(r) == false).forEach(realm -> {
-                if (InternalRealms.isStandardRealm(realm.type())) {
-                    Security.STANDARD_REALMS_FEATURE.stopTracking(licenseStateSnapshot, realm.name());
-                } else {
-                    Security.ALL_REALMS_FEATURE.stopTracking(licenseStateSnapshot, realm.name());
-                }
+                final LicensedFeature.Persistent feature = getLicensedFeatureForRealm(realm.type());
+                assert feature != null : "Realm ["
+                    + realm
+                    + "] with no licensed feature became inactive due to change to license mode ["
+                    + licenseStateSnapshot.getOperationMode()
+                    + "]";
+                feature.stopTracking(licenseStateSnapshot, realm.name());
             });
         }
-
-        activeRealms = licensedRealms;
     }
 
     @Override
@@ -194,27 +203,29 @@ private boolean hasUserRealm(List<Realm> licensedRealms) {
     }
 
     private static boolean checkLicense(Realm realm, XPackLicenseState licenseState) {
-        if (isBasicLicensedRealm(realm.type())) {
+        final LicensedFeature.Persistent feature = getLicensedFeatureForRealm(realm.type());
+        if (feature == null) {
             return true;
         }
-        if (InternalRealms.isStandardRealm(realm.type())) {
-            return Security.STANDARD_REALMS_FEATURE.checkAndStartTracking(licenseState, realm.name());
-        }
-        return Security.ALL_REALMS_FEATURE.checkAndStartTracking(licenseState, realm.name());
+        return feature.checkAndStartTracking(licenseState, realm.name());
     }
 
     public static boolean isRealmTypeAvailable(XPackLicenseState licenseState, String type) {
-        if (Security.ALL_REALMS_FEATURE.checkWithoutTracking(licenseState)) {
+        final LicensedFeature.Persistent feature = getLicensedFeatureForRealm(type);
+        if (feature == null) {
             return true;
-        } else if (Security.STANDARD_REALMS_FEATURE.checkWithoutTracking(licenseState)) {
-            return InternalRealms.isStandardRealm(type) || ReservedRealm.TYPE.equals(type);
-        } else {
-            return isBasicLicensedRealm(type);
         }
+        return feature.checkWithoutTracking(licenseState);
     }
 
-    private static boolean isBasicLicensedRealm(String type) {
-        return ReservedRealm.TYPE.equals(type) || InternalRealms.isBuiltinRealm(type);
+    @Nullable
+    private static LicensedFeature.Persistent getLicensedFeatureForRealm(String realmType) {
+        assert Strings.hasText(realmType) : "Realm type must be provided (received [" + realmType + "])";
+        if (InternalRealms.isInternalRealm(realmType)) {
+            return InternalRealms.getLicensedFeature(realmType);
+        } else {
+            return Security.CUSTOM_REALMS_FEATURE;
+        }
     }
 
     public Realm realm(String name) {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/RestDelegatePkiAuthenticationAction.java

@@ -56,7 +56,7 @@ protected Exception checkFeatureAvailable(RestRequest request) {
         Exception failedFeature = super.checkFeatureAvailable(request);
         if (failedFeature != null) {
             return failedFeature;
-        } else if (Security.STANDARD_REALMS_FEATURE.checkWithoutTracking(licenseState)) {
+        } else if (Security.PKI_REALM_FEATURE.checkWithoutTracking(licenseState)) {
             return null;
         } else {
             logger.info("The '{}' realm is not available under the current license", PkiRealmSettings.TYPE);

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java

@@ -52,6 +52,7 @@
 import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.license.License;
+import org.elasticsearch.license.LicensedFeature;
 import org.elasticsearch.license.MockLicenseState;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.license.XPackLicenseState.Feature;
@@ -218,9 +219,14 @@ public void init() throws Exception {
             .put(XPackSettings.API_KEY_SERVICE_ENABLED_SETTING.getKey(), true)
             .build();
         MockLicenseState licenseState = mock(MockLicenseState.class);
-        when(licenseState.isAllowed(Security.ALL_REALMS_FEATURE)).thenReturn(true);
         when(licenseState.isSecurityEnabled()).thenReturn(true);
-        when(licenseState.isAllowed(Security.STANDARD_REALMS_FEATURE)).thenReturn(true);
+        for (String realmType : InternalRealms.getConfigurableRealmsTypes()) {
+            final LicensedFeature.Persistent feature = InternalRealms.getLicensedFeature(realmType);
+            if (feature != null) {
+                when(licenseState.isAllowed(feature)).thenReturn(true);
+            }
+        }
+        when(licenseState.isAllowed(Security.CUSTOM_REALMS_FEATURE)).thenReturn(true);
         when(licenseState.checkFeature(Feature.SECURITY_TOKEN_SERVICE)).thenReturn(true);
         when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState);
         when(licenseState.checkFeature(Feature.SECURITY_AUDITING)).thenReturn(true);
@@ -229,9 +235,10 @@ public void init() throws Exception {
         ReservedRealm reservedRealm = mock(ReservedRealm.class);
         when(reservedRealm.type()).thenReturn("reserved");
         when(reservedRealm.name()).thenReturn("reserved_realm");
-        realms = spy(new TestRealms(Settings.EMPTY, TestEnvironment.newEnvironment(settings), Collections.<String, Realm.Factory>emptyMap(),
-                licenseState, threadContext, reservedRealm, Arrays.asList(firstRealm, secondRealm),
-                Collections.singletonList(firstRealm)));
+        realms = spy(new TestRealms(Settings.EMPTY, TestEnvironment.newEnvironment(settings),
+            Collections.<String, Realm.Factory>emptyMap(),
+            licenseState, threadContext, reservedRealm, Arrays.asList(firstRealm, secondRealm),
+            Arrays.asList(firstRealm)));
 
         // Needed because this is calculated in the constructor, which means the override doesn't get called correctly
         realms.recomputeActiveRealms();
@@ -428,6 +435,7 @@ public void testAuthenticateBothSupportSecondSucceeds() throws Exception {
         verify(realms, atLeastOnce()).recomputeActiveRealms();
         verify(realms, atLeastOnce()).calculateLicensedRealms(any(XPackLicenseState.class));
         verify(realms, atLeastOnce()).getActiveRealms();
+        verify(realms, atLeastOnce()).stopTrackingInactiveRealms(any(XPackLicenseState.class), any());
         // ^^ We don't care how many times these methods are called, we just check it here so that we can verify no more interactions below.
         verifyNoMoreInteractions(realms);
     }
@@ -2171,7 +2179,9 @@ protected List<Realm> calculateLicensedRealms(XPackLicenseState licenseState) {
                 // This can happen because the realms are recalculated during construction
                 return super.calculateLicensedRealms(licenseState);
             }
-            if (Security.STANDARD_REALMS_FEATURE.checkWithoutTracking(licenseState)) {
+
+            // Use custom as a placeholder for all non-internal realm
+            if (Security.CUSTOM_REALMS_FEATURE.checkWithoutTracking(licenseState)) {
                 return allRealms;
             } else {
                 return internalRealms;
@@ -2182,6 +2192,11 @@ protected List<Realm> calculateLicensedRealms(XPackLicenseState licenseState) {
         public void recomputeActiveRealms() {
             super.recomputeActiveRealms();
         }
+
+        @Override
+        protected void stopTrackingInactiveRealms(XPackLicenseState licenseStateSnapshot, List<Realm> licensedRealms) {
+            // Ignore
+        }
     }
 
     private void logAndFail(Exception e) {