Make reserved built-in roles queryable (elastic#117581)

This PR makes reserved [built-in
roles](https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-roles.html)
queryable via [Query Role
API](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-query-role.html)
by indexing them into the `.security` index.

Currently, the built-in roles were only available via [Get Role
API](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-get-role.html).

The built-in roles are synced into the `.security` index on cluster
recovery. The `.security` index will be created (if it's not existing)
before built-in roles are synced. In order to avoid concurrent updates,
the built-in roles will only be synced by a master node. 

Once the built-in roles are synced, the information about indexed roles
is kept in the cluster state as part of the `.security` index's
metadata. The map containing role names and their digests is persisted
as part of `queryable_built_in_roles_digest` property:

```
GET /_cluster/state/metadata/.security
"queryable_built_in_roles_digest": {
   "superuser": "lRRmA3kPO1/ztr3ESAlTetOuDjgUC3fKcGS3ZCqM+6k=",
    ...
}
```

Important: The reserved roles stored in the `.security` index are only
intended to be used for querying and retrieving. The role resolution and
mapping during authentication will remain the same and give a priority
to static/file role definitions. This is ensured by the [order in which
role providers (built-in, file and native) are
invoked](https://github.com/elastic/elasticsearch/blob/71c252c274aa967d5a66f7d081291ac5d87d27a9/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/RoleProviders.java#L77-L81).
It’s important to note this because there can be a short period of time
where we have a temporary inconsistency between actual built-in role
definitions and what is stored in the `.security` index.

---

Note: The functionality is temporarily hidden behind the
`es.queryable_built_in_roles_enabled` system property. By default, the
flag is disabled and will become enabled in a followup PR. The reason
for this is to keep this PR as small as possible and to avoid the need
to adjust a large number of tests that don't expect `.security` index to
exist. 

Testing: To run and test locally execute `./gradlew run
-Dtests.jvm.argline="-Des.queryable_built_in_roles_enabled=true"`. To
query all reserved built-in roles execute:

```
POST /_security/_query/role
{
  "query": {
    "bool": {
      "must": {
        "term": {
          "metadata._reserved": true
        }
      }
    }
  }
}
```

docs/changelog/117581.yaml

@@ -0,0 +1,5 @@
+pr: 117581
+summary: Make reserved built-in roles queryable
+area: Authorization
+type: enhancement
+issues: []

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -1138,7 +1138,7 @@ protected static void wipeAllIndices(boolean preserveSecurityIndices) throws IOE
         }
     }
 
-    private static boolean ignoreSystemIndexAccessWarnings(List<String> warnings) {
+    protected static boolean ignoreSystemIndexAccessWarnings(List<String> warnings) {
         for (String warning : warnings) {
             if (warning.startsWith("this request accesses system indices:")) {
                 SUITE_LOGGER.warn("Ignoring system index access warning during test cleanup: {}", warning);

x-pack/plugin/security/qa/security-basic/build.gradle

@@ -4,20 +4,31 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-
+apply plugin: 'elasticsearch.base-internal-es-plugin'
 apply plugin: 'elasticsearch.internal-java-rest-test'
 
+
+esplugin {
+  name 'queryable-reserved-roles-test'
+  description 'A test plugin for testing that changes to reserved roles are made queryable'
+  classname 'org.elasticsearch.xpack.security.role.QueryableBuiltInRolesTestPlugin'
+  extendedPlugins = ['x-pack-core', 'x-pack-security']
+}
 dependencies {
   javaRestTestImplementation(testArtifact(project(xpackModule('security'))))
   javaRestTestImplementation(testArtifact(project(xpackModule('core'))))
+  compileOnly project(':x-pack:plugin:core')
+  compileOnly project(':x-pack:plugin:security')
+  clusterPlugins project(':x-pack:plugin:security:qa:security-basic')
 }
 
 tasks.named('javaRestTest') {
   usesDefaultDistribution()
 }
 
+tasks.named("javadoc").configure { enabled = false }
 
-if (buildParams.inFipsJvm){
+if (buildParams.inFipsJvm) {
   // This test cluster is using a BASIC license and FIPS 140 mode is not supported in BASIC
-  tasks.named("javaRestTest").configure{enabled = false }
+  tasks.named("javaRestTest").configure { enabled = false }
 }

x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryRoleIT.java

@@ -496,7 +496,7 @@ private RoleDescriptor createRole(
         );
     }
 
-    private void assertQuery(String body, int total, Consumer<List<Map<String, Object>>> roleVerifier) throws IOException {
+    static void assertQuery(String body, int total, Consumer<List<Map<String, Object>>> roleVerifier) throws IOException {
         assertQuery(client(), body, total, roleVerifier);
     }
 

x-pack/plugin/security/qa/security-basic/src/main/java/module-info.java

@@ -0,0 +1,6 @@
+module org.elasticsearch.internal.security {
+    requires org.elasticsearch.base;
+    requires org.elasticsearch.server;
+    requires org.elasticsearch.xcore;
+    requires org.elasticsearch.security;
+}

x-pack/plugin/security/qa/security-basic/src/main/java/org/elasticsearch/xpack/security/role/QueryableBuiltInRolesTestPlugin.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.role;
+
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
+
+import java.util.List;
+
+public class QueryableBuiltInRolesTestPlugin extends Plugin {
+
+    @Override
+    public List<Setting<?>> getSettings() {
+        return List.of(ReservedRolesStore.INCLUDED_RESERVED_ROLES_SETTING);
+    }
+}

x-pack/plugin/security/src/main/java/module-info.java

@@ -70,6 +70,8 @@
     exports org.elasticsearch.xpack.security.slowlog to org.elasticsearch.server;
     exports org.elasticsearch.xpack.security.authc.support to org.elasticsearch.internal.security;
     exports org.elasticsearch.xpack.security.rest.action.apikey to org.elasticsearch.internal.security;
+    exports org.elasticsearch.xpack.security.support to org.elasticsearch.internal.security;
+    exports org.elasticsearch.xpack.security.authz.store to org.elasticsearch.internal.security;
 
     provides org.elasticsearch.index.SlowLogFieldProvider with org.elasticsearch.xpack.security.slowlog.SecuritySlowLogFieldProvider;
 

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -411,6 +411,8 @@
 import org.elasticsearch.xpack.security.rest.action.user.RestSetEnabledAction;
 import org.elasticsearch.xpack.security.support.CacheInvalidatorRegistry;
 import org.elasticsearch.xpack.security.support.ExtensionComponents;
+import org.elasticsearch.xpack.security.support.QueryableBuiltInRolesProviderFactory;
+import org.elasticsearch.xpack.security.support.QueryableBuiltInRolesSynchronizer;
 import org.elasticsearch.xpack.security.support.ReloadableSecurityComponent;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 import org.elasticsearch.xpack.security.support.SecurityMigrationExecutor;
@@ -461,6 +463,7 @@
 import static org.elasticsearch.xpack.core.security.SecurityField.FIELD_LEVEL_SECURITY_FEATURE;
 import static org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore.INCLUDED_RESERVED_ROLES_SETTING;
 import static org.elasticsearch.xpack.security.operator.OperatorPrivileges.OPERATOR_PRIVILEGES_ENABLED;
+import static org.elasticsearch.xpack.security.support.QueryableBuiltInRolesSynchronizer.QUERYABLE_BUILT_IN_ROLES_ENABLED;
 import static org.elasticsearch.xpack.security.transport.SSLEngineUtils.extractClientCertificates;
 
 public class Security extends Plugin
@@ -631,7 +634,7 @@ public class Security extends Plugin
     private final SetOnce<ReservedRoleNameChecker.Factory> reservedRoleNameCheckerFactory = new SetOnce<>();
     private final SetOnce<FileRoleValidator> fileRoleValidator = new SetOnce<>();
     private final SetOnce<SecondaryAuthActions> secondaryAuthActions = new SetOnce<>();
-
+    private final SetOnce<QueryableBuiltInRolesProviderFactory> queryableRolesProviderFactory = new SetOnce<>();
     private final SetOnce<SecurityMigrationExecutor> securityMigrationExecutor = new SetOnce<>();
 
     // Node local retry count for migration jobs that's checked only on the master node to make sure
@@ -1202,6 +1205,23 @@ Collection<Object> createComponents(
 
         reservedRoleMappingAction.set(new ReservedRoleMappingAction());
 
+        if (QUERYABLE_BUILT_IN_ROLES_ENABLED) {
+            if (queryableRolesProviderFactory.get() == null) {
+                queryableRolesProviderFactory.set(new QueryableBuiltInRolesProviderFactory.Default());
+            }
+            components.add(
+                new QueryableBuiltInRolesSynchronizer(
+                    clusterService,
+                    featureService,
+                    queryableRolesProviderFactory.get(),
+                    nativeRolesStore,
+                    reservedRolesStore,
+                    fileRolesStore.get(),
+                    threadPool
+                )
+            );
+        }
+
         cacheInvalidatorRegistry.validate();
 
         final List<ReloadableSecurityComponent> reloadableComponents = new ArrayList<>();
@@ -2317,6 +2337,7 @@ public void loadExtensions(ExtensionLoader loader) {
         loadSingletonExtensionAndSetOnce(loader, grantApiKeyRequestTranslator, RestGrantApiKeyAction.RequestTranslator.class);
         loadSingletonExtensionAndSetOnce(loader, fileRoleValidator, FileRoleValidator.class);
         loadSingletonExtensionAndSetOnce(loader, secondaryAuthActions, SecondaryAuthActions.class);
+        loadSingletonExtensionAndSetOnce(loader, queryableRolesProviderFactory, QueryableBuiltInRolesProviderFactory.class);
     }
 
     private <T> void loadSingletonExtensionAndSetOnce(ExtensionLoader loader, SetOnce<T> setOnce, Class<T> clazz) {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatures.java

@@ -12,6 +12,7 @@
 
 import java.util.Set;
 
+import static org.elasticsearch.xpack.security.support.QueryableBuiltInRolesSynchronizer.QUERYABLE_BUILT_IN_ROLES_FEATURE;
 import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MIGRATION_FRAMEWORK;
 import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_ROLES_METADATA_FLATTENED;
 import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_ROLE_MAPPING_CLEANUP;
@@ -20,6 +21,11 @@ public class SecurityFeatures implements FeatureSpecification {
 
     @Override
     public Set<NodeFeature> getFeatures() {
-        return Set.of(SECURITY_ROLE_MAPPING_CLEANUP, SECURITY_ROLES_METADATA_FLATTENED, SECURITY_MIGRATION_FRAMEWORK);
+        return Set.of(
+            SECURITY_ROLE_MAPPING_CLEANUP,
+            SECURITY_ROLES_METADATA_FLATTENED,
+            SECURITY_MIGRATION_FRAMEWORK,
+            QUERYABLE_BUILT_IN_ROLES_FEATURE
+        );
     }
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportGetRolesAction.java

@@ -20,11 +20,9 @@
 import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
 import org.elasticsearch.xpack.security.authz.store.NativeRolesStore;
 
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
+import java.util.LinkedHashSet;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -51,8 +49,8 @@ protected void doExecute(Task task, final GetRolesRequest request, final ActionL
             return;
         }
 
-        final Set<String> rolesToSearchFor = new HashSet<>();
-        final List<RoleDescriptor> reservedRoles = new ArrayList<>();
+        final Set<String> rolesToSearchFor = new LinkedHashSet<>();
+        final Set<RoleDescriptor> reservedRoles = new LinkedHashSet<>();
         if (specificRolesRequested) {
             for (String role : requestedRoles) {
                 if (ReservedRolesStore.isReserved(role)) {
@@ -80,10 +78,10 @@ protected void doExecute(Task task, final GetRolesRequest request, final ActionL
     }
 
     private void getNativeRoles(Set<String> rolesToSearchFor, ActionListener<GetRolesResponse> listener) {
-        getNativeRoles(rolesToSearchFor, new ArrayList<>(), listener);
+        getNativeRoles(rolesToSearchFor, new LinkedHashSet<>(), listener);
     }
 
-    private void getNativeRoles(Set<String> rolesToSearchFor, List<RoleDescriptor> foundRoles, ActionListener<GetRolesResponse> listener) {
+    private void getNativeRoles(Set<String> rolesToSearchFor, Set<RoleDescriptor> foundRoles, ActionListener<GetRolesResponse> listener) {
         nativeRolesStore.getRoleDescriptors(rolesToSearchFor, ActionListener.wrap((retrievalResult) -> {
             if (retrievalResult.isSuccess()) {
                 foundRoles.addAll(retrievalResult.getDescriptors());

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/FileRolesStore.java

@@ -44,6 +44,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -173,6 +174,14 @@ public Path getFile() {
         return file;
     }
 
+    /**
+     * @return a map of all file role definitions. The returned map is unmodifiable.
+     */
+    public Map<String, RoleDescriptor> getAllRoleDescriptors() {
+        final Map<String, RoleDescriptor> localPermissions = permissions;
+        return Collections.unmodifiableMap(localPermissions);
+    }
+
     // package private for testing
     Set<String> getAllRoleNames() {
         return permissions.keySet();

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java

@@ -63,13 +63,13 @@
 import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivileges;
 import org.elasticsearch.xpack.core.security.authz.store.RoleRetrievalResult;
 import org.elasticsearch.xpack.core.security.authz.support.DLSRoleQueryValidator;
-import org.elasticsearch.xpack.core.security.support.NativeRealmValidationUtil;
 import org.elasticsearch.xpack.security.authz.ReservedRoleNameChecker;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -169,6 +169,10 @@ public NativeRolesStore(
         this.enabled = settings.getAsBoolean(NATIVE_ROLES_ENABLED, true);
     }
 
+    public boolean isEnabled() {
+        return enabled;
+    }
+
     @Override
     public void accept(Set<String> names, ActionListener<RoleRetrievalResult> listener) {
         getRoleDescriptors(names, listener);
@@ -263,6 +267,10 @@ public boolean isMetadataSearchable() {
     }
 
     public void queryRoleDescriptors(SearchSourceBuilder searchSourceBuilder, ActionListener<QueryRoleResult> listener) {
+        if (enabled == false) {
+            listener.onResponse(QueryRoleResult.EMPTY);
+            return;
+        }
         SearchRequest searchRequest = new SearchRequest(new String[] { SECURITY_MAIN_ALIAS }, searchSourceBuilder);
         SecurityIndexManager frozenSecurityIndex = securityIndex.defensiveCopy();
         if (frozenSecurityIndex.indexExists() == false) {
@@ -345,6 +353,15 @@ public void deleteRoles(
         final List<String> roleNames,
         WriteRequest.RefreshPolicy refreshPolicy,
         final ActionListener<BulkRolesResponse> listener
+    ) {
+        deleteRoles(roleNames, refreshPolicy, true, listener);
+    }
+
+    public void deleteRoles(
+        final Collection<String> roleNames,
+        WriteRequest.RefreshPolicy refreshPolicy,
+        boolean validateRoleNames,
+        final ActionListener<BulkRolesResponse> listener
     ) {
         if (enabled == false) {
             listener.onFailure(new IllegalStateException("Native role management is disabled"));
@@ -355,7 +372,7 @@ public void deleteRoles(
         Map<String, Exception> validationErrorByRoleName = new HashMap<>();
 
         for (String roleName : roleNames) {
-            if (reservedRoleNameChecker.isReserved(roleName)) {
+            if (validateRoleNames && reservedRoleNameChecker.isReserved(roleName)) {
                 validationErrorByRoleName.put(
                     roleName,
                     new IllegalArgumentException("role [" + roleName + "] is reserved and cannot be deleted")
@@ -402,7 +419,7 @@ public void onFailure(Exception e) {
     }
 
     private void bulkResponseAndRefreshRolesCache(
-        List<String> roleNames,
+        Collection<String> roleNames,
         BulkResponse bulkResponse,
         Map<String, Exception> validationErrorByRoleName,
         ActionListener<BulkRolesResponse> listener
@@ -430,7 +447,7 @@ private void bulkResponseAndRefreshRolesCache(
     }
 
     private void bulkResponseWithOnlyValidationErrors(
-        List<String> roleNames,
+        Collection<String> roleNames,
         Map<String, Exception> validationErrorByRoleName,
         ActionListener<BulkRolesResponse> listener
     ) {
@@ -542,7 +559,16 @@ public void onFailure(Exception e) {
 
     public void putRoles(
         final WriteRequest.RefreshPolicy refreshPolicy,
-        final List<RoleDescriptor> roles,
+        final Collection<RoleDescriptor> roles,
+        final ActionListener<BulkRolesResponse> listener
+    ) {
+        putRoles(refreshPolicy, roles, true, listener);
+    }
+
+    public void putRoles(
+        final WriteRequest.RefreshPolicy refreshPolicy,
+        final Collection<RoleDescriptor> roles,
+        boolean validateRoleDescriptors,
         final ActionListener<BulkRolesResponse> listener
     ) {
         if (enabled == false) {
@@ -555,7 +581,7 @@ public void putRoles(
         for (RoleDescriptor role : roles) {
             Exception validationException;
             try {
-                validationException = validateRoleDescriptor(role);
+                validationException = validateRoleDescriptors ? validateRoleDescriptor(role) : null;
             } catch (Exception e) {
                 validationException = e;
             }
@@ -621,8 +647,6 @@ private DeleteRequest createRoleDeleteRequest(final String roleName) {
 
     // Package private for testing
     XContentBuilder createRoleXContentBuilder(RoleDescriptor role) throws IOException {
-        assert NativeRealmValidationUtil.validateRoleName(role.getName(), false) == null
-            : "Role name was invalid or reserved: " + role.getName();
         assert false == role.hasRestriction() : "restriction is not supported for native roles";
 
         XContentBuilder builder = jsonBuilder().startObject();
@@ -671,7 +695,11 @@ public void usageStats(ActionListener<Map<String, Object>> listener) {
                     client.prepareMultiSearch()
                         .add(
                             client.prepareSearch(SECURITY_MAIN_ALIAS)
-                                .setQuery(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE))
+                                .setQuery(
+                                    QueryBuilders.boolQuery()
+                                        .must(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE))
+                                        .mustNot(QueryBuilders.termQuery("metadata_flattened._reserved", true))
+                                )
                                 .setTrackTotalHits(true)
                                 .setSize(0)
                         )
@@ -680,6 +708,7 @@ public void usageStats(ActionListener<Map<String, Object>> listener) {
                                 .setQuery(
                                     QueryBuilders.boolQuery()
                                         .must(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE))
+                                        .mustNot(QueryBuilders.termQuery("metadata_flattened._reserved", true))
                                         .must(
                                             QueryBuilders.boolQuery()
                                                 .should(existsQuery("indices.field_security.grant"))
@@ -697,6 +726,7 @@ public void usageStats(ActionListener<Map<String, Object>> listener) {
                                 .setQuery(
                                     QueryBuilders.boolQuery()
                                         .must(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE))
+                                        .mustNot(QueryBuilders.termQuery("metadata_flattened._reserved", true))
                                         .filter(existsQuery("indices.query"))
                                 )
                                 .setTrackTotalHits(true)
@@ -708,6 +738,7 @@ public void usageStats(ActionListener<Map<String, Object>> listener) {
                                 .setQuery(
                                     QueryBuilders.boolQuery()
                                         .must(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE))
+                                        .mustNot(QueryBuilders.termQuery("metadata_flattened._reserved", true))
                                         .filter(existsQuery("remote_indices"))
                                 )
                                 .setTrackTotalHits(true)
@@ -718,6 +749,7 @@ public void usageStats(ActionListener<Map<String, Object>> listener) {
                                 .setQuery(
                                     QueryBuilders.boolQuery()
                                         .must(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE))
+                                        .mustNot(QueryBuilders.termQuery("metadata_flattened._reserved", true))
                                         .filter(existsQuery("remote_cluster"))
                                 )
                                 .setTrackTotalHits(true)