Make reserved built-in roles queryable

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java

@@ -50,6 +50,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.TreeMap;
 
 import static org.elasticsearch.common.xcontent.XContentHelper.createParserNotCompressed;
 import static org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions.ROLE_REMOTE_CLUSTER_PRIVS;
@@ -189,9 +190,9 @@ public RoleDescriptor(
         this.indicesPrivileges = indicesPrivileges != null ? indicesPrivileges : IndicesPrivileges.NONE;
         this.applicationPrivileges = applicationPrivileges != null ? applicationPrivileges : ApplicationResourcePrivileges.NONE;
         this.runAs = runAs != null ? runAs : Strings.EMPTY_ARRAY;
-        this.metadata = metadata != null ? Collections.unmodifiableMap(metadata) : Collections.emptyMap();
+        this.metadata = metadata != null ? Collections.unmodifiableMap(new TreeMap<>(metadata)) : Collections.emptyMap();
         this.transientMetadata = transientMetadata != null
-            ? Collections.unmodifiableMap(transientMetadata)
+            ? Collections.unmodifiableMap(new TreeMap<>(transientMetadata))
             : Collections.singletonMap("enabled", true);
         this.remoteIndicesPrivileges = remoteIndicesPrivileges != null ? remoteIndicesPrivileges : RemoteIndicesPrivileges.NONE;
         this.remoteClusterPermissions = remoteClusterPermissions != null && remoteClusterPermissions.hasAnyPrivileges()

x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryRoleIT.java

@@ -49,15 +49,16 @@ public final class QueryRoleIT extends SecurityInBasicRestTestCase {
 
     private static final String READ_SECURITY_USER_AUTH_HEADER = "Basic cmVhZF9zZWN1cml0eV91c2VyOnJlYWQtc2VjdXJpdHktcGFzc3dvcmQ=";
 
-    public void testSimpleQueryAllRoles() throws IOException {
-        assertQuery("", 0, roles -> assertThat(roles, emptyIterable()));
-        RoleDescriptor createdRole = createRandomRole();
-        assertQuery("", 1, roles -> {
-            assertThat(roles, iterableWithSize(1));
-            assertRoleMap(roles.get(0), createdRole);
+    public void testSimpleQueryAllRoles() throws Exception {
+        createRandomRole();
+
+        // 31 built-in reserved roles + 1 random role
+        assertQuery("", 1 + 31, roles -> {
+            // default size is 10
+            assertThat(roles, iterableWithSize(10));
         });
         assertQuery("""
-            {"query":{"match_all":{}},"from":1}""", 1, roles -> assertThat(roles, emptyIterable()));
+            {"query":{"match_all":{}},"from":32}""", 1 + 31, roles -> assertThat(roles, emptyIterable()));
     }
 
     public void testDisallowedFields() throws Exception {
@@ -496,7 +497,7 @@ private RoleDescriptor createRole(
         );
     }
 
-    private void assertQuery(String body, int total, Consumer<List<Map<String, Object>>> roleVerifier) throws IOException {
+    static void assertQuery(String body, int total, Consumer<List<Map<String, Object>>> roleVerifier) throws IOException {
         assertQuery(client(), body, total, roleVerifier);
     }
 

x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryableReservedRolesIT.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
+import org.elasticsearch.xpack.security.support.SecurityMigrations;
+import org.junit.BeforeClass;
+
+import static org.elasticsearch.xpack.security.QueryRoleIT.assertQuery;
+import static org.elasticsearch.xpack.security.QueryRoleIT.waitForMigrationCompletion;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.iterableWithSize;
+import static org.hamcrest.Matchers.oneOf;
+
+public class QueryableReservedRolesIT extends SecurityInBasicRestTestCase {
+
+    @BeforeClass
+    public static void setup() {
+        new ReservedRolesStore();
+    }
+
+    public void testQueryDeleteOrUpdateReservedRoles() throws Exception {
+        waitForMigrationCompletion(adminClient(), SecurityMigrations.ROLE_METADATA_FLATTENED_MIGRATION_VERSION);
+
+        final String[] allReservedRoles = ReservedRolesStore.names().toArray(new String[0]);
+        assertQuery("""
+            { "query": { "bool": { "must": { "term": { "metadata._reserved": true } } } }, "size": 100 }
+            """, 31, roles -> {
+            assertThat(roles, iterableWithSize(31));
+            for (var role : roles) {
+                assertThat((String) role.get("name"), is(oneOf(allReservedRoles)));
+            }
+        });
+
+        final String roleName = randomFrom(allReservedRoles);
+        assertQuery(String.format("""
+            { "query": { "bool": { "must": { "term": { "name": "%s" } } } } }
+            """, roleName), 1, roles -> {
+            assertThat(roles, iterableWithSize(1));
+            assertThat((String) roles.get(0).get("name"), equalTo(roleName));
+        });
+
+        assertDeleteReservedRole(roleName);
+        assertCreateOrUpdateReservedRole(roleName);
+    }
+
+    private void assertDeleteReservedRole(String roleName) throws Exception {
+        Request request = new Request("DELETE", "/_security/role/" + roleName);
+        var e = expectThrows(ResponseException.class, () -> adminClient().performRequest(request));
+        assertThat(e.getMessage(), containsString("role [" + roleName + "] is reserved and cannot be deleted"));
+    }
+
+    private void assertCreateOrUpdateReservedRole(String roleName) throws Exception {
+        Request request = new Request(randomBoolean() ? "PUT" : "POST", "/_security/role/" + roleName);
+        request.setJsonEntity("""
+            {
+              "cluster": ["all"],
+              "indices": [
+                {
+                  "names": ["*"],
+                  "privileges": ["all"]
+                }
+              ]
+            }
+            """);
+        var e = expectThrows(ResponseException.class, () -> adminClient().performRequest(request));
+        assertThat(e.getMessage(), containsString("Role [" + roleName + "] is reserved and may not be used."));
+    }
+
+}

x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/SecurityInBasicRestTestCase.java

@@ -54,6 +54,7 @@ public abstract class SecurityInBasicRestTestCase extends ESRestTestCase {
         .user(API_KEY_USER, API_KEY_USER_PASSWORD.toString(), "api_key_user_role", false)
         .user(API_KEY_ADMIN_USER, API_KEY_ADMIN_USER_PASSWORD.toString(), "api_key_admin_role", false)
         .user(READ_SECURITY_USER, READ_SECURITY_PASSWORD.toString(), "read_security_user_role", false)
+        .systemProperty("es.queryable_built_in_roles_enabled", "true")
         .build();
 
     @Override

x-pack/plugin/security/src/main/java/module-info.java

@@ -70,6 +70,8 @@
     exports org.elasticsearch.xpack.security.slowlog to org.elasticsearch.server;
     exports org.elasticsearch.xpack.security.authc.support to org.elasticsearch.internal.security;
     exports org.elasticsearch.xpack.security.rest.action.apikey to org.elasticsearch.internal.security;
+    exports org.elasticsearch.xpack.security.support to org.elasticsearch.internal.security;
+    exports org.elasticsearch.xpack.security.authz.store to org.elasticsearch.internal.security;
 
     provides org.elasticsearch.index.SlowLogFieldProvider with org.elasticsearch.xpack.security.slowlog.SecuritySlowLogFieldProvider;
 

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -337,6 +337,7 @@
 import org.elasticsearch.xpack.security.authz.store.FileRolesStore;
 import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore;
 import org.elasticsearch.xpack.security.authz.store.NativeRolesStore;
+import org.elasticsearch.xpack.security.authz.store.QueryableBuiltInRolesStore;
 import org.elasticsearch.xpack.security.authz.store.RoleProviders;
 import org.elasticsearch.xpack.security.ingest.SetSecurityUserProcessor;
 import org.elasticsearch.xpack.security.operator.DefaultOperatorOnlyRegistry;
@@ -411,6 +412,8 @@
 import org.elasticsearch.xpack.security.rest.action.user.RestSetEnabledAction;
 import org.elasticsearch.xpack.security.support.CacheInvalidatorRegistry;
 import org.elasticsearch.xpack.security.support.ExtensionComponents;
+import org.elasticsearch.xpack.security.support.QueryableBuiltInRolesProviderFactory;
+import org.elasticsearch.xpack.security.support.QueryableBuiltInRolesSynchronizer;
 import org.elasticsearch.xpack.security.support.ReloadableSecurityComponent;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 import org.elasticsearch.xpack.security.support.SecurityMigrationExecutor;
@@ -461,6 +464,7 @@
 import static org.elasticsearch.xpack.core.security.SecurityField.FIELD_LEVEL_SECURITY_FEATURE;
 import static org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore.INCLUDED_RESERVED_ROLES_SETTING;
 import static org.elasticsearch.xpack.security.operator.OperatorPrivileges.OPERATOR_PRIVILEGES_ENABLED;
+import static org.elasticsearch.xpack.security.support.QueryableBuiltInRolesSynchronizer.QUERYABLE_BUILT_IN_ROLES_ENABLED;
 import static org.elasticsearch.xpack.security.transport.SSLEngineUtils.extractClientCertificates;
 
 public class Security extends Plugin
@@ -631,7 +635,7 @@ public class Security extends Plugin
     private final SetOnce<ReservedRoleNameChecker.Factory> reservedRoleNameCheckerFactory = new SetOnce<>();
     private final SetOnce<FileRoleValidator> fileRoleValidator = new SetOnce<>();
     private final SetOnce<SecondaryAuthActions> secondaryAuthActions = new SetOnce<>();
-
+    private final SetOnce<QueryableBuiltInRolesProviderFactory> queryableRolesProviderFactory = new SetOnce<>();
     private final SetOnce<SecurityMigrationExecutor> securityMigrationExecutor = new SetOnce<>();
 
     // Node local retry count for migration jobs that's checked only on the master node to make sure
@@ -1202,6 +1206,24 @@ Collection<Object> createComponents(
 
         reservedRoleMappingAction.set(new ReservedRoleMappingAction());
 
+        if (QUERYABLE_BUILT_IN_ROLES_ENABLED) {
+            if (queryableRolesProviderFactory.get() == null) {
+                queryableRolesProviderFactory.set(new QueryableBuiltInRolesProviderFactory.Default());
+            }
+            components.add(
+                new QueryableBuiltInRolesSynchronizer(
+                    clusterService,
+                    featureService,
+                    queryableRolesProviderFactory.get(),
+                    new QueryableBuiltInRolesStore(nativeRolesStore),
+                    reservedRolesStore,
+                    fileRolesStore.get(),
+                    systemIndices.getMainIndexManager(),
+                    threadPool
+                )
+            );
+        }
+
         cacheInvalidatorRegistry.validate();
 
         final List<ReloadableSecurityComponent> reloadableComponents = new ArrayList<>();
@@ -2317,6 +2339,7 @@ public void loadExtensions(ExtensionLoader loader) {
         loadSingletonExtensionAndSetOnce(loader, grantApiKeyRequestTranslator, RestGrantApiKeyAction.RequestTranslator.class);
         loadSingletonExtensionAndSetOnce(loader, fileRoleValidator, FileRoleValidator.class);
         loadSingletonExtensionAndSetOnce(loader, secondaryAuthActions, SecondaryAuthActions.class);
+        loadSingletonExtensionAndSetOnce(loader, queryableRolesProviderFactory, QueryableBuiltInRolesProviderFactory.class);
     }
 
     private <T> void loadSingletonExtensionAndSetOnce(ExtensionLoader loader, SetOnce<T> setOnce, Class<T> clazz) {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatures.java

@@ -12,6 +12,7 @@
 
 import java.util.Set;
 
+import static org.elasticsearch.xpack.security.support.QueryableBuiltInRolesSynchronizer.QUERYABLE_BUILT_IN_ROLES_FEATURE;
 import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MIGRATION_FRAMEWORK;
 import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_ROLES_METADATA_FLATTENED;
 import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_ROLE_MAPPING_CLEANUP;
@@ -20,6 +21,11 @@ public class SecurityFeatures implements FeatureSpecification {
 
     @Override
     public Set<NodeFeature> getFeatures() {
-        return Set.of(SECURITY_ROLE_MAPPING_CLEANUP, SECURITY_ROLES_METADATA_FLATTENED, SECURITY_MIGRATION_FRAMEWORK);
+        return Set.of(
+            SECURITY_ROLE_MAPPING_CLEANUP,
+            SECURITY_ROLES_METADATA_FLATTENED,
+            SECURITY_MIGRATION_FRAMEWORK,
+            QUERYABLE_BUILT_IN_ROLES_FEATURE
+        );
     }
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/FileRolesStore.java

@@ -44,6 +44,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -173,6 +174,14 @@ public Path getFile() {
         return file;
     }
 
+    /**
+     * @return a map of all file role definitions. The returned map is unmodifiable.
+     */
+    public Map<String, RoleDescriptor> getAllRoleDescriptors() {
+        final Map<String, RoleDescriptor> localPermissions = permissions;
+        return Collections.unmodifiableMap(localPermissions);
+    }
+
     // package private for testing
     Set<String> getAllRoleNames() {
         return permissions.keySet();