Introduce new audit record for security configuration changes via API (…

…elastic#62916)

This PR introduces a new event.type category for audit records,
namely the `security_config_change`, in the existing audit trail.
Events in this category record that a security configuration has been
set (eg user/role created/updated) or cleared (eg user/role deleted).
The events are emitted by default, but can be explicitly toggled by
the `security_config_changed` handler. The record contains all the
change details, (e.g. the rules of the particular role mapping that
has been created or updated), but all credentials are redacted out.
The change details are formatted as a JSON object are are part of
audit record structure (i.e. they are not JSON-escaped and put
in a string field).

Co-authored-by: Yang Wang <[email protected]>
Co-authored-by: Tim Vernum <[email protected]>

x-pack/plugin/core/src/main/config/log4j2.properties

@@ -36,13 +36,17 @@ appender.audit_rolling.layout.pattern = {\
                 %varsNotEmpty{, "x_forwarded_for":"%enc{%map{x_forwarded_for}}{JSON}"}\
                 %varsNotEmpty{, "transport.profile":"%enc{%map{transport.profile}}{JSON}"}\
                 %varsNotEmpty{, "rule":"%enc{%map{rule}}{JSON}"}\
-                %varsNotEmpty{, "event.category":"%enc{%map{event.category}}{JSON}"}\
+                %varsNotEmpty{, "put":%map{put}}\
+                %varsNotEmpty{, "delete":%map{delete}}\
+                %varsNotEmpty{, "change":%map{change}}\
+                %varsNotEmpty{, "create":%map{create}}\
+                %varsNotEmpty{, "invalidate":%map{invalidate}}\
                 }%n
 # "node.name" node name from the `elasticsearch.yml` settings
 # "node.id" node id which should not change between cluster restarts
 # "host.name" unresolved hostname of the local node
 # "host.ip" the local bound ip (i.e. the ip listening for connections)
-# "event.type" a received REST request is translated into one or more transport requests. This indicates which processing layer generated the event "rest" or "transport" (internal)
+# "origin.type" a received REST request is translated into one or more transport requests. This indicates which processing layer generated the event "rest" or "transport" (internal)
 # "event.action" the name of the audited event, eg. "authentication_failed", "access_granted", "run_as_granted", etc.
 # "authentication.type" one of "realm", "api_key", "token", "anonymous" or "internal"
 # "user.name" the subject name as authenticated by a realm
@@ -54,7 +58,7 @@ appender.audit_rolling.layout.pattern = {\
 # "user.roles" the roles array of the user; these are the roles that are granting privileges
 # "apikey.id" this field is present if and only if the "authentication.type" is "api_key"
 # "apikey.name" this field is present if and only if the "authentication.type" is "api_key"
-# "origin.type" it is "rest" if the event is originating (is in relation to) a REST request; possible other values are "transport" and "ip_filter"
+# "event.type" informs about what internal system generated the event; possible values are "rest", "transport", "ip_filter" and "security_config_change"
 # "origin.address" the remote address and port of the first network hop, i.e. a REST proxy or another cluster node
 # "realm" name of a realm that has generated an "authentication_failed" or an "authentication_successful"; the subject is not yet authenticated
 # "url.path" the URI component between the port and the query string; it is percent (URL) encoded
@@ -69,7 +73,8 @@ appender.audit_rolling.layout.pattern = {\
 # "x_forwarded_for" the addresses from the "X-Forwarded-For" request header, as a verbatim string value (not an array)
 # "transport.profile" name of the transport profile in case this is a "connection_granted" or "connection_denied" event
 # "rule" name of the applied rule if the "origin.type" is "ip_filter"
-# "event.category" fixed value "elasticsearch-audit"
+# the "put", "delete", "change", "create", "invalidate" fields are only present
+# when the "event.type" is "security_config_change" and contain the security config change (as an object) taking effect
 
 appender.audit_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_audit-%d{yyyy-MM-dd}.json
 appender.audit_rolling.policies.type = Policies

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequest.java

@@ -122,11 +122,11 @@ public void cluster(String... clusterPrivileges) {
         this.clusterPrivileges = clusterPrivileges;
     }
 
-    void conditionalCluster(ConfigurableClusterPrivilege... configurableClusterPrivileges) {
+    public void conditionalCluster(ConfigurableClusterPrivilege... configurableClusterPrivileges) {
         this.configurableClusterPrivileges = configurableClusterPrivileges;
     }
 
-    void addIndex(RoleDescriptor.IndicesPrivileges... privileges) {
+    public void addIndex(RoleDescriptor.IndicesPrivileges... privileges) {
         this.indicesPrivileges.addAll(Arrays.asList(privileges));
     }
 
@@ -142,7 +142,7 @@ public void addIndex(String[] indices, String[] privileges, String[] grantedFiel
                 .build());
     }
 
-    void addApplicationPrivileges(RoleDescriptor.ApplicationResourcePrivileges... privileges) {
+    public void addApplicationPrivileges(RoleDescriptor.ApplicationResourcePrivileges... privileges) {
         this.applicationPrivileges.addAll(Arrays.asList(privileges));
     }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/esnative/NativeRealmSettings.java

@@ -14,6 +14,7 @@
 
 public final class NativeRealmSettings {
     public static final String TYPE = "native";
+    public static final String DEFAULT_NAME = "default_native";
 
     private NativeRealmSettings() {}
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/file/FileRealmSettings.java

@@ -14,6 +14,7 @@
 
 public final class FileRealmSettings {
     public static final String TYPE = "file";
+    public static final String DEFAULT_NAME = "default_file";
 
     private FileRealmSettings() {}
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java

@@ -686,11 +686,11 @@ public boolean allowRestrictedIndices() {
             return allowRestrictedIndices;
         }
 
-        private boolean hasDeniedFields() {
+        public boolean hasDeniedFields() {
             return deniedFields != null && deniedFields.length > 0;
         }
 
-        private boolean hasGrantedFields() {
+        public boolean hasGrantedFields() {
             if (grantedFields != null && grantedFields.length >= 0) {
                 // we treat just '*' as no FLS since that's what the UI defaults to
                 if (grantedFields.length == 1 && "*".equals(grantedFields[0])) {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditLevel.java

@@ -12,7 +12,6 @@
 
 public enum AuditLevel {
 
-
     ANONYMOUS_ACCESS_DENIED,
     AUTHENTICATION_FAILED,
     REALM_AUTHENTICATION_FAILED,
@@ -22,6 +21,7 @@ public enum AuditLevel {
     CONNECTION_GRANTED,
     CONNECTION_DENIED,
     SYSTEM_ACCESS_GRANTED,
+    SECURITY_CONFIG_CHANGE,
     AUTHENTICATION_SUCCESS,
     RUN_AS_GRANTED,
     RUN_AS_DENIED;
@@ -61,6 +61,9 @@ static EnumSet<AuditLevel> parse(List<String> levels) {
                 case "system_access_granted":
                     enumSet.add(SYSTEM_ACCESS_GRANTED);
                     break;
+                case "security_config_change":
+                    enumSet.add(SECURITY_CONFIG_CHANGE);
+                    break;
                 case "authentication_success":
                     enumSet.add(AUTHENTICATION_SUCCESS);
                     break;