Add new slowlog fields to filebeat

CHANGELOG.next.asciidoc

@@ -204,6 +204,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d
 - Improve rate limit handling by HTTPJSON {issue}36207[36207] {pull}38161[38161] {pull}38237[38237]
 - Add parseDateInTZ value template for the HTTPJSON input. {pull}37738[37738]
 - Add support for complex event objects in the HTTP Endpoint input. {issue}37910[37910] {pull}38193[38193]
+- Parse more fields from Elasticsearch slowlogs {pull}38295[38295]
 
 *Auditbeat*
 

filebeat/docs/fields.asciidoc

@@ -50730,6 +50730,61 @@ type: keyword
 
 --
 
+*`elasticsearch.slowlog.user.realm`*::
++
+--
+The authentication realm the user was authenticated against
+
+type: keyword
+
+example: default_file
+
+--
+
+*`elasticsearch.slowlog.user.effective.realm`*::
++
+--
+The authentication realm the effective user was authenticated against
+
+type: keyword
+
+example: default_file
+
+--
+
+*`elasticsearch.slowlog.auth.type`*::
++
+--
+The authentication type used to authenticate the user. One of TOKEN | REALM | API_KEY
+
+type: keyword
+
+example: REALM
+
+--
+
+*`elasticsearch.slowlog.apikey.id`*::
++
+--
+The id of the API key used
+
+type: keyword
+
+example: WzL_kb6VSvOhAq0twPvHOQ
+
+--
+
+*`elasticsearch.slowlog.apikey.name`*::
++
+--
+The name of the API key used
+
+type: keyword
+
+example: my-api-key
+
+--
+
 [[exported-fields-envoyproxy]]
 == Envoyproxy fields
 

filebeat/module/elasticsearch/slowlog/_meta/fields.yml

@@ -54,3 +54,23 @@
   - name: source
     description: Source of document that was indexed
     type: keyword
+  - name: user.realm
+    description: The authentication realm the user was authenticated against
+    example: "default_file"
+    type: keyword
+  - name: user.effective.realm
+    description: The authentication realm the effective user was authenticated against
+    example: "default_file"
+    type: keyword
+  - name: auth.type
+    description: The authentication type used to authenticate the user. One of TOKEN | REALM | API_KEY
+    example: REALM
+    type: keyword
+  - name: apikey.id
+    description: The id of the API key used
+    example: "WzL_kb6VSvOhAq0twPvHOQ"
+    type: keyword
+  - name: apikey.name
+    description: The name of the API key used
+    example: "my-api-key"
+    type: keyword

filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml

@@ -10,3 +10,23 @@ processors:
   - pipeline:
       if: 'ctx.message.contains("ecs.version")'
       name: '{< IngestPipeline "pipeline-json-8" >}'
+  - rename:
+      field: auth.type
+      target_field: elasticsearch.slowlog.auth.type
+      ignore_missing: true
+  - rename:
+      field: user.realm
+      target_field: elasticsearch.slowlog.user.realm
+      ignore_missing: true
+  - rename:
+      field: user.effective.realm
+      target_field: elasticsearch.slowlog.user.effective.realm
+      ignore_missing: true
+  - rename:
+      field: apikey.id
+      target_field: elasticsearch.slowlog.user.apikey.id
+      ignore_missing: true
+  - rename:
+      field: apikey.name
+      target_field: elasticsearch.slowlog.user.apikey.name
+      ignore_missing: true

filebeat/module/elasticsearch/slowlog/test/es814_index_indexing_slowlog-json.log

@@ -0,0 +1,4 @@
+{"@timestamp":"2024-03-13T10:34:33.289Z", "log.level": "WARN",  "auth.type":"REALM","elasticsearch.slowlog.id":"2","elasticsearch.slowlog.message":"[my-index/stZSoQ12R56VZORRItBKjA]","elasticsearch.slowlog.source":"{\\\"indices\\\":{\\\"field_security\\\":{\\\"grant\\\":\\\"read\\\",\\\"except\\\":\\\"confidential\\\"},\\\"names\\\":[\\\"foo\\\",\\\"bar\\\"],\\\"privileges\\\":\\\"admin\\\",\\\"query\\\":\\\"example\\\",\\\"allow_restricted_indices\\\":true}}","elasticsearch.slowlog.took":"12.3ms","elasticsearch.slowlog.took_millis":"12","user.name":"elastic","user.realm":"reserved" , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_indexing_slowlog","process.thread.name":"elasticsearch[runTask-0][write][T#7]","log.logger":"index.indexing.slowlog.index","elasticsearch.cluster.uuid":"0d2MZYNKR7Wqr2U6Cvpp7g","elasticsearch.node.id":"a8BUD2RfQSu4aqtpePX7BA","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"}
+{"@timestamp":"2024-03-13T10:34:36.139Z", "log.level": "WARN",  "auth.type":"REALM","elasticsearch.slowlog.id":"3","elasticsearch.slowlog.message":"[my-index/stZSoQ12R56VZORRItBKjA]","elasticsearch.slowlog.source":"{\\\"indices\\\":{\\\"field_security\\\":{\\\"grant\\\":\\\"read\\\",\\\"except\\\":\\\"confidential\\\"},\\\"names\\\":[\\\"foo\\\",\\\"bar\\\"],\\\"privileges\\\":\\\"admin\\\",\\\"query\\\":\\\"example\\\",\\\"allow_restricted_indices\\\":true}}","elasticsearch.slowlog.took":"5.9ms","elasticsearch.slowlog.took_millis":"5","user.name":"elastic","user.realm":"reserved" , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_indexing_slowlog","process.thread.name":"elasticsearch[runTask-0][write][T#9]","log.logger":"index.indexing.slowlog.index","elasticsearch.cluster.uuid":"0d2MZYNKR7Wqr2U6Cvpp7g","elasticsearch.node.id":"a8BUD2RfQSu4aqtpePX7BA","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"}
+{"@timestamp":"2024-03-13T10:34:37.257Z", "log.level": "WARN",  "auth.type":"REALM","elasticsearch.slowlog.id":"4","elasticsearch.slowlog.message":"[my-index/stZSoQ12R56VZORRItBKjA]","elasticsearch.slowlog.source":"{\\\"indices\\\":{\\\"field_security\\\":{\\\"grant\\\":\\\"read\\\",\\\"except\\\":\\\"confidential\\\"},\\\"names\\\":[\\\"foo\\\",\\\"bar\\\"],\\\"privileges\\\":\\\"admin\\\",\\\"query\\\":\\\"example\\\",\\\"allow_restricted_indices\\\":true}}","elasticsearch.slowlog.took":"2.5ms","elasticsearch.slowlog.took_millis":"2","user.name":"elastic","user.realm":"reserved" , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_indexing_slowlog","process.thread.name":"elasticsearch[runTask-0][write][T#12]","log.logger":"index.indexing.slowlog.index","elasticsearch.cluster.uuid":"0d2MZYNKR7Wqr2U6Cvpp7g","elasticsearch.node.id":"a8BUD2RfQSu4aqtpePX7BA","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"}
+{"@timestamp":"2024-03-13T10:34:38.373Z", "log.level": "WARN",  "auth.type":"REALM","elasticsearch.slowlog.id":"5","elasticsearch.slowlog.message":"[my-index/stZSoQ12R56VZORRItBKjA]","elasticsearch.slowlog.source":"{\\\"indices\\\":{\\\"field_security\\\":{\\\"grant\\\":\\\"read\\\",\\\"except\\\":\\\"confidential\\\"},\\\"names\\\":[\\\"foo\\\",\\\"bar\\\"],\\\"privileges\\\":\\\"admin\\\",\\\"query\\\":\\\"example\\\",\\\"allow_restricted_indices\\\":true}}","elasticsearch.slowlog.took":"2.2ms","elasticsearch.slowlog.took_millis":"2","user.name":"elastic","user.realm":"reserved" , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_indexing_slowlog","process.thread.name":"elasticsearch[runTask-0][write][T#3]","log.logger":"index.indexing.slowlog.index","elasticsearch.cluster.uuid":"0d2MZYNKR7Wqr2U6Cvpp7g","elasticsearch.node.id":"a8BUD2RfQSu4aqtpePX7BA","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"}