Drop not audit logs in elasticsearch/audit fileset ingest pipeline (#…

…30164) * adjust elasticsearch/audit fileset Signed-off-by: Tetiana Kravchenko <[email protected]> * fix formatting Signed-off-by: Tetiana Kravchenko <[email protected]> * adjust drop condition; revert test case for docker Signed-off-by: Tetiana Kravchenko <[email protected]> * fix typos Signed-off-by: Tetiana Kravchenko <[email protected]> * remove 'remove' of user.name field in pipeline-json for elasticsearch/audit Signed-off-by: Tetiana Kravchenko <[email protected]> (cherry picked from commit 1e72124)
elastic · Feb 7, 2022 · 76dffa7 · 76dffa7
1 parent 1eb1652
commit 76dffa7
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 24 deletions.
diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml
@@ -3,6 +3,13 @@ processors:
   - json:
       field: message
       target_field: elasticsearch.audit
+  - dot_expander:
+      field: event.type
+      path: elasticsearch.audit
+  - drop:
+      if: ctx.elasticsearch.audit.containsKey('type') && ctx.elasticsearch.audit.type != 'audit'
+  - drop:
+      if: '!ctx.elasticsearch.audit.containsKey("type") && !["rest", "transport", "ip_filter", "security_config_change"].contains(ctx.elasticsearch?.audit?.event?.type)'
   - remove:
       field: elasticsearch.audit.type
       ignore_missing: true
@@ -25,13 +32,13 @@ processors:
   - dot_expander:
       field: event.action
       path: elasticsearch.audit
+  - remove:
+      field: event.action
+      ignore_missing: true
   - rename:
       field: elasticsearch.audit.event.action
       target_field: event.action
       ignore_missing: true
-  - dot_expander:
-      field: event.type
-      path: elasticsearch.audit
   - rename:
       field: elasticsearch.audit.event.type
       target_field: elasticsearch.audit.layer
@@ -45,6 +52,9 @@ processors:
         - \[%{IPORHOST:source.ip}\]:%{INT:source.port:int}
         - "%{IPORHOST:source.ip}:%{INT:source.port:int}"
       ignore_missing: true
+  - remove:
+      field: source.address
+      ignore_missing: true
   - rename:
       field: elasticsearch.audit.origin.address
       target_field: source.address
@@ -76,6 +86,9 @@ processors:
   - dot_expander:
       field: node.name
       path: elasticsearch.audit
+  - remove:
+      field: elasticsearch.node
+      ignore_missing: true
   - rename:
       field: elasticsearch.audit.node
       target_field: elasticsearch.node

diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-docker.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit-docker.log-expected.json
@@ -24,27 +24,6 @@
         "source.port": 40380,
         "url.original": "/"
     },
-    {
-        "@timestamp": "2019-06-11T15:03:32.777Z",
-        "elasticsearch.audit.component": "o.e.x.s.a.AuthenticationService",
-        "elasticsearch.audit.message": "Authentication of [elastic] was terminated by realm [reserved] - failed to authenticate user [elastic]",
-        "elasticsearch.cluster.name": "docker-cluster",
-        "elasticsearch.cluster.uuid": "xEiKc6ipRiyzU8_8czXrJw",
-        "elasticsearch.node.id": "Xaq2BFVcQ1OhyMrjL8gNOg",
-        "elasticsearch.node.name": "dff7befc418f",
-        "event.category": "database",
-        "event.dataset": "elasticsearch.audit",
-        "event.kind": "event",
-        "event.module": "elasticsearch",
-        "event.outcome": "failure",
-        "fileset.name": "audit",
-        "host.id": "Xaq2BFVcQ1OhyMrjL8gNOg",
-        "input.type": "log",
-        "log.level": "INFO",
-        "log.offset": 299,
-        "message": "{\"type\": \"server\", \"timestamp\": \"2019-06-11T15:03:32,777+0000\", \"level\": \"INFO\", \"component\": \"o.e.x.s.a.AuthenticationService\", \"cluster.name\": \"docker-cluster\", \"node.name\": \"dff7befc418f\", \"cluster.uuid\": \"xEiKc6ipRiyzU8_8czXrJw\", \"node.id\": \"Xaq2BFVcQ1OhyMrjL8gNOg\",  \"message\": \"Authentication of [elastic] was terminated by realm [reserved] - failed to authenticate user [elastic]\"  }",
-        "service.type": "elasticsearch"
-    },
     {
         "@timestamp": "2019-06-11T15:03:32.778Z",
         "elasticsearch.audit.layer": "rest",