[Filebeat] Improve ECS field mapping for auditd module (#16280) (#16355)

* Improve ECS field mapping for auditd module - event.kind - event.type - event.category - container.name - container.runtime - process.args_count - process.exit_code - process.working_directory Closes #16153 (cherry picked from commit 43463f1)
elastic · Feb 19, 2020 · 9f9340c · 9f9340c
1 parent edd9766
commit 9f9340c
Show file tree

Hide file tree

Showing 6 changed files with 475 additions and 78 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -197,7 +197,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add ECS tls and categorization fields to apache module. {issue}16032[16032] {pull}16121[16121]
 - Add MQTT input. {issue}15602[15602] {pull}16204[16204]
 - Add a TLS test and more debug output to httpjson input {pull}16315[16315]
-- Add an SSL config example in config.yml for filebeat MISP module. {pull}16320[16320] 
+- Add an SSL config example in config.yml for filebeat MISP module. {pull}16320[16320]
+- Improve ECS categorization, container & process field mappings in auditd module. {issue}16153[16153] {pull}16280[16280]
 
 *Heartbeat*
 

diff --git a/filebeat/module/auditd/log/ingest/pipeline.yml b/filebeat/module/auditd/log/ingest/pipeline.yml
@@ -11,7 +11,7 @@ processors:
     - "%{AUDIT_PREFIX} %{AUDIT_KEY_VALUES:auditd.log.kv} old auid=%{NUMBER:auditd.log.old_auid}
       new auid=%{NUMBER:auditd.log.new_auid} old ses=%{NUMBER:auditd.log.old_ses}
       new ses=%{NUMBER:auditd.log.new_ses}"
-    - '%{AUDIT_PREFIX} %{AUDIT_KEY_VALUES:auditd.log.kv} msg=[''"](%{DATA:auditd.log.msg}\s+)?%{AUDIT_KEY_VALUES:auditd.log.sub_kv}[''"]'
+    - "%{AUDIT_PREFIX} %{AUDIT_KEY_VALUES:auditd.log.kv} msg=['\"]%{AUDIT_KEY_VALUES:auditd.log.sub_kv}['\"]"
     - "%{AUDIT_PREFIX} %{AUDIT_KEY_VALUES:auditd.log.kv}"
     - "%{AUDIT_PREFIX}"
     - "%{AUDIT_TYPE} %{AUDIT_KEY_VALUES:auditd.log.kv}"
@@ -132,6 +132,61 @@ processors:
     params:
       single_quote: "'"
       double_quote: "\""
+- set:
+    field: event.kind
+    value: event
+- set:
+    if: "ctx.auditd.log?.record_type == 'USER_AUTH'"
+    field: event.type
+    value: authentication
+- set:
+    if: "ctx.auditd.log?.record_type == 'KERN_MODULE'"
+    field: event.type
+    value: driver
+- set:
+    if: "ctx.auditd.log?.record_type == 'SOFTWARE_UPDATE'"
+    field: event.type
+    value: package
+- set:
+    if: "ctx.auditd.log?.record_type == 'SYSTEM_BOOT' || ctx.auditd.log?.record_type == 'SYSTEM_SHUTDOWN'"
+    field: event.type
+    value: host
+- set:
+    if: "ctx.auditd.log?.record_type == 'SYSCALL' && ctx.auditd.log?.syscall == 'execve'"
+    field: event.type
+    value: process
+- set:
+    if: "ctx.auditd.log?.record_type == 'VIRT_CONTROL' || ctx.auditd.log?.record_type == 'VIRT_MACHINE_ID'"
+    field: event.category
+    value: host
+- set:
+    if: "ctx.auditd.log?.record_type == 'VIRT_CONTROL' && ctx.auditd.log?.op == 'start'"
+    field: event.type
+    value: start
+- set:
+    if: "ctx.auditd.log?.record_type == 'VIRT_CONTROL' && ctx.auditd.log?.op == 'stop'"
+    field: event.type
+    value: end
+- set:
+    if: "ctx.auditd.log?.record_type == 'VIRT_CONTROL' && ctx.auditd.log?.op == 'create'"
+    field: event.type
+    value: creation
+- set:
+    if: "ctx.auditd.log?.record_type == 'VIRT_CONTROL' && ctx.auditd.log?.op == 'delete'"
+    field: event.type
+    value: deletion
+- set:
+    if: "ctx.auditd.log?.record_type == 'VIRT_MACHINE_ID'"
+    field: event.type
+    value: creation
+- set:
+    if: "ctx.auditd.log?.record_type == 'VIRT_MACHINE_ID' && ctx.auditd.log?.vm != null"
+    field: container.name
+    value: "{{ auditd.log.vm }}"
+- set:
+    if: "ctx.auditd.log?.record_type == 'VIRT_MACHINE_ID' && ctx.auditd.log?.virt != null"
+    field: container.runtime
+    value: "{{ auditd.log.virt }}"
 - rename:
     ignore_failure: true
     field: auditd.log.arch
@@ -220,6 +275,25 @@ processors:
     ignore_failure: true
     field: process.args
     separator: "\\s+"
+- script:
+    if: "ctx?.process?.args != null"
+    lang: painless
+    source: >-
+      if (ctx.process.args instanceof List) {
+        ctx.process.args_count = ctx.process.args.length;
+      }
+- rename:
+    ignore_failure: true
+    field: auditd.log.exit
+    target_field: process.exit_code
+- convert:
+    ignore_missing: true
+    field: process.exit_code
+    type: long
+- rename:
+    ignore_missing: true
+    field: auditd.log.cwd
+    target_field: process.working_directory
 - rename:
     ignore_failure: true
     field: auditd.log.terminal

diff --git a/filebeat/module/auditd/log/test/audit-rhel6.log-expected.json b/filebeat/module/auditd/log/test/audit-rhel6.log-expected.json
@@ -1,16 +1,17 @@
 [
     {
         "@timestamp": "2017-03-14T19:20:30.178Z",
+        "auditd.log.op": "PAM:session_close",
         "auditd.log.sequence": 19600327,
         "auditd.log.ses": "11988",
         "event.action": "user_end",
         "event.dataset": "auditd.log",
+        "event.kind": "event",
         "event.module": "auditd",
         "event.outcome": "success",
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 0,
-        "message": "op=PAM:session_close",
         "process.executable": "/usr/bin/sudo",
         "process.pid": 4121,
         "service.type": "auditd",
@@ -20,16 +21,17 @@
     },
     {
         "@timestamp": "2017-03-14T19:20:30.178Z",
+        "auditd.log.op": "PAM:setcred",
         "auditd.log.sequence": 19600328,
         "auditd.log.ses": "11988",
         "event.action": "cred_disp",
         "event.dataset": "auditd.log",
+        "event.kind": "event",
         "event.module": "auditd",
         "event.outcome": "success",
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 189,
-        "message": "op=PAM:setcred",
         "process.executable": "/usr/bin/sudo",
         "process.pid": 4121,
         "service.type": "auditd",
@@ -43,34 +45,37 @@
         "auditd.log.ses": "11988",
         "event.action": "user_cmd",
         "event.dataset": "auditd.log",
+        "event.kind": "event",
         "event.module": "auditd",
         "event.outcome": "success",
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 373,
-        "message": "cwd=\"/",
         "process.args": [
             "/usr/lib64/nagios/plugins/check_asterisk_sip_peers",
             "-p",
             "202"
         ],
+        "process.args_count": 3,
         "process.pid": 4151,
+        "process.working_directory": "/",
         "service.type": "auditd",
         "user.audit.id": "700",
         "user.id": "497"
     },
     {
         "@timestamp": "2017-03-14T19:20:56.193Z",
+        "auditd.log.op": "PAM:setcred",
         "auditd.log.sequence": 19600330,
         "auditd.log.ses": "11988",
         "event.action": "cred_acq",
         "event.dataset": "auditd.log",
+        "event.kind": "event",
         "event.module": "auditd",
         "event.outcome": "success",
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 620,
-        "message": "op=PAM:setcred",
         "process.executable": "/usr/bin/sudo",
         "process.pid": 4151,
         "service.type": "auditd",
@@ -80,16 +85,17 @@
     },
     {
         "@timestamp": "2017-03-14T19:20:56.193Z",
+        "auditd.log.op": "PAM:session_open",
         "auditd.log.sequence": 19600331,
         "auditd.log.ses": "11988",
         "event.action": "user_start",
         "event.dataset": "auditd.log",
+        "event.kind": "event",
         "event.module": "auditd",
         "event.outcome": "success",
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 803,
-        "message": "op=PAM:session_open",
         "process.executable": "/usr/bin/sudo",
         "process.pid": 4151,
         "service.type": "auditd",
@@ -107,6 +113,7 @@
         "destination.address": "10.100.4.0",
         "event.action": "mac_ipsec_event",
         "event.dataset": "auditd.log",
+        "event.kind": "event",
         "event.module": "auditd",
         "event.outcome": "1",
         "fileset.name": "log",
@@ -123,7 +130,6 @@
         "auditd.log.a1": "7f564ee6d2a0",
         "auditd.log.a2": "b8",
         "auditd.log.a3": "0",
-        "auditd.log.exit": "184",
         "auditd.log.items": "0",
         "auditd.log.sequence": 19600354,
         "auditd.log.ses": "4294967295",
@@ -132,12 +138,14 @@
         "auditd.log.tty": "(none)",
         "event.action": "syscall",
         "event.dataset": "auditd.log",
+        "event.kind": "event",
         "event.module": "auditd",
         "fileset.name": "log",
         "host.architecture": "x86_64",
         "input.type": "log",
         "log.offset": 1162,
         "process.executable": "/usr/libexec/strongswan/charon (deleted)",
+        "process.exit_code": 184,
         "process.name": "charon",
         "process.pid": 1275,
         "process.ppid": 1240,
@@ -161,6 +169,7 @@
         "auditd.log.sequence": 19623791,
         "event.action": "login",
         "event.dataset": "auditd.log",
+        "event.kind": "event",
         "event.module": "auditd",
         "fileset.name": "log",
         "input.type": "log",
@@ -176,18 +185,19 @@
         "auditd.log.kind": "session",
         "auditd.log.laddr": "107.170.139.210",
         "auditd.log.lport": "50022",
+        "auditd.log.op": "destroy",
         "auditd.log.rport": "58994",
         "auditd.log.sequence": 19623788,
         "auditd.log.ses": "6793",
         "auditd.log.spid": "28282",
         "event.action": "crypto_key_user",
         "event.dataset": "auditd.log",
+        "event.kind": "event",
         "event.module": "auditd",
         "event.outcome": "success",
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 1640,
-        "message": "op=destroy",
         "process.executable": "/usr/sbin/sshd",
         "process.pid": 28281,
         "service.type": "auditd",
@@ -198,16 +208,18 @@
     {
         "@timestamp": "2017-03-16T04:02:40.072Z",
         "auditd.log.addr": "96.241.146.97",
+        "auditd.log.op": "success",
         "auditd.log.sequence": 19623789,
         "auditd.log.ses": "6793",
         "event.action": "user_auth",
         "event.dataset": "auditd.log",
+        "event.kind": "event",
         "event.module": "auditd",
         "event.outcome": "success",
+        "event.type": "authentication",
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 1926,
-        "message": "op=success",
         "process.executable": "/usr/sbin/sshd",
         "process.pid": 28281,
         "service.type": "auditd",
@@ -218,16 +230,18 @@
     },
     {
         "@timestamp": "2017-03-16T04:02:57.804Z",
+        "auditd.log.op": "PAM:authentication",
         "auditd.log.sequence": 19623807,
         "auditd.log.ses": "12286",
         "event.action": "user_auth",
         "event.dataset": "auditd.log",
+        "event.kind": "event",
         "event.module": "auditd",
         "event.outcome": "success",
+        "event.type": "authentication",
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 2122,
-        "message": "op=PAM:authentication",
         "process.executable": "/bin/su",
         "process.pid": 28395,
         "service.type": "auditd",
@@ -238,16 +252,17 @@
     },
     {
         "@timestamp": "2017-03-16T04:02:57.805Z",
+        "auditd.log.op": "PAM:accounting",
         "auditd.log.sequence": 19623808,
         "auditd.log.ses": "12286",
         "event.action": "user_acct",
         "event.dataset": "auditd.log",
+        "event.kind": "event",
         "event.module": "auditd",
         "event.outcome": "success",
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 2312,
-        "message": "op=PAM:accounting",
         "process.executable": "/bin/su",
         "process.pid": 28395,
         "service.type": "auditd",