Addressing review comments

- Renamed system.audit.type to system.audit.record_type - Renamed system.audit.counter to system.audit.sequence - Dropped system.audit.epoch after parsing it into @timestamp - Dropped message field. - Removed “and others” from the supported OSes listed in docs. - Updated dashboards after renaming and deleting fields. - Updated script to drop fields where value was `(null)`.
elastic · Mar 31, 2017 · ececd94 · ececd94
1 parent c945fb7
commit ececd94
Show file tree

Hide file tree

Showing 10 changed files with 37 additions and 37 deletions.
diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -873,23 +873,17 @@ Fields from the Linux audit logs. Not all fields are documented here because the
 
 
 [float]
-=== system.audit.type
+=== system.audit.record_type
 
 The audit event type.
 
 
 [float]
-=== system.audit.epoch
-
-The unix epoch timestamp from the audit log.
-
-
-[float]
-=== system.audit.counter
+=== system.audit.sequence
 
 type: long
 
-The audit event counter.
+The audit event sequence number.
 
 
 [float]

diff --git a/filebeat/docs/modules/system.asciidoc b/filebeat/docs/modules/system.asciidoc
@@ -11,8 +11,8 @@ Unix/Linux based distributions.
 [float]
 === Compatibility
 
-This module was tested with logs from OSes like Ubuntu 12.04, Centos 7, macOS
-Sierra, and others.
+This module was tested with logs from OSes like Ubuntu 12.04, Centos 7, and
+macOS Sierra.
 
 This module is not available for Windows.
 

diff --git a/filebeat/module/system/_meta/docs.asciidoc b/filebeat/module/system/_meta/docs.asciidoc
@@ -6,8 +6,8 @@ Unix/Linux based distributions.
 [float]
 === Compatibility
 
-This module was tested with logs from OSes like Ubuntu 12.04, Centos 7, macOS
-Sierra, and others.
+This module was tested with logs from OSes like Ubuntu 12.04, Centos 7, and
+macOS Sierra.
 
 This module is not available for Windows.
 

diff --git a/filebeat/module/system/audit/_meta/fields.yml b/filebeat/module/system/audit/_meta/fields.yml
@@ -4,16 +4,13 @@
     Fields from the Linux audit logs. Not all fields are documented here because
     they are dynamic and vary by audit event type.
   fields:
-    - name: type
+    - name: record_type
       description: >
         The audit event type.
-    - name: epoch
-      description: >
-        The unix epoch timestamp from the audit log.
-    - name: counter
+    - name: sequence
       type: long
       description: >
-        The audit event counter.
+        The audit event sequence number.
     - name: pid
       type: long
       description: >

diff --git a/...beat/module/system/audit/_meta/kibana/dashboard/dfbb49f0-0a0f-11e7-8a62-2d05eaaac5cb.json b/...beat/module/system/audit/_meta/kibana/dashboard/dfbb49f0-0a0f-11e7-8a62-2d05eaaac5cb.json
@@ -4,7 +4,7 @@
   "description": "", 
   "title": "Filebeat System Audit", 
   "uiStateJSON": "{\"P-2\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", 
-  "panelsJSON": "[{\"col\":1,\"id\":\"6295bdd0-0a0e-11e7-825f-6748cda7d858\",\"panelIndex\":1,\"row\":1,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":9,\"id\":\"5ebdbe50-0a0f-11e7-825f-6748cda7d858\",\"panelIndex\":2,\"row\":1,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"2bb0fa70-0a11-11e7-9e84-43da493ad0c7\",\"panelIndex\":3,\"row\":5,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"columns\":[\"message\"],\"id\":\"4ac0a370-0a11-11e7-8b04-eb22a5669f27\",\"panelIndex\":4,\"row\":8,\"size_x\":12,\"size_y\":3,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":7,\"id\":\"d1726930-0a7f-11e7-8b04-eb22a5669f27\",\"panelIndex\":5,\"row\":5,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"size_x\":4,\"size_y\":4,\"panelIndex\":6,\"type\":\"visualization\",\"id\":\"c5411910-0a87-11e7-8b04-eb22a5669f27\",\"col\":5,\"row\":1}]", 
+  "panelsJSON": "[{\"col\":1,\"id\":\"6295bdd0-0a0e-11e7-825f-6748cda7d858\",\"panelIndex\":1,\"row\":1,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":9,\"id\":\"5ebdbe50-0a0f-11e7-825f-6748cda7d858\",\"panelIndex\":2,\"row\":1,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"2bb0fa70-0a11-11e7-9e84-43da493ad0c7\",\"panelIndex\":3,\"row\":5,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"d1726930-0a7f-11e7-8b04-eb22a5669f27\",\"panelIndex\":5,\"row\":5,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"c5411910-0a87-11e7-8b04-eb22a5669f27\",\"panelIndex\":6,\"row\":1,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"size_x\":12,\"size_y\":3,\"panelIndex\":7,\"type\":\"search\",\"id\":\"4ac0a370-0a11-11e7-8b04-eb22a5669f27\",\"col\":1,\"row\":8,\"columns\":[\"system.audit.record_type\",\"system.audit.sequence\",\"system.audit.acct\"],\"sort\":[\"@timestamp\",\"desc\"]}]", 
   "optionsJSON": "{\"darkTheme\":false}", 
   "version": 1, 
   "kibanaSavedObjectMeta": {

diff --git a/filebeat/module/system/audit/_meta/kibana/search/4ac0a370-0a11-11e7-8b04-eb22a5669f27.json b/filebeat/module/system/audit/_meta/kibana/search/4ac0a370-0a11-11e7-8b04-eb22a5669f27.json
@@ -11,6 +11,8 @@
     "searchSourceJSON": "{\"index\":\"filebeat-*\",\"highlightAll\":true,\"version\":true,\"query\":{\"query_string\":{\"query\":\"_exists_:system.audit\",\"analyze_wildcard\":true}},\"filter\":[]}"
   }, 
   "columns": [
-    "message"
+    "system.audit.record_type", 
+    "system.audit.sequence", 
+    "system.audit.acct"
   ]
 }
diff --git a/.../module/system/audit/_meta/kibana/visualization/5ebdbe50-0a0f-11e7-825f-6748cda7d858.json b/.../module/system/audit/_meta/kibana/visualization/5ebdbe50-0a0f-11e7-825f-6748cda7d858.json
@@ -5,6 +5,6 @@
   "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", 
   "version": 1, 
   "kibanaSavedObjectMeta": {
-    "searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"query_string\":{\"query\":\"system.audit.type:EXECVE\",\"analyze_wildcard\":true}},\"filter\":[]}"
+    "searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"query_string\":{\"query\":\"system.audit.record_type:EXECVE\",\"analyze_wildcard\":true}},\"filter\":[]}"
   }
 }
diff --git a/.../module/system/audit/_meta/kibana/visualization/6295bdd0-0a0e-11e7-825f-6748cda7d858.json b/.../module/system/audit/_meta/kibana/visualization/6295bdd0-0a0e-11e7-825f-6748cda7d858.json
@@ -1,10 +1,10 @@
 {
-  "visState": "{\"title\":\"Audit Event Types\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"system.audit.type\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", 
+  "visState": "{\n  \"title\": \"Audit Event Types\",\n  \"type\": \"pie\",\n  \"params\": {\n    \"addTooltip\": true,\n    \"addLegend\": true,\n    \"legendPosition\": \"right\",\n    \"isDonut\": true\n  },\n  \"aggs\": [\n    {\n      \"id\": \"1\",\n      \"enabled\": true,\n      \"type\": \"count\",\n      \"schema\": \"metric\",\n      \"params\": {}\n    },\n    {\n      \"id\": \"2\",\n      \"enabled\": true,\n      \"type\": \"terms\",\n      \"schema\": \"segment\",\n      \"params\": {\n        \"field\": \"system.audit.record_type\",\n        \"size\": 50,\n        \"order\": \"desc\",\n        \"orderBy\": \"1\"\n      }\n    }\n  ],\n  \"listeners\": {}\n}", 
   "description": "", 
   "title": "Audit Event Types", 
   "uiStateJSON": "{}", 
   "version": 1, 
   "kibanaSavedObjectMeta": {
-    "searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+    "searchSourceJSON": "{\n  \"index\": \"filebeat-*\",\n  \"query\": {\n    \"query_string\": {\n      \"query\": \"*\",\n      \"analyze_wildcard\": true\n    }\n  },\n  \"filter\": []\n}"
   }
 }
diff --git a/filebeat/module/system/audit/ingest/pipeline.json b/filebeat/module/system/audit/ingest/pipeline.json
@@ -5,8 +5,8 @@
             "grok": {
                 "field": "message",
                 "pattern_definitions": {
-                    "AUDIT_TYPE": "^type=%{NOTSPACE:system.audit.type}",
-                    "AUDIT_PREFIX": "%{AUDIT_TYPE} msg=audit\\(%{NUMBER:system.audit.epoch}:%{NUMBER:system.audit.counter}\\):(%{DATA})?",
+                    "AUDIT_TYPE": "^type=%{NOTSPACE:system.audit.record_type}",
+                    "AUDIT_PREFIX": "%{AUDIT_TYPE} msg=audit\\(%{NUMBER:system.audit.epoch}:%{NUMBER:system.audit.sequence}\\):(%{DATA})?",
                     "AUDIT_KEY_VALUES": "%{WORD}=%{GREEDYDATA}"
                 },
                 "patterns": [
@@ -47,6 +47,12 @@
                 "ignore_failure": true
             }
         },
+        {
+            "remove": {
+                "field": "message",
+                "ignore_failure": true
+            }
+        },
         {
             "date": {
                 "field": "system.audit.epoch",
@@ -57,9 +63,15 @@
                 "ignore_failure": true
             }
         },
+        {
+            "remove": {
+                "field": "system.audit.epoch",
+                "ignore_failure": true
+            }
+        },
         {
             "convert": {
-                "field" : "system.audit.counter",
+                "field" : "system.audit.sequence",
                 "type": "integer",
                 "ignore_missing": true
             }
@@ -95,7 +107,7 @@
         {
             "script": {
                 "lang": "painless",
-                "inline": "    String trimQuotes(def v) {\n        if (v.startsWith(\"'\") || v.startsWith('\"')) {\n          v = v.substring(1, v.length());\n        }\n        if (v.endsWith(\"'\") || v.endsWith('\"')) {\n          v = v.substring(0, v.length()-1);\n        }  \n        return v;\n    }\n    \n    boolean isHexAscii(String v) {\n      def len = v.length();\n      if (len == 0 || len % 2 != 0) {\n        return false; \n      }\n  \n      for (int i = 0 ; i < len ; i++) {\n        if (Character.digit(v.charAt(i), 16) == -1) {\n          return false;\n        }\n      }\n\n      return true;\n    }\n    \n    String convertHexToString(String hex) {\n\t    StringBuilder sb = new StringBuilder();\n\n      for (int i=0; i < hex.length() - 1; i+=2) {\n          String output = hex.substring(i, (i + 2));\n          int decimal = Integer.parseInt(output, 16);\n          sb.append((char)decimal);\n      }\n\n      return sb.toString();\n    }\n    \n    def possibleHexKeys = ['exe', 'cmd'];\n    \n    def audit = ctx.system.get(\"audit\");\n    Iterator entries = audit.entrySet().iterator();\n    while (entries.hasNext()) {\n      def e = entries.next();\n      def k = e.getKey();\n      def v = e.getValue(); \n\n      // Remove entries whose value is ?\n      if (v == \"?\") {\n        entries.remove();\n        continue;\n      }\n      \n      // Convert hex values to ASCII.\n      if (possibleHexKeys.contains(k) && isHexAscii(v)) {\n        v = convertHexToString(v);\n        audit.put(k, v);\n      }\n      \n      // Trim quotes.\n      if (v instanceof String) {\n        v = trimQuotes(v);\n        audit.put(k, v);\n      }\n      \n      // Convert arch.\n      if (k == \"arch\" && v == \"c000003e\") {\n        audit.put(k, \"x86_64\");\n      }\n    }"
+                "inline": "    String trimQuotes(def v) {\n        if (v.startsWith(\"'\") || v.startsWith('\"')) {\n          v = v.substring(1, v.length());\n        }\n        if (v.endsWith(\"'\") || v.endsWith('\"')) {\n          v = v.substring(0, v.length()-1);\n        }  \n        return v;\n    }\n    \n    boolean isHexAscii(String v) {\n      def len = v.length();\n      if (len == 0 || len % 2 != 0) {\n        return false; \n      }\n  \n      for (int i = 0 ; i < len ; i++) {\n        if (Character.digit(v.charAt(i), 16) == -1) {\n          return false;\n        }\n      }\n\n      return true;\n    }\n    \n    String convertHexToString(String hex) {\n\t    StringBuilder sb = new StringBuilder();\n\n      for (int i=0; i < hex.length() - 1; i+=2) {\n          String output = hex.substring(i, (i + 2));\n          int decimal = Integer.parseInt(output, 16);\n          sb.append((char)decimal);\n      }\n\n      return sb.toString();\n    }\n    \n    def possibleHexKeys = ['exe', 'cmd'];\n    \n    def audit = ctx.system.get(\"audit\");\n    Iterator entries = audit.entrySet().iterator();\n    while (entries.hasNext()) {\n      def e = entries.next();\n      def k = e.getKey();\n      def v = e.getValue(); \n\n      // Remove entries whose value is ?\n      if (v == \"?\" || v == \"(null)\" || v == \"\") {\n        entries.remove();\n        continue;\n      }\n      \n      // Convert hex values to ASCII.\n      if (possibleHexKeys.contains(k) && isHexAscii(v)) {\n        v = convertHexToString(v);\n        audit.put(k, v);\n      }\n      \n      // Trim quotes.\n      if (v instanceof String) {\n        v = trimQuotes(v);\n        audit.put(k, v);\n      }\n      \n      // Convert arch.\n      if (k == \"arch\" && v == \"c000003e\") {\n        audit.put(k, \"x86_64\");\n      }\n    }"
             }
         },
         {

diff --git a/filebeat/module/system/audit/test/test.log-expected.json b/filebeat/module/system/audit/test/test.log-expected.json
@@ -13,7 +13,6 @@
                 "module": "system",
                 "name": "audit"
             },
-            "message": "type=MAC_IPSEC_EVENT msg=audit(1485893834.891:18877201): op=SPD-delete auid=4294967295 ses=4294967295 res=1 src=192.168.2.0 src_prefixlen=24 dst=192.168.0.0 dst_prefixlen=16",
             "type": "log",
             "error": "",
             "@timestamp": "2017-01-31T20:18:46.912Z",
@@ -25,10 +24,9 @@
                     "auid": "4294967295",
                     "dst": "192.168.0.0",
                     "src": "192.168.2.0",
-                    "epoch": "1485893834.891",
-                    "counter": 18877201,
+                    "sequence": 18877201,
                     "src_prefixlen": "24",
-                    "type": "MAC_IPSEC_EVENT",
+                    "record_type": "MAC_IPSEC_EVENT",
                     "dst_prefixlen": "16"
                 }
             },
@@ -53,7 +51,6 @@
                 "module": "system",
                 "name": "audit"
             },
-            "message": "type=SYSCALL msg=audit(1485893834.891:18877199): arch=c000003e syscall=44 success=yes exit=184 a0=9 a1=7f564b2672a0 a2=b8 a3=0 items=0 ppid=1240 pid=1281 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"charon\" exe=2F7573722F6C6962657865632F7374726F6E677377616E2F636861726F6E202864656C6574656429 key=(null)",
             "type": "log",
             "error": "",
             "@timestamp": "2017-01-31T20:18:46.912Z",
@@ -62,20 +59,18 @@
                     "syscall": "44",
                     "gid": "0",
                     "fsgid": "0",
-                    "epoch": "1485893834.891",
                     "pid": 1281,
                     "suid": "0",
-                    "type": "SYSCALL",
+                    "record_type": "SYSCALL",
                     "uid": "0",
                     "egid": "0",
                     "exe": "/usr/libexec/strongswan/charon (deleted)",
                     "sgid": "0",
-                    "key": "(null)",
                     "ses": "4294967295",
                     "auid": "4294967295",
                     "comm": "charon",
                     "euid": "0",
-                    "counter": 18877199,
+                    "sequence": 18877199,
                     "a0": "9",
                     "ppid": 1240,
                     "a1": "7f564b2672a0",