Adding categorization fields for the system/auth module (elastic#11334)

* Adding categorization fields for the system/auth module This PR adds the following fields for the SSH login events: * `event.category: authentication` * `event.action: ssh_login` * `event.type` either `authentication_success` or `authentication_failure` The `event.outcome` is currently not quite ECS compliant, but I didn't touch it to avoid a breaking change. The PR doesn't attempt to categorize other logs besides the SSH login attempts, so it's a subset of elastic#9905, but it's what we need for the UI. * Normalized event.outcome and brought back `system.auth.ssh.event`. * changelog (cherry picked from commit a9f567b)
tsg · Mar 21, 2019 · cc8c923 · cc8c923
1 parent f0ba5d6
commit cc8c923
Show file tree

Hide file tree

Showing 7 changed files with 49 additions and 18 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -77,6 +77,8 @@ https://github.com/elastic/beats/compare/v7.0.0-beta1...master[Check the HEAD di
 - Fix a bug when converting NetFlow fields to snake_case. {pull}10950[10950]
 - Add on_failure handler for Zeek ingest pipelines. Fix one field name error for notice and add an additional test case. {issue}11004[11004] {pull}11105[11105]
 - Fix issue preventing docker container events to be stored if the container has a network interface without ip address. {issue}11225[11225] {pull}11247[11247]
+- Add on_failure handler for Zeek ingest pipelines. Fix one field name error for notice and add an additional test
+  case. {issue}11004[11004] {pull}11105[11105]
 - Change URLPATH grok pattern to support brackets. {issue}11135[11135] {pull}11252[11252]
 - Add support for iis log with different address format. {issue}11255[11255] {pull}11256[11256]
 
@@ -120,6 +122,15 @@ https://github.com/elastic/beats/compare/v7.0.0-beta1...master[Check the HEAD di
 
 - Add ISO8601 timestamp support in syslog metricset. {issue}8716[8716] {pull}10736[10736]
 - Add support for loading custom NetFlow and IPFIX field definitions to netflow input. {pull}10945[10945] {pull}11223[11223]
+- Add more info to message logged when a duplicated symlink file is found {pull}10845[10845]
+- Add option to configure docker input with paths {pull}10687[10687]
+- Add Netflow module to enrich flow events with geoip data. {pull}10877[10877]
+- Set `event.category: network_traffic` for Suricata. {pull}10882[10882]
+- Add configuration knob for auto-discover hints to control whether log harvesting is enabled for the pod/container. {issue}10811[10811] {pull}10911[10911]
+- Change Suricata module pipeline to handle `destination.domain` being set if a reverse DNS processor is used. {issue}10510[10510]
+- Add the `network.community_id` flow identifier to field to the IPTables, Suricata, and Zeek modules. {pull}11005[11005]
+- Add support for loading custom NetFlow and IPFIX field definitions to netflow input. {pull}10945[10945]
+- Added categorization fields for SSH login events in the system/auth fileset. {pull}11334[11334]
 
 *Heartbeat*
 

diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml
@@ -486,11 +486,6 @@
   alias: true
   beat: filebeat
 
-- from: system.auth.ssh.event
-  to: event.action
-  alias: true
-  beat: filebeat
-
 - from: system.auth.program
   to: process.name
   alias: true

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -13084,9 +13084,10 @@ The client IP from SSH connections that are open and immediately dropped.
 *`system.auth.ssh.event`*::
 +
 --
-type: alias
+example: Accepted
+
+The SSH event as found in the logs (Accepted, Invalid, Failed, etc.)
 
-alias to: event.action
 
 --
 

diff --git a/filebeat/module/system/auth/_meta/fields.yml b/filebeat/module/system/auth/_meta/fields.yml
@@ -43,9 +43,10 @@
           The client IP from SSH connections that are open and immediately dropped.
 
       - name: event
-        type: alias
-        path: event.action
-        migration: true
+        example: Accepted
+        description: >
+          The SSH event as found in the logs (Accepted, Invalid, Failed, etc.)
+
       - name: ip
         type: alias
         path: source.ip

diff --git a/filebeat/module/system/auth/ingest/pipeline.json b/filebeat/module/system/auth/ingest/pipeline.json
@@ -9,8 +9,8 @@
                     "GREEDYMULTILINE" : "(.|\n)*"
                 },
                 "patterns": [
-                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{DATA:event.outcome} %{DATA:system.auth.ssh.method} for (invalid user )?%{DATA:user.name} from %{IPORHOST:source.ip} port %{NUMBER:source.port:long} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?",
-                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{DATA:event.outcome} user %{DATA:user.name} from %{IPORHOST:source.ip}",
+                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{DATA:system.auth.ssh.event} %{DATA:system.auth.ssh.method} for (invalid user )?%{DATA:user.name} from %{IPORHOST:source.ip} port %{NUMBER:source.port:long} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?",
+                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{DATA:system.auth.ssh.event} user %{DATA:user.name} from %{IPORHOST:source.ip}",
                     "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: Did not receive identification string from %{IPORHOST:system.auth.ssh.dropped_ip}",
                     "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: \\s*%{DATA:user.name} :( %{DATA:system.auth.sudo.error} ;)? TTY=%{DATA:system.auth.sudo.tty} ; PWD=%{DATA:system.auth.sudo.pwd} ; USER=%{DATA:system.auth.sudo.user} ; COMMAND=%{GREEDYDATA:system.auth.sudo.command}",
                     "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: new group: name=%{DATA:group.name}, GID=%{NUMBER:group.id}",
@@ -61,6 +61,13 @@
                 "target_field": "source.geo",
                 "ignore_failure": true
             }
+        },
+        {
+            "script": {
+                "lang": "painless",
+                "ignore_failure": true,
+                "source": "if (ctx.system.auth.ssh.event == \"Accepted\") {                  if (!ctx.containsKey(\"event\")) {                    ctx.event = [:];                  }                  ctx.event.type = \"authentication_success\";                  ctx.event.category = \"authentication\";                  ctx.event.action = \"ssh_login\";                  ctx.event.outcome = \"success\";                } else if (ctx.system.auth.ssh.event == \"Invalid\" || ctx.system.auth.ssh.event == \"Failed\") {                  if (!ctx.containsKey(\"event\")) {                    ctx.event = [:];                  }                  ctx.event.type = \"authentication_failure\";                  ctx.event.category = \"authentication\";                  ctx.event.action = \"ssh_login\";                  ctx.event.outcome = \"failure\";                }"
+            }
         }
     ],
     "on_failure" : [{

diff --git a/filebeat/module/system/auth/test/test.log-expected.json b/filebeat/module/system/auth/test/test.log-expected.json
@@ -3,7 +3,11 @@
         "ecs.version": "1.0.0",
         "event.dataset": "system.auth",
         "event.module": "system",
-        "event.outcome": "Accepted",
+        "system.auth.ssh.event": "Accepted",
+        "event.outcome": "success",
+        "event.category": "authentication",
+        "event.action": "ssh_login",
+        "event.type": "authentication_success",
         "fileset.name": "auth",
         "host.hostname": "localhost",
         "input.type": "log",
@@ -21,7 +25,11 @@
         "ecs.version": "1.0.0",
         "event.dataset": "system.auth",
         "event.module": "system",
-        "event.outcome": "Accepted",
+        "system.auth.ssh.event": "Accepted",
+        "event.outcome": "success",
+        "event.category": "authentication",
+        "event.action": "ssh_login",
+        "event.type": "authentication_success",
         "fileset.name": "auth",
         "host.hostname": "localhost",
         "input.type": "log",
@@ -38,7 +46,11 @@
         "ecs.version": "1.0.0",
         "event.dataset": "system.auth",
         "event.module": "system",
-        "event.outcome": "Invalid",
+        "system.auth.ssh.event": "Invalid",
+        "event.outcome": "failure",
+        "event.category": "authentication",
+        "event.action": "ssh_login",
+        "event.type": "authentication_failure",
         "fileset.name": "auth",
         "host.hostname": "localhost",
         "input.type": "log",
@@ -53,7 +65,11 @@
         "ecs.version": "1.0.0",
         "event.dataset": "system.auth",
         "event.module": "system",
-        "event.outcome": "Failed",
+        "system.auth.ssh.event": "Failed",
+        "event.outcome": "failure",
+        "event.category": "authentication",
+        "event.action": "ssh_login",
+        "event.type": "authentication_failure",
         "fileset.name": "auth",
         "host.hostname": "slave22",
         "input.type": "log",
@@ -170,4 +186,4 @@
         "user.id": "48",
         "user.name": "apache"
     }
-]
+]
diff --git a/filebeat/module/system/fields.go b/filebeat/module/system/fields.go