Filebeat syslog module: support for lines without a program name

Closes elastic#3913.
tsg · Apr 10, 2017 · 3098911 · 3098911
1 parent 7dadfe7
commit 3098911
Show file tree

Hide file tree

Showing 4 changed files with 32 additions and 1 deletion.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -56,6 +56,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff]
 - Fix empty registry file on machine crash. {issue}3537[3537]
 - Allow `-` in Apache access log byte count. {pull}3863[3863]
 - Downgrade Elasticsearch per batch item failure log to debug level. {issue}3953[3953]
+- Allow log lines without a program name in the Syslog fileset. {pull}3944[3944]
 
 *Heartbeat*
 

diff --git a/filebeat/module/system/syslog/ingest/pipeline.json b/filebeat/module/system/syslog/ingest/pipeline.json
@@ -5,7 +5,8 @@
 			"grok": {
 				"field": "message",
 				"patterns": [
-					"%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{SYSLOGHOST:system.syslog.hostname} %{DATA:system.syslog.program}(?:\\[%{POSINT:system.syslog.pid}\\])?: %{GREEDYMULTILINE:system.syslog.message}"
+					"%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{SYSLOGHOST:system.syslog.hostname} %{DATA:system.syslog.program}(?:\\[%{POSINT:system.syslog.pid}\\])?: %{GREEDYMULTILINE:system.syslog.message}",
+					"%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{GREEDYMULTILINE:system.syslog.message}"
 				],
         "pattern_definitions" : {
           "GREEDYMULTILINE" : "(.|\n)*"

diff --git a/filebeat/module/system/syslog/test/darwin-syslog-sample.log b/filebeat/module/system/syslog/test/darwin-syslog-sample.log
@@ -18,3 +18,4 @@ Dec 13 11:35:28 a-mac-with-esc-key GoogleSoftwareUpdateAgent[21412]: 2016-12-13
 		errors=0
 	>
 Dec 13 11:35:28 a-mac-with-esc-key GoogleSoftwareUpdateAgent[21412]: 2016-12-13 11:35:28.421 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSUpdateEngine updateAllExceptProduct:] KSUpdateEngine updating all installed products, except:'com.google.Keystone'.
+Apr 4 03:39:57 --- last message repeated 1 time ---
diff --git a/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json b/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json
@@ -58,5 +58,33 @@
       },
       "type" : "log"
     }
+  },
+  {
+    "_index": "test-filebeat-modules",
+    "_type": "doc",
+    "_id": "AVtFVFY1nbkdi71WgGxo",
+    "_score": 1,
+    "_source": {
+      "@timestamp": "2017-04-04T03:39:57.000Z",
+      "system": {
+        "syslog": {
+          "message": "--- last message repeated 1 time ---",
+          "timestamp": "Apr 4 03:39:57"
+        }
+      },
+      "offset": 1228,
+      "beat": {
+        "hostname": "a-mac-with-esc-key-2.local",
+        "name": "a-mac-with-esc-key-2.local",
+        "version": "6.0.0-alpha1"
+      },
+      "input_type": "log",
+      "source": "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/syslog/test/darwin-syslog-sample.log",
+      "fileset": {
+        "module": "system",
+        "name": "syslog"
+      },
+      "type": "log"
+    }
   }
 ]