Ignore trailing spaces in CEF messages

x-pack/filebeat/module/cef/log/test/cef.log

@@ -1,3 +1,4 @@
 CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Web request|low|eventId=3457 requestMethod=POST slat=38.915 slong=-77.511 proto=TCP sourceServiceName=httpd requestContext=https://www.google.com src=6.7.8.9 spt=33876 dst=192.168.10.1 dpt=443 request=https://www.example.com/cart
 CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|eventId=123 src=6.7.8.9 spt=33876 dst=1.2.3.4 dpt=443 duser=alice suser=bob destinationTranslatedAddress=10.10.10.10 fileHash=bc8bbe52f041fd17318f08a0f73762ce oldFileHash=a9796280592f86b74b27e370662d41eb
 CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|spriv=user dpriv=root
+CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|message=This event is padded with whitespace dst=192.168.1.2 src=192.168.3.4     

x-pack/filebeat/module/cef/log/test/cef.log-expected.json

@@ -150,5 +150,39 @@
         "tags": [
             "cef"
         ]
+    },
+    {
+        "cef.device.event_class_id": "18",
+        "cef.device.product": "Vaporware",
+        "cef.device.vendor": "Elastic",
+        "cef.device.version": "1.0.0-alpha",
+        "cef.extensions.destinationAddress": "192.168.1.2",
+        "cef.extensions.message": "This event is padded with whitespace",
+        "cef.extensions.sourceAddress": "192.168.3.4",
+        "cef.name": "Authentication",
+        "cef.severity": "low",
+        "cef.version": "0",
+        "destination.ip": "192.168.1.2",
+        "event.code": "18",
+        "event.dataset": "cef.log",
+        "event.module": "cef",
+        "event.original": "CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|message=This event is padded with whitespace dst=192.168.1.2 src=192.168.3.4     ",
+        "event.severity": 0,
+        "fileset.name": "log",
+        "input.type": "log",
+        "log.offset": 611,
+        "message": "This event is padded with whitespace",
+        "observer.product": "Vaporware",
+        "observer.vendor": "Elastic",
+        "observer.version": "1.0.0-alpha",
+        "related.ip": [
+            "192.168.1.2",
+            "192.168.3.4"
+        ],
+        "service.type": "cef",
+        "source.ip": "192.168.3.4",
+        "tags": [
+            "cef"
+        ]
     }
 ]

x-pack/filebeat/processors/decode_cef/cef/cef.go

@@ -14,7 +14,10 @@ import (
 // Parser is generated from a ragel state machine using the following command:
 //go:generate ragel -Z -G1 cef.rl -o parser.go
 //go:generate goimports -l -w parser.go
-
+//
+// Run go vet and remove any unreachable code in the generated parser.go.
+// The go generator outputs duplicated goto statements sometimes.
+//
 // An SVG rendering of the state machine can be viewed by opening cef.svg in
 // Chrome / Firefox.
 //go:generate ragel -V -p cef.rl -o cef.dot

x-pack/filebeat/processors/decode_cef/cef/cef.rl

@@ -124,13 +124,13 @@ func (e *Event) unpack(data string) error {
         extension_key_start_chars = alnum | '_';
         extension_key_chars = extension_key_start_chars | '.' | ',' | '[' | ']';
         extension_key_pattern = extension_key_start_chars extension_key_chars*;
-        extension_value_chars = backslash | escape_equal | (any -- equal -- escape);
+        extension_value_chars_nospace = backslash | escape_equal | (any -- equal -- escape -- space);
 
         # Extension fields.
         extension_key = extension_key_pattern >mark %extension_key;
-        extension_value = (extension_value_chars @extension_value_mark)* >extension_value_start $err(extension_err);
-        extension = extension_key equal extension_value %/extension_eof;
-        extensions = " "* extension (" " extension)*;
+        extension_value = (space* extension_value_chars_nospace @extension_value_mark)* >extension_value_start $err(extension_err);
+        extension = extension_key equal extension_value;
+        extensions = " "* extension (space* " " extension)* space* %/extension_eof;
 
         # gobble_extension attempts recovery from a malformed value by trying to
         # advance to the next extension key and re-entering the main state machine.

x-pack/filebeat/processors/decode_cef/cef/cef_test.go

@@ -44,6 +44,14 @@ const (
 	malformedExtensionEscape = `CEF:0|FooBar|Web Gateway|1.2.3.45.67|200|Success|2|rt=Sep 07 2018 14:50:39 cat=Access Log dst=1.1.1.1 dhost=foo.example.com suser=redacted src=2.2.2.2 requestMethod=POST request='https://foo.example.com/bar/bingo/1' requestClientApplication='Foo-Bar/2018.1.7; =Email:[email protected]; Guid:test=' cs1= cs1Label=Foo Bar`
 
 	multipleMalformedExtensionValues = `CEF:0|vendor|product|version|event_id|name|Very-High| msg=Hello World error=Failed because id==old_id user=root angle=106.7<=180`
+
+	paddedMessage = `CEF:0|security|threatmanager|1.0|100|message is padded|10|spt=1232 msg=Trailing space in non-final extensions is  preserved    src=10.0.0.192 `
+
+	crlfMessage = "CEF:0|security|threatmanager|1.0|100|message is padded|10|spt=1232 msg=Trailing space in final extensions is not preserved\t \r\n"
+
+	tabMessage = "CEF:0|security|threatmanager|1.0|100|message is padded|10|spt=1232 msg=Tabs\tand\rcontrol\ncharacters are preserved\t src=127.0.0.1"
+
+	tabNoSepMessage = "CEF:0|security|threatmanager|1.0|100|message has tabs|10|spt=1232 msg=Tab is not a separator\tsrc=127.0.0.1"
 )
 
 var testMessages = []string{
@@ -60,6 +68,9 @@ var testMessages = []string{
 	escapesInExtension,
 	malformedExtensionEscape,
 	multipleMalformedExtensionValues,
+	paddedMessage,
+	crlfMessage,
+	tabMessage,
 }
 
 func TestGenerateFuzzCorpus(t *testing.T) {
@@ -322,6 +333,47 @@ func TestEventUnpack(t *testing.T) {
 		err := e.Unpack("CEF:0|||||||a=")
 		assert.NoError(t, err)
 	})
+
+	t.Run("padded", func(t *testing.T) {
+		var e Event
+		err := e.Unpack(paddedMessage)
+		assert.NoError(t, err)
+		assert.Equal(t, map[string]*Field{
+			"src": IPField("10.0.0.192"),
+			"spt": IntegerField(1232),
+			"msg": StringField("Trailing space in non-final extensions is  preserved   "),
+		}, e.Extensions)
+	})
+
+	t.Run("padded with extra whitespace chars", func(t *testing.T) {
+		var e Event
+		err := e.Unpack(crlfMessage)
+		assert.NoError(t, err)
+		assert.Equal(t, map[string]*Field{
+			"spt": IntegerField(1232),
+			"msg": StringField("Trailing space in final extensions is not preserved"),
+		}, e.Extensions)
+	})
+
+	t.Run("internal whitespace chars", func(t *testing.T) {
+		var e Event
+		err := e.Unpack(tabMessage)
+		assert.NoError(t, err)
+		assert.Equal(t, map[string]*Field{
+			"spt": IntegerField(1232),
+			"src": IPField("127.0.0.1"),
+			"msg": StringField("Tabs\tand\rcontrol\ncharacters are preserved\t"),
+		}, e.Extensions)
+	})
+
+	t.Run("No tab as separator", func(t *testing.T) {
+		var e Event
+		err := e.Unpack(tabNoSepMessage)
+		assert.Error(t, err)
+		assert.Equal(t, map[string]*Field{
+			"spt": IntegerField(1232),
+		}, e.Extensions)
+	})
 }
 
 func TestEventUnpackWithFullExtensionNames(t *testing.T) {