Adds support for RFC3339 timestamp and nanoseconds parsing (elastic#7046

) (elastic#7063)

* Adds support for RFC3339 timestamp

Syslog formatted events format sometime differ from the official specs,
it's possible to receive timestamp formatted using RFC3339.

This commits fixes a few things:

- Correctly support the 2018-05-08T10:31:24 format;
- Fix nanosecond parsing to respect the actual numerical representation
(microsecond vs. nanosecond)
- Fix an issue when partial Syslog events sent to the publisher,
now when invalid message is received we send the raw format.

* Update changelog

(cherry picked from commit a9616bd)

CHANGELOG.asciidoc

@@ -68,6 +68,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Fix memory leak in log prospector when files cannot be read. {issue}6797[6797]
 - Add raw JSON to message field when JSON parsing fails. {issue}6516[6516]
 - Commit registry writes to stable storage to avoid corrupt registry files. {pull}6877[6877]
+- Fix a parsing issue in the syslog input for RFC3339 timestamp and time with nanoseconds. {pull}7046[7046]
 
 *Heartbeat*
 - Fix race due to updates of shared a map, that was not supposed to be shared between multiple go-routines. {issue}6616[6616]

filebeat/input/syslog/event.go

@@ -1,6 +1,7 @@
 package syslog
 
 import (
+	"math"
 	"time"
 )
 
@@ -22,6 +23,22 @@ var month = map[string]time.Month{
 	"Dec": time.December,
 }
 
+var monthIndexed = []time.Month{
+	0,
+	time.January,
+	time.February,
+	time.March,
+	time.April,
+	time.May,
+	time.June,
+	time.July,
+	time.August,
+	time.September,
+	time.October,
+	time.November,
+	time.December,
+}
+
 // event is a parsed syslog event, validation of the format is done at the parser level.
 type event struct {
 	message    string
@@ -35,6 +52,7 @@ type event struct {
 	minute     int
 	second     int
 	nanosecond int
+	year       int
 	loc        *time.Location
 }
 
@@ -48,9 +66,15 @@ func newEvent() *event {
 		hour:     -1,
 		minute:   -1,
 		second:   -1,
+		year:     time.Now().Year(),
 	}
 }
 
+// SetMonthNumeric sets the month with a number.
+func (s *event) SetMonthNumeric(b []byte) {
+	s.month = monthIndexed[bytesToInt(skipLeadZero(b))]
+}
+
 // SetMonth sets the month.
 func (s *event) SetMonth(b []byte) {
 	var k string
@@ -110,9 +134,14 @@ func (s *event) Second() int {
 	return s.second
 }
 
+// SetYear sets the current year.
+func (s *event) SetYear(b []byte) {
+	s.year = bytesToInt(b)
+}
+
 // Year returns the current year, since syslog events don't include that.
 func (s *event) Year() int {
-	return time.Now().Year()
+	return s.year
 }
 
 // SetMessage sets the message.
@@ -192,7 +221,13 @@ func (s *event) HasPid() bool {
 
 // SetNanoSecond sets the nanosecond.
 func (s *event) SetNanosecond(b []byte) {
-	s.nanosecond = bytesToInt(skipLeadZero(b))
+	// We assume that we receive a byte array representing a nanosecond, this might not be
+	// always the case, so we have to pad it.
+	if len(b) < 7 {
+		s.nanosecond = bytesToInt(skipLeadZero(b)) * int(math.Pow10((7 - len(b))))
+	} else {
+		s.nanosecond = bytesToInt(skipLeadZero(b))
+	}
 }
 
 // NanoSecond returns the nanosecond.

filebeat/input/syslog/input.go

@@ -109,11 +109,27 @@ func NewInput(
 	cb := func(data []byte, metadata inputsource.NetworkMetadata) {
 		ev := newEvent()
 		Parse(data, ev)
+		var d *util.Data
 		if !ev.IsValid() {
 			log.Errorw("can't not parse event as syslog rfc3164", "message", string(data))
+			// On error revert to the raw bytes content, we need a better way to communicate this kind of
+			// error upstream this should be a global effort.
+			d = &util.Data{
+				Event: beat.Event{
+					Timestamp: time.Now(),
+					Meta: common.MapStr{
+						"truncated": metadata.Truncated,
+					},
+					Fields: common.MapStr{
+						"message": string(data),
+					},
+				},
+			}
+		} else {
+			event := createEvent(ev, metadata, time.Local, log)
+			d = &util.Data{Event: *event}
 		}
-		event := createEvent(ev, metadata, time.Local, log)
-		d := &util.Data{Event: *event}
+
 		forwarder.Send(d)
 	}