diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 086ce5a6278b..14c4fcebf265 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -67,6 +67,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Fix memory leak in log prospector when files cannot be read. {issue}6797[6797] - Add raw JSON to message field when JSON parsing fails. {issue}6516[6516] - Commit registry writes to stable storage to avoid corrupt registry files. {pull}6877[6877] +- Fix a parsing issue in the syslog input for RFC3339 timestamp and time with nanoseconds. {pull}7046[7046] *Heartbeat* - Fix race due to updates of shared a map, that was not supposed to be shared between multiple go-routines. {issue}6616[6616] diff --git a/filebeat/input/syslog/event.go b/filebeat/input/syslog/event.go index dd5b31ac6224..e7ff847d8ec5 100644 --- a/filebeat/input/syslog/event.go +++ b/filebeat/input/syslog/event.go @@ -1,6 +1,7 @@ package syslog import ( + "math" "time" ) @@ -22,6 +23,22 @@ var month = map[string]time.Month{ "Dec": time.December, } +var monthIndexed = []time.Month{ + 0, + time.January, + time.February, + time.March, + time.April, + time.May, + time.June, + time.July, + time.August, + time.September, + time.October, + time.November, + time.December, +} + // event is a parsed syslog event, validation of the format is done at the parser level. type event struct { message string @@ -35,6 +52,7 @@ type event struct { minute int second int nanosecond int + year int loc *time.Location } @@ -48,9 +66,15 @@ func newEvent() *event { hour: -1, minute: -1, second: -1, + year: time.Now().Year(), } } +// SetMonthNumeric sets the month with a number. +func (s *event) SetMonthNumeric(b []byte) { + s.month = monthIndexed[bytesToInt(skipLeadZero(b))] +} + // SetMonth sets the month. func (s *event) SetMonth(b []byte) { var k string @@ -110,9 +134,14 @@ func (s *event) Second() int { return s.second } +// SetYear sets the current year. +func (s *event) SetYear(b []byte) { + s.year = bytesToInt(b) +} + // Year returns the current year, since syslog events don't include that. func (s *event) Year() int { - return time.Now().Year() + return s.year } // SetMessage sets the message. @@ -192,7 +221,13 @@ func (s *event) HasPid() bool { // SetNanoSecond sets the nanosecond. func (s *event) SetNanosecond(b []byte) { - s.nanosecond = bytesToInt(skipLeadZero(b)) + // We assume that we receive a byte array representing a nanosecond, this might not be + // always the case, so we have to pad it. + if len(b) < 7 { + s.nanosecond = bytesToInt(skipLeadZero(b)) * int(math.Pow10((7 - len(b)))) + } else { + s.nanosecond = bytesToInt(skipLeadZero(b)) + } } // NanoSecond returns the nanosecond. diff --git a/filebeat/input/syslog/input.go b/filebeat/input/syslog/input.go index 7208c0f2cad2..bdff0c8a21e2 100644 --- a/filebeat/input/syslog/input.go +++ b/filebeat/input/syslog/input.go @@ -109,11 +109,27 @@ func NewInput( cb := func(data []byte, metadata inputsource.NetworkMetadata) { ev := newEvent() Parse(data, ev) + var d *util.Data if !ev.IsValid() { log.Errorw("can't not parse event as syslog rfc3164", "message", string(data)) + // On error revert to the raw bytes content, we need a better way to communicate this kind of + // error upstream this should be a global effort. + d = &util.Data{ + Event: beat.Event{ + Timestamp: time.Now(), + Meta: common.MapStr{ + "truncated": metadata.Truncated, + }, + Fields: common.MapStr{ + "message": string(data), + }, + }, + } + } else { + event := createEvent(ev, metadata, time.Local, log) + d = &util.Data{Event: *event} } - event := createEvent(ev, metadata, time.Local, log) - d := &util.Data{Event: *event} + forwarder.Send(d) } diff --git a/filebeat/input/syslog/parser.go b/filebeat/input/syslog/parser.go index 7e6e29fbe387..7eac6054eaae 100644 --- a/filebeat/input/syslog/parser.go +++ b/filebeat/input/syslog/parser.go @@ -186,28 +186,51 @@ func Parse(data []byte, event *event) { goto st_case_75 case 76: goto st_case_76 + case 77: + goto st_case_77 + case 78: + goto st_case_78 + case 79: + goto st_case_79 + case 80: + goto st_case_80 + case 81: + goto st_case_81 + case 82: + goto st_case_82 + case 83: + goto st_case_83 + case 84: + goto st_case_84 + case 85: + goto st_case_85 + case 86: + goto st_case_86 } goto st_out st_case_0: switch data[(p)] { case 60: - goto tr1 - case 65: goto tr2 - case 70: + case 65: goto tr3 - case 74: + case 70: goto tr4 - case 77: + case 74: goto tr5 - case 78: + case 77: goto tr6 - case 79: + case 78: goto tr7 - case 83: + case 79: goto tr8 - case 101: + case 83: goto tr9 + case 101: + goto tr10 + } + if 48 <= data[(p)] && data[(p)] <= 57 { + goto tr1 } goto tr0 tr0: @@ -220,7 +243,7 @@ func Parse(data []byte, event *event) { goto _test_eof1 } st_case_1: -//line parser.go:228 +//line parser.go:251 goto st1 tr1: //line parser.rl:20 @@ -232,25 +255,16 @@ func Parse(data []byte, event *event) { goto _test_eof2 } st_case_2: -//line parser.go:241 +//line parser.go:264 if 48 <= data[(p)] && data[(p)] <= 57 { - goto tr11 + goto st3 } goto st1 - tr11: -//line parser.rl:20 - tok = p - - goto st3 st3: if (p)++; (p) == (pe) { goto _test_eof3 } st_case_3: -//line parser.go:257 - if data[(p)] == 62 { - goto tr13 - } if 48 <= data[(p)] && data[(p)] <= 57 { goto st4 } @@ -260,9 +274,6 @@ func Parse(data []byte, event *event) { goto _test_eof4 } st_case_4: - if data[(p)] == 62 { - goto tr13 - } if 48 <= data[(p)] && data[(p)] <= 57 { goto st5 } @@ -272,67 +283,52 @@ func Parse(data []byte, event *event) { goto _test_eof5 } st_case_5: - if data[(p)] == 62 { - goto tr13 - } - if 48 <= data[(p)] && data[(p)] <= 57 { - goto st6 + if data[(p)] == 45 { + goto tr15 } goto st1 + tr15: +//line parser.rl:36 + event.SetYear(data[tok:p]) + + goto st6 st6: if (p)++; (p) == (pe) { goto _test_eof6 } st_case_6: - if data[(p)] == 62 { - goto tr13 - } +//line parser.go:307 if 48 <= data[(p)] && data[(p)] <= 57 { - goto st7 + goto tr16 } goto st1 + tr16: +//line parser.rl:20 + tok = p + + goto st7 st7: if (p)++; (p) == (pe) { goto _test_eof7 } st_case_7: - if data[(p)] == 62 { - goto tr13 +//line parser.go:323 + if 48 <= data[(p)] && data[(p)] <= 57 { + goto st8 } goto st1 - tr13: -//line parser.rl:24 - event.SetPriority(data[tok:p]) - - goto st8 st8: if (p)++; (p) == (pe) { goto _test_eof8 } st_case_8: -//line parser.go:321 - switch data[(p)] { - case 65: - goto tr2 - case 70: - goto tr3 - case 74: - goto tr4 - case 77: - goto tr5 - case 78: - goto tr6 - case 79: - goto tr7 - case 83: - goto tr8 - case 101: - goto tr9 + if data[(p)] == 45 { + goto tr18 } - goto tr0 - tr2: -//line parser.rl:20 - tok = p + goto st1 + tr18: +//line parser.rl:40 + event.SetMonthNumeric(data[tok:p]) goto st9 st9: @@ -340,20 +336,23 @@ func Parse(data []byte, event *event) { goto _test_eof9 } st_case_9: -//line parser.go:352 - switch data[(p)] { - case 112: - goto st10 - case 117: - goto st41 +//line parser.go:348 + if 48 <= data[(p)] && data[(p)] <= 51 { + goto tr19 } goto st1 + tr19: +//line parser.rl:20 + tok = p + + goto st10 st10: if (p)++; (p) == (pe) { goto _test_eof10 } st_case_10: - if data[(p)] == 114 { +//line parser.go:364 + if 48 <= data[(p)] && data[(p)] <= 57 { goto st11 } goto st1 @@ -362,19 +361,13 @@ func Parse(data []byte, event *event) { goto _test_eof11 } st_case_11: - switch data[(p)] { - case 32: - goto tr20 - case 105: - goto st39 - } - if 9 <= data[(p)] && data[(p)] <= 13 { - goto tr20 + if data[(p)] == 84 { + goto tr21 } goto st1 - tr20: -//line parser.rl:32 - event.SetMonth(data[tok:p]) + tr21: +//line parser.rl:44 + event.SetDay(data[tok:p]) goto st12 st12: @@ -382,56 +375,41 @@ func Parse(data []byte, event *event) { goto _test_eof12 } st_case_12: -//line parser.go:395 - switch data[(p)] { - case 32: - goto st13 - case 51: - goto tr24 - } - switch { - case data[(p)] < 49: - if 9 <= data[(p)] && data[(p)] <= 13 { - goto st13 - } - case data[(p)] > 50: - if 52 <= data[(p)] && data[(p)] <= 57 { - goto tr25 - } - default: +//line parser.go:389 + if data[(p)] == 50 { goto tr23 } + if 48 <= data[(p)] && data[(p)] <= 49 { + goto tr22 + } goto st1 + tr22: +//line parser.rl:20 + tok = p + + goto st13 st13: if (p)++; (p) == (pe) { goto _test_eof13 } st_case_13: - if 49 <= data[(p)] && data[(p)] <= 57 { - goto tr25 +//line parser.go:408 + if 48 <= data[(p)] && data[(p)] <= 57 { + goto st14 } goto st1 - tr25: -//line parser.rl:20 - tok = p - - goto st14 st14: if (p)++; (p) == (pe) { goto _test_eof14 } st_case_14: -//line parser.go:435 - if data[(p)] == 32 { - goto tr26 - } - if 9 <= data[(p)] && data[(p)] <= 13 { - goto tr26 + if data[(p)] == 58 { + goto tr25 } goto st1 - tr26: -//line parser.rl:36 - event.SetDay(data[tok:p]) + tr25: +//line parser.rl:48 + event.SetHour(data[tok:p]) goto st15 st15: @@ -439,15 +417,12 @@ func Parse(data []byte, event *event) { goto _test_eof15 } st_case_15: -//line parser.go:454 - if data[(p)] == 50 { - goto tr28 - } - if 48 <= data[(p)] && data[(p)] <= 49 { - goto tr27 +//line parser.go:433 + if 48 <= data[(p)] && data[(p)] <= 53 { + goto tr26 } goto st1 - tr27: + tr26: //line parser.rl:20 tok = p @@ -457,7 +432,7 @@ func Parse(data []byte, event *event) { goto _test_eof16 } st_case_16: -//line parser.go:473 +//line parser.go:449 if 48 <= data[(p)] && data[(p)] <= 57 { goto st17 } @@ -468,12 +443,12 @@ func Parse(data []byte, event *event) { } st_case_17: if data[(p)] == 58 { - goto tr30 + goto tr28 } goto st1 - tr30: -//line parser.rl:40 - event.SetHour(data[tok:p]) + tr28: +//line parser.rl:52 + event.SetMinute(data[tok:p]) goto st18 st18: @@ -481,12 +456,12 @@ func Parse(data []byte, event *event) { goto _test_eof18 } st_case_18: -//line parser.go:498 +//line parser.go:474 if 48 <= data[(p)] && data[(p)] <= 53 { - goto tr31 + goto tr29 } goto st1 - tr31: + tr29: //line parser.rl:20 tok = p @@ -496,7 +471,7 @@ func Parse(data []byte, event *event) { goto _test_eof19 } st_case_19: -//line parser.go:514 +//line parser.go:490 if 48 <= data[(p)] && data[(p)] <= 57 { goto st20 } @@ -506,113 +481,79 @@ func Parse(data []byte, event *event) { goto _test_eof20 } st_case_20: - if data[(p)] == 58 { - goto tr33 - } - goto st1 - tr33: -//line parser.rl:44 - event.SetMinute(data[tok:p]) - - goto st21 - st21: - if (p)++; (p) == (pe) { - goto _test_eof21 - } - st_case_21: -//line parser.go:539 - if 48 <= data[(p)] && data[(p)] <= 53 { - goto tr34 - } - goto st1 - tr34: -//line parser.rl:20 - tok = p - - goto st22 - st22: - if (p)++; (p) == (pe) { - goto _test_eof22 - } - st_case_22: -//line parser.go:555 - if 48 <= data[(p)] && data[(p)] <= 57 { - goto st23 - } - goto st1 - st23: - if (p)++; (p) == (pe) { - goto _test_eof23 - } - st_case_23: switch data[(p)] { case 32: - goto tr36 + goto tr31 case 46: - goto tr37 + goto tr32 } if 9 <= data[(p)] && data[(p)] <= 13 { - goto tr36 + goto tr31 } goto st1 - tr36: -//line parser.rl:48 + tr31: +//line parser.rl:56 event.SetSecond(data[tok:p]) - goto st24 - st24: + goto st21 + tr46: +//line parser.rl:60 + event.SetNanosecond(data[tok:p]) + + goto st21 + st21: if (p)++; (p) == (pe) { - goto _test_eof24 + goto _test_eof21 } - st_case_24: -//line parser.go:586 + st_case_21: +//line parser.go:527 switch { case data[(p)] > 95: if 97 <= data[(p)] && data[(p)] <= 122 { - goto tr38 + goto tr33 } case data[(p)] >= 46: - goto tr38 + goto tr33 } goto tr0 - tr38: + tr33: //line parser.rl:20 tok = p - goto st25 - st25: + goto st22 + st22: if (p)++; (p) == (pe) { - goto _test_eof25 + goto _test_eof22 } - st_case_25: -//line parser.go:607 + st_case_22: +//line parser.go:548 if data[(p)] == 32 { - goto tr39 + goto tr34 } switch { case data[(p)] < 46: if 9 <= data[(p)] && data[(p)] <= 13 { - goto tr39 + goto tr34 } case data[(p)] > 95: if 97 <= data[(p)] && data[(p)] <= 122 { - goto st25 + goto st22 } default: - goto st25 + goto st22 } goto st1 - tr39: -//line parser.rl:56 + tr34: +//line parser.rl:64 event.SetHostname(data[tok:p]) - goto st26 - st26: + goto st23 + st23: if (p)++; (p) == (pe) { - goto _test_eof26 + goto _test_eof23 } - st_case_26: -//line parser.go:635 + st_case_23: +//line parser.go:576 switch data[(p)] { case 32: goto tr0 @@ -624,81 +565,126 @@ func Parse(data []byte, event *event) { if 9 <= data[(p)] && data[(p)] <= 13 { goto tr0 } - goto tr41 - tr41: + goto tr36 + tr36: //line parser.rl:20 tok = p - goto st27 - st27: + goto st24 + st24: if (p)++; (p) == (pe) { - goto _test_eof27 + goto _test_eof24 } - st_case_27: -//line parser.go:659 + st_case_24: +//line parser.go:600 switch data[(p)] { case 32: goto st1 case 58: - goto tr43 + goto tr38 case 91: - goto tr44 + goto tr39 case 93: goto st1 } if 9 <= data[(p)] && data[(p)] <= 13 { goto st1 } - goto st27 - tr43: -//line parser.rl:60 + goto st24 + tr38: +//line parser.rl:68 event.SetProgram(data[tok:p]) - goto st28 - st28: + goto st25 + st25: if (p)++; (p) == (pe) { - goto _test_eof28 + goto _test_eof25 } - st_case_28: -//line parser.go:685 + st_case_25: +//line parser.go:626 switch data[(p)] { case 32: - goto st29 + goto st26 case 58: - goto tr43 + goto tr38 case 91: - goto tr44 + goto tr39 case 93: goto st1 } if 9 <= data[(p)] && data[(p)] <= 13 { - goto st29 + goto st26 + } + goto st24 + st26: + if (p)++; (p) == (pe) { + goto _test_eof26 } + st_case_26: + goto tr0 + tr39: +//line parser.rl:68 + event.SetProgram(data[tok:p]) + goto st27 + st27: + if (p)++; (p) == (pe) { + goto _test_eof27 + } + st_case_27: +//line parser.go:658 + if 48 <= data[(p)] && data[(p)] <= 57 { + goto tr41 + } + goto st1 + tr41: +//line parser.rl:20 + tok = p + + goto st28 + st28: + if (p)++; (p) == (pe) { + goto _test_eof28 + } + st_case_28: +//line parser.go:674 + if data[(p)] == 93 { + goto tr43 + } + if 48 <= data[(p)] && data[(p)] <= 57 { + goto st28 + } + goto st1 + tr43: +//line parser.rl:72 + event.SetPid(data[tok:p]) + + goto st29 st29: if (p)++; (p) == (pe) { goto _test_eof29 } st_case_29: - goto tr0 - tr44: -//line parser.rl:60 - event.SetProgram(data[tok:p]) - - goto st30 +//line parser.go:693 + if data[(p)] == 58 { + goto st30 + } + goto st1 st30: if (p)++; (p) == (pe) { goto _test_eof30 } st_case_30: -//line parser.go:717 - if 48 <= data[(p)] && data[(p)] <= 57 { - goto tr46 + if data[(p)] == 32 { + goto st26 + } + if 9 <= data[(p)] && data[(p)] <= 13 { + goto st26 } goto st1 - tr46: -//line parser.rl:20 - tok = p + tr32: +//line parser.rl:56 + event.SetSecond(data[tok:p]) goto st31 st31: @@ -706,17 +692,14 @@ func Parse(data []byte, event *event) { goto _test_eof31 } st_case_31: -//line parser.go:733 - if data[(p)] == 93 { - goto tr48 - } +//line parser.go:721 if 48 <= data[(p)] && data[(p)] <= 57 { - goto st31 + goto tr45 } goto st1 - tr48: -//line parser.rl:64 - event.SetPid(data[tok:p]) + tr45: +//line parser.rl:20 + tok = p goto st32 st32: @@ -724,26 +707,37 @@ func Parse(data []byte, event *event) { goto _test_eof32 } st_case_32: -//line parser.go:752 - if data[(p)] == 58 { - goto st33 +//line parser.go:737 + if data[(p)] == 32 { + goto tr46 + } + switch { + case data[(p)] > 13: + if 48 <= data[(p)] && data[(p)] <= 57 { + goto st32 + } + case data[(p)] >= 9: + goto tr46 } goto st1 + tr23: +//line parser.rl:20 + tok = p + + goto st33 st33: if (p)++; (p) == (pe) { goto _test_eof33 } st_case_33: - if data[(p)] == 32 { - goto st29 - } - if 9 <= data[(p)] && data[(p)] <= 13 { - goto st29 +//line parser.go:761 + if 48 <= data[(p)] && data[(p)] <= 51 { + goto st14 } goto st1 - tr37: -//line parser.rl:48 - event.SetSecond(data[tok:p]) + tr2: +//line parser.rl:20 + tok = p goto st34 st34: @@ -751,87 +745,63 @@ func Parse(data []byte, event *event) { goto _test_eof34 } st_case_34: -//line parser.go:780 +//line parser.go:777 if 48 <= data[(p)] && data[(p)] <= 57 { - goto st35 + goto tr48 } goto st1 + tr48: +//line parser.rl:20 + tok = p + + goto st35 st35: if (p)++; (p) == (pe) { goto _test_eof35 } st_case_35: - if data[(p)] == 32 { - goto st24 +//line parser.go:793 + if data[(p)] == 62 { + goto tr50 } - switch { - case data[(p)] > 13: - if 48 <= data[(p)] && data[(p)] <= 57 { - goto st35 - } - case data[(p)] >= 9: - goto st24 + if 48 <= data[(p)] && data[(p)] <= 57 { + goto st36 } goto st1 - tr28: -//line parser.rl:20 - tok = p - - goto st36 st36: if (p)++; (p) == (pe) { goto _test_eof36 } st_case_36: -//line parser.go:813 - if 48 <= data[(p)] && data[(p)] <= 51 { - goto st17 + if data[(p)] == 62 { + goto tr50 + } + if 48 <= data[(p)] && data[(p)] <= 57 { + goto st37 } goto st1 - tr23: -//line parser.rl:20 - tok = p - - goto st37 st37: if (p)++; (p) == (pe) { goto _test_eof37 } st_case_37: -//line parser.go:829 - if data[(p)] == 32 { - goto tr26 + if data[(p)] == 62 { + goto tr50 } - switch { - case data[(p)] > 13: - if 48 <= data[(p)] && data[(p)] <= 57 { - goto st14 - } - case data[(p)] >= 9: - goto tr26 + if 48 <= data[(p)] && data[(p)] <= 57 { + goto st38 } goto st1 - tr24: -//line parser.rl:20 - tok = p - - goto st38 st38: if (p)++; (p) == (pe) { goto _test_eof38 } st_case_38: -//line parser.go:853 - if data[(p)] == 32 { - goto tr26 + if data[(p)] == 62 { + goto tr50 } - switch { - case data[(p)] > 13: - if 48 <= data[(p)] && data[(p)] <= 49 { - goto st14 - } - case data[(p)] >= 9: - goto tr26 + if 48 <= data[(p)] && data[(p)] <= 57 { + goto st39 } goto st1 st39: @@ -839,29 +809,59 @@ func Parse(data []byte, event *event) { goto _test_eof39 } st_case_39: - if data[(p)] == 108 { - goto st40 + if data[(p)] == 62 { + goto tr50 } goto st1 + tr50: +//line parser.rl:24 + event.SetPriority(data[tok:p]) + + goto st40 st40: if (p)++; (p) == (pe) { goto _test_eof40 } st_case_40: - if data[(p)] == 32 { - goto tr20 +//line parser.go:857 + switch data[(p)] { + case 65: + goto tr3 + case 70: + goto tr4 + case 74: + goto tr5 + case 77: + goto tr6 + case 78: + goto tr7 + case 79: + goto tr8 + case 83: + goto tr9 + case 101: + goto tr10 } - if 9 <= data[(p)] && data[(p)] <= 13 { - goto tr20 + if 48 <= data[(p)] && data[(p)] <= 57 { + goto tr1 } - goto st1 + goto tr0 + tr3: +//line parser.rl:20 + tok = p + + goto st41 st41: if (p)++; (p) == (pe) { goto _test_eof41 } st_case_41: - if data[(p)] == 103 { +//line parser.go:891 + switch data[(p)] { + case 112: goto st42 + case 117: + goto st51 } goto st1 st42: @@ -869,80 +869,126 @@ func Parse(data []byte, event *event) { goto _test_eof42 } st_case_42: - switch data[(p)] { - case 32: - goto tr20 - case 117: + if data[(p)] == 114 { goto st43 } - if 9 <= data[(p)] && data[(p)] <= 13 { - goto tr20 - } goto st1 st43: if (p)++; (p) == (pe) { goto _test_eof43 } st_case_43: - if data[(p)] == 115 { - goto st44 + switch data[(p)] { + case 32: + goto tr57 + case 105: + goto st49 + } + if 9 <= data[(p)] && data[(p)] <= 13 { + goto tr57 } goto st1 + tr57: +//line parser.rl:32 + event.SetMonth(data[tok:p]) + + goto st44 st44: if (p)++; (p) == (pe) { goto _test_eof44 } st_case_44: - if data[(p)] == 116 { - goto st40 +//line parser.go:934 + switch data[(p)] { + case 32: + goto st45 + case 51: + goto tr61 + } + switch { + case data[(p)] < 49: + if 9 <= data[(p)] && data[(p)] <= 13 { + goto st45 + } + case data[(p)] > 50: + if 52 <= data[(p)] && data[(p)] <= 57 { + goto tr62 + } + default: + goto tr60 } goto st1 - tr3: -//line parser.rl:20 - tok = p - - goto st45 st45: if (p)++; (p) == (pe) { goto _test_eof45 } st_case_45: -//line parser.go:940 - if data[(p)] == 101 { - goto st46 + if 49 <= data[(p)] && data[(p)] <= 57 { + goto tr62 } goto st1 + tr62: +//line parser.rl:20 + tok = p + + goto st46 st46: if (p)++; (p) == (pe) { goto _test_eof46 } st_case_46: - if data[(p)] == 98 { - goto st47 +//line parser.go:974 + if data[(p)] == 32 { + goto tr21 + } + if 9 <= data[(p)] && data[(p)] <= 13 { + goto tr21 } goto st1 + tr60: +//line parser.rl:20 + tok = p + + goto st47 st47: if (p)++; (p) == (pe) { goto _test_eof47 } st_case_47: - switch data[(p)] { - case 32: - goto tr20 - case 114: - goto st48 +//line parser.go:993 + if data[(p)] == 32 { + goto tr21 } - if 9 <= data[(p)] && data[(p)] <= 13 { - goto tr20 + switch { + case data[(p)] > 13: + if 48 <= data[(p)] && data[(p)] <= 57 { + goto st46 + } + case data[(p)] >= 9: + goto tr21 } goto st1 + tr61: +//line parser.rl:20 + tok = p + + goto st48 st48: if (p)++; (p) == (pe) { goto _test_eof48 } st_case_48: - if data[(p)] == 117 { - goto st49 +//line parser.go:1017 + if data[(p)] == 32 { + goto tr21 + } + switch { + case data[(p)] > 13: + if 48 <= data[(p)] && data[(p)] <= 49 { + goto st46 + } + case data[(p)] >= 9: + goto tr21 } goto st1 st49: @@ -950,7 +996,7 @@ func Parse(data []byte, event *event) { goto _test_eof49 } st_case_49: - if data[(p)] == 97 { + if data[(p)] == 108 { goto st50 } goto st1 @@ -959,8 +1005,11 @@ func Parse(data []byte, event *event) { goto _test_eof50 } st_case_50: - if data[(p)] == 114 { - goto st51 + if data[(p)] == 32 { + goto tr57 + } + if 9 <= data[(p)] && data[(p)] <= 13 { + goto tr57 } goto st1 st51: @@ -968,26 +1017,23 @@ func Parse(data []byte, event *event) { goto _test_eof51 } st_case_51: - if data[(p)] == 121 { - goto st40 + if data[(p)] == 103 { + goto st52 } goto st1 - tr4: -//line parser.rl:20 - tok = p - - goto st52 st52: if (p)++; (p) == (pe) { goto _test_eof52 } st_case_52: -//line parser.go:1016 switch data[(p)] { - case 97: - goto st53 + case 32: + goto tr57 case 117: - goto st55 + goto st53 + } + if 9 <= data[(p)] && data[(p)] <= 13 { + goto tr57 } goto st1 st53: @@ -995,7 +1041,7 @@ func Parse(data []byte, event *event) { goto _test_eof53 } st_case_53: - if data[(p)] == 110 { + if data[(p)] == 115 { goto st54 } goto st1 @@ -1004,26 +1050,23 @@ func Parse(data []byte, event *event) { goto _test_eof54 } st_case_54: - switch data[(p)] { - case 32: - goto tr20 - case 117: - goto st49 - } - if 9 <= data[(p)] && data[(p)] <= 13 { - goto tr20 + if data[(p)] == 116 { + goto st50 } goto st1 + tr4: +//line parser.rl:20 + tok = p + + goto st55 st55: if (p)++; (p) == (pe) { goto _test_eof55 } st_case_55: - switch data[(p)] { - case 108: +//line parser.go:1104 + if data[(p)] == 101 { goto st56 - case 110: - goto st57 } goto st1 st56: @@ -1031,14 +1074,8 @@ func Parse(data []byte, event *event) { goto _test_eof56 } st_case_56: - switch data[(p)] { - case 32: - goto tr20 - case 121: - goto st40 - } - if 9 <= data[(p)] && data[(p)] <= 13 { - goto tr20 + if data[(p)] == 98 { + goto st57 } goto st1 st57: @@ -1048,26 +1085,20 @@ func Parse(data []byte, event *event) { st_case_57: switch data[(p)] { case 32: - goto tr20 - case 101: - goto st40 + goto tr57 + case 114: + goto st58 } if 9 <= data[(p)] && data[(p)] <= 13 { - goto tr20 + goto tr57 } goto st1 - tr5: -//line parser.rl:20 - tok = p - - goto st58 st58: if (p)++; (p) == (pe) { goto _test_eof58 } st_case_58: -//line parser.go:1101 - if data[(p)] == 97 { + if data[(p)] == 117 { goto st59 } goto st1 @@ -1076,16 +1107,8 @@ func Parse(data []byte, event *event) { goto _test_eof59 } st_case_59: - switch data[(p)] { - case 32: - goto tr20 - case 114: + if data[(p)] == 97 { goto st60 - case 121: - goto st40 - } - if 9 <= data[(p)] && data[(p)] <= 13 { - goto tr20 } goto st1 st60: @@ -1093,26 +1116,20 @@ func Parse(data []byte, event *event) { goto _test_eof60 } st_case_60: - switch data[(p)] { - case 32: - goto tr20 - case 99: + if data[(p)] == 114 { goto st61 } - if 9 <= data[(p)] && data[(p)] <= 13 { - goto tr20 - } goto st1 st61: if (p)++; (p) == (pe) { goto _test_eof61 } st_case_61: - if data[(p)] == 104 { - goto st40 + if data[(p)] == 121 { + goto st50 } goto st1 - tr6: + tr5: //line parser.rl:20 tok = p @@ -1122,9 +1139,12 @@ func Parse(data []byte, event *event) { goto _test_eof62 } st_case_62: -//line parser.go:1158 - if data[(p)] == 111 { +//line parser.go:1180 + switch data[(p)] { + case 97: goto st63 + case 117: + goto st65 } goto st1 st63: @@ -1132,7 +1152,7 @@ func Parse(data []byte, event *event) { goto _test_eof63 } st_case_63: - if data[(p)] == 118 { + if data[(p)] == 110 { goto st64 } goto st1 @@ -1143,12 +1163,12 @@ func Parse(data []byte, event *event) { st_case_64: switch data[(p)] { case 32: - goto tr20 - case 101: - goto st65 + goto tr57 + case 117: + goto st59 } if 9 <= data[(p)] && data[(p)] <= 13 { - goto tr20 + goto tr57 } goto st1 st65: @@ -1156,8 +1176,11 @@ func Parse(data []byte, event *event) { goto _test_eof65 } st_case_65: - if data[(p)] == 109 { + switch data[(p)] { + case 108: goto st66 + case 110: + goto st67 } goto st1 st66: @@ -1165,8 +1188,14 @@ func Parse(data []byte, event *event) { goto _test_eof66 } st_case_66: - if data[(p)] == 98 { - goto st67 + switch data[(p)] { + case 32: + goto tr57 + case 121: + goto st50 + } + if 9 <= data[(p)] && data[(p)] <= 13 { + goto tr57 } goto st1 st67: @@ -1174,32 +1203,46 @@ func Parse(data []byte, event *event) { goto _test_eof67 } st_case_67: - if data[(p)] == 101 { - goto st68 + switch data[(p)] { + case 32: + goto tr57 + case 101: + goto st50 + } + if 9 <= data[(p)] && data[(p)] <= 13 { + goto tr57 } goto st1 + tr6: +//line parser.rl:20 + tok = p + + goto st68 st68: if (p)++; (p) == (pe) { goto _test_eof68 } st_case_68: - if data[(p)] == 114 { - goto st40 +//line parser.go:1265 + if data[(p)] == 97 { + goto st69 } goto st1 - tr7: -//line parser.rl:20 - tok = p - - goto st69 st69: if (p)++; (p) == (pe) { goto _test_eof69 } st_case_69: -//line parser.go:1234 - if data[(p)] == 99 { + switch data[(p)] { + case 32: + goto tr57 + case 114: goto st70 + case 121: + goto st50 + } + if 9 <= data[(p)] && data[(p)] <= 13 { + goto tr57 } goto st1 st70: @@ -1207,26 +1250,26 @@ func Parse(data []byte, event *event) { goto _test_eof70 } st_case_70: - if data[(p)] == 116 { + switch data[(p)] { + case 32: + goto tr57 + case 99: goto st71 } + if 9 <= data[(p)] && data[(p)] <= 13 { + goto tr57 + } goto st1 st71: if (p)++; (p) == (pe) { goto _test_eof71 } st_case_71: - switch data[(p)] { - case 32: - goto tr20 - case 111: - goto st66 - } - if 9 <= data[(p)] && data[(p)] <= 13 { - goto tr20 + if data[(p)] == 104 { + goto st50 } goto st1 - tr8: + tr7: //line parser.rl:20 tok = p @@ -1236,8 +1279,8 @@ func Parse(data []byte, event *event) { goto _test_eof72 } st_case_72: -//line parser.go:1274 - if data[(p)] == 101 { +//line parser.go:1322 + if data[(p)] == 111 { goto st73 } goto st1 @@ -1246,7 +1289,7 @@ func Parse(data []byte, event *event) { goto _test_eof73 } st_case_73: - if data[(p)] == 112 { + if data[(p)] == 118 { goto st74 } goto st1 @@ -1257,12 +1300,12 @@ func Parse(data []byte, event *event) { st_case_74: switch data[(p)] { case 32: - goto tr20 - case 116: + goto tr57 + case 101: goto st75 } if 9 <= data[(p)] && data[(p)] <= 13 { - goto tr20 + goto tr57 } goto st1 st75: @@ -1270,23 +1313,137 @@ func Parse(data []byte, event *event) { goto _test_eof75 } st_case_75: + if data[(p)] == 109 { + goto st76 + } + goto st1 + st76: + if (p)++; (p) == (pe) { + goto _test_eof76 + } + st_case_76: + if data[(p)] == 98 { + goto st77 + } + goto st1 + st77: + if (p)++; (p) == (pe) { + goto _test_eof77 + } + st_case_77: if data[(p)] == 101 { - goto st65 + goto st78 + } + goto st1 + st78: + if (p)++; (p) == (pe) { + goto _test_eof78 + } + st_case_78: + if data[(p)] == 114 { + goto st50 + } + goto st1 + tr8: +//line parser.rl:20 + tok = p + + goto st79 + st79: + if (p)++; (p) == (pe) { + goto _test_eof79 + } + st_case_79: +//line parser.go:1398 + if data[(p)] == 99 { + goto st80 + } + goto st1 + st80: + if (p)++; (p) == (pe) { + goto _test_eof80 + } + st_case_80: + if data[(p)] == 116 { + goto st81 + } + goto st1 + st81: + if (p)++; (p) == (pe) { + goto _test_eof81 + } + st_case_81: + switch data[(p)] { + case 32: + goto tr57 + case 111: + goto st76 + } + if 9 <= data[(p)] && data[(p)] <= 13 { + goto tr57 } goto st1 tr9: //line parser.rl:20 tok = p - goto st76 - st76: + goto st82 + st82: if (p)++; (p) == (pe) { - goto _test_eof76 + goto _test_eof82 } - st_case_76: -//line parser.go:1323 + st_case_82: +//line parser.go:1438 + if data[(p)] == 101 { + goto st83 + } + goto st1 + st83: + if (p)++; (p) == (pe) { + goto _test_eof83 + } + st_case_83: + if data[(p)] == 112 { + goto st84 + } + goto st1 + st84: + if (p)++; (p) == (pe) { + goto _test_eof84 + } + st_case_84: + switch data[(p)] { + case 32: + goto tr57 + case 116: + goto st85 + } + if 9 <= data[(p)] && data[(p)] <= 13 { + goto tr57 + } + goto st1 + st85: + if (p)++; (p) == (pe) { + goto _test_eof85 + } + st_case_85: + if data[(p)] == 101 { + goto st75 + } + goto st1 + tr10: +//line parser.rl:20 + tok = p + + goto st86 + st86: + if (p)++; (p) == (pe) { + goto _test_eof86 + } + st_case_86: +//line parser.go:1487 if data[(p)] == 99 { - goto st64 + goto st74 } goto st1 st_out: @@ -1518,21 +1675,51 @@ func Parse(data []byte, event *event) { _test_eof76: cs = 76 goto _test_eof + _test_eof77: + cs = 77 + goto _test_eof + _test_eof78: + cs = 78 + goto _test_eof + _test_eof79: + cs = 79 + goto _test_eof + _test_eof80: + cs = 80 + goto _test_eof + _test_eof81: + cs = 81 + goto _test_eof + _test_eof82: + cs = 82 + goto _test_eof + _test_eof83: + cs = 83 + goto _test_eof + _test_eof84: + cs = 84 + goto _test_eof + _test_eof85: + cs = 85 + goto _test_eof + _test_eof86: + cs = 86 + goto _test_eof _test_eof: { } if (p) == eof { switch cs { - case 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76: + case 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86: //line parser.rl:28 event.SetMessage(data[tok:p]) -//line parser.go:1414 +//line parser.go:1588 } } } -//line parser.rl:72 +//line parser.rl:80 } diff --git a/filebeat/input/syslog/parser.rl b/filebeat/input/syslog/parser.rl index bbf4629258e6..9052e1701f88 100644 --- a/filebeat/input/syslog/parser.rl +++ b/filebeat/input/syslog/parser.rl @@ -33,6 +33,14 @@ func Parse(data []byte, event *event) { event.SetMonth(data[tok:p]) } + action year{ + event.SetYear(data[tok:p]) + } + + action month_numeric { + event.SetMonthNumeric(data[tok:p]) + } + action day { event.SetDay(data[tok:p]) } diff --git a/filebeat/input/syslog/parser_test.go b/filebeat/input/syslog/parser_test.go index 3cfd581d148c..1d12eaf76c58 100644 --- a/filebeat/input/syslog/parser_test.go +++ b/filebeat/input/syslog/parser_test.go @@ -13,6 +13,58 @@ func TestParseSyslog(t *testing.T) { log []byte syslog event }{ + { + title: "priority and timestamp defined as 2018-05-08T10:31:24 (rfc3339)", + log: []byte("<38>2018-05-08T10:31:24 localhost prg00000[1234]: seq: 0000000000, thread: 0000, runid: 1525768284, stamp: 2018-05-08T10:31:24 PADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPAD DPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADD"), + syslog: event{ + priority: 38, + message: "seq: 0000000000, thread: 0000, runid: 1525768284, stamp: 2018-05-08T10:31:24 PADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPAD DPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADD", + hostname: "localhost", + program: "prg00000", + pid: 1234, + month: 5, + day: 8, + hour: 10, + minute: 31, + second: 24, + year: 2016, + }, + }, + { + title: "timestamp defined as 2018-05-08T10:31:24 (rfc3339)", + log: []byte("2016-05-08T10:31:24 localhost prg00000[1234]: seq: 0000000000, thread: 0000, runid: 1525768284"), + syslog: event{ + priority: -1, + message: "seq: 0000000000, thread: 0000, runid: 1525768284", + hostname: "localhost", + program: "prg00000", + pid: 1234, + month: 5, + day: 8, + hour: 10, + minute: 31, + second: 24, + year: 2016, + }, + }, + { + title: "timestamp with nanosecond defined as 2018-05-08T10:31:24.0004 (rfc3339)", + log: []byte("2016-05-08T10:31:24.0004 localhost prg00000[1234]: seq: 0000000000, thread: 0000, runid: 1525768284"), + syslog: event{ + priority: -1, + message: "seq: 0000000000, thread: 0000, runid: 1525768284", + hostname: "localhost", + program: "prg00000", + pid: 1234, + month: 5, + day: 8, + hour: 10, + minute: 31, + second: 24, + year: 2016, + nanosecond: 4000, + }, + }, { title: "message only", log: []byte("--- last message repeated 1 time ---"), @@ -45,6 +97,23 @@ func TestParseSyslog(t *testing.T) { second: 15, }, }, + { + title: "time with nanosecond", + log: []byte("Oct 11 22:14:15.0000005 --- last message repeated 1 time ---"), + syslog: event{ + priority: -1, + message: "--- last message repeated 1 time ---", + hostname: "", + program: "", + pid: -1, + month: 10, + day: 11, + hour: 22, + minute: 14, + second: 15, + nanosecond: 5, + }, + }, { title: "No priority defined", log: []byte("Oct 11 22:14:15 mymachine su[230]: 'su root' failed for lonvick on /dev/pts/8"), @@ -89,7 +158,7 @@ func TestParseSyslog(t *testing.T) { hour: 22, minute: 14, second: 15, - nanosecond: 57643, + nanosecond: 5764300, }, }, { @@ -243,6 +312,7 @@ func TestParseSyslog(t *testing.T) { assert.Equal(t, test.syslog.Hour(), l.Hour()) assert.Equal(t, test.syslog.Minute(), l.Minute()) assert.Equal(t, test.syslog.Second(), l.Second()) + assert.Equal(t, test.syslog.Nanosecond(), l.Nanosecond()) }) } } diff --git a/filebeat/input/syslog/syslog_rfc3164.rl b/filebeat/input/syslog/syslog_rfc3164.rl index ea0f632bba7c..5133c4dbd233 100644 --- a/filebeat/input/syslog/syslog_rfc3164.rl +++ b/filebeat/input/syslog/syslog_rfc3164.rl @@ -25,9 +25,18 @@ hour = ([01][0-9]|"2"[0-3])>tok %hour; minute = ([0-5][0-9])>tok %minute; second = ([0-5][0-9])>tok %second; - nanosecond = digit+; + nanosecond = digit+>tok %nanosecond; time = hour ":" minute ":" second ("." nanosecond)?; - timestamp = month space day space time; + + # Some BSD style actually uses rfc3339 formatted date. + year = digit{4}>tok %year; + month_numeric = digit{2}>tok %month_numeric; + day_two_digits = ([0-3][0-9])>tok %day; + + # common timestamp format + timestamp_rfc3164 = month space day space time; + timestamp_rfc3339 = year "-" month_numeric "-" day_two_digits "T" time; + timestamp = timestamp_rfc3339 | timestamp_rfc3164; hostname = [a-zA-Z0-9.-_:]+>tok %hostname; header = timestamp space hostname space;