From e23ccb01c7026787f6fa57ab19c3d7a645f8aa2c Mon Sep 17 00:00:00 2001 From: jsirianni Date: Thu, 4 Feb 2021 19:34:23 -0500 Subject: [PATCH 1/2] Add optional location parameter for setting the timezone for rfc3164 syslog. Original implementation can be found here: https://github.com/observIQ/stanza/pull/247/commits --- docs/operators/syslog_parser.md | 1 + operator/builtin/parser/syslog/syslog.go | 18 +++++- operator/builtin/parser/syslog/syslog_test.go | 63 ++++++++++++++++++- 3 files changed, 78 insertions(+), 4 deletions(-) diff --git a/docs/operators/syslog_parser.md b/docs/operators/syslog_parser.md index 860b5bff..9e66c49a 100644 --- a/docs/operators/syslog_parser.md +++ b/docs/operators/syslog_parser.md @@ -13,6 +13,7 @@ The `syslog_parser` operator parses the string-type field selected by `parse_fro | `preserve_to` | | Preserves the unparsed value at the specified [field](/docs/types/field.md) | | `on_error` | `send` | The behavior of the operator if it encounters an error. See [on_error](/docs/types/on_error.md) | | `protocol` | required | The protocol to parse the syslog messages as. Options are `rfc3164` and `rfc5424` | +| `location` | `UTC` | The geographic location (timezone) to use when parsing the timestamp (Syslog RFC 3164 only). The available locations depend on the local IANA Time Zone database. [This page](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) contains many examples, such as `America/New_York`. | | `timestamp` | `nil` | An optional [timestamp](/docs/types/timestamp.md) block which will parse a timestamp field before passing the entry to the output operator | | `severity` | `nil` | An optional [severity](/docs/types/severity.md) block which will parse a severity field before passing the entry to the output operator | | `if` | | An [expression](/docs/types/expression.md) that, when set, will be evaluated to determine whether this operator should be used for the given entry. This allows you to do easy conditional parsing without branching logic with routers. | diff --git a/operator/builtin/parser/syslog/syslog.go b/operator/builtin/parser/syslog/syslog.go index ad54ae1f..eeb2f749 100644 --- a/operator/builtin/parser/syslog/syslog.go +++ b/operator/builtin/parser/syslog/syslog.go @@ -43,6 +43,7 @@ type SyslogParserConfig struct { helper.ParserConfig `yaml:",inline"` Protocol string `json:"protocol,omitempty" yaml:"protocol,omitempty"` + Location string `json:"location,omitempty" yaml:"location,omitempty"` } // Build will build a JSON parser operator. @@ -64,18 +65,28 @@ func (c SyslogParserConfig) Build(context operator.BuildContext) ([]operator.Ope return nil, fmt.Errorf("missing field 'protocol'") } + if c.Location == "" { + c.Location = "UTC" + } + + location, err := time.LoadLocation(c.Location) + if err != nil { + return nil, err + } + syslogParser := &SyslogParser{ ParserOperator: parserOperator, protocol: c.Protocol, + location: location, } return []operator.Operator{syslogParser}, nil } -func buildMachine(protocol string) (sl.Machine, error) { +func buildMachine(protocol string, location *time.Location) (sl.Machine, error) { switch protocol { case "rfc3164": - return rfc3164.NewMachine(), nil + return rfc3164.NewMachine(rfc3164.WithLocaleTimezone(location)), nil case "rfc5424": return rfc5424.NewMachine(), nil default: @@ -87,6 +98,7 @@ func buildMachine(protocol string) (sl.Machine, error) { type SyslogParser struct { helper.ParserOperator protocol string + location *time.Location } // Process will parse an entry field as syslog. @@ -104,7 +116,7 @@ func (s *SyslogParser) parse(value interface{}) (interface{}, error) { return nil, err } - machine, err := buildMachine(s.protocol) + machine, err := buildMachine(s.protocol, s.location) if err != nil { return nil, err } diff --git a/operator/builtin/parser/syslog/syslog_test.go b/operator/builtin/parser/syslog/syslog_test.go index 77dc9b66..4d85e4c7 100644 --- a/operator/builtin/parser/syslog/syslog_test.go +++ b/operator/builtin/parser/syslog/syslog_test.go @@ -25,6 +25,23 @@ import ( "github.com/stretchr/testify/require" ) +func testLocations() (map[string]*time.Location, error) { + locations := map[string]string{ + "utc": "UTC", + "detroit": "America/Detroit", + "athens": "Europe/Athens", + } + + l := make(map[string]*time.Location) + for k, v := range locations { + var err error + if l[k], err = time.LoadLocation(v); err != nil { + return nil, err + } + } + return l, nil +} + func TestSyslogParser(t *testing.T) { basicConfig := func() *SyslogParserConfig { cfg := NewSyslogParserConfig("test_operator_id") @@ -32,6 +49,9 @@ func TestSyslogParser(t *testing.T) { return cfg } + location, err := testLocations() + require.NoError(t, err) + cases := []struct { name string config *SyslogParserConfig @@ -46,10 +66,51 @@ func TestSyslogParser(t *testing.T) { func() *SyslogParserConfig { cfg := basicConfig() cfg.Protocol = "rfc3164" + cfg.Location = location["utc"].String() return cfg }(), "<34>Jan 12 06:30:00 1.2.3.4 apache_server: test message", - time.Date(time.Now().Year(), 1, 12, 6, 30, 0, 0, time.UTC), + time.Date(time.Now().Year(), 1, 12, 6, 30, 0, 0, location["utc"]), + map[string]interface{}{ + "appname": "apache_server", + "facility": 4, + "hostname": "1.2.3.4", + "message": "test message", + "priority": 34, + }, + entry.Critical, + "crit", + }, + { + "RFC3164Detroit", + func() *SyslogParserConfig { + cfg := basicConfig() + cfg.Protocol = "rfc3164" + cfg.Location = location["detroit"].String() + return cfg + }(), + "<34>Jan 12 06:30:00 1.2.3.4 apache_server: test message", + time.Date(time.Now().Year(), 1, 12, 6, 30, 0, 0, location["detroit"]), + map[string]interface{}{ + "appname": "apache_server", + "facility": 4, + "hostname": "1.2.3.4", + "message": "test message", + "priority": 34, + }, + entry.Critical, + "crit", + }, + { + "RFC3164Athens", + func() *SyslogParserConfig { + cfg := basicConfig() + cfg.Protocol = "rfc3164" + cfg.Location = location["athens"].String() + return cfg + }(), + "<34>Jan 12 06:30:00 1.2.3.4 apache_server: test message", + time.Date(time.Now().Year(), 1, 12, 6, 30, 0, 0, location["athens"]), map[string]interface{}{ "appname": "apache_server", "facility": 4, From 56eb283bda20648a2d2a1fdc681972ebd3f4df13 Mon Sep 17 00:00:00 2001 From: Joseph Sirianni Date: Fri, 5 Feb 2021 10:31:14 -0500 Subject: [PATCH 2/2] update changelog to include 0.14.0 release and unreleased syslog location parameter --- CHANGELOG.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f744030..04a7be46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,20 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Added +- Syslog operator RFC 3164 location parameter ([PR11](https://github.com/open-telemetry/opentelemetry-log-collection/pull/11)) + +## [0.14.0] - 2020-02-02 + +### Changed +- Remove standalone agent functionality +- Simplifies modules + - Combines `parser/syslog` and `input/windows` modules into the primary module + - Removes output operators that were previously separate modules + - Leaves `input/k8sevent` and `transformer/k8smetadata` as separate modules for now. These two have extensive dependencies and their usefulness in the collector needs to be discussed before merging or removing. + ## [0.13.12] - 2020-01-26 ### Changed