Support configuring a default timezone in JSON parser (influxdata#5472)

internal/config/config.go

@@ -1362,6 +1362,14 @@ func getParserConfig(name string, tbl *ast.Table) (*parsers.Config, error) {
 		}
 	}
 
+	if node, ok := tbl.Fields["json_timezone"]; ok {
+		if kv, ok := node.(*ast.KeyValue); ok {
+			if str, ok := kv.Value.(*ast.String); ok {
+				c.JSONTimezone = str.Value
+			}
+		}
+	}
+
 	if node, ok := tbl.Fields["data_type"]; ok {
 		if kv, ok := node.(*ast.KeyValue); ok {
 			if str, ok := kv.Value.(*ast.String); ok {
@@ -1637,6 +1645,7 @@ func getParserConfig(name string, tbl *ast.Table) (*parsers.Config, error) {
 	delete(tbl.Fields, "json_string_fields")
 	delete(tbl.Fields, "json_time_format")
 	delete(tbl.Fields, "json_time_key")
+	delete(tbl.Fields, "json_timezone")
 	delete(tbl.Fields, "data_type")
 	delete(tbl.Fields, "collectd_auth_file")
 	delete(tbl.Fields, "collectd_security_level")

internal/internal.go

@@ -333,13 +333,18 @@ func CompressWithGzip(data io.Reader) (io.Reader, error) {
 	return pipeReader, err
 }
 
+// ParseTimestamp with no location provided parses a timestamp value as UTC
+func ParseTimestamp(timestamp interface{}, format string) (time.Time, error) {
+	return ParseTimestampWithLocation(timestamp, format, "UTC")
+}
+
 // ParseTimestamp parses a timestamp value as a unix epoch of various precision.
 //
 // format = "unix": epoch is assumed to be in seconds and can come as number or string. Can have a decimal part.
 // format = "unix_ms": epoch is assumed to be in milliseconds and can come as number or string. Cannot have a decimal part.
 // format = "unix_us": epoch is assumed to be in microseconds and can come as number or string. Cannot have a decimal part.
 // format = "unix_ns": epoch is assumed to be in nanoseconds and can come as number or string. Cannot have a decimal part.
-func ParseTimestamp(timestamp interface{}, format string) (time.Time, error) {
+func ParseTimestampWithLocation(timestamp interface{}, format string, location string) (time.Time, error) {
 	timeInt, timeFractional := int64(0), int64(0)
 	timeEpochStr, ok := timestamp.(string)
 	var err error
@@ -355,7 +360,11 @@ func ParseTimestamp(timestamp interface{}, format string) (time.Time, error) {
 		splitted := regexp.MustCompile("[.,]").Split(timeEpochStr, 2)
 		timeInt, err = strconv.ParseInt(splitted[0], 10, 64)
 		if err != nil {
-			return time.Parse(format, timeEpochStr)
+			loc, err := time.LoadLocation(location)
+			if err != nil {
+				return time.Time{}, fmt.Errorf("location: %s could not be loaded as a location", location)
+			}
+			return time.ParseInLocation(format, timeEpochStr, loc)
 		}
 
 		if len(splitted) == 2 {

internal/internal_test.go

@@ -270,3 +270,34 @@ func TestAlignDuration(t *testing.T) {
 		})
 	}
 }
+
+func TestParseTimestamp(t *testing.T) {
+	time, err := ParseTimestamp("2019-02-20 21:50:34.029665", "2006-01-02 15:04:05.000000")
+	assert.Nil(t, err)
+	assert.EqualValues(t, int64(1550699434029665000), time.UnixNano())
+
+	time, err = ParseTimestamp("2019-02-20 21:50:34.029665-04:00", "2006-01-02 15:04:05.000000-07:00")
+	assert.Nil(t, err)
+	assert.EqualValues(t, int64(1550713834029665000), time.UnixNano())
+
+	time, err = ParseTimestamp("2019-02-20 21:50:34.029665", "2006-01-02 15:04:05.000000-06:00")
+	assert.NotNil(t, err)
+}
+
+func TestParseTimestampWithLocation(t *testing.T) {
+	time, err := ParseTimestampWithLocation("2019-02-20 21:50:34.029665", "2006-01-02 15:04:05.000000", "UTC")
+	assert.Nil(t, err)
+	assert.EqualValues(t, int64(1550699434029665000), time.UnixNano())
+
+	time, err = ParseTimestampWithLocation("2019-02-20 21:50:34.029665", "2006-01-02 15:04:05.000000", "America/New_York")
+	assert.Nil(t, err)
+	assert.EqualValues(t, int64(1550717434029665000), time.UnixNano())
+
+	//Provided location is ignored if an offset is successfully parsed
+	time, err = ParseTimestampWithLocation("2019-02-20 21:50:34.029665-07:00", "2006-01-02 15:04:05.000000-07:00", "America/New_York")
+	assert.Nil(t, err)
+	assert.EqualValues(t, int64(1550724634029665000), time.UnixNano())
+
+	time, err = ParseTimestampWithLocation("2019-02-20 21:50:34.029665", "2006-01-02 15:04:05.000000", "InvalidTimeZone")
+	assert.NotNil(t, err)
+}

plugins/parsers/json/README.md

@@ -49,9 +49,21 @@ ignored unless specified in the `tag_key` or `json_string_fields` options.
   ## https://golang.org/pkg/time/#Time.Format
   ##   ex: json_time_format = "Mon Jan 2 15:04:05 -0700 MST 2006"
   ##       json_time_format = "2006-01-02T15:04:05Z07:00"
+  ##       json_time_format = "01/02/2006 15:04:05"
   ##       json_time_format = "unix"
   ##       json_time_format = "unix_ms"
   json_time_format = ""
+
+  ## Timezone allows you to provide an override for timestamps that
+  ## don't already include an offset
+  ## e.g. 04/06/2016 12:41:45
+  ##
+  ## Default: "" which renders UTC
+  ## Options are as follows:
+  ##   1. Local               -- interpret based on machine localtime
+  ##   2. "America/New_York"  -- Unix TZ values like those found in https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
+  ##   3. UTC                 -- or blank/unspecified, will return timestamp in UTC
+  json_timezone = ""
 ```
 
 #### json_query
@@ -62,7 +74,7 @@ query should contain a JSON object or an array of objects.
 
 Consult the GJSON [path syntax][gjson syntax] for details and examples.
 
-#### json_time_key, json_time_format
+#### json_time_key, json_time_format, json_timezone
 
 By default the current time will be used for all created metrics, to set the
 time using the JSON document you can use the `json_time_key` and
@@ -77,6 +89,12 @@ the Go "reference time" which is defined to be the specific time:
 Consult the Go [time][time parse] package for details and additional examples
 on how to set the time format.
 
+When parsing times that don't include a timezone specifier, times are assumed
+to be UTC. To default to another timezone, or to local time, specify the
+`json_timezone` option.  This option should be set to a
+[Unix TZ value](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones),
+such as `America/New_York`, to `Local` to utilize the system timezone, or to `UTC`.
+
 ### Examples
 
 #### Basic Parsing

plugins/parsers/json/parser.go

@@ -28,6 +28,7 @@ type JSONParser struct {
 	JSONQuery      string
 	JSONTimeKey    string
 	JSONTimeFormat string
+	JSONTimezone   string
 	DefaultTags    map[string]string
 }
 
@@ -82,7 +83,7 @@ func (p *JSONParser) parseObject(metrics []telegraf.Metric, jsonOut map[string]i
 			return nil, err
 		}
 
-		nTime, err = internal.ParseTimestamp(f.Fields[p.JSONTimeKey], p.JSONTimeFormat)
+		nTime, err = internal.ParseTimestampWithLocation(f.Fields[p.JSONTimeKey], p.JSONTimeFormat, p.JSONTimezone)
 		if err != nil {
 			return nil, err
 		}

plugins/parsers/json/parser_test.go

@@ -599,6 +599,23 @@ func TestTimeParser(t *testing.T) {
 	require.Equal(t, false, metrics[0].Time() == metrics[1].Time())
 }
 
+func TestTimeParserWithTimezone(t *testing.T) {
+	testString := `{
+		"time": "04 Jan 06 15:04"
+	}`
+
+	parser := JSONParser{
+		MetricName:     "json_test",
+		JSONTimeKey:    "time",
+		JSONTimeFormat: "02 Jan 06 15:04",
+		JSONTimezone:   "America/New_York",
+	}
+	metrics, err := parser.Parse([]byte(testString))
+	require.NoError(t, err)
+	require.Equal(t, 1, len(metrics))
+	require.EqualValues(t, int64(1136405040000000000), metrics[0].Time().UnixNano())
+}
+
 func TestUnixTimeParser(t *testing.T) {
 	testString := `[
 		{

plugins/parsers/registry.go

@@ -85,6 +85,9 @@ type Config struct {
 	// time format
 	JSONTimeFormat string `toml:"json_time_format"`
 
+	// default timezone
+	JSONTimezone string `toml:"json_timezone"`
+
 	// Authentication file for collectd
 	CollectdAuthFile string `toml:"collectd_auth_file"`
 	// One of none (default), sign, or encrypt
@@ -152,6 +155,7 @@ func NewParser(config *Config) (Parser, error) {
 			config.JSONQuery,
 			config.JSONTimeKey,
 			config.JSONTimeFormat,
+			config.JSONTimezone,
 			config.DefaultTags)
 	case "value":
 		parser, err = NewValueParser(config.MetricName,
@@ -275,6 +279,7 @@ func newJSONParser(
 	jsonQuery string,
 	timeKey string,
 	timeFormat string,
+	timezone string,
 	defaultTags map[string]string,
 ) Parser {
 	parser := &json.JSONParser{
@@ -285,6 +290,7 @@ func newJSONParser(
 		JSONQuery:      jsonQuery,
 		JSONTimeKey:    timeKey,
 		JSONTimeFormat: timeFormat,
+		JSONTimezone:   timezone,
 		DefaultTags:    defaultTags,
 	}
 	return parser