Use UTC time when formatting timestamps (#3744)

Solves issue #3744

Now the when the timestamp is stored in the context passed to the formatter is to converted to UTC. Some unit test (formatevents_test.go and datetime_test.go) had to be adapted to this behavior.

Modified GetIndex unit tests (elasticsearch/client_test.go) to expose this bug, now they use a timezone and a time that falls in between localtime and UTC to make sure the date in the index name is based on UTC.

libbeat/common/datetime_test.go

@@ -67,9 +67,9 @@ func TestTimeMarshal(t *testing.T) {
 	tests := []inputOutput{
 		{
 			Input: MapStr{
-				"@timestamp": Time(time.Date(2015, time.March, 01, 11, 19, 05, 112*1e6, time.UTC)),
+				"@timestamp": Time(time.Date(2015, time.March, 01, 11, 19, 05, 112*1e6, time.FixedZone("GMT-5", -3600*5))),
 			},
-			Output: `{"@timestamp":"2015-03-01T11:19:05.112Z"}`,
+			Output: `{"@timestamp":"2015-03-01T16:19:05.112Z"}`, // expect GMT
 		},
 		{
 			Input: MapStr{

libbeat/common/fmtstr/formatevents.go

@@ -243,7 +243,7 @@ func (fs *EventFormatString) collectFields(
 	}
 
 	if fs.timestamp {
-		ctx.ts = event.Timestamp
+		ctx.ts = event.Timestamp.UTC()
 	}
 
 	return nil

libbeat/common/fmtstr/formatevents_test.go

@@ -85,24 +85,24 @@ func TestEventFormatString(t *testing.T) {
 			"test timestamp formatter",
 			"%{[key]}: %{+YYYY.MM.dd}",
 			beat.Event{
-				Timestamp: time.Date(2015, 5, 1, 20, 12, 34, 0, time.Local),
+				Timestamp: time.Date(2015, 5, 1, 20, 12, 34, 0, time.FixedZone("GMT-5", -3600*5)),
 				Fields: common.MapStr{
 					"key": "timestamp",
 				},
 			},
-			"timestamp: 2015.05.01",
+			"timestamp: 2015.05.02", // expect UTC time
 			[]string{"key"},
 		},
 		{
 			"test timestamp formatter",
 			"%{[@timestamp]}: %{+YYYY.MM.dd}",
 			beat.Event{
-				Timestamp: time.Date(2015, 5, 1, 20, 12, 34, 0, time.Local),
+				Timestamp: time.Date(2015, 5, 1, 20, 12, 34, 0, time.FixedZone("GMT-5", -3600*5)),
 				Fields: common.MapStr{
 					"key": "timestamp",
 				},
 			},
-			"2015-05-01T20:12:34.000Z: 2015.05.01",
+			"2015-05-01T20:12:34.000Z: 2015.05.02", // expect UTC time
 			[]string{"@timestamp"},
 		},
 	}

libbeat/outputs/elasticsearch/client_test.go

@@ -169,8 +169,10 @@ func TestCollectPipelinePublishFail(t *testing.T) {
 }
 
 func TestGetIndexStandard(t *testing.T) {
-	ts := time.Now().UTC()
-	extension := fmt.Sprintf("%d.%02d.%02d", ts.Year(), ts.Month(), ts.Day())
+	// use a time that falls in between UTC and local time midnight
+	// this is to make sure index pattern uses UTC date and not local date
+	ts := time.Date(2018, 01, 01, 22, 00, 00, 00, time.FixedZone("GMT-5", -5*3600))
+	extension := "2018.01.02" // in UTC it is the next day already
 	fields := common.MapStr{"field": 1}
 
 	pattern := "beatname-%{+yyyy.MM.dd}"
@@ -183,8 +185,10 @@ func TestGetIndexStandard(t *testing.T) {
 }
 
 func TestGetIndexOverwrite(t *testing.T) {
-	time := time.Now().UTC()
-	extension := fmt.Sprintf("%d.%02d.%02d", time.Year(), time.Month(), time.Day())
+	// use a time that falls in between UTC and local time midnight
+	// this is to make sure index pattern uses UTC date and not local date
+	time := time.Date(2018, 01, 01, 22, 00, 00, 00, time.FixedZone("GMT-5", -5*3600))
+	extension := "2018.01.02" // in UTC it is the next day already
 
 	fields := common.MapStr{
 		"@timestamp": common.Time(time),