From a019e18d782412da95cbf5d781ffe90035b8fe41 Mon Sep 17 00:00:00 2001
From: fjgal <fernando.galandrini@gmail.com>
Date: Tue, 27 Feb 2018 18:47:25 -0500
Subject: [PATCH] Use UTC time when formatting timestamps (#3744)

Solves issue #3744

Now the when the timestamp is stored in the context passed to the formatter is to converted to UTC. Some unit test (formatevents_test.go and datetime_test.go) had to be adapted to this behavior.

Modified GetIndex unit tests (elasticsearch/client_test.go) to expose this bug, now they use a timezone and a time that falls in between localtime and UTC to make sure the date in the index name is based on UTC.
---
 libbeat/common/datetime_test.go              |  4 ++--
 libbeat/common/fmtstr/formatevents.go        |  2 +-
 libbeat/common/fmtstr/formatevents_test.go   |  8 ++++----
 libbeat/outputs/elasticsearch/client_test.go | 12 ++++++++----
 4 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/libbeat/common/datetime_test.go b/libbeat/common/datetime_test.go
index 2ab899bdc97..3619b0328ba 100644
--- a/libbeat/common/datetime_test.go
+++ b/libbeat/common/datetime_test.go
@@ -67,9 +67,9 @@ func TestTimeMarshal(t *testing.T) {
 	tests := []inputOutput{
 		{
 			Input: MapStr{
-				"@timestamp": Time(time.Date(2015, time.March, 01, 11, 19, 05, 112*1e6, time.UTC)),
+				"@timestamp": Time(time.Date(2015, time.March, 01, 11, 19, 05, 112*1e6, time.FixedZone("GMT-5", -3600*5))),
 			},
-			Output: `{"@timestamp":"2015-03-01T11:19:05.112Z"}`,
+			Output: `{"@timestamp":"2015-03-01T16:19:05.112Z"}`, // expect GMT
 		},
 		{
 			Input: MapStr{
diff --git a/libbeat/common/fmtstr/formatevents.go b/libbeat/common/fmtstr/formatevents.go
index 724d66e3f42..9493ef54050 100644
--- a/libbeat/common/fmtstr/formatevents.go
+++ b/libbeat/common/fmtstr/formatevents.go
@@ -243,7 +243,7 @@ func (fs *EventFormatString) collectFields(
 	}
 
 	if fs.timestamp {
-		ctx.ts = event.Timestamp
+		ctx.ts = event.Timestamp.UTC()
 	}
 
 	return nil
diff --git a/libbeat/common/fmtstr/formatevents_test.go b/libbeat/common/fmtstr/formatevents_test.go
index 3d2cdeb4628..b10d8e54658 100644
--- a/libbeat/common/fmtstr/formatevents_test.go
+++ b/libbeat/common/fmtstr/formatevents_test.go
@@ -85,24 +85,24 @@ func TestEventFormatString(t *testing.T) {
 			"test timestamp formatter",
 			"%{[key]}: %{+YYYY.MM.dd}",
 			beat.Event{
-				Timestamp: time.Date(2015, 5, 1, 20, 12, 34, 0, time.Local),
+				Timestamp: time.Date(2015, 5, 1, 20, 12, 34, 0, time.FixedZone("GMT-5", -3600*5)),
 				Fields: common.MapStr{
 					"key": "timestamp",
 				},
 			},
-			"timestamp: 2015.05.01",
+			"timestamp: 2015.05.02", // expect UTC time
 			[]string{"key"},
 		},
 		{
 			"test timestamp formatter",
 			"%{[@timestamp]}: %{+YYYY.MM.dd}",
 			beat.Event{
-				Timestamp: time.Date(2015, 5, 1, 20, 12, 34, 0, time.Local),
+				Timestamp: time.Date(2015, 5, 1, 20, 12, 34, 0, time.FixedZone("GMT-5", -3600*5)),
 				Fields: common.MapStr{
 					"key": "timestamp",
 				},
 			},
-			"2015-05-01T20:12:34.000Z: 2015.05.01",
+			"2015-05-01T20:12:34.000Z: 2015.05.02", // expect UTC time
 			[]string{"@timestamp"},
 		},
 	}
diff --git a/libbeat/outputs/elasticsearch/client_test.go b/libbeat/outputs/elasticsearch/client_test.go
index bd38e329152..7b32f303b76 100644
--- a/libbeat/outputs/elasticsearch/client_test.go
+++ b/libbeat/outputs/elasticsearch/client_test.go
@@ -169,8 +169,10 @@ func TestCollectPipelinePublishFail(t *testing.T) {
 }
 
 func TestGetIndexStandard(t *testing.T) {
-	ts := time.Now().UTC()
-	extension := fmt.Sprintf("%d.%02d.%02d", ts.Year(), ts.Month(), ts.Day())
+	// use a time that falls in between UTC and local time midnight
+	// this is to make sure index pattern uses UTC date and not local date
+	ts := time.Date(2018, 01, 01, 22, 00, 00, 00, time.FixedZone("GMT-5", -5*3600))
+	extension := "2018.01.02" // in UTC it is the next day already
 	fields := common.MapStr{"field": 1}
 
 	pattern := "beatname-%{+yyyy.MM.dd}"
@@ -183,8 +185,10 @@ func TestGetIndexStandard(t *testing.T) {
 }
 
 func TestGetIndexOverwrite(t *testing.T) {
-	time := time.Now().UTC()
-	extension := fmt.Sprintf("%d.%02d.%02d", time.Year(), time.Month(), time.Day())
+	// use a time that falls in between UTC and local time midnight
+	// this is to make sure index pattern uses UTC date and not local date
+	time := time.Date(2018, 01, 01, 22, 00, 00, 00, time.FixedZone("GMT-5", -5*3600))
+	extension := "2018.01.02" // in UTC it is the next day already
 
 	fields := common.MapStr{
 		"@timestamp": common.Time(time),