Skip to content

Commit

Permalink
Use time.Duration directly in GetStartTimeEndTime function (elastic#1…
Browse files Browse the repository at this point in the history 
…2033)

* Remove convertPeriodToDuration and use duration directly in GetStartTimeEndTime

* Pass period in time.Duration type
  • Loading branch information
@kaiyan-sheng @ph
kaiyan-sheng authored and ph committed May 21, 2019
1 parent 112be65 commit dcdf2a3
Show file tree
Hide file tree
Showing 10 changed files with 60 additions and 125 deletions.
27 changes: 6 additions & 21 deletions x-pack/metricbeat/module/aws/aws.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,9 @@ type Config struct {
// MetricSet is the base metricset for all aws metricsets
type MetricSet struct {
mb.BaseMetricSet
RegionsList []string
DurationString string
PeriodInSec int
AwsConfig *awssdk.Config
RegionsList []string
Period time.Duration
AwsConfig *awssdk.Config
}

// ModuleName is the name of this module.
Expand Down Expand Up @@ -77,16 +76,10 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {

awsConfig.Region = config.DefaultRegion

durationString, periodSec := convertPeriodToDuration(config.Period)
if err != nil {
return nil, err
}

metricSet := MetricSet{
BaseMetricSet: base,
DurationString: durationString,
PeriodInSec: periodSec,
AwsConfig: &awsConfig,
BaseMetricSet: base,
Period: config.Period,
AwsConfig: &awsConfig,
}

// Construct MetricSet with a full regions list
Expand Down Expand Up @@ -120,14 +113,6 @@ func getRegions(svc ec2iface.EC2API) (completeRegionsList []string, err error) {
return
}

func convertPeriodToDuration(period time.Duration) (string, int) {
// Set starttime double the default frequency earlier than the endtime in order to make sure
// GetMetricDataRequest gets the latest data point for each metric.
duration := "-" + (period * 2).String()
numberPeriod := int(period.Seconds())
return duration, numberPeriod
}

// StringInSlice checks if a string is already exists in list
func StringInSlice(str string, list []string) bool {
for _, v := range list {
Expand Down
36 changes: 0 additions & 36 deletions x-pack/metricbeat/module/aws/aws_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ package aws
import (
"fmt"
"testing"
"time"

awssdk "github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/ec2"
Expand Down Expand Up @@ -48,38 +47,3 @@ func TestGetRegions(t *testing.T) {
assert.Equal(t, 1, len(regionsList))
assert.Equal(t, regionName, regionsList[0])
}

func TestConvertPeriodToDuration(t *testing.T) {
cases := []struct {
period time.Duration
expectedDuration string
expectedPeriodNumber int
}{
{
period: time.Duration(300) * time.Second,
expectedDuration: "-10m0s",
expectedPeriodNumber: 300,
},
{
period: time.Duration(10) * time.Minute,
expectedDuration: "-20m0s",
expectedPeriodNumber: 600,
},
{
period: time.Duration(30) * time.Second,
expectedDuration: "-1m0s",
expectedPeriodNumber: 30,
},
{
period: time.Duration(60) * time.Second,
expectedDuration: "-2m0s",
expectedPeriodNumber: 60,
},
}

for _, c := range cases {
duration, periodSec := convertPeriodToDuration(c.period)
assert.Equal(t, c.expectedDuration, duration)
assert.Equal(t, c.expectedPeriodNumber, periodSec)
}
}
20 changes: 9 additions & 11 deletions x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,18 +91,15 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
// of an error set the Error field of mb.Event or simply call report.Error().
func (m *MetricSet) Fetch(report mb.ReporterV2) error {
// Get startTime and endTime
startTime, endTime, err := aws.GetStartTimeEndTime(m.DurationString)
if err != nil {
return errors.Wrap(err, "error GetStartTimeEndTime")
}
startTime, endTime := aws.GetStartTimeEndTime(m.Period)

// Get listMetrics and namespacesTotal from configuration
listMetrics, namespacesTotal := readCloudwatchConfig(m.CloudwatchConfigs)
for _, regionName := range m.MetricSet.RegionsList {
awsConfig := m.MetricSet.AwsConfig.Copy()
awsConfig.Region = regionName
svcCloudwatch := cloudwatch.New(awsConfig)
err := createEvents(svcCloudwatch, listMetrics, regionName, m.PeriodInSec, startTime, endTime, report)
err := createEvents(svcCloudwatch, listMetrics, regionName, m.Period, startTime, endTime, report)
if err != nil {
return errors.Wrap(err, "createEvents failed")
}
Expand All @@ -124,7 +121,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
continue
}

err = createEvents(svcCloudwatch, listMetricsOutput, regionName, m.PeriodInSec, startTime, endTime, report)
err = createEvents(svcCloudwatch, listMetricsOutput, regionName, m.Period, startTime, endTime, report)
if err != nil {
return errors.Wrap(err, "createEvents failed for region "+regionName)
}
Expand All @@ -150,7 +147,7 @@ func readCloudwatchConfig(cloudwatchConfigs []Config) ([]cloudwatch.Metric, []st
return listMetrics, namespacesTotal
}

func constructMetricQueries(listMetricsOutput []cloudwatch.Metric, period int64) []cloudwatch.MetricDataQuery {
func constructMetricQueries(listMetricsOutput []cloudwatch.Metric, period time.Duration) []cloudwatch.MetricDataQuery {
var metricDataQueries []cloudwatch.MetricDataQuery
for i, listMetric := range listMetricsOutput {
metricDataQuery := createMetricDataQuery(listMetric, i, period)
Expand Down Expand Up @@ -181,15 +178,16 @@ func constructLabel(metric cloudwatch.Metric) string {
return label
}

func createMetricDataQuery(metric cloudwatch.Metric, index int, period int64) (metricDataQuery cloudwatch.MetricDataQuery) {
func createMetricDataQuery(metric cloudwatch.Metric, index int, period time.Duration) (metricDataQuery cloudwatch.MetricDataQuery) {
statistic := "Average"
id := "cw" + strconv.Itoa(index)
label := constructLabel(metric)
periodInSec := int64(period.Seconds())

metricDataQuery = cloudwatch.MetricDataQuery{
Id: &id,
MetricStat: &cloudwatch.MetricStat{
Period: &period,
Period: &periodInSec,
Stat: &statistic,
Metric: &metric,
},
Expand Down Expand Up @@ -267,7 +265,7 @@ func convertConfigToListMetrics(cloudwatchConfig Config, namespace string) cloud
return listMetricsOutput
}

func createEvents(svc cloudwatchiface.CloudWatchAPI, listMetricsTotal []cloudwatch.Metric, regionName string, period int, startTime time.Time, endTime time.Time, report mb.ReporterV2) error {
func createEvents(svc cloudwatchiface.CloudWatchAPI, listMetricsTotal []cloudwatch.Metric, regionName string, period time.Duration, startTime time.Time, endTime time.Time, report mb.ReporterV2) error {
identifiers := getIdentifiers(listMetricsTotal)
// Initialize events map per region, which stores one event per identifierValue
events := map[string]mb.Event{}
Expand All @@ -280,7 +278,7 @@ func createEvents(svc cloudwatchiface.CloudWatchAPI, listMetricsTotal []cloudwat
var eventsNoIdentifier []mb.Event

// Construct metricDataQueries
metricDataQueries := constructMetricQueries(listMetricsTotal, int64(period))
metricDataQueries := constructMetricQueries(listMetricsTotal, period)
if len(metricDataQueries) == 0 {
return nil
}
Expand Down
22 changes: 10 additions & 12 deletions x-pack/metricbeat/module/aws/ec2/ec2.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"fmt"
"strconv"
"strings"
"time"

"github.com/aws/aws-sdk-go-v2/service/cloudwatch"
"github.com/aws/aws-sdk-go-v2/service/ec2"
Expand Down Expand Up @@ -51,8 +52,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
}

// Check if period is set to be multiple of 60s or 300s
remainder300 := metricSet.PeriodInSec % 300
remainder60 := metricSet.PeriodInSec % 60
remainder300 := int(metricSet.Period.Seconds()) % 300
remainder60 := int(metricSet.Period.Seconds()) % 60
if remainder300 != 0 || remainder60 != 0 {
err := errors.New("period needs to be set to 60s (or a multiple of 60s) if detailed monitoring is " +
"enabled for EC2 instances or set to 300s (or a multiple of 300s) if EC2 instances has basic monitoring. " +
Expand All @@ -70,10 +71,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
// of an error set the Error field of mb.Event or simply call report.Error().
func (m *MetricSet) Fetch(report mb.ReporterV2) error {
// Get startTime and endTime
startTime, endTime, err := aws.GetStartTimeEndTime(m.DurationString)
if err != nil {
return errors.Wrap(err, "Error ParseDuration")
}
startTime, endTime := aws.GetStartTimeEndTime(m.Period)

for _, regionName := range m.MetricSet.RegionsList {
awsConfig := m.MetricSet.AwsConfig.Copy()
Expand Down Expand Up @@ -102,7 +100,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {

var metricDataQueriesTotal []cloudwatch.MetricDataQuery
for _, instanceID := range instanceIDs {
metricDataQueriesTotal = append(metricDataQueriesTotal, constructMetricQueries(listMetricsOutput, instanceID, m.PeriodInSec)...)
metricDataQueriesTotal = append(metricDataQueriesTotal, constructMetricQueries(listMetricsOutput, instanceID, m.Period)...)
}

var metricDataOutput []cloudwatch.MetricDataResult
Expand Down Expand Up @@ -139,11 +137,11 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
return nil
}

func constructMetricQueries(listMetricsOutput []cloudwatch.Metric, instanceID string, periodInSec int) []cloudwatch.MetricDataQuery {
func constructMetricQueries(listMetricsOutput []cloudwatch.Metric, instanceID string, period time.Duration) []cloudwatch.MetricDataQuery {
var metricDataQueries []cloudwatch.MetricDataQuery
metricDataQueryEmpty := cloudwatch.MetricDataQuery{}
for i, listMetric := range listMetricsOutput {
metricDataQuery := createMetricDataQuery(listMetric, instanceID, i, periodInSec)
metricDataQuery := createMetricDataQuery(listMetric, instanceID, i, period)
if metricDataQuery == metricDataQueryEmpty {
continue
}
Expand Down Expand Up @@ -260,9 +258,9 @@ func getInstancesPerRegion(svc ec2iface.EC2API) (instanceIDs []string, instances
return
}

func createMetricDataQuery(metric cloudwatch.Metric, instanceID string, index int, periodInSec int) (metricDataQuery cloudwatch.MetricDataQuery) {
func createMetricDataQuery(metric cloudwatch.Metric, instanceID string, index int, period time.Duration) (metricDataQuery cloudwatch.MetricDataQuery) {
statistic := "Average"
period := int64(periodInSec)
periodInSeconds := int64(period.Seconds())
id := metricsetName + strconv.Itoa(index)
metricDims := metric.Dimensions

Expand All @@ -273,7 +271,7 @@ func createMetricDataQuery(metric cloudwatch.Metric, instanceID string, index in
metricDataQuery = cloudwatch.MetricDataQuery{
Id: &id,
MetricStat: &cloudwatch.MetricStat{
Period: &period,
Period: &periodInSeconds,
Stat: &statistic,
Metric: &metric,
},
Expand Down
2 changes: 1 addition & 1 deletion x-pack/metricbeat/module/aws/ec2/ec2_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ func TestConstructMetricQueries(t *testing.T) {
}

listMetricsOutput := []cloudwatch.Metric{listMetric}
metricDataQuery := constructMetricQueries(listMetricsOutput, instanceID, 300)
metricDataQuery := constructMetricQueries(listMetricsOutput, instanceID, 5*time.Minute)
assert.Equal(t, 1, len(metricDataQuery))
assert.Equal(t, "i-123 CPUUtilization", *metricDataQuery[0].Label)
assert.Equal(t, "Average", *metricDataQuery[0].MetricStat.Stat)
Expand Down
20 changes: 9 additions & 11 deletions x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"fmt"
"strconv"
"strings"
"time"

"github.com/aws/aws-sdk-go-v2/service/cloudwatch"
"github.com/pkg/errors"
Expand Down Expand Up @@ -55,7 +56,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
}

// Check if period is set to be multiple of 86400s
remainder := metricSet.PeriodInSec % 86400
remainder := int(metricSet.Period.Seconds()) % 86400
if remainder != 0 {
err := errors.New("period needs to be set to 86400s (or a multiple of 86400s). " +
"To avoid data missing or extra costs, please make sure period is set correctly " +
Expand All @@ -74,10 +75,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
func (m *MetricSet) Fetch(report mb.ReporterV2) error {
namespace := "AWS/S3"
// Get startTime and endTime
startTime, endTime, err := aws.GetStartTimeEndTime(m.DurationString)
if err != nil {
return errors.Wrap(err, "Error ParseDuration")
}
startTime, endTime := aws.GetStartTimeEndTime(m.Period)

// GetMetricData for AWS S3 from Cloudwatch
for _, regionName := range m.MetricSet.RegionsList {
Expand All @@ -96,7 +94,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
continue
}

metricDataQueries := constructMetricQueries(listMetricsOutputs, m.PeriodInSec)
metricDataQueries := constructMetricQueries(listMetricsOutputs, m.Period)
// Use metricDataQueries to make GetMetricData API calls
metricDataOutputs, err := aws.GetMetricDataResults(metricDataQueries, svcCloudwatch, startTime, endTime)
if err != nil {
Expand Down Expand Up @@ -142,7 +140,7 @@ func getBucketNames(listMetricsOutputs []cloudwatch.Metric) (bucketNames []strin
return
}

func constructMetricQueries(listMetricsOutputs []cloudwatch.Metric, periodInSec int) []cloudwatch.MetricDataQuery {
func constructMetricQueries(listMetricsOutputs []cloudwatch.Metric, period time.Duration) []cloudwatch.MetricDataQuery {
var metricDataQueries []cloudwatch.MetricDataQuery
metricDataQueryEmpty := cloudwatch.MetricDataQuery{}
metricNames := []string{"NumberOfObjects", "BucketSizeBytes"}
Expand All @@ -151,7 +149,7 @@ func constructMetricQueries(listMetricsOutputs []cloudwatch.Metric, periodInSec
continue
}

metricDataQuery := createMetricDataQuery(listMetric, periodInSec, i)
metricDataQuery := createMetricDataQuery(listMetric, period, i)
if metricDataQuery == metricDataQueryEmpty {
continue
}
Expand All @@ -160,9 +158,9 @@ func constructMetricQueries(listMetricsOutputs []cloudwatch.Metric, periodInSec
return metricDataQueries
}

func createMetricDataQuery(metric cloudwatch.Metric, periodInSec int, index int) (metricDataQuery cloudwatch.MetricDataQuery) {
func createMetricDataQuery(metric cloudwatch.Metric, period time.Duration, index int) (metricDataQuery cloudwatch.MetricDataQuery) {
statistic := "Average"
period := int64(periodInSec)
periodInSec := int64(period.Seconds())
id := "s3d" + strconv.Itoa(index)
metricDims := metric.Dimensions
bucketName := ""
Expand All @@ -180,7 +178,7 @@ func createMetricDataQuery(metric cloudwatch.Metric, periodInSec int, index int)
metricDataQuery = cloudwatch.MetricDataQuery{
Id: &id,
MetricStat: &cloudwatch.MetricStat{
Period: &period,
Period: &periodInSec,
Stat: &statistic,
Metric: &metric,
},
Expand Down
Loading

0 comments on commit dcdf2a3

Please sign in to comment.