From 938e66ccf352e2925b99fc988b28c46fc3ee304e Mon Sep 17 00:00:00 2001 From: Mariana Date: Thu, 23 Jul 2020 11:01:08 +0200 Subject: [PATCH 1/4] mofidy doc --- .../module/azure/app_insights/_meta/docs.asciidoc | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/x-pack/metricbeat/module/azure/app_insights/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/app_insights/_meta/docs.asciidoc index 2ba1150078b..2b587acbbdd 100644 --- a/x-pack/metricbeat/module/azure/app_insights/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/azure/app_insights/_meta/docs.asciidoc @@ -45,17 +45,15 @@ This value is only valid when segment is specified. `filter`:: (_string_) An expression used to filter the results. This value should be a valid OData filter expression where the keys of each clause should be applicable dimensions for the metric you are retrieving. -Users can select the options to retrieve all metrics from a specific namespace using the following: +Example configuration: ["source","yaml"] ---- - metrics: - - id: ["*"] - timespan: "Microsoft.Storage/storageAccounts" +metrics: + - id: ["requests/count", "requests/failed"] + segment: "request/name" + aggregation: ["sum"] ---- -A default non configurable timegrain of 5 min is set so users are advised to configure an interval of 300s or a multiply of it. - - From f7a57de3331bc592f1219521d32ad16003864f6c Mon Sep 17 00:00:00 2001 From: Mariana Date: Mon, 3 Aug 2020 18:04:43 +0200 Subject: [PATCH 2/4] work on data --- .../module/azure/app_insights/data.go | 66 +++++++++++++++---- 1 file changed, 52 insertions(+), 14 deletions(-) diff --git a/x-pack/metricbeat/module/azure/app_insights/data.go b/x-pack/metricbeat/module/azure/app_insights/data.go index 62afa32163f..41d45b3814c 100644 --- a/x-pack/metricbeat/module/azure/app_insights/data.go +++ b/x-pack/metricbeat/module/azure/app_insights/data.go @@ -6,6 +6,7 @@ package app_insights import ( "fmt" + "github.com/Azure/go-autorest/autorest/date" "strings" "github.com/Azure/azure-sdk-for-go/services/preview/appinsights/v1/insights" @@ -19,45 +20,80 @@ func EventsMapping(metricValues insights.ListMetricsResultsItem, applicationId s if metricValues.Value == nil { return events } + groupedAddProp := make(map[string][]insights.MetricsResultInfo) for _, item := range *metricValues.Value { if item.Body != nil && item.Body.Value != nil { if item.Body.Value.AdditionalProperties != nil { - events = append(events, createEvent(*item.Body.Value, insights.MetricsSegmentInfo{}, applicationId)) + groupedAddProp[fmt.Sprintf("%sTO%s", item.Body.Value.Start, item.Body.Value.End)] = + append(groupedAddProp[fmt.Sprintf("%sTO%s", item.Body.Value.Start, item.Body.Value.End)], *item.Body.Value) } else if item.Body.Value.Segments != nil { for _, segment := range *item.Body.Value.Segments { - events = append(events, createEvent(*item.Body.Value, segment, applicationId)) + event, ok := createSegmentEvent(*item.Body.Value.Start, *item.Body.Value.End, segment, applicationId) + if ok { + events = append(events, event) + } } } } } + if len(groupedAddProp) > 0 { + for _, val := range groupedAddProp { + event, ok := createEvent(val, applicationId) + if ok { + events = append(events, event) + } + } + } return events } -func createEvent(value insights.MetricsResultInfo, segment insights.MetricsSegmentInfo, applicationId string) mb.Event { +func createSegmentEvent(start date.Time, end date.Time, segment insights.MetricsSegmentInfo, applicationId string) (mb.Event, bool) { + metricList := common.MapStr{} + metrics := getMetric(segment.AdditionalProperties) + if len(metrics) == 0 { + return mb.Event{}, false + } + for key, metric := range metrics { + metricList.Put(key, metric) + } + event := mb.Event{ + MetricSetFields: common.MapStr{ + "start_date": start, + "end_date": end, + "application_id": applicationId, + }, + Timestamp: end.Time, + } + event.RootFields = common.MapStr{} + event.RootFields.Put("cloud.provider", "azure") + event.MetricSetFields.Put("metrics", metricList) + return event, true +} + +func createEvent(values []insights.MetricsResultInfo, applicationId string) (mb.Event, bool) { metricList := common.MapStr{} - if value.AdditionalProperties != nil { + for _, value := range values { metrics := getMetric(value.AdditionalProperties) for key, metric := range metrics { metricList.Put(key, metric) } - } else { - metrics := getMetric(segment.AdditionalProperties) - for key, metric := range metrics { - metricList.Put(key, metric) - } } + if len(metricList) == 0 { + return mb.Event{}, false + } + event := mb.Event{ MetricSetFields: common.MapStr{ - "start_date": value.Start, - "end_date": value.End, + "start_date": values[0].Start, + "end_date": values[0].End, "application_id": applicationId, }, - Timestamp: value.End.Time, + Timestamp: values[0].End.Time, } event.RootFields = common.MapStr{} event.RootFields.Put("cloud.provider", "azure") event.MetricSetFields.Put("metrics", metricList) - return event + return event, true } func getMetric(addProp map[string]interface{}) map[string]interface{} { @@ -66,7 +102,9 @@ func getMetric(addProp map[string]interface{}) map[string]interface{} { switch val.(type) { case map[string]interface{}: for subKey, subVal := range val.(map[string]interface{}) { - metricNames[cleanMetricNames(fmt.Sprintf("%s.%s", key, subKey))] = subVal + if subVal != nil { + metricNames[cleanMetricNames(fmt.Sprintf("%s.%s", key, subKey))] = subVal + } } default: metricNames[cleanMetricNames(key)] = val From 5897911c70b4d93765ac96e5251d29ffec28c497 Mon Sep 17 00:00:00 2001 From: Mariana Date: Mon, 3 Aug 2020 18:10:01 +0200 Subject: [PATCH 3/4] changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c12c2a9574f..8ac52afec6b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -302,6 +302,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix k8s scheduler compatibility issue. {pull}19699[19699] - Fix SQL module mapping NULL values as string {pull}18955[18955] {issue}18898[18898 - Modify doc for app_insights metricset to contain example of config. {pull}20185[20185] +- Groups same timestamp metric values to one event in the app_insights metricset. {pull}20403[20403] *Packetbeat* From fcd363722747fd193a74291d3bc42ab13aaaff4e Mon Sep 17 00:00:00 2001 From: Mariana Date: Mon, 3 Aug 2020 19:13:45 +0200 Subject: [PATCH 4/4] mage fmt update --- x-pack/metricbeat/module/azure/app_insights/data.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/azure/app_insights/data.go b/x-pack/metricbeat/module/azure/app_insights/data.go index 41d45b3814c..df7efdbeaba 100644 --- a/x-pack/metricbeat/module/azure/app_insights/data.go +++ b/x-pack/metricbeat/module/azure/app_insights/data.go @@ -6,9 +6,10 @@ package app_insights import ( "fmt" - "github.com/Azure/go-autorest/autorest/date" "strings" + "github.com/Azure/go-autorest/autorest/date" + "github.com/Azure/azure-sdk-for-go/services/preview/appinsights/v1/insights" "github.com/elastic/beats/v7/libbeat/common"