diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 6289c16ef15..5feac0d3837 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -542,6 +542,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add support for wildcard `*` in dimension value of AWS CloudWatch metrics config. {issue}18050[18050] {pull}19660[19660] - The `elasticsearch/index` metricset now collects metrics for hidden indices as well. {issue}18639[18639] {pull}18703[18703] - The `elasticsearch-xpack/index` metricset now reports hidden indices as such. {issue}18639[18639] {pull}18706[18706] +- Adds support for app insights metrics in the azure module. {issue}18570[18570] {pull}18940[18940] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index d41da1de5bf..2ad60932343 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -4516,6 +4516,53 @@ type: keyword Azure metric dimensions. +type: object + +-- + +[float] +=== app_insights + +application insights + + + +*`azure.app_insights.application_id`*:: ++ +-- +The application ID + + +type: keyword + +-- + +*`azure.app_insights.start_date`*:: ++ +-- +The start date + + +type: date + +-- + +*`azure.app_insights.end_date`*:: ++ +-- +The end date + + +type: date + +-- + +*`azure.app_insights.metrics.*.*`*:: ++ +-- +The metrics + + type: object -- diff --git a/metricbeat/docs/modules/azure.asciidoc b/metricbeat/docs/modules/azure.asciidoc index 301e76b3c03..32cbe864201 100644 --- a/metricbeat/docs/modules/azure.asciidoc +++ b/metricbeat/docs/modules/azure.asciidoc @@ -105,6 +105,9 @@ so the `period` for `container_service` metricset should be `300s` or multiples This metricset will collect relevant metrics from specified database accounts, these metrics will have a timegrain every 5 minutes, so the `period` for `database_account` metricset should be `300s` or multiples of `300s`. +[float] +=== `app_insights` +This metricset will collect application insights metrics, the `period` (interval) for the `app-insights` metricset is set by default at `300s`. [float] == Additional notes about metrics and costs @@ -198,7 +201,6 @@ metricbeat.modules: tenant_id: '${AZURE_TENANT_ID:""}' subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' - - module: azure metricsets: - database_account @@ -209,6 +211,14 @@ metricbeat.modules: tenant_id: '${AZURE_TENANT_ID:""}' subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +- module: azure + metricsets: + - app_insights + enabled: true + period: 300s + application_id: '' + api_key: '' + ---- [float] @@ -216,6 +226,8 @@ metricbeat.modules: The following metricsets are available: +* <> + * <> * <> @@ -232,6 +244,8 @@ The following metricsets are available: * <> +include::azure/app_insights.asciidoc[] + include::azure/compute_vm.asciidoc[] include::azure/compute_vm_scaleset.asciidoc[] diff --git a/metricbeat/docs/modules/azure/app_insights.asciidoc b/metricbeat/docs/modules/azure/app_insights.asciidoc new file mode 100644 index 00000000000..2cc018c73f5 --- /dev/null +++ b/metricbeat/docs/modules/azure/app_insights.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-azure-app_insights]] +=== Azure app_insights metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/azure/app_insights/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/azure/app_insights/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 917f9818abd..3080c24b7ed 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -33,7 +33,8 @@ This file is generated! See scripts/mage/docs_collector.go |<> beta[] |<> beta[] |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.8+| .8+| |<> +.9+| .9+| |<> beta[] +|<> |<> |<> |<> diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index a46368da408..9f9704d067e 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -19,6 +19,7 @@ import ( _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws/s3_request" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws/sqs" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/app_insights" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/compute_vm" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/compute_vm_scaleset" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/monitor" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 0242e23f7c9..43325499040 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -313,7 +313,6 @@ metricbeat.modules: tenant_id: '${AZURE_TENANT_ID:""}' subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' - - module: azure metricsets: - database_account @@ -324,6 +323,14 @@ metricbeat.modules: tenant_id: '${AZURE_TENANT_ID:""}' subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +- module: azure + metricsets: + - app_insights + enabled: true + period: 300s + application_id: '' + api_key: '' + #--------------------------------- Beat Module --------------------------------- - module: beat diff --git a/x-pack/metricbeat/module/azure/_meta/config.reference.yml b/x-pack/metricbeat/module/azure/_meta/config.reference.yml index 5823b66c671..6127c2123d6 100644 --- a/x-pack/metricbeat/module/azure/_meta/config.reference.yml +++ b/x-pack/metricbeat/module/azure/_meta/config.reference.yml @@ -73,7 +73,6 @@ tenant_id: '${AZURE_TENANT_ID:""}' subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' - - module: azure metricsets: - database_account @@ -84,3 +83,11 @@ tenant_id: '${AZURE_TENANT_ID:""}' subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +- module: azure + metricsets: + - app_insights + enabled: true + period: 300s + application_id: '' + api_key: '' + diff --git a/x-pack/metricbeat/module/azure/_meta/config.yml b/x-pack/metricbeat/module/azure/_meta/config.yml index 3f3b29713e9..6217294e5bf 100644 --- a/x-pack/metricbeat/module/azure/_meta/config.yml +++ b/x-pack/metricbeat/module/azure/_meta/config.yml @@ -90,3 +90,11 @@ # tenant_id: '${AZURE_TENANT_ID:""}' # subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' # refresh_list_interval: 600s + +#- module: azure +# metricsets: +# - app_insights +# enabled: true +# period: 300s +# application_id: '' +# api_key: '' diff --git a/x-pack/metricbeat/module/azure/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/_meta/docs.asciidoc index 20459d962e2..571181dde7a 100644 --- a/x-pack/metricbeat/module/azure/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/azure/_meta/docs.asciidoc @@ -97,6 +97,9 @@ so the `period` for `container_service` metricset should be `300s` or multiples This metricset will collect relevant metrics from specified database accounts, these metrics will have a timegrain every 5 minutes, so the `period` for `database_account` metricset should be `300s` or multiples of `300s`. +[float] +=== `app_insights` +This metricset will collect application insights metrics, the `period` (interval) for the `app-insights` metricset is set by default at `300s`. [float] == Additional notes about metrics and costs diff --git a/x-pack/metricbeat/module/azure/app_insights/_meta/data.json b/x-pack/metricbeat/module/azure/app_insights/_meta/data.json new file mode 100644 index 00000000000..3d1f07c1ac1 --- /dev/null +++ b/x-pack/metricbeat/module/azure/app_insights/_meta/data.json @@ -0,0 +1,31 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "azure" : { + "app_insights" : { + "metrics" : { + "requests_failed" : { + "sum" : 182 + }, + "request_name" : "GET /favicon.ico" + }, + "start_date" : "2020-07-12T10:52:11.831Z", + "end_date" : "2020-07-12T12:52:11.831Z", + "application_id" : "42cb59a9-d5be-400b-a5c4-69b0a0026ac6" + } + }, + "cloud": { + "provider": "azure" + }, + "event": { + "dataset": "azure.app_insights", + "duration": 115000, + "module": "azure" + }, + "metricset": { + "name": "app_insights", + "period": 10000 + }, + "service": { + "type": "azure" + } +} diff --git a/x-pack/metricbeat/module/azure/app_insights/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/app_insights/_meta/docs.asciidoc new file mode 100644 index 00000000000..2ba1150078b --- /dev/null +++ b/x-pack/metricbeat/module/azure/app_insights/_meta/docs.asciidoc @@ -0,0 +1,61 @@ +This is the app_insights metricset. + +This metricset allows users to retrieve application insights metrics from specified applications. + +[float] +==== Config options to identify resources + +`application_id`:: (_[]string_) ID of the application. This is Application ID from the API Access settings blade in the Azure portal. + +`api_key`:: (_[]string_) The API key which will be generated, more on the steps here https://dev.applicationinsights.io/documentation/Authorization/API-key-and-App-ID. + +[float] +==== App insights metric configurations + +`metrics`:: List of different metrics to collect information + +`id`:: (_[]string_) IDs of the metrics that's being reported. Usually, the id is descriptive enough to help identify what's measured. +A list of metric names can be entered as well. +Default metricsets include: `requests/count` `requests/duration` `requests/failed` `users/count``users/authenticated` +`pageViews/count` `pageViews/duration` `customEvents/count` +`browserTimings/processingDuration` `browserTimings/receiveDuration` `browserTimings/networkDuration` `browserTimings/sendDuration` +`browserTimings/totalDuration` `dependencies/count` `dependencies/duration` `dependencies/failed` +`exceptions/count` `exceptions/browser` `exceptions/server` `sessions/count` `performanceCounters/requestExecutionTime` `performanceCounters/requestsPerSecond` +`performanceCounters/requestsInQueue` `performanceCounters/memoryAvailableBytes` `performanceCounters/exceptionsPerSecond` `performanceCounters/processCpuPercentage` +`performanceCounters/processIOBytesPerSecond` `performanceCounters/processPrivateBytes` `performanceCounters/processorCpuPercentage` `availabilityResults/count` +`availabilityResults/availabilityPercentage` `availabilityResults/duration` + +`interval`:: (_string_) The time interval to use when retrieving metric values. This is an ISO8601 duration. +If interval is omitted, the metric value is aggregated across the entire timespan. +If interval is supplied, the result may adjust the interval to a more appropriate size based on the timespan used for the query. + +`aggregation`:: (_[]string_) The aggregation to use when computing the metric values. +To retrieve more than one aggregation at a time, separate them with a comma. +If no aggregation is specified, then the default aggregation for the metric is used. + +`segment`:: (_[]string_) The name of the dimension to segment the metric values by. +This dimension must be applicable to the metric you are retrieving. +In this case, the metric data will be segmented in the order the dimensions are listed in the parameter. + +`top`:: (_int_) The number of segments to return. This value is only valid when segment is specified. + +`order_by`:: (_string_) The aggregation function and direction to sort the segments by. +This value is only valid when segment is specified. + +`filter`:: (_string_) An expression used to filter the results. +This value should be a valid OData filter expression where the keys of each clause should be applicable dimensions for the metric you are retrieving. + +Users can select the options to retrieve all metrics from a specific namespace using the following: + +["source","yaml"] +---- + metrics: + - id: ["*"] + timespan: "Microsoft.Storage/storageAccounts" +---- + + + +A default non configurable timegrain of 5 min is set so users are advised to configure an interval of 300s or a multiply of it. + + diff --git a/x-pack/metricbeat/module/azure/app_insights/_meta/fields.yml b/x-pack/metricbeat/module/azure/app_insights/_meta/fields.yml new file mode 100644 index 00000000000..40ab8560827 --- /dev/null +++ b/x-pack/metricbeat/module/azure/app_insights/_meta/fields.yml @@ -0,0 +1,24 @@ +- name: app_insights + type: group + release: beta + description: > + application insights + fields: + - name: application_id + type: keyword + description: > + The application ID + - name: start_date + type: date + description: > + The start date + - name: end_date + type: date + description: > + The end date + - name: metrics.*.* + type: object + object_type: float + object_type_mapping_type: "*" + description: > + The metrics diff --git a/x-pack/metricbeat/module/azure/app_insights/app_insights.go b/x-pack/metricbeat/module/azure/app_insights/app_insights.go new file mode 100644 index 00000000000..e3a087ad7ce --- /dev/null +++ b/x-pack/metricbeat/module/azure/app_insights/app_insights.go @@ -0,0 +1,81 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package app_insights + +import ( + "time" + + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/metricbeat/mb/parse" + + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/metricbeat/mb" +) + +const metricsetName = "app_insights" + +// Config options +type Config struct { + ApplicationId string `config:"application_id" validate:"required"` + ApiKey string `config:"api_key" validate:"required"` + Period time.Duration `config:"period" validate:"nonzero,required"` + Metrics []Metric `config:"metrics"` +} + +// Metric struct used for configuration options +type Metric struct { + ID []string `config:"id" validate:"required"` + Interval string `config:"interval"` + Aggregation []string `config:"aggregation"` + Segment []string `config:"segment"` + Top int32 `config:"top"` + OrderBy string `config:"order_by"` + Filter string `config:"filter"` +} + +func init() { + mb.Registry.MustAddMetricSet("azure", metricsetName, New, mb.WithHostParser(parse.EmptyHostParser)) +} + +// MetricSet struct used for app insights. +type MetricSet struct { + mb.BaseMetricSet + log *logp.Logger + client *Client +} + +// New creates a new instance of the MetricSet. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + var config Config + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + client, err := NewClient(config) + if err != nil { + return nil, errors.Wrapf(err, "error initializing the monitor client: module azure - %s metricset", metricsetName) + } + return &MetricSet{ + BaseMetricSet: base, + log: logp.NewLogger(metricsetName), + client: client, + }, nil +} + +// Fetch fetches events and reports them upstream +func (m *MetricSet) Fetch(report mb.ReporterV2) error { + results, err := m.client.GetMetricValues() + if err != nil { + return errors.Wrap(err, "error retrieving metric values") + } + events := EventsMapping(results, m.client.Config.ApplicationId) + for _, event := range events { + isOpen := report.Event(event) + if !isOpen { + break + } + } + return nil +} diff --git a/x-pack/metricbeat/module/azure/app_insights/app_insights_integration_test.go b/x-pack/metricbeat/module/azure/app_insights/app_insights_integration_test.go new file mode 100644 index 00000000000..3cb93663007 --- /dev/null +++ b/x-pack/metricbeat/module/azure/app_insights/app_insights_integration_test.go @@ -0,0 +1,31 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration +// +build azure + +package app_insights + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" +) + +func TestFetchMetricset(t *testing.T) { + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + mbtest.TestMetricsetFieldsDocumented(t, metricSet, events) +} + +func TestData(t *testing.T) { + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") +} diff --git a/x-pack/metricbeat/module/azure/app_insights/client.go b/x-pack/metricbeat/module/azure/app_insights/client.go new file mode 100644 index 00000000000..b78f2257d3f --- /dev/null +++ b/x-pack/metricbeat/module/azure/app_insights/client.go @@ -0,0 +1,84 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package app_insights + +import ( + "fmt" + "time" + + "github.com/gofrs/uuid" + + "github.com/Azure/azure-sdk-for-go/services/preview/appinsights/v1/insights" + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/logp" +) + +// Client represents the azure client which will make use of the azure sdk go metrics related clients +type Client struct { + Service Service + Config Config + Log *logp.Logger +} + +type MetricValue struct { +} + +// NewClient instantiates the an Azure monitoring client +func NewClient(config Config) (*Client, error) { + service, err := NewService(config) + if err != nil { + return nil, err + } + client := &Client{ + Service: service, + Config: config, + } + return client, nil +} + +// GetMetricValues returns the specified app insights metric data points. +func (client *Client) GetMetricValues() (insights.ListMetricsResultsItem, error) { + var bodyMetrics []insights.MetricsPostBodySchema + var result insights.ListMetricsResultsItem + for _, metrics := range client.Config.Metrics { + var aggregations []insights.MetricsAggregation + var segments []insights.MetricsSegment + for _, agg := range metrics.Aggregation { + aggregations = append(aggregations, insights.MetricsAggregation(agg)) + } + for _, seg := range metrics.Segment { + segments = append(segments, insights.MetricsSegment(seg)) + } + for _, metric := range metrics.ID { + bodyMetric := insights.MetricsPostBodySchemaParameters{ + MetricID: insights.MetricID(metric), + Timespan: calculateTimespan(client.Config.Period), + Aggregation: &aggregations, + Interval: &metrics.Interval, + Segment: &segments, + Top: &metrics.Top, + Orderby: &metrics.OrderBy, + Filter: &metrics.Filter, + } + id, err := uuid.NewV4() + if err != nil { + return result, errors.Wrap(err, "could not generate identifier in client") + } + strId := id.String() + bodyMetrics = append(bodyMetrics, insights.MetricsPostBodySchema{ID: &strId, Parameters: &bodyMetric}) + } + } + result, err := client.Service.GetMetricValues(client.Config.ApplicationId, bodyMetrics) + if err == nil { + return result, nil + } + return result, errors.Wrap(err, "could not retrieve app insights metrics from service") +} + +func calculateTimespan(duration time.Duration) *string { + timespan := fmt.Sprintf("PT%fM", duration.Minutes()) + return ×pan +} diff --git a/x-pack/metricbeat/module/azure/app_insights/client_test.go b/x-pack/metricbeat/module/azure/app_insights/client_test.go new file mode 100644 index 00000000000..ca62e9a82d2 --- /dev/null +++ b/x-pack/metricbeat/module/azure/app_insights/client_test.go @@ -0,0 +1,52 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package app_insights + +import ( + "testing" + + "github.com/Azure/azure-sdk-for-go/services/preview/appinsights/v1/insights" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +var ( + config = Config{ + ApplicationId: "", + ApiKey: "", + Metrics: []Metric{ + { + ID: []string{"requests/count"}, + }, + }, + } +) + +func TestClient(t *testing.T) { + t.Run("return error not valid query", func(t *testing.T) { + client := NewMockClient() + client.Config = config + m := &MockService{} + m.On("GetMetricValues", mock.Anything, mock.Anything).Return(insights.ListMetricsResultsItem{}, errors.New("invalid query")) + client.Service = m + results, err := client.GetMetricValues() + assert.Error(t, err) + assert.Nil(t, results.Value) + m.AssertExpectations(t) + }) + t.Run("return results", func(t *testing.T) { + client := NewMockClient() + client.Config = config + m := &MockService{} + metrics := []insights.MetricsResultsItem{{}, {}} + m.On("GetMetricValues", mock.Anything, mock.Anything).Return(insights.ListMetricsResultsItem{Value: &metrics}, nil) + client.Service = m + results, err := client.GetMetricValues() + assert.NoError(t, err) + assert.Equal(t, len(*results.Value), 2) + m.AssertExpectations(t) + }) +} diff --git a/x-pack/metricbeat/module/azure/app_insights/data.go b/x-pack/metricbeat/module/azure/app_insights/data.go new file mode 100644 index 00000000000..62afa32163f --- /dev/null +++ b/x-pack/metricbeat/module/azure/app_insights/data.go @@ -0,0 +1,80 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package app_insights + +import ( + "fmt" + "strings" + + "github.com/Azure/azure-sdk-for-go/services/preview/appinsights/v1/insights" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/metricbeat/mb" +) + +func EventsMapping(metricValues insights.ListMetricsResultsItem, applicationId string) []mb.Event { + var events []mb.Event + if metricValues.Value == nil { + return events + } + for _, item := range *metricValues.Value { + if item.Body != nil && item.Body.Value != nil { + if item.Body.Value.AdditionalProperties != nil { + events = append(events, createEvent(*item.Body.Value, insights.MetricsSegmentInfo{}, applicationId)) + } else if item.Body.Value.Segments != nil { + for _, segment := range *item.Body.Value.Segments { + events = append(events, createEvent(*item.Body.Value, segment, applicationId)) + } + } + } + } + return events +} + +func createEvent(value insights.MetricsResultInfo, segment insights.MetricsSegmentInfo, applicationId string) mb.Event { + metricList := common.MapStr{} + if value.AdditionalProperties != nil { + metrics := getMetric(value.AdditionalProperties) + for key, metric := range metrics { + metricList.Put(key, metric) + } + } else { + metrics := getMetric(segment.AdditionalProperties) + for key, metric := range metrics { + metricList.Put(key, metric) + } + } + event := mb.Event{ + MetricSetFields: common.MapStr{ + "start_date": value.Start, + "end_date": value.End, + "application_id": applicationId, + }, + Timestamp: value.End.Time, + } + event.RootFields = common.MapStr{} + event.RootFields.Put("cloud.provider", "azure") + event.MetricSetFields.Put("metrics", metricList) + return event +} + +func getMetric(addProp map[string]interface{}) map[string]interface{} { + metricNames := make(map[string]interface{}) + for key, val := range addProp { + switch val.(type) { + case map[string]interface{}: + for subKey, subVal := range val.(map[string]interface{}) { + metricNames[cleanMetricNames(fmt.Sprintf("%s.%s", key, subKey))] = subVal + } + default: + metricNames[cleanMetricNames(key)] = val + } + } + return metricNames +} + +func cleanMetricNames(metric string) string { + return strings.Replace(metric, "/", "_", -1) +} diff --git a/x-pack/metricbeat/module/azure/app_insights/data_test.go b/x-pack/metricbeat/module/azure/app_insights/data_test.go new file mode 100644 index 00000000000..ebe4e7d98aa --- /dev/null +++ b/x-pack/metricbeat/module/azure/app_insights/data_test.go @@ -0,0 +1,58 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package app_insights + +import ( + "testing" + + "github.com/Azure/azure-sdk-for-go/services/preview/appinsights/v1/insights" + "github.com/Azure/go-autorest/autorest/date" + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/common" +) + +func TestEventMapping(t *testing.T) { + startDate := date.Time{} + id := "123" + var info = insights.MetricsResultInfo{ + AdditionalProperties: map[string]interface{}{ + "requests/count": map[string]interface{}{"sum": 12}, + "requests/failed": map[string]interface{}{"sum": 10}, + }, + Start: &startDate, + End: &startDate, + } + var metricResult = insights.MetricsResult{ + Value: &info, + } + metrics := []insights.MetricsResultsItem{ + { + ID: &id, + Status: nil, + Body: &metricResult, + }, + } + var result = insights.ListMetricsResultsItem{ + Value: &metrics, + } + applicationId := "abc" + events := EventsMapping(result, applicationId) + assert.Equal(t, len(events), 1) + for _, event := range events { + val1, _ := event.MetricSetFields.GetValue("start_date") + assert.Equal(t, val1, &startDate) + val2, _ := event.MetricSetFields.GetValue("end_date") + assert.Equal(t, val2, &startDate) + val3, _ := event.MetricSetFields.GetValue("metrics.requests_count") + assert.Equal(t, val3, common.MapStr{"sum": 12}) + val5, _ := event.MetricSetFields.GetValue("metrics.requests_failed") + assert.Equal(t, val5, common.MapStr{"sum": 10}) + val4, _ := event.MetricSetFields.GetValue("application_id") + assert.Equal(t, val4, applicationId) + + } + +} diff --git a/x-pack/metricbeat/module/azure/app_insights/mock_service.go b/x-pack/metricbeat/module/azure/app_insights/mock_service.go new file mode 100644 index 00000000000..97c47f5ff00 --- /dev/null +++ b/x-pack/metricbeat/module/azure/app_insights/mock_service.go @@ -0,0 +1,37 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package app_insights + +import ( + "github.com/Azure/azure-sdk-for-go/services/preview/appinsights/v1/insights" + "github.com/stretchr/testify/mock" + + "github.com/elastic/beats/v7/libbeat/logp" +) + +// Service interface for the azure monitor service and mock for testing +type Service interface { + GetMetricValues(applicationId string, bodyMetrics []insights.MetricsPostBodySchema) (insights.ListMetricsResultsItem, error) +} + +// MockService mock for the azure monitor services +type MockService struct { + mock.Mock +} + +// NewMockClient instantiates a new client with the mock billing service +func NewMockClient() *Client { + return &Client{ + new(MockService), + Config{}, + logp.NewLogger("test azure appinsights"), + } +} + +// GetMetricValues will return specified app insights metrics +func (service *MockService) GetMetricValues(applicationId string, bodyMetrics []insights.MetricsPostBodySchema) (insights.ListMetricsResultsItem, error) { + args := service.Called(applicationId, bodyMetrics) + return args.Get(0).(insights.ListMetricsResultsItem), args.Error(1) +} diff --git a/x-pack/metricbeat/module/azure/app_insights/service.go b/x-pack/metricbeat/module/azure/app_insights/service.go new file mode 100644 index 00000000000..92c04a76854 --- /dev/null +++ b/x-pack/metricbeat/module/azure/app_insights/service.go @@ -0,0 +1,41 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package app_insights + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/services/preview/appinsights/v1/insights" + "github.com/Azure/go-autorest/autorest" + + "github.com/elastic/beats/v7/libbeat/logp" +) + +// AppInsightsService service wrapper to the azure sdk for go +type AppInsightsService struct { + metricsClient *insights.MetricsClient + eventClient *insights.EventsClient + context context.Context + log *logp.Logger +} + +// NewService instantiates the Azure monitoring service +func NewService(config Config) (*AppInsightsService, error) { + metricsClient := insights.NewMetricsClient() + metricsClient.Authorizer = autorest.NewAPIKeyAuthorizerWithHeaders(map[string]interface{}{ + "x-api-key": config.ApiKey, + }) + service := &AppInsightsService{ + metricsClient: &metricsClient, + context: context.Background(), + log: logp.NewLogger("app insights service"), + } + return service, nil +} + +// GetMetricValues will return specified app insights metrics +func (service *AppInsightsService) GetMetricValues(applicationId string, bodyMetrics []insights.MetricsPostBodySchema) (insights.ListMetricsResultsItem, error) { + return service.metricsClient.GetMultiple(service.context, applicationId, bodyMetrics) +} diff --git a/x-pack/metricbeat/module/azure/azure.go b/x-pack/metricbeat/module/azure/azure.go index bc70de81bcc..44f098c9619 100644 --- a/x-pack/metricbeat/module/azure/azure.go +++ b/x-pack/metricbeat/module/azure/azure.go @@ -15,10 +15,10 @@ import ( // Config options type Config struct { - ClientId string `config:"client_id" validate:"required"` - ClientSecret string `config:"client_secret" validate:"required"` - TenantId string `config:"tenant_id" validate:"required"` - SubscriptionId string `config:"subscription_id" validate:"required"` + ClientId string `config:"client_id"` + ClientSecret string `config:"client_secret"` + TenantId string `config:"tenant_id"` + SubscriptionId string `config:"subscription_id"` Period time.Duration `config:"period" validate:"nonzero,required"` Resources []ResourceConfig `config:"resources"` RefreshListInterval time.Duration `config:"refresh_list_interval"` @@ -60,10 +60,6 @@ func init() { // newModule adds validation that hosts is non-empty, a requirement to use the // azure module. func newModule(base mb.BaseModule) (mb.Module, error) { - var config Config - if err := base.UnpackConfig(&config); err != nil { - return nil, errors.Wrap(err, "error unpack raw module config using UnpackConfig") - } return &base, nil } @@ -164,3 +160,19 @@ func hasConfigOptions(config []string) bool { } return true } + +func (conf *Config) Validate() error { + if conf.SubscriptionId == "" { + return errors.New("no subscription ID has been configured") + } + if conf.ClientSecret == "" { + return errors.New("no client secret has been configured") + } + if conf.ClientId == "" { + return errors.New("no client ID has been configured") + } + if conf.TenantId == "" { + return errors.New("no tenant ID has been configured") + } + return nil +} diff --git a/x-pack/metricbeat/module/azure/fields.go b/x-pack/metricbeat/module/azure/fields.go index fe169f09d27..654ab4f460b 100644 --- a/x-pack/metricbeat/module/azure/fields.go +++ b/x-pack/metricbeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzUlsGOmzAQhu95ilGOkTYPwKHSSr300FvvyDETOl2wrZlhK/r0FQt4TSCwafeQ+BAptvn///MMFk/wgm0G5k/DuANQ0goz2D93//c7gALFMgUl7zL4sgOAfi/Uvmiq7hHGCo1gBqXZAZwJq0Kyt41P4EyN7+Ld0DZ0W9k3YZhZcJjKpFJKNZZsyMWVUfIF29+ei2R+UbgfP34iPPcYqEx2QXd0ZBTfsMWZYcrwAbtRBySgpTNhGvUSd4LcBpwsXCfeiDFG6R4HfwZNYi1aXyJ+gnc8hrl2BDalHA+Ltv70C61eLPWT+VqwZEtemxDIlcP+/WF/G0TfNhHjLeysabpfCWaha25u0ygFghVaTfpmdJPmFCVyKv7fMxWEb19nhgXV6IS8m9bpSo026vPR2qxknrzKSbhZcOvr0Cjmr/XxMIk+vcRuATpX3lxZ/Fec95QrALlYU6GgPgZJjBvXF9icGnLIOTlR4yzeN9qQFsa0K0CMJYly+xhAY9oVIEF+pUcp0BB2fo0ZNScjmBtrfePu+k0as8KQdQZTe0fqefMzZZlqxXkuvPa10l/CcnGUKwe3eXi3HOAGSje+9wGBURt2WByvX0iink15100+RIxd8TcAAP//NYAvfg==" + return "eJzUlb9u2zAQxnc/xcFjgPgBPBQI0KVDt+7CmTor10gkwTulcJ++kPUnlCVTSuPB1mDAIvl9v093JJ/hjU57wL91oA2Aspa0h+1L83+7AchJTGCv7Owevm0AoJ0LlcvrslkSqCQU2kOBG4AjU5nL/jzxGSxW9CHePHryzdTgat+9mXEYy8RSyhUVAdkOI73kG53+uJBH72eF2+fXK8FLG4M0sJnR7R0DiauDoYlhnGGFXa8D4snwkSlGvYw7inzyNBq4nngBo0dploM7gkZYs9aXEW/gPXyGqfYQGAvZPc3ausNvMnox1L7MUmDRlKxC79kW3fzt0/ZzIdq2GWKcYSdN0/yKx5mu+XSbDlIgVJLRqG96N6kPg0TG+dc9Y0H48X1imHNFVtjZcZ2u1GihPmtrk2AebeUIbgKO3mdshYtXlcUNPRxsB1JcB4Lel2zw/N1mfFLbPFo6rmGqjgs4fT1jrKicsb0oBs1y1PmzZmZghe9ZdLq29ySb39qRbH7dr20Q2T399+lyLB0mJnzxbGkCdIyTzjWu8rVS9l5d4I+v34VAyTA32IgflIkAmRgsSUgfI8mAO4zPZLOKbCk0h4uiNXTf0Tpa6GkTgQIVLBpOjxGop00EEgrv/CgF6mCnFzAqHlAoQ2Ncbe96J/Ws0LFOwlTOsrqw/j4uVt7GU+HUBXzvtwP8bAEhkNbBUr67fiCJuoDFXTd5hzh0xb8AAAD///4f3bg=" } diff --git a/x-pack/metricbeat/modules.d/azure.yml.disabled b/x-pack/metricbeat/modules.d/azure.yml.disabled index a0ca31b5bc3..abc4594d857 100644 --- a/x-pack/metricbeat/modules.d/azure.yml.disabled +++ b/x-pack/metricbeat/modules.d/azure.yml.disabled @@ -93,3 +93,11 @@ # tenant_id: '${AZURE_TENANT_ID:""}' # subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' # refresh_list_interval: 600s + +#- module: azure +# metricsets: +# - app_insights +# enabled: true +# period: 300s +# application_id: '' +# api_key: ''