Add autodiscover for aws_ec2

CHANGELOG.next.asciidoc

@@ -98,7 +98,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 *Affecting all Beats*
 
 - Add document_id setting to decode_json_fields processor. {pull}15859[15859]
-
+- Add `aws_ec2` provider for autodiscover. {issue}12518[12518] {pull}14823[14823]
 
 *Auditbeat*
 

libbeat/docs/shared-autodiscover.asciidoc

@@ -362,6 +362,51 @@ ifdef::autodiscoverHints[]
 include::../../{beatname_lc}/docs/autodiscover-hints.asciidoc[]
 endif::autodiscoverHints[]
 
+ifdef::autodiscoverAWSEC2[]
+[float]
+===== Amazon EC2s
+
+*Note: This provider is experimental*
+
+The Amazon EC2 autodiscover provider discovers https://aws.amazon.com/ec2/[EC2 instances].
+This is useful for users to launch Metricbeat modules to monitor services running on AWS EC2 instances.
+For example, to gather MySQL metrics from mysql servers running on EC2 instances with specific tag `service: mysql`.
+
+This provider will load AWS credentials using the standard AWS environment variables and shared credentials files
+see https://docs.aws.amazon.com/general/latest/gr/aws-access-keys-best-practices.html[Best Practices for Managing AWS Access Keys]
+for more information. If you do not wish to use these, you may explicitly set the `access_key_id` and
+`secret_access_key` variables.
+
+These are the available fields during within config templating.
+The `aws.ec2.*` fields and `cloud.*` fields will be available on each emitted event.
+
+* cloud.availability_zone
+* cloud.instance.id
+* cloud.machine.type
+* cloud.provider
+* cloud.region
+
+* aws.ec2.architecture
+* aws.ec2.image.id
+* aws.ec2.kernel.id
+* aws.ec2.monitoring.state
+* aws.ec2.private.dns_name
+* aws.ec2.private.ip
+* aws.ec2.public.dns_name
+* aws.ec2.public.ip
+* aws.ec2.root_device_name
+* aws.ec2.state.code
+* aws.ec2.state.name
+* aws.ec2.subnet.id
+* aws.ec2.tags
+* aws.ec2.vpc.id
+
+include::../../{beatname_lc}/docs/autodiscover-aws-ec2-config.asciidoc[]
+
+This autodiscover provider takes our standard <<aws-credentials-config,AWS credentials options>>.
+
+endif::autodiscoverAWSEC2[]
+
 [[configuration-autodiscover-advanced]]
 === Advanced usage
 

metricbeat/docs/autodiscover-aws-ec2-config.asciidoc

@@ -0,0 +1,25 @@
+{beatname_uc} supports templates for modules:
+
+["source","yaml",subs="attributes"]
+-------------------------------------------------------------------------------------
+metricbeat.autodiscover:
+  providers:
+    - type: aws_ec2
+      period: 1m
+      credential_profile_name: elastic-beats
+      templates:
+        - condition:
+            equals:
+              aws.ec2.tags.service: "mysql"
+          config:
+            - module: mysql
+              metricsets: ["status", "galera_status"]
+              period: 10s
+              hosts: ["root:password@tcp(${data.aws.ec2.public.ip}:3306)/"]
+              username: root
+              password: password
+-------------------------------------------------------------------------------------
+
+This autodiscover provider takes our standard AWS credentials options.
+With this configuration, `mysql` metricbeat module will be launched for all EC2
+instances that have `service: mysql` as a tag.

metricbeat/docs/configuring-howto.asciidoc

@@ -77,7 +77,9 @@ include::{libbeat-dir}/shared-env-vars.asciidoc[]
 
 :autodiscoverJolokia:
 :autodiscoverHints:
+:autodiscoverAWSEC2:
 include::{libbeat-dir}/shared-autodiscover.asciidoc[]
+:autodiscoverAWSEC2!:
 
 :standalone:
 include::{libbeat-dir}/yaml.asciidoc[]

x-pack/libbeat/autodiscover/providers/aws/aws.go

@@ -3,3 +3,38 @@
 // you may not use this file except in compliance with the Elastic License.
 
 package aws
+
+import (
+	"context"
+
+	"github.com/aws/aws-sdk-go-v2/service/ec2"
+	"github.com/aws/aws-sdk-go-v2/service/ec2/ec2iface"
+	"github.com/pkg/errors"
+)
+
+// SafeString makes handling AWS *string types easier.
+// The AWS lib never returns plain strings, always using pointers, probably for memory efficiency reasons.
+// This is a bit odd, because strings are just pointers into byte arrays, however this is the choice they've made.
+// This will return the plain version of the given string or an empty string if the pointer is null
+func SafeString(str *string) string {
+	if str == nil {
+		return ""
+	}
+
+	return *str
+}
+
+// GetRegions makes DescribeRegions API call to list all regions from AWS
+func GetRegions(svc ec2iface.ClientAPI) (completeRegionsList []string, err error) {
+	input := &ec2.DescribeRegionsInput{}
+	req := svc.DescribeRegionsRequest(input)
+	output, err := req.Send(context.TODO())
+	if err != nil {
+		err = errors.Wrap(err, "Failed DescribeRegions")
+		return
+	}
+	for _, region := range output.Regions {
+		completeRegionsList = append(completeRegionsList, *region.RegionName)
+	}
+	return
+}

x-pack/libbeat/autodiscover/providers/aws/elb/config.go → x-pack/libbeat/autodiscover/providers/aws/config.go

@@ -2,39 +2,31 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
-package elb
+package aws
 
 import (
 	"time"
 
 	"github.com/elastic/beats/x-pack/libbeat/common/aws"
 
 	"github.com/elastic/beats/libbeat/autodiscover/template"
-	"github.com/elastic/beats/libbeat/common"
 )
 
-// Config for the aws_elb autodiscover provider.
+// Config for all aws autodiscover providers.
 type Config struct {
-	Type string `config:"type"`
-
-	// Standard autodiscover fields.
-
-	// Hints are currently not supported, but may be implemented in a later release
-	HintsEnabled bool                    `config:"hints.enabled"`
-	Builders     []*common.Config        `config:"builders"`
-	Appenders    []*common.Config        `config:"appenders"`
-	Templates    template.MapperSettings `config:"templates"`
+	Type      string                  `config:"type"`
+	Templates template.MapperSettings `config:"templates"`
 
 	// Period defines how often to poll the AWS API.
 	Period time.Duration `config:"period" validate:"nonzero,required"`
 
 	// AWS Specific autodiscover fields
-
-	Regions   []string      `config:"regions" validate:"required"`
+	Regions   []string      `config:"regions"`
 	AWSConfig aws.ConfigAWS `config:",inline"`
 }
 
-func defaultConfig() *Config {
+// DefaultConfig for all aws autodiscover providers.
+func DefaultConfig() *Config {
 	return &Config{
 		Period: time.Minute,
 	}

x-pack/libbeat/autodiscover/providers/aws/ec2/_meta/fields.yml

@@ -0,0 +1,11 @@
+- key: ec2_listener
+  title: "EC2 Listener"
+  description: >
+    AWS EC2 Listeners
+  short_config: false
+  release: experimental
+  fields:
+    - name: ec2_listener
+      type: group
+      description: >
+        Represents an AWS EC2 Listener, e.g. state of an EC2.

x-pack/libbeat/autodiscover/providers/aws/ec2/ec2.go

@@ -0,0 +1,134 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package ec2
+
+import (
+	"github.com/aws/aws-sdk-go-v2/service/ec2"
+	"github.com/pkg/errors"
+
+	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/logp"
+	awsauto "github.com/elastic/beats/x-pack/libbeat/autodiscover/providers/aws"
+)
+
+type ec2Instance struct {
+	ec2Instance ec2.Instance
+}
+
+// toMap converts this ec2Instance into the form consumed as metadata in the autodiscovery process.
+func (i *ec2Instance) toMap() common.MapStr {
+	architecture, err := i.ec2Instance.Architecture.MarshalValue()
+	if err != nil {
+		logp.Error(errors.Wrap(err, "MarshalValue failed for architecture: "))
+	}
+
+	m := common.MapStr{
+		"image":            i.toImage(),
+		"vpc":              i.toVpc(),
+		"subnet":           i.toSubnet(),
+		"private":          i.toPrivate(),
+		"public":           i.toPublic(),
+		"monitoring":       i.toMonitoringState(),
+		"kernel":           i.toKernel(),
+		"state":            i.stateMap(),
+		"architecture":     architecture,
+		"root_device_name": awsauto.SafeString(i.ec2Instance.RootDeviceName),
+	}
+
+	for _, tag := range i.ec2Instance.Tags {
+		m.Put("tags."+awsauto.SafeString(tag.Key), awsauto.SafeString(tag.Value))
+	}
+	return m
+}
+
+func (i *ec2Instance) instanceID() string {
+	return awsauto.SafeString(i.ec2Instance.InstanceId)
+}
+
+func (i *ec2Instance) toImage() common.MapStr {
+	m := common.MapStr{}
+	m["id"] = awsauto.SafeString(i.ec2Instance.ImageId)
+	return m
+}
+
+func (i *ec2Instance) toMonitoringState() common.MapStr {
+	monitoringState, err := i.ec2Instance.Monitoring.State.MarshalValue()
+	if err != nil {
+		logp.Error(errors.Wrap(err, "MarshalValue failed for monitoring state: "))
+	}
+
+	m := common.MapStr{}
+	m["state"] = monitoringState
+	return m
+}
+
+func (i *ec2Instance) toPrivate() common.MapStr {
+	m := common.MapStr{}
+	m["ip"] = awsauto.SafeString(i.ec2Instance.PrivateIpAddress)
+	m["dns_name"] = awsauto.SafeString(i.ec2Instance.PrivateDnsName)
+	return m
+}
+
+func (i *ec2Instance) toPublic() common.MapStr {
+	m := common.MapStr{}
+	m["ip"] = awsauto.SafeString(i.ec2Instance.PublicIpAddress)
+	m["dns_name"] = awsauto.SafeString(i.ec2Instance.PublicDnsName)
+	return m
+}
+
+func (i *ec2Instance) toVpc() common.MapStr {
+	m := common.MapStr{}
+	m["id"] = awsauto.SafeString(i.ec2Instance.VpcId)
+	return m
+}
+
+func (i *ec2Instance) toSubnet() common.MapStr {
+	m := common.MapStr{}
+	m["id"] = awsauto.SafeString(i.ec2Instance.SubnetId)
+	return m
+}
+
+func (i *ec2Instance) toKernel() common.MapStr {
+	m := common.MapStr{}
+	m["id"] = awsauto.SafeString(i.ec2Instance.KernelId)
+	return m
+}
+
+func (i *ec2Instance) toCloudMap() common.MapStr {
+	m := common.MapStr{}
+	availabilityZone := awsauto.SafeString(i.ec2Instance.Placement.AvailabilityZone)
+	m["availability_zone"] = availabilityZone
+	m["provider"] = "aws"
+
+	// The region is just an AZ with the last character removed
+	m["region"] = availabilityZone[:len(availabilityZone)-1]
+
+	instance := common.MapStr{}
+	instance["id"] = i.instanceID()
+	m["instance"] = instance
+
+	instanceType, err := i.ec2Instance.InstanceType.MarshalValue()
+	if err != nil {
+		logp.Error(errors.Wrap(err, "MarshalValue failed for instance type: "))
+	}
+	machine := common.MapStr{}
+	machine["type"] = instanceType
+	m["machine"] = machine
+	return m
+}
+
+// stateMap converts the State part of the ec2 struct into a friendlier map with 'reason' and 'code' fields.
+func (i *ec2Instance) stateMap() (stateMap common.MapStr) {
+	state := i.ec2Instance.State
+	stateMap = common.MapStr{}
+	nameString, err := state.Name.MarshalValue()
+	if err != nil {
+		logp.Error(errors.Wrap(err, "MarshalValue failed for instance state name: "))
+	}
+
+	stateMap["name"] = nameString
+	stateMap["code"] = state.Code
+	return stateMap
+}