Add host inventory metrics to ec2 metricset (#20171) (#20673)

* Add host inventory metrics to ec2 metricset
* move host fields into metricbeat/_meta/fields.common.yml

(cherry picked from commit 54d2268)

CHANGELOG.next.asciidoc

@@ -722,6 +722,7 @@ field. You can revert this change by configuring tags for the module and omittin
 - Added cache and connection_errors metrics to status metricset of MySQL module {issue}16955[16955] {pull}19844[19844]
 - Update MySQL dashboard with connection errors and cache metrics {pull}19913[19913] {issue}16955[16955]
 - Add cloud.instance.name into aws ec2 metricset. {pull}20077[20077]
+- Add host inventory metrics into aws ec2 metricset. {pull}20171[20171]
 - Add `scope` setting for elasticsearch module, allowing it to monitor an Elasticsearch cluster behind a load-balancing proxy. {issue}18539[18539] {pull}18547[18547]
 
 *Packetbeat*

metricbeat/_meta/fields.common.yml

@@ -41,3 +41,28 @@
     - name: systemd.unit
       type: keyword
       description: the unit name of the systemd service
+
+    - name: host
+      type: group
+      fields:
+        - name: cpu.pct
+          type: scaled_float
+          description: Percent CPU used. This value is normalized by the number of CPU cores and it ranges from 0 to 1.
+        - name: network.in.bytes
+          type: scaled_float
+          description: The number of bytes received on all network interfaces by the host in a given period of time.
+        - name: network.out.bytes
+          type: scaled_float
+          description: The number of bytes sent out on all network interfaces by the host in a given period of time.
+        - name: network.in.packets
+          type: scaled_float
+          description: The number of packets received on all network interfaces by the host in a given period of time.
+        - name: network.out.packets
+          type: scaled_float
+          description: The number of packets sent out on all network interfaces by the host in a given period of time.
+        - name: disk.read.bytes
+          type: scaled_float
+          description: The total number of bytes read successfully in a given period of time.
+        - name: disk.write.bytes
+          type: scaled_float
+          description: The total number of bytes write successfully in a given period of time.

metricbeat/docs/fields.asciidoc

@@ -6393,6 +6393,70 @@ type: keyword
 
 --
 
+
+*`host.cpu.pct`*::
++
+--
+Percent CPU used. This value is normalized by the number of CPU cores and it ranges from 0 to 1.
+
+type: scaled_float
+
+--
+
+*`host.network.in.bytes`*::
++
+--
+The number of bytes received on all network interfaces by the host in a given period of time.
+
+type: scaled_float
+
+--
+
+*`host.network.out.bytes`*::
++
+--
+The number of bytes sent out on all network interfaces by the host in a given period of time.
+
+type: scaled_float
+
+--
+
+*`host.network.in.packets`*::
++
+--
+The number of packets received on all network interfaces by the host in a given period of time.
+
+type: scaled_float
+
+--
+
+*`host.network.out.packets`*::
++
+--
+The number of packets sent out on all network interfaces by the host in a given period of time.
+
+type: scaled_float
+
+--
+
+*`host.disk.read.bytes`*::
++
+--
+The total number of bytes read successfully in a given period of time.
+
+type: scaled_float
+
+--
+
+*`host.disk.write.bytes`*::
++
+--
+The total number of bytes write successfully in a given period of time.
+
+type: scaled_float
+
+--
+
 [[exported-fields-consul]]
 == Consul fields
 

x-pack/metricbeat/module/aws/ec2/_meta/data.json

@@ -4,11 +4,11 @@
         "ec2": {
             "cpu": {
                 "credit_balance": 144,
-                "credit_usage": 0.061395,
+                "credit_usage": 0.058005,
                 "surplus_credit_balance": 0,
                 "surplus_credits_charged": 0,
                 "total": {
-                    "pct": 1.1651199407241788
+                    "pct": 1.1631008613503082
                 }
             },
             "diskio": {
@@ -51,16 +51,16 @@
             },
             "network": {
                 "in": {
-                    "bytes": 7375.4,
-                    "bytes_per_sec": 24.584666666666667,
-                    "packets": 49,
-                    "packets_per_sec": 0.16333333333333333
+                    "bytes": 786.6,
+                    "bytes_per_sec": 2.622,
+                    "packets": 5.8,
+                    "packets_per_sec": 0.019333333333333334
                 },
                 "out": {
-                    "bytes": 11567,
-                    "bytes_per_sec": 38.556666666666665,
-                    "packets": 44.8,
-                    "packets_per_sec": 0.14933333333333332
+                    "bytes": 627,
+                    "bytes_per_sec": 2.09,
+                    "packets": 5.8,
+                    "packets_per_sec": 0.019333333333333334
                 }
             },
             "status": {
@@ -95,6 +95,31 @@
         "duration": 115000,
         "module": "aws"
     },
+    "host": {
+        "cpu": {
+            "pct": 0.011631008613503082
+        },
+        "disk": {
+            "read": {
+                "bytes": 0
+            },
+            "write": {
+                "bytes": 0
+            }
+        },
+        "id": "i-0516ddaca5c1d231f",
+        "name": "mysql-test",
+        "network": {
+            "in": {
+                "bytes": 786.6,
+                "packets": 5.8
+            },
+            "out": {
+                "bytes": 627,
+                "packets": 5.8
+            }
+        }
+    },
     "metricset": {
         "name": "ec2",
         "period": 10000

x-pack/metricbeat/module/aws/ec2/ec2.go

@@ -204,6 +204,7 @@ func (m *MetricSet) createCloudWatchEvents(getMetricDataResults []cloudwatch.Met
 					// add cloud.instance.name and host.name into ec2 events
 					if *tag.Key == "Name" {
 						events[instanceID].RootFields.Put("cloud.instance.name", *tag.Value)
+						events[instanceID].RootFields.Put("host.name", *tag.Value)
 					}
 				}
 
@@ -269,6 +270,10 @@ func (m *MetricSet) createCloudWatchEvents(getMetricDataResults []cloudwatch.Met
 				return events, errors.Wrap(err, "EventMapping failed")
 			}
 
+			// add host cpu/network/disk fields and host.id
+			hostFields := addHostFields(resultMetricsetFields, events[instanceID].RootFields, instanceID)
+			events[instanceID].RootFields.Update(hostFields)
+
 			// add rate metrics
 			calculateRate(resultMetricsetFields, monitoringStates[instanceID])
 
@@ -309,6 +314,40 @@ func calculateRate(resultMetricsetFields common.MapStr, monitoringState string)
 	}
 }
 
+func addHostFields(resultMetricsetFields common.MapStr, rootFields common.MapStr, instanceID string) common.MapStr {
+	hostRootFields := common.MapStr{}
+	hostRootFields.Put("host.id", instanceID)
+
+	// If there is no instance name, use instance ID as the host.name
+	hostName, err := rootFields.GetValue("host.name")
+	if err == nil && hostName != nil {
+		hostRootFields.Put("host.name", hostName)
+	} else {
+		hostRootFields.Put("host.name", instanceID)
+	}
+
+	hostFieldTable := map[string]string{
+		"cpu.total.pct":       "host.cpu.pct",
+		"network.in.bytes":    "host.network.in.bytes",
+		"network.out.bytes":   "host.network.out.bytes",
+		"network.in.packets":  "host.network.in.packets",
+		"network.out.packets": "host.network.out.packets",
+		"diskio.read.bytes":   "host.disk.read.bytes",
+		"diskio.write.bytes":  "host.disk.write.bytes",
+	}
+
+	for ec2MetricName, hostMetricName := range hostFieldTable {
+		metricValue, err := resultMetricsetFields.GetValue(ec2MetricName)
+		if ec2MetricName == "cpu.total.pct" {
+			metricValue = metricValue.(float64) / 100
+		}
+		if err == nil && metricValue != nil {
+			hostRootFields.Put(hostMetricName, metricValue)
+		}
+	}
+	return hostRootFields
+}
+
 func getInstancesPerRegion(svc ec2iface.ClientAPI) (instanceIDs []string, instancesOutputs map[string]ec2.Instance, err error) {
 	instancesOutputs = map[string]ec2.Instance{}
 	output := ec2.DescribeInstancesOutput{NextToken: nil}

x-pack/metricbeat/module/aws/ec2/ec2_test.go

@@ -82,6 +82,10 @@ func (m *MockEC2Client) DescribeInstancesRequest(input *ec2.DescribeInstancesInp
 			Key:   awssdk.String("helm.sh/chart"),
 			Value: awssdk.String("foo-chart"),
 		},
+		{
+			Key:   awssdk.String("Name"),
+			Value: awssdk.String("test-instance"),
+		},
 	}
 
 	instance := ec2.Instance{
@@ -144,10 +148,15 @@ func TestCreateCloudWatchEventsDedotTags(t *testing.T) {
 			"cloud": common.MapStr{
 				"region":            regionName,
 				"provider":          "aws",
-				"instance":          common.MapStr{"id": "i-123"},
+				"instance":          common.MapStr{"id": "i-123", "name": "test-instance"},
 				"machine":           common.MapStr{"type": "t2.medium"},
 				"availability_zone": "us-west-1a",
 			},
+			"host": common.MapStr{
+				"cpu":  common.MapStr{"pct": 0.0025},
+				"id":   "i-123",
+				"name": "test-instance",
+			},
 		},
 		MetricSetFields: common.MapStr{
 			"cpu": common.MapStr{
@@ -171,6 +180,7 @@ func TestCreateCloudWatchEventsDedotTags(t *testing.T) {
 			"tags": common.MapStr{
 				"app_kubernetes_io/name": "foo",
 				"helm_sh/chart":          "foo-chart",
+				"Name":                   "test-instance",
 			},
 		},
 	}
@@ -227,10 +237,15 @@ func TestCreateCloudWatchEventsWithTagsFilter(t *testing.T) {
 			"cloud": common.MapStr{
 				"region":            regionName,
 				"provider":          "aws",
-				"instance":          common.MapStr{"id": "i-123"},
+				"instance":          common.MapStr{"id": "i-123", "name": "test-instance"},
 				"machine":           common.MapStr{"type": "t2.medium"},
 				"availability_zone": "us-west-1a",
 			},
+			"host": common.MapStr{
+				"cpu":  common.MapStr{"pct": 0.0025},
+				"id":   "i-123",
+				"name": "test-instance",
+			},
 		},
 		MetricSetFields: common.MapStr{
 			"cpu": common.MapStr{
@@ -254,6 +269,7 @@ func TestCreateCloudWatchEventsWithTagsFilter(t *testing.T) {
 			"tags": common.MapStr{
 				"app_kubernetes_io/name": "foo",
 				"helm_sh/chart":          "foo-chart",
+				"Name":                   "test-instance",
 			},
 		},
 	}
@@ -455,3 +471,62 @@ func TestCalculateRate(t *testing.T) {
 		assert.Equal(t, c.rateMetricValueDetailed, output)
 	}
 }
+
+func TestCreateCloudWatchEventsWithInstanceName(t *testing.T) {
+	expectedEvent := mb.Event{
+		RootFields: common.MapStr{
+			"cloud": common.MapStr{
+				"region":            regionName,
+				"provider":          "aws",
+				"instance":          common.MapStr{"id": "i-123", "name": "test-instance"},
+				"machine":           common.MapStr{"type": "t2.medium"},
+				"availability_zone": "us-west-1a",
+			},
+			"host": common.MapStr{
+				"cpu": common.MapStr{"pct": 0.25},
+				"id":  "i-123",
+			},
+		},
+		MetricSetFields: common.MapStr{
+			"tags": common.MapStr{
+				"app_kubernetes_io/name": "foo",
+				"helm_sh/chart":          "foo-chart",
+				"Name":                   "test-instance",
+			},
+		},
+	}
+	svcEC2Mock := &MockEC2Client{}
+	instanceIDs, instancesOutputs, err := getInstancesPerRegion(svcEC2Mock)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(instanceIDs))
+	instanceID := instanceIDs[0]
+	assert.Equal(t, instanceID, instanceID)
+	timestamp := time.Now()
+
+	getMetricDataOutput := []cloudwatch.MetricDataResult{
+		{
+			Id:         &id1,
+			Label:      &label1,
+			Values:     []float64{0.25},
+			Timestamps: []time.Time{timestamp},
+		},
+	}
+
+	metricSet := MetricSet{
+		&aws.MetricSet{},
+	}
+
+	events, err := metricSet.createCloudWatchEvents(getMetricDataOutput, instancesOutputs, "us-west-1")
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(events))
+
+	assert.Equal(t, expectedEvent.MetricSetFields["tags"], events[instanceID].ModuleFields["tags"])
+
+	hostID, err := events[instanceID].RootFields.GetValue("host.id")
+	assert.NoError(t, err)
+	assert.Equal(t, "i-123", hostID)
+
+	instanceName, err := events[instanceID].RootFields.GetValue("cloud.instance.name")
+	assert.NoError(t, err)
+	assert.Equal(t, "test-instance", instanceName)
+}