diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b7950f8a336..52e7ed1c942 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -722,6 +722,7 @@ field. You can revert this change by configuring tags for the module and omittin - Added cache and connection_errors metrics to status metricset of MySQL module {issue}16955[16955] {pull}19844[19844] - Update MySQL dashboard with connection errors and cache metrics {pull}19913[19913] {issue}16955[16955] - Add cloud.instance.name into aws ec2 metricset. {pull}20077[20077] +- Add host inventory metrics into aws ec2 metricset. {pull}20171[20171] - Add `scope` setting for elasticsearch module, allowing it to monitor an Elasticsearch cluster behind a load-balancing proxy. {issue}18539[18539] {pull}18547[18547] *Packetbeat* diff --git a/metricbeat/_meta/fields.common.yml b/metricbeat/_meta/fields.common.yml index 44bab6f614e..7a0fb0057ff 100644 --- a/metricbeat/_meta/fields.common.yml +++ b/metricbeat/_meta/fields.common.yml @@ -41,3 +41,28 @@ - name: systemd.unit type: keyword description: the unit name of the systemd service + + - name: host + type: group + fields: + - name: cpu.pct + type: scaled_float + description: Percent CPU used. This value is normalized by the number of CPU cores and it ranges from 0 to 1. + - name: network.in.bytes + type: scaled_float + description: The number of bytes received on all network interfaces by the host in a given period of time. + - name: network.out.bytes + type: scaled_float + description: The number of bytes sent out on all network interfaces by the host in a given period of time. + - name: network.in.packets + type: scaled_float + description: The number of packets received on all network interfaces by the host in a given period of time. + - name: network.out.packets + type: scaled_float + description: The number of packets sent out on all network interfaces by the host in a given period of time. + - name: disk.read.bytes + type: scaled_float + description: The total number of bytes read successfully in a given period of time. + - name: disk.write.bytes + type: scaled_float + description: The total number of bytes write successfully in a given period of time. diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index dab188450c4..a988fa9f452 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -6393,6 +6393,70 @@ type: keyword -- + +*`host.cpu.pct`*:: ++ +-- +Percent CPU used. This value is normalized by the number of CPU cores and it ranges from 0 to 1. + +type: scaled_float + +-- + +*`host.network.in.bytes`*:: ++ +-- +The number of bytes received on all network interfaces by the host in a given period of time. + +type: scaled_float + +-- + +*`host.network.out.bytes`*:: ++ +-- +The number of bytes sent out on all network interfaces by the host in a given period of time. + +type: scaled_float + +-- + +*`host.network.in.packets`*:: ++ +-- +The number of packets received on all network interfaces by the host in a given period of time. + +type: scaled_float + +-- + +*`host.network.out.packets`*:: ++ +-- +The number of packets sent out on all network interfaces by the host in a given period of time. + +type: scaled_float + +-- + +*`host.disk.read.bytes`*:: ++ +-- +The total number of bytes read successfully in a given period of time. + +type: scaled_float + +-- + +*`host.disk.write.bytes`*:: ++ +-- +The total number of bytes write successfully in a given period of time. + +type: scaled_float + +-- + [[exported-fields-consul]] == Consul fields diff --git a/x-pack/metricbeat/module/aws/ec2/_meta/data.json b/x-pack/metricbeat/module/aws/ec2/_meta/data.json index 8b1c43a1d19..6b8e8bcb720 100644 --- a/x-pack/metricbeat/module/aws/ec2/_meta/data.json +++ b/x-pack/metricbeat/module/aws/ec2/_meta/data.json @@ -4,11 +4,11 @@ "ec2": { "cpu": { "credit_balance": 144, - "credit_usage": 0.061395, + "credit_usage": 0.058005, "surplus_credit_balance": 0, "surplus_credits_charged": 0, "total": { - "pct": 1.1651199407241788 + "pct": 1.1631008613503082 } }, "diskio": { @@ -51,16 +51,16 @@ }, "network": { "in": { - "bytes": 7375.4, - "bytes_per_sec": 24.584666666666667, - "packets": 49, - "packets_per_sec": 0.16333333333333333 + "bytes": 786.6, + "bytes_per_sec": 2.622, + "packets": 5.8, + "packets_per_sec": 0.019333333333333334 }, "out": { - "bytes": 11567, - "bytes_per_sec": 38.556666666666665, - "packets": 44.8, - "packets_per_sec": 0.14933333333333332 + "bytes": 627, + "bytes_per_sec": 2.09, + "packets": 5.8, + "packets_per_sec": 0.019333333333333334 } }, "status": { @@ -95,6 +95,31 @@ "duration": 115000, "module": "aws" }, + "host": { + "cpu": { + "pct": 0.011631008613503082 + }, + "disk": { + "read": { + "bytes": 0 + }, + "write": { + "bytes": 0 + } + }, + "id": "i-0516ddaca5c1d231f", + "name": "mysql-test", + "network": { + "in": { + "bytes": 786.6, + "packets": 5.8 + }, + "out": { + "bytes": 627, + "packets": 5.8 + } + } + }, "metricset": { "name": "ec2", "period": 10000 diff --git a/x-pack/metricbeat/module/aws/ec2/ec2.go b/x-pack/metricbeat/module/aws/ec2/ec2.go index f3374de8a3e..c61ca5ad08b 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2.go @@ -204,6 +204,7 @@ func (m *MetricSet) createCloudWatchEvents(getMetricDataResults []cloudwatch.Met // add cloud.instance.name and host.name into ec2 events if *tag.Key == "Name" { events[instanceID].RootFields.Put("cloud.instance.name", *tag.Value) + events[instanceID].RootFields.Put("host.name", *tag.Value) } } @@ -269,6 +270,10 @@ func (m *MetricSet) createCloudWatchEvents(getMetricDataResults []cloudwatch.Met return events, errors.Wrap(err, "EventMapping failed") } + // add host cpu/network/disk fields and host.id + hostFields := addHostFields(resultMetricsetFields, events[instanceID].RootFields, instanceID) + events[instanceID].RootFields.Update(hostFields) + // add rate metrics calculateRate(resultMetricsetFields, monitoringStates[instanceID]) @@ -309,6 +314,40 @@ func calculateRate(resultMetricsetFields common.MapStr, monitoringState string) } } +func addHostFields(resultMetricsetFields common.MapStr, rootFields common.MapStr, instanceID string) common.MapStr { + hostRootFields := common.MapStr{} + hostRootFields.Put("host.id", instanceID) + + // If there is no instance name, use instance ID as the host.name + hostName, err := rootFields.GetValue("host.name") + if err == nil && hostName != nil { + hostRootFields.Put("host.name", hostName) + } else { + hostRootFields.Put("host.name", instanceID) + } + + hostFieldTable := map[string]string{ + "cpu.total.pct": "host.cpu.pct", + "network.in.bytes": "host.network.in.bytes", + "network.out.bytes": "host.network.out.bytes", + "network.in.packets": "host.network.in.packets", + "network.out.packets": "host.network.out.packets", + "diskio.read.bytes": "host.disk.read.bytes", + "diskio.write.bytes": "host.disk.write.bytes", + } + + for ec2MetricName, hostMetricName := range hostFieldTable { + metricValue, err := resultMetricsetFields.GetValue(ec2MetricName) + if ec2MetricName == "cpu.total.pct" { + metricValue = metricValue.(float64) / 100 + } + if err == nil && metricValue != nil { + hostRootFields.Put(hostMetricName, metricValue) + } + } + return hostRootFields +} + func getInstancesPerRegion(svc ec2iface.ClientAPI) (instanceIDs []string, instancesOutputs map[string]ec2.Instance, err error) { instancesOutputs = map[string]ec2.Instance{} output := ec2.DescribeInstancesOutput{NextToken: nil} diff --git a/x-pack/metricbeat/module/aws/ec2/ec2_test.go b/x-pack/metricbeat/module/aws/ec2/ec2_test.go index 49757a621c6..b98e3758c65 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2_test.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2_test.go @@ -82,6 +82,10 @@ func (m *MockEC2Client) DescribeInstancesRequest(input *ec2.DescribeInstancesInp Key: awssdk.String("helm.sh/chart"), Value: awssdk.String("foo-chart"), }, + { + Key: awssdk.String("Name"), + Value: awssdk.String("test-instance"), + }, } instance := ec2.Instance{ @@ -144,10 +148,15 @@ func TestCreateCloudWatchEventsDedotTags(t *testing.T) { "cloud": common.MapStr{ "region": regionName, "provider": "aws", - "instance": common.MapStr{"id": "i-123"}, + "instance": common.MapStr{"id": "i-123", "name": "test-instance"}, "machine": common.MapStr{"type": "t2.medium"}, "availability_zone": "us-west-1a", }, + "host": common.MapStr{ + "cpu": common.MapStr{"pct": 0.0025}, + "id": "i-123", + "name": "test-instance", + }, }, MetricSetFields: common.MapStr{ "cpu": common.MapStr{ @@ -171,6 +180,7 @@ func TestCreateCloudWatchEventsDedotTags(t *testing.T) { "tags": common.MapStr{ "app_kubernetes_io/name": "foo", "helm_sh/chart": "foo-chart", + "Name": "test-instance", }, }, } @@ -227,10 +237,15 @@ func TestCreateCloudWatchEventsWithTagsFilter(t *testing.T) { "cloud": common.MapStr{ "region": regionName, "provider": "aws", - "instance": common.MapStr{"id": "i-123"}, + "instance": common.MapStr{"id": "i-123", "name": "test-instance"}, "machine": common.MapStr{"type": "t2.medium"}, "availability_zone": "us-west-1a", }, + "host": common.MapStr{ + "cpu": common.MapStr{"pct": 0.0025}, + "id": "i-123", + "name": "test-instance", + }, }, MetricSetFields: common.MapStr{ "cpu": common.MapStr{ @@ -254,6 +269,7 @@ func TestCreateCloudWatchEventsWithTagsFilter(t *testing.T) { "tags": common.MapStr{ "app_kubernetes_io/name": "foo", "helm_sh/chart": "foo-chart", + "Name": "test-instance", }, }, } @@ -455,3 +471,62 @@ func TestCalculateRate(t *testing.T) { assert.Equal(t, c.rateMetricValueDetailed, output) } } + +func TestCreateCloudWatchEventsWithInstanceName(t *testing.T) { + expectedEvent := mb.Event{ + RootFields: common.MapStr{ + "cloud": common.MapStr{ + "region": regionName, + "provider": "aws", + "instance": common.MapStr{"id": "i-123", "name": "test-instance"}, + "machine": common.MapStr{"type": "t2.medium"}, + "availability_zone": "us-west-1a", + }, + "host": common.MapStr{ + "cpu": common.MapStr{"pct": 0.25}, + "id": "i-123", + }, + }, + MetricSetFields: common.MapStr{ + "tags": common.MapStr{ + "app_kubernetes_io/name": "foo", + "helm_sh/chart": "foo-chart", + "Name": "test-instance", + }, + }, + } + svcEC2Mock := &MockEC2Client{} + instanceIDs, instancesOutputs, err := getInstancesPerRegion(svcEC2Mock) + assert.NoError(t, err) + assert.Equal(t, 1, len(instanceIDs)) + instanceID := instanceIDs[0] + assert.Equal(t, instanceID, instanceID) + timestamp := time.Now() + + getMetricDataOutput := []cloudwatch.MetricDataResult{ + { + Id: &id1, + Label: &label1, + Values: []float64{0.25}, + Timestamps: []time.Time{timestamp}, + }, + } + + metricSet := MetricSet{ + &aws.MetricSet{}, + } + + events, err := metricSet.createCloudWatchEvents(getMetricDataOutput, instancesOutputs, "us-west-1") + assert.NoError(t, err) + assert.Equal(t, 1, len(events)) + + assert.Equal(t, expectedEvent.MetricSetFields["tags"], events[instanceID].ModuleFields["tags"]) + + hostID, err := events[instanceID].RootFields.GetValue("host.id") + assert.NoError(t, err) + assert.Equal(t, "i-123", hostID) + + instanceName, err := events[instanceID].RootFields.GetValue("cloud.instance.name") + assert.NoError(t, err) + assert.Equal(t, "test-instance", instanceName) +}