From 2ce54817dc3b4b0d786af1ab9074ee42ad0addcb Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 24 Aug 2020 06:44:02 -0600 Subject: [PATCH] Add host inventory metrics to googlecloud compute metricset (#20391) (#20724) * Add host inventory metrics to googlecloud compute metricset (cherry picked from commit cbb2f372a5e1adab399a17627d2e3b38dcacff1d) --- CHANGELOG.next.asciidoc | 1 + .../googlecloud/compute/_meta/data.json | 50 +++++------- .../googlecloud/compute/_meta/data_cpu.json | 50 +++++------- .../googlecloud/compute/_meta/data_disk.json | 30 ++++--- .../compute/_meta/data_firewall.json | 48 +++++------- .../compute/_meta/data_memory.json | 78 ------------------- .../compute/_meta/data_network.json | 32 +++++--- .../module/googlecloud/metrics/metricset.go | 38 ++++++++- .../timeseries_metadata_collector.go | 16 ++++ .../timeseries_metadata_collector_test.go | 17 ++++ 10 files changed, 168 insertions(+), 192 deletions(-) delete mode 100644 x-pack/metricbeat/module/googlecloud/compute/_meta/data_memory.json create mode 100644 x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 00a3647657b..3239d9c04cc 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -726,6 +726,7 @@ field. You can revert this change by configuring tags for the module and omittin - Add cloud.instance.name into aws ec2 metricset. {pull}20077[20077] - Add host inventory metrics into aws ec2 metricset. {pull}20171[20171] - Add `scope` setting for elasticsearch module, allowing it to monitor an Elasticsearch cluster behind a load-balancing proxy. {issue}18539[18539] {pull}18547[18547] +- Add host inventory metrics to googlecloud compute metricset. {pull}20391[20391] - Add host inventory metrics to azure compute_vm metricset. {pull}20641[20641] *Packetbeat* diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data.json index 13c4958d13f..36c46744b50 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data.json +++ b/x-pack/metricbeat/module/googlecloud/compute/_meta/data.json @@ -5,15 +5,16 @@ "id": "elastic-observability" }, "instance": { - "id": "7692260917184259934", - "name": "instance-test-ks" + "id": "2438671519308240892", + "name": "heartbeat-mumbai" }, "machine": { - "type": "e2-standard-2" + "type": "f1-micro" }, "provider": "googlecloud" }, - "cloud.availability_zone": "us-central1-a", + "cloud.availability_zone": "asia-south1-c", + "cloud.region": "asia-south1", "event": { "dataset": "googlecloud.compute", "duration": 115000, @@ -23,50 +24,37 @@ "compute": { "firewall": { "dropped_bytes_count": { - "value": 171 + "value": 435 }, "dropped_packets_count": { - "value": 4 + "value": 7 } }, "instance": { "cpu": { "reserved_cores": { - "value": 2 + "value": 0.2 }, "usage_time": { - "value": 0.1803964574089818 + "value": 0.8653667340986431 }, "utilization": { - "value": 0.001503303811741515 - } - }, - "memory": { - "balloon": { - "ram_size": { - "value": 7896264704 - }, - "ram_used": { - "value": 416518144 - }, - "swap_in_bytes_count": { - "value": 0 - }, - "swap_out_bytes_count": { - "value": 0 - } + "value": 0.07211389450822024 } }, "uptime": { - "value": 60.000000000000455 + "value": 60 } } }, - "labels": { - "user": { - "created-by": "ks" - } - } + "labels": {} + }, + "host": { + "cpu": { + "pct": 0.07211389450822024 + }, + "id": "2438671519308240892", + "name": "heartbeat-mumbai" }, "metricset": { "name": "compute", diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_cpu.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_cpu.json index 13c4958d13f..b43490d1915 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_cpu.json +++ b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_cpu.json @@ -5,15 +5,16 @@ "id": "elastic-observability" }, "instance": { - "id": "7692260917184259934", - "name": "instance-test-ks" + "id": "7925831082803123728", + "name": "heartbeat-tokyo" }, "machine": { - "type": "e2-standard-2" + "type": "f1-micro" }, "provider": "googlecloud" }, - "cloud.availability_zone": "us-central1-a", + "cloud.availability_zone": "asia-northeast1-b", + "cloud.region": "asia-northeast1", "event": { "dataset": "googlecloud.compute", "duration": 115000, @@ -23,50 +24,37 @@ "compute": { "firewall": { "dropped_bytes_count": { - "value": 171 + "value": 422 }, "dropped_packets_count": { - "value": 4 + "value": 7 } }, "instance": { "cpu": { "reserved_cores": { - "value": 2 + "value": 0.2 }, "usage_time": { - "value": 0.1803964574089818 + "value": 0.8921228461549617 }, "utilization": { - "value": 0.001503303811741515 - } - }, - "memory": { - "balloon": { - "ram_size": { - "value": 7896264704 - }, - "ram_used": { - "value": 416518144 - }, - "swap_in_bytes_count": { - "value": 0 - }, - "swap_out_bytes_count": { - "value": 0 - } + "value": 0.07434357051291347 } }, "uptime": { - "value": 60.000000000000455 + "value": 60 } } }, - "labels": { - "user": { - "created-by": "ks" - } - } + "labels": {} + }, + "host": { + "cpu": { + "pct": 0.07434357051291347 + }, + "id": "7925831082803123728", + "name": "heartbeat-tokyo" }, "metricset": { "name": "compute", diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk.json index 38b0e2a5b4f..7a8f923569e 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk.json +++ b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk.json @@ -5,15 +5,16 @@ "id": "elastic-observability" }, "instance": { - "id": "7692260917184259934", - "name": "instance-test-ks" + "id": "7925831082803123728", + "name": "heartbeat-tokyo" }, "machine": { - "type": "e2-standard-2" + "type": "f1-micro" }, "provider": "googlecloud" }, - "cloud.availability_zone": "us-central1-a", + "cloud.availability_zone": "asia-northeast1-b", + "cloud.region": "asia-northeast1", "event": { "dataset": "googlecloud.compute", "duration": 115000, @@ -30,25 +31,34 @@ "value": 0 }, "write_bytes_count": { - "value": 158506 + "value": 0 }, "write_ops_count": { - "value": 25 + "value": 0 } } } }, "labels": { "metrics": { - "device_name": "instance-test-ks", + "device_name": "disk-1", "device_type": "permanent", "storage_type": "pd-standard" - }, - "user": { - "created-by": "ks" } } }, + "host": { + "disk": { + "read": { + "bytes": 0 + }, + "write": { + "bytes": 0 + } + }, + "id": "7925831082803123728", + "name": "heartbeat-tokyo" + }, "metricset": { "name": "compute", "period": 10000 diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_firewall.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_firewall.json index f147f276f32..b43490d1915 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_firewall.json +++ b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_firewall.json @@ -5,15 +5,16 @@ "id": "elastic-observability" }, "instance": { - "id": "7692260917184259934", - "name": "instance-test-ks" + "id": "7925831082803123728", + "name": "heartbeat-tokyo" }, "machine": { - "type": "e2-standard-2" + "type": "f1-micro" }, "provider": "googlecloud" }, - "cloud.availability_zone": "us-central1-a", + "cloud.availability_zone": "asia-northeast1-b", + "cloud.region": "asia-northeast1", "event": { "dataset": "googlecloud.compute", "duration": 115000, @@ -23,38 +24,22 @@ "compute": { "firewall": { "dropped_bytes_count": { - "value": 277 + "value": 422 }, "dropped_packets_count": { - "value": 5 + "value": 7 } }, "instance": { "cpu": { "reserved_cores": { - "value": 2 + "value": 0.2 }, "usage_time": { - "value": 0.18404532833979204 + "value": 0.8921228461549617 }, "utilization": { - "value": 0.001533711069498267 - } - }, - "memory": { - "balloon": { - "ram_size": { - "value": 7896264704 - }, - "ram_used": { - "value": 416260096 - }, - "swap_in_bytes_count": { - "value": 0 - }, - "swap_out_bytes_count": { - "value": 0 - } + "value": 0.07434357051291347 } }, "uptime": { @@ -62,11 +47,14 @@ } } }, - "labels": { - "user": { - "created-by": "ks" - } - } + "labels": {} + }, + "host": { + "cpu": { + "pct": 0.07434357051291347 + }, + "id": "7925831082803123728", + "name": "heartbeat-tokyo" }, "metricset": { "name": "compute", diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_memory.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_memory.json deleted file mode 100644 index f147f276f32..00000000000 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_memory.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "@timestamp": "2017-10-12T08:05:34.853Z", - "cloud": { - "account": { - "id": "elastic-observability" - }, - "instance": { - "id": "7692260917184259934", - "name": "instance-test-ks" - }, - "machine": { - "type": "e2-standard-2" - }, - "provider": "googlecloud" - }, - "cloud.availability_zone": "us-central1-a", - "event": { - "dataset": "googlecloud.compute", - "duration": 115000, - "module": "googlecloud" - }, - "googlecloud": { - "compute": { - "firewall": { - "dropped_bytes_count": { - "value": 277 - }, - "dropped_packets_count": { - "value": 5 - } - }, - "instance": { - "cpu": { - "reserved_cores": { - "value": 2 - }, - "usage_time": { - "value": 0.18404532833979204 - }, - "utilization": { - "value": 0.001533711069498267 - } - }, - "memory": { - "balloon": { - "ram_size": { - "value": 7896264704 - }, - "ram_used": { - "value": 416260096 - }, - "swap_in_bytes_count": { - "value": 0 - }, - "swap_out_bytes_count": { - "value": 0 - } - } - }, - "uptime": { - "value": 60 - } - } - }, - "labels": { - "user": { - "created-by": "ks" - } - } - }, - "metricset": { - "name": "compute", - "period": 10000 - }, - "service": { - "type": "googlecloud" - } -} \ No newline at end of file diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_network.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_network.json index 1b86e390e16..22257a87521 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_network.json +++ b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_network.json @@ -5,15 +5,16 @@ "id": "elastic-observability" }, "instance": { - "id": "7692260917184259934", - "name": "instance-test-ks" + "id": "7925831082803123728", + "name": "heartbeat-tokyo" }, "machine": { - "type": "e2-standard-2" + "type": "f1-micro" }, "provider": "googlecloud" }, - "cloud.availability_zone": "us-central1-a", + "cloud.availability_zone": "asia-northeast1-b", + "cloud.region": "asia-northeast1", "event": { "dataset": "googlecloud.compute", "duration": 115000, @@ -24,16 +25,16 @@ "instance": { "network": { "received_bytes_count": { - "value": 13073 + "value": 2866053 }, "received_packets_count": { - "value": 95 + "value": 2449 }, "sent_bytes_count": { - "value": 0 + "value": 119338 }, "sent_packets_count": { - "value": 0 + "value": 695 } } } @@ -41,9 +42,20 @@ "labels": { "metrics": { "loadbalanced": "false" + } + } + }, + "host": { + "id": "7925831082803123728", + "name": "heartbeat-tokyo", + "network": { + "in": { + "bytes": 119338, + "packets": 695 }, - "user": { - "created-by": "ks" + "out": { + "bytes": 2866053, + "packets": 2449 } } }, diff --git a/x-pack/metricbeat/module/googlecloud/metrics/metricset.go b/x-pack/metricbeat/module/googlecloud/metrics/metricset.go index c5abd87426e..b0823feeace 100644 --- a/x-pack/metricbeat/module/googlecloud/metrics/metricset.go +++ b/x-pack/metricbeat/module/googlecloud/metrics/metricset.go @@ -172,8 +172,7 @@ func (m *MetricSet) eventMapping(ctx context.Context, tss []timeSeriesWithAligne events := make([]mb.Event, 0) for _, groupedEvents := range tsGrouped { event := mb.Event{ - Timestamp: groupedEvents[0].Timestamp, - RootFields: groupedEvents[0].ECS, + Timestamp: groupedEvents[0].Timestamp, ModuleFields: common.MapStr{ "labels": groupedEvents[0].Labels, }, @@ -184,6 +183,12 @@ func (m *MetricSet) eventMapping(ctx context.Context, tss []timeSeriesWithAligne event.MetricSetFields.Put(singleEvent.Key, singleEvent.Value) } + if serviceName == "compute" { + event.RootFields = addHostFields(groupedEvents) + } else { + event.RootFields = groupedEvents[0].ECS + } + events = append(events, event) } @@ -269,3 +274,32 @@ func (m *MetricSet) getMetadata(out *metric.MetricDescriptor, metricsWithMeta ma metricsWithMeta[out.Type] = meta return metricsWithMeta } + +func addHostFields(groupedEvents []KeyValuePoint) common.MapStr { + hostRootFields := groupedEvents[0].ECS + // add host.id and host.name + if hostID, err := groupedEvents[0].ECS.GetValue("cloud.instance.id"); err == nil { + hostRootFields.Put("host.id", hostID) + } + + if hostName, err := groupedEvents[0].ECS.GetValue("cloud.instance.name"); err == nil { + hostRootFields.Put("host.name", hostName) + } + + hostFieldTable := map[string]string{ + "instance.cpu.utilization.value": "host.cpu.pct", + "instance.network.sent_bytes_count.value": "host.network.in.bytes", + "instance.network.received_bytes_count.value": "host.network.out.bytes", + "instance.network.sent_packets_count.value": "host.network.in.packets", + "instance.network.received_packets_count.value": "host.network.out.packets", + "instance.disk.read_bytes_count.value": "host.disk.read.bytes", + "instance.disk.write_bytes_count.value": "host.disk.write.bytes", + } + + for _, singleEvent := range groupedEvents { + if hostMetricName, ok := hostFieldTable[singleEvent.Key]; ok { + hostRootFields.Put(hostMetricName, singleEvent.Value) + } + } + return hostRootFields +} diff --git a/x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector.go b/x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector.go index 90603389426..fe14aff5966 100644 --- a/x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector.go +++ b/x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector.go @@ -6,6 +6,7 @@ package googlecloud import ( "context" + "strings" "time" "github.com/golang/protobuf/ptypes" @@ -62,6 +63,12 @@ func (s *StackdriverTimeSeriesMetadataCollector) Metadata(ctx context.Context, i if availabilityZone != "" { ecs[ECSCloud+"."+ECSCloudAvailabilityZone] = availabilityZone + + // Get region name from availability zone name + region := getRegionName(availabilityZone) + if region != "" { + ecs[ECSCloud+"."+ECSCloudRegion] = region + } } //Remove keys from resource that refers to ECS fields @@ -169,3 +176,12 @@ func (s *StackdriverTimeSeriesMetadataCollector) getTimestamp(p *monitoringpb.Po return time.Time{}, errors.New("error trying to extract the timestamp from the point data") } + +func getRegionName(availabilityZone string) (region string) { + azSplit := strings.Split(availabilityZone, "-") + if len(azSplit) != 3 { + return "" + } + region = azSplit[0] + "-" + azSplit[1] + return +} diff --git a/x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector_test.go b/x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector_test.go new file mode 100644 index 00000000000..38ac903e2e3 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector_test.go @@ -0,0 +1,17 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package googlecloud + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetRegionName(t *testing.T) { + availabilityZone := "us-central1-a" + region := getRegionName(availabilityZone) + assert.Equal(t, "us-central1", region) +}