Add memory metrics into compute googlecloud (#18802)

* Add memory metrics into compute googlecloud * update unit test and data.json for memory metrics
elastic · May 28, 2020 · 1eb4e79 · 1eb4e79
1 parent df6f216
commit 1eb4e79
Show file tree

Hide file tree

Showing 14 changed files with 274 additions and 59 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -206,6 +206,7 @@ field. You can revert this change by configuring tags for the module and omittin
 - Remove specific win32 api errors from events in perfmon. {issue}18292[18292] {pull}18361[18361]
 - Fix application_pool metricset after pdh changes. {pull}18477[18477]
 - Fix tags_filter for cloudwatch metricset in aws. {pull}18524[18524]
+- Add missing network.sent_packets_count metric into compute metricset in googlecloud module. {pull}18802[18802]
 
 *Packetbeat*
 
@@ -440,6 +441,7 @@ field. You can revert this change by configuring tags for the module and omittin
 - Update MSSQL module to fix some SSPI authentication and add brackets to USE statements {pull}17862[17862]]
 - Add client address to events from http server module {pull}18336[18336]
 - Remove required for region/zone and make stackdriver a metricset in googlecloud. {issue}16785[16785] {pull}18398[18398]
+- Add memory metrics into compute googlecloud. {pull}18802[18802]
 
 *Packetbeat*
 

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -18232,6 +18232,9 @@ GCP module
 *`googlecloud.labels`*::
 +
 --
+Google cloud monitoring metrics labels
+
+
 type: object
 
 --
@@ -18383,6 +18386,43 @@ type: long
 
 --
 
+
+*`googlecloud.compute.instance.memory.balloon.ram_size.value`*::
++
+--
+The total amount of memory in the VM. This metric is only available for VMs that belong to the e2 family.
+
+type: long
+
+--
+
+*`googlecloud.compute.instance.memory.balloon.ram_used.value`*::
++
+--
+Memory currently used in the VM. This metric is only available for VMs that belong to the e2 family.
+
+type: long
+
+--
+
+*`googlecloud.compute.instance.memory.balloon.swap_in_bytes_count.value`*::
++
+--
+The amount of memory read into the guest from its own swap space. This metric is only available for VMs that belong to the e2 family.
+
+type: long
+
+--
+
+*`googlecloud.compute.instance.memory.balloon.swap_out_bytes_count.value`*::
++
+--
+The amount of memory written from the guest to its own swap space. This metric is only available for VMs that belong to the e2 family.
+
+type: long
+
+--
+
 [float]
 === loadbalancing
 

diff --git a/x-pack/metricbeat/module/googlecloud/_meta/fields.yml b/x-pack/metricbeat/module/googlecloud/_meta/fields.yml
@@ -9,15 +9,21 @@
       fields:
         - name: labels
           type: object
+          description: >
+            Google cloud monitoring metrics labels
           fields:
             - name: user.*
               type: object
+              object_type: keyword
             - name: metadata.*
               type: object
+              object_type: keyword
             - name: metrics.*
               type: object
+              object_type: keyword
             - name: system.*
               type: object
+              object_type: keyword
         - name: "stackdriver.*.*.*.*"
           type: object
           object_type: double

diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data.json
@@ -5,47 +5,66 @@
             "id": "elastic-observability"
         },
         "instance": {
-            "id": "1174463293187628268",
-            "name": "gke-observability-8--observability-8--bc1afd95-ngmh"
+            "id": "7692260917184259934",
+            "name": "instance-test-ks"
         },
         "machine": {
-            "type": "n1-standard-4"
+            "type": "e2-standard-2"
         },
         "provider": "googlecloud"
     },
-    "cloud.availability_zone": "europe-west1-c",
+    "cloud.availability_zone": "us-central1-a",
     "event": {
         "dataset": "googlecloud.compute",
         "duration": 115000,
         "module": "googlecloud"
     },
     "googlecloud": {
         "compute": {
+            "firewall": {
+                "dropped_bytes_count": {
+                    "value": 171
+                },
+                "dropped_packets_count": {
+                    "value": 4
+                }
+            },
             "instance": {
-                "disk": {
-                    "read_bytes_count": {
-                        "value": 0
-                    },
-                    "read_ops_count": {
-                        "value": 0
+                "cpu": {
+                    "reserved_cores": {
+                        "value": 2
                     },
-                    "write_bytes_count": {
-                        "value": 0
+                    "usage_time": {
+                        "value": 0.1803964574089818
                     },
-                    "write_ops_count": {
-                        "value": 0
+                    "utilization": {
+                        "value": 0.001503303811741515
                     }
+                },
+                "memory": {
+                    "balloon": {
+                        "ram_size": {
+                            "value": 7896264704
+                        },
+                        "ram_used": {
+                            "value": 416518144
+                        },
+                        "swap_in_bytes_count": {
+                            "value": 0
+                        },
+                        "swap_out_bytes_count": {
+                            "value": 0
+                        }
+                    }
+                },
+                "uptime": {
+                    "value": 60.000000000000455
                 }
             }
         },
         "labels": {
-            "metrics": {
-                "device_name": "disk-2",
-                "device_type": "permanent",
-                "storage_type": "pd-standard"
-            },
             "user": {
-                "goog-gke-node": ""
+                "created-by": "ks"
             }
         }
     },

diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_cpu.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_cpu.json
@@ -5,15 +5,15 @@
             "id": "elastic-observability"
         },
         "instance": {
-            "id": "1174463293187628268",
-            "name": "gke-observability-8--observability-8--bc1afd95-ngmh"
+            "id": "7692260917184259934",
+            "name": "instance-test-ks"
         },
         "machine": {
-            "type": "n1-standard-4"
+            "type": "e2-standard-2"
         },
         "provider": "googlecloud"
     },
-    "cloud.availability_zone": "europe-west1-c",
+    "cloud.availability_zone": "us-central1-a",
     "event": {
         "dataset": "googlecloud.compute",
         "duration": 115000,
@@ -23,32 +23,48 @@
         "compute": {
             "firewall": {
                 "dropped_bytes_count": {
-                    "value": 181
+                    "value": 171
                 },
                 "dropped_packets_count": {
-                    "value": 3
+                    "value": 4
                 }
             },
             "instance": {
                 "cpu": {
                     "reserved_cores": {
-                        "value": 4
+                        "value": 2
                     },
                     "usage_time": {
-                        "value": 63.478293027728796
+                        "value": 0.1803964574089818
                     },
                     "utilization": {
-                        "value": 0.26449288761553663
+                        "value": 0.001503303811741515
+                    }
+                },
+                "memory": {
+                    "balloon": {
+                        "ram_size": {
+                            "value": 7896264704
+                        },
+                        "ram_used": {
+                            "value": 416518144
+                        },
+                        "swap_in_bytes_count": {
+                            "value": 0
+                        },
+                        "swap_out_bytes_count": {
+                            "value": 0
+                        }
                     }
                 },
                 "uptime": {
-                    "value": 60
+                    "value": 60.000000000000455
                 }
             }
         },
         "labels": {
             "user": {
-                "goog-gke-node": ""
+                "created-by": "ks"
             }
         }
     },

diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk.json
@@ -5,15 +5,15 @@
             "id": "elastic-observability"
         },
         "instance": {
-            "id": "8390997210852978465",
-            "name": "gke-observability-7--observability-7--3dd3e39b-0jm5"
+            "id": "7692260917184259934",
+            "name": "instance-test-ks"
         },
         "machine": {
-            "type": "n1-standard-4"
+            "type": "e2-standard-2"
         },
         "provider": "googlecloud"
     },
-    "cloud.availability_zone": "europe-west1-c",
+    "cloud.availability_zone": "us-central1-a",
     "event": {
         "dataset": "googlecloud.compute",
         "duration": 115000,
@@ -30,22 +30,22 @@
                         "value": 0
                     },
                     "write_bytes_count": {
-                        "value": 0
+                        "value": 158506
                     },
                     "write_ops_count": {
-                        "value": 0
+                        "value": 25
                     }
                 }
             }
         },
         "labels": {
             "metrics": {
-                "device_name": "gke-observability-7-1--pvc-65581044-7d5d-11ea-8cd9-42010af0011c",
+                "device_name": "instance-test-ks",
                 "device_type": "permanent",
                 "storage_type": "pd-standard"
             },
             "user": {
-                "goog-gke-node": ""
+                "created-by": "ks"
             }
         }
     },

diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_firewall.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_firewall.json
@@ -5,15 +5,15 @@
             "id": "elastic-observability"
         },
         "instance": {
-            "id": "2528596280375797115",
-            "name": "gke-dev-next-oblt-dev-next-oblt-pool-404d7f0c-cpj6"
+            "id": "7692260917184259934",
+            "name": "instance-test-ks"
         },
         "machine": {
-            "type": "n1-standard-4"
+            "type": "e2-standard-2"
         },
         "provider": "googlecloud"
     },
-    "cloud.availability_zone": "europe-west1-c",
+    "cloud.availability_zone": "us-central1-a",
     "event": {
         "dataset": "googlecloud.compute",
         "duration": 115000,
@@ -23,22 +23,38 @@
         "compute": {
             "firewall": {
                 "dropped_bytes_count": {
-                    "value": 386
+                    "value": 277
                 },
                 "dropped_packets_count": {
-                    "value": 7
+                    "value": 5
                 }
             },
             "instance": {
                 "cpu": {
                     "reserved_cores": {
-                        "value": 4
+                        "value": 2
                     },
                     "usage_time": {
-                        "value": 106.88293868489563
+                        "value": 0.18404532833979204
                     },
                     "utilization": {
-                        "value": 0.4453455778537318
+                        "value": 0.001533711069498267
+                    }
+                },
+                "memory": {
+                    "balloon": {
+                        "ram_size": {
+                            "value": 7896264704
+                        },
+                        "ram_used": {
+                            "value": 416260096
+                        },
+                        "swap_in_bytes_count": {
+                            "value": 0
+                        },
+                        "swap_out_bytes_count": {
+                            "value": 0
+                        }
                     }
                 },
                 "uptime": {
@@ -48,7 +64,7 @@
         },
         "labels": {
             "user": {
-                "goog-gke-node": ""
+                "created-by": "ks"
             }
         }
     },