Change disk and network metrics to use Sum statistic method (#20680)

* Change disk and network metrics to use Sum statistic method
elastic · Aug 21, 2020 · adeb6f3 · adeb6f3
1 parent 5050283
commit adeb6f3
Show file tree

Hide file tree

Showing 9 changed files with 219 additions and 134 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -326,6 +326,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Groups same timestamp metric values to one event in the app_insights metricset. {pull}20403[20403]
 - Updates vm_compute metricset with more info on guest metrics. {pull}20448[20448]
 - Fix resource tags in aws cloudwatch metricset {issue}20326[20326]  {pull}20385[20385]
+- Fix ec2 disk and network metrics to use Sum statistic method. {pull}20680[20680]
 
 *Packetbeat*
 

diff --git a/metricbeat/_meta/fields.common.yml b/metricbeat/_meta/fields.common.yml
@@ -47,22 +47,27 @@
       fields:
         - name: cpu.pct
           type: scaled_float
+          format: percent
           description: Percent CPU used. This value is normalized by the number of CPU cores and it ranges from 0 to 1.
         - name: network.in.bytes
-          type: scaled_float
+          type: long
+          format: bytes
           description: The number of bytes received on all network interfaces by the host in a given period of time.
         - name: network.out.bytes
-          type: scaled_float
+          type: long
+          format: bytes
           description: The number of bytes sent out on all network interfaces by the host in a given period of time.
         - name: network.in.packets
-          type: scaled_float
+          type: long
           description: The number of packets received on all network interfaces by the host in a given period of time.
         - name: network.out.packets
-          type: scaled_float
+          type: long
           description: The number of packets sent out on all network interfaces by the host in a given period of time.
         - name: disk.read.bytes
-          type: scaled_float
+          type: long
+          format: bytes
           description: The total number of bytes read successfully in a given period of time.
         - name: disk.write.bytes
-          type: scaled_float
+          type: long
+          format: bytes
           description: The total number of bytes write successfully in a given period of time.
diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -2041,7 +2041,7 @@ type: long
 *`aws.ec2.network.in.packets`*::
 +
 --
-The number of packets received on all network interfaces by the instance.
+The total number of packets received on all network interfaces by the instance in collection period.
 
 
 type: long
@@ -2054,14 +2054,14 @@ type: long
 The number of packets per second sent out on all network interfaces by the instance.
 
 
-type: long
+type: scaled_float
 
 --
 
 *`aws.ec2.network.out.packets`*::
 +
 --
-The number of packets sent out on all network interfaces by the instance.
+The total number of packets sent out on all network interfaces by the instance in collection period.
 
 
 type: long
@@ -2074,14 +2074,14 @@ type: long
 The number of packets per second sent out on all network interfaces by the instance.
 
 
-type: long
+type: scaled_float
 
 --
 
 *`aws.ec2.network.in.bytes`*::
 +
 --
-The number of bytes received on all network interfaces by the instance.
+The total number of bytes received on all network interfaces by the instance in collection period.
 
 
 type: long
@@ -2096,14 +2096,14 @@ format: bytes
 The number of bytes per second received on all network interfaces by the instance.
 
 
-type: long
+type: scaled_float
 
 --
 
 *`aws.ec2.network.out.bytes`*::
 +
 --
-The number of bytes sent out on all network interfaces by the instance.
+The total number of bytes sent out on all network interfaces by the instance in collection period.
 
 
 type: long
@@ -2118,14 +2118,14 @@ format: bytes
 The number of bytes per second sent out on all network interfaces by the instance.
 
 
-type: long
+type: scaled_float
 
 --
 
 *`aws.ec2.diskio.read.bytes`*::
 +
 --
-Bytes read from all instance store volumes available to the instance.
+Total bytes read from all instance store volumes available to the instance in collection period.
 
 
 type: long
@@ -2140,14 +2140,14 @@ format: bytes
 Bytes read per second from all instance store volumes available to the instance.
 
 
-type: long
+type: scaled_float
 
 --
 
 *`aws.ec2.diskio.write.bytes`*::
 +
 --
-Bytes written to all instance store volumes available to the instance.
+Total bytes written to all instance store volumes available to the instance in collection period.
 
 
 type: long
@@ -2162,14 +2162,14 @@ format: bytes
 Bytes written per second to all instance store volumes available to the instance.
 
 
-type: long
+type: scaled_float
 
 --
 
 *`aws.ec2.diskio.read.ops`*::
 +
 --
-Completed read operations from all instance store volumes available to the instance in a specified period of time.
+Total completed read operations from all instance store volumes available to the instance in collection period.
 
 
 type: long
@@ -2189,7 +2189,7 @@ type: long
 *`aws.ec2.diskio.write.ops`*::
 +
 --
-Completed write operations to all instance store volumes available to the instance in a specified period of time.
+Total completed write operations to all instance store volumes available to the instance in collection period.
 
 
 type: long
@@ -6403,14 +6403,18 @@ Percent CPU used. This value is normalized by the number of CPU cores and it ran
 
 type: scaled_float
 
+format: percent
+
 --
 
 *`host.network.in.bytes`*::
 +
 --
 The number of bytes received on all network interfaces by the host in a given period of time.
 
-type: scaled_float
+type: long
+
+format: bytes
 
 --
 
@@ -6419,7 +6423,9 @@ type: scaled_float
 --
 The number of bytes sent out on all network interfaces by the host in a given period of time.
 
-type: scaled_float
+type: long
+
+format: bytes
 
 --
 
@@ -6428,7 +6434,7 @@ type: scaled_float
 --
 The number of packets received on all network interfaces by the host in a given period of time.
 
-type: scaled_float
+type: long
 
 --
 
@@ -6437,7 +6443,7 @@ type: scaled_float
 --
 The number of packets sent out on all network interfaces by the host in a given period of time.
 
-type: scaled_float
+type: long
 
 --
 
@@ -6446,7 +6452,9 @@ type: scaled_float
 --
 The total number of bytes read successfully in a given period of time.
 
-type: scaled_float
+type: long
+
+format: bytes
 
 --
 
@@ -6455,7 +6463,9 @@ type: scaled_float
 --
 The total number of bytes write successfully in a given period of time.
 
-type: scaled_float
+type: long
+
+format: bytes
 
 --
 

diff --git a/x-pack/metricbeat/module/aws/ec2/_meta/data.json b/x-pack/metricbeat/module/aws/ec2/_meta/data.json
@@ -3,12 +3,12 @@
     "aws": {
         "ec2": {
             "cpu": {
-                "credit_balance": 144,
-                "credit_usage": 0.058005,
+                "credit_balance": 1944,
+                "credit_usage": 0.019738,
                 "surplus_credit_balance": 0,
                 "surplus_credits_charged": 0,
                 "total": {
-                    "pct": 1.1631008613503082
+                    "pct": 0.054166666666484745
                 }
             },
             "diskio": {
@@ -27,21 +27,21 @@
             },
             "instance": {
                 "core": {
-                    "count": 1
+                    "count": 8
                 },
                 "image": {
-                    "id": "ami-04bc3da8f14823e88"
+                    "id": "ami-0b418580298265d5c"
                 },
                 "monitoring": {
                     "state": "disabled"
                 },
                 "private": {
-                    "dns_name": "ip-172-31-9-119.us-west-1.compute.internal",
-                    "ip": "172.31.9.119"
+                    "dns_name": "ip-172-31-47-161.eu-central-1.compute.internal",
+                    "ip": "172.31.47.161"
                 },
                 "public": {
-                    "dns_name": "ec2-13-52-163-56.us-west-1.compute.amazonaws.com",
-                    "ip": "13.52.163.56"
+                    "dns_name": "ec2-3-126-207-95.eu-central-1.compute.amazonaws.com",
+                    "ip": "3.126.207.95"
                 },
                 "state": {
                     "code": 16,
@@ -51,54 +51,46 @@
             },
             "network": {
                 "in": {
-                    "bytes": 786.6,
-                    "bytes_per_sec": 2.622,
-                    "packets": 5.8,
-                    "packets_per_sec": 0.019333333333333334
+                    "bytes": 420,
+                    "bytes_per_sec": 1.4,
+                    "packets": 10,
+                    "packets_per_sec": 0.03333333333333333
                 },
                 "out": {
-                    "bytes": 627,
-                    "bytes_per_sec": 2.09,
-                    "packets": 5.8,
-                    "packets_per_sec": 0.019333333333333334
+                    "bytes": 280,
+                    "bytes_per_sec": 0.9333333333333333,
+                    "packets": 10,
+                    "packets_per_sec": 0.03333333333333333
                 }
             },
             "status": {
                 "check_failed": 0,
                 "check_failed_instance": 0,
                 "check_failed_system": 0
             }
-        },
-        "tags": {
-            "Name": "mysql-test",
-            "created-by": "ks"
         }
     },
     "cloud": {
         "account": {
             "id": "428152502467",
             "name": "elastic-beats"
         },
-        "availability_zone": "us-west-1b",
+        "availability_zone": "eu-central-1b",
         "instance": {
-            "id": "i-0516ddaca5c1d231f",
-            "name": "mysql-test"
+            "id": "i-061884169c1e2ba3f"
         },
         "machine": {
-            "type": "t2.micro"
+            "type": "t2.2xlarge"
         },
         "provider": "aws",
-        "region": "us-west-1"
+        "region": "eu-central-1"
     },
     "event": {
         "dataset": "aws.ec2",
         "duration": 115000,
         "module": "aws"
     },
     "host": {
-        "cpu": {
-            "pct": 0.011631008613503082
-        },
         "disk": {
             "read": {
                 "bytes": 0
@@ -107,16 +99,16 @@
                 "bytes": 0
             }
         },
-        "id": "i-0516ddaca5c1d231f",
-        "name": "mysql-test",
+        "id": "i-061884169c1e2ba3f",
+        "name": "i-061884169c1e2ba3f",
         "network": {
             "in": {
-                "bytes": 786.6,
-                "packets": 5.8
+                "bytes": 420,
+                "packets": 10
             },
             "out": {
-                "bytes": 627,
-                "packets": 5.8
+                "bytes": 280,
+                "packets": 10
             }
         }
     },