[vSphere][datastore] Add support for additional metrics

packages/vsphere/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.14.0"
+  changes:
+    - description: Add additional metrics for datastore datastream
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10911
 - version: "1.13.0"
   changes:
     - description: ECS version updated to 8.11.0. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template.

packages/vsphere/data_stream/datastore/fields/fields.yml

@@ -6,33 +6,85 @@
       type: keyword
       # Reason to add as a dimension field: to uniquely identify the datastore
       dimension: true
-      description: Datastore name
+      description: Datastore name.
     - name: fstype
       type: keyword
       # Reason to add as a dimension field: in case datastore name is duplicated across different type.
       dimension: true
-      description: Filesystem type
+      description: Filesystem type.
     - name: capacity.total.bytes
       type: long
       metric_type: gauge
       unit: byte
-      description: Total bytes of the datastore
+      description: Total bytes of the datastore.
       format: bytes
     - name: capacity.free.bytes
       type: long
       metric_type: gauge
       unit: byte
-      description: Free bytes of the datastore
+      description: Free bytes of the datastore.
       format: bytes
     - name: capacity.used.bytes
       type: long
       metric_type: gauge
       unit: byte
-      description: Used bytes of the datastore
+      description: Used bytes of the datastore.
       format: bytes
     - name: capacity.used.pct
       type: scaled_float
       metric_type: gauge
       unit: percent
-      description: Used percent of the datastore
+      description: Percentage of datastore capacity used.
       format: percent
+    - name: host.count
+      type: long
+      description: >
+        Number of hosts.
+
+    - name: host.names
+      type: keyword
+      description: >
+        List of all the host names.
+
+    - name: iops
+      type: long
+      description: >
+        Storage I/O Control aggregated Input/Output Operations Per Second.
+
+    - name: read.bytes
+      type: long
+      description: >
+        Rate of reading data from the datastore.
+
+      format: bytes
+    - name: read.latency.total.ms
+      type: long
+      description: >
+        Average amount of time for a read operation from the datastore in milliseconds.
+
+    - name: status
+      type: keyword
+      description: >
+        Status of the datastore.
+
+    - name: vm.count
+      type: long
+      description: >
+        Number of VMs.
+
+    - name: vm.names
+      type: keyword
+      description: >
+        List of all the VM names.
+
+    - name: write.bytes
+      type: long
+      description: >
+        Rate of writing data to the datastore.
+
+      format: bytes
+    - name: write.latency.total.ms
+      type: long
+      description: >
+        Average amount of time for a write operation from the datastore in milliseconds.
+

packages/vsphere/data_stream/datastore/sample_event.json

@@ -1,5 +1,5 @@
 {
-    "@timestamp": "2023-06-29T08:03:30.114Z",
+    "@timestamp": "2017-10-12T08:05:34.853Z",
     "agent": {
         "ephemeral_id": "8b019ff3-cbda-41fa-b1ff-974d482b9694",
         "id": "5096d7cc-1e4b-4959-abea-7355be2913a7",
@@ -54,25 +54,55 @@
         "period": 10000
     },
     "service": {
-        "address": "https://elastic-package-service_vsphere-metrics_1:8989/sdk",
+        "address": "127.0.0.1:33365",
         "type": "vsphere"
     },
     "vsphere": {
         "datastore": {
+            "iops": 0,
+            "host": {
+                "count": 1,
+                "names": [
+                    "DC3_H0"
+                ]
+            },
+            "status": "green",
+            "vm": {
+                "count": 6,
+                "names": [
+                    "DC3_H0_VM0"
+                ]
+            },
             "capacity": {
                 "free": {
-                    "bytes": 47869427712
+                    "bytes": 37120094208
                 },
                 "total": {
-                    "bytes": 62725623808
+                    "bytes": 74686664704
                 },
                 "used": {
-                    "bytes": 14856196096,
-                    "pct": 0.237
+                    "bytes": 37566570496,
+                    "pct": 0.502988996026061
+                }
+            },
+            "read": {
+                "bytes": 0,
+                "latency": {
+                    "total": {
+                        "ms": 0
+                    }
+                }
+            },
+            "write": {
+                "bytes": 337000,
+                "latency": {
+                    "total": {
+                        "ms": 0
+                    }
                 }
             },
-            "fstype": "OTHER",
+            "fstype": "local",
             "name": "LocalDS_0"
         }
     }
-}
+}

packages/vsphere/docs/README.md

@@ -299,7 +299,7 @@ An example event for `datastore` looks as following:
 
 ```json
 {
-    "@timestamp": "2023-06-29T08:03:30.114Z",
+    "@timestamp": "2017-10-12T08:05:34.853Z",
     "agent": {
         "ephemeral_id": "8b019ff3-cbda-41fa-b1ff-974d482b9694",
         "id": "5096d7cc-1e4b-4959-abea-7355be2913a7",
@@ -354,28 +354,59 @@ An example event for `datastore` looks as following:
         "period": 10000
     },
     "service": {
-        "address": "https://elastic-package-service_vsphere-metrics_1:8989/sdk",
+        "address": "127.0.0.1:33365",
         "type": "vsphere"
     },
     "vsphere": {
         "datastore": {
+            "iops": 0,
+            "host": {
+                "count": 1,
+                "names": [
+                    "DC3_H0"
+                ]
+            },
+            "status": "green",
+            "vm": {
+                "count": 6,
+                "names": [
+                    "DC3_H0_VM0"
+                ]
+            },
             "capacity": {
                 "free": {
-                    "bytes": 47869427712
+                    "bytes": 37120094208
                 },
                 "total": {
-                    "bytes": 62725623808
+                    "bytes": 74686664704
                 },
                 "used": {
-                    "bytes": 14856196096,
-                    "pct": 0.237
+                    "bytes": 37566570496,
+                    "pct": 0.502988996026061
                 }
             },
-            "fstype": "OTHER",
+            "read": {
+                "bytes": 0,
+                "latency": {
+                    "total": {
+                        "ms": 0
+                    }
+                }
+            },
+            "write": {
+                "bytes": 337000,
+                "latency": {
+                    "total": {
+                        "ms": 0
+                    }
+                }
+            },
+            "fstype": "local",
             "name": "LocalDS_0"
         }
     }
 }
+
 ```
 
 **ECS Field Reference**
@@ -404,12 +435,22 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur
 | host.os.build | OS build information. | keyword |  |  |
 | host.os.codename | OS codename, if any. | keyword |  |  |
 | service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword |  |  |
-| vsphere.datastore.capacity.free.bytes | Free bytes of the datastore | long | byte | gauge |
-| vsphere.datastore.capacity.total.bytes | Total bytes of the datastore | long | byte | gauge |
-| vsphere.datastore.capacity.used.bytes | Used bytes of the datastore | long | byte | gauge |
-| vsphere.datastore.capacity.used.pct | Used percent of the datastore | scaled_float | percent | gauge |
-| vsphere.datastore.fstype | Filesystem type | keyword |  |  |
-| vsphere.datastore.name | Datastore name | keyword |  |  |
+| vsphere.datastore.capacity.free.bytes | Free bytes of the datastore. | long | byte | gauge |
+| vsphere.datastore.capacity.total.bytes | Total bytes of the datastore. | long | byte | gauge |
+| vsphere.datastore.capacity.used.bytes | Used bytes of the datastore. | long | byte | gauge |
+| vsphere.datastore.capacity.used.pct | Percentage of datastore capacity used. | scaled_float | percent | gauge |
+| vsphere.datastore.fstype | Filesystem type. | keyword |  |  |
+| vsphere.datastore.host.count | Number of hosts. | long |  |  |
+| vsphere.datastore.host.names | List of all the host names. | keyword |  |  |
+| vsphere.datastore.iops | Storage I/O Control aggregated Input/Output Operations Per Second. | long |  |  |
+| vsphere.datastore.name | Datastore name. | keyword |  |  |
+| vsphere.datastore.read.bytes | Rate of reading data from the datastore. | long |  |  |
+| vsphere.datastore.read.latency.total.ms | Average amount of time for a read operation from the datastore in milliseconds. | long |  |  |
+| vsphere.datastore.status | Status of the datastore. | keyword |  |  |
+| vsphere.datastore.vm.count | Number of VMs. | long |  |  |
+| vsphere.datastore.vm.names | List of all the VM names. | keyword |  |  |
+| vsphere.datastore.write.bytes | Rate of writing data to the datastore. | long |  |  |
+| vsphere.datastore.write.latency.total.ms | Average amount of time for a write operation from the datastore in milliseconds. | long |  |  |
 
 
 ## Logs

packages/vsphere/manifest.yml

@@ -1,7 +1,7 @@
 title: VMware vSphere
 format_version: "3.0.2"
 name: vsphere
-version: "1.13.0"
+version: "1.14.0"
 description: This Elastic integration collects metrics and logs from vSphere/vCenter servers
 type: integration
 categories: