From 4c9360b1472d4a2ff7c7fb143ab69fa89618edf0 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 1 Mar 2021 14:48:55 +0100 Subject: [PATCH] Store cloudfoundry.container.cpu.pct in decimal form (#24219) (#24234) Percentages should be stored in decimal form, and can be stored as scaled floats. Some code is refactored to remove previous helper that makes less sense now. (cherry picked from commit 2cf650ccc84cc1a85b672eb202167b47392a9c71) --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/fields.asciidoc | 2 +- .../cloudfoundry/container/_meta/fields.yml | 2 +- .../cloudfoundry/container/container.go | 14 ++-- .../cloudfoundry/container/container_test.go | 4 +- .../metricbeat/module/cloudfoundry/fields.go | 2 +- x-pack/metricbeat/module/cloudfoundry/util.go | 27 ------- .../module/cloudfoundry/util_test.go | 70 ------------------- .../module/cloudfoundry/value/value.go | 5 +- 9 files changed, 19 insertions(+), 108 deletions(-) delete mode 100644 x-pack/metricbeat/module/cloudfoundry/util.go delete mode 100644 x-pack/metricbeat/module/cloudfoundry/util_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1ce13d908f2..e85258eb618 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -72,6 +72,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Change types of numeric metrics from Kubelet summary api to double so as to cover big numbers. {pull}23335[23335] - Add container.image.name and containe.name ECS fields for state_container. {pull}23802[23802] - Add support for the MemoryPressure, DiskPressure, OutOfDisk and PIDPressure status conditions in state_node. {pull}[23905] +- Store `cloudfoundry.container.cpu.pct` in decimal form and as `scaled_float`. {pull}24219[24219] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index e27ea0c6462..21106df1b68 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -7078,7 +7078,7 @@ type: long CPU usage percentage. -type: float +type: scaled_float -- diff --git a/x-pack/metricbeat/module/cloudfoundry/container/_meta/fields.yml b/x-pack/metricbeat/module/cloudfoundry/container/_meta/fields.yml index 1cd1228ab25..1e75d664253 100644 --- a/x-pack/metricbeat/module/cloudfoundry/container/_meta/fields.yml +++ b/x-pack/metricbeat/module/cloudfoundry/container/_meta/fields.yml @@ -9,7 +9,7 @@ description: > Index of the instance the metric belongs to. - name: cpu.pct - type: float + type: scaled_float description: > CPU usage percentage. - name: memory.bytes diff --git a/x-pack/metricbeat/module/cloudfoundry/container/container.go b/x-pack/metricbeat/module/cloudfoundry/container/container.go index 50287ef8c49..a6f8d18191e 100644 --- a/x-pack/metricbeat/module/cloudfoundry/container/container.go +++ b/x-pack/metricbeat/module/cloudfoundry/container/container.go @@ -6,6 +6,7 @@ package container import ( "fmt" + "math" "github.com/elastic/beats/v7/x-pack/metricbeat/module/cloudfoundry" @@ -53,12 +54,17 @@ type containerReporter struct { func (r *containerReporter) Event(event mb.Event) bool { cpuPctKey := "cloudfoundry.container.cpu.pct" - found, err := cloudfoundry.HasNonNumericFloat(event.RootFields, cpuPctKey) + value, err := event.RootFields.GetValue(cpuPctKey) if err != nil { r.logger.Debugf("Unexpected failure while checking for non-numeric values: %v", err) - } - if found { - event.RootFields.Delete(cpuPctKey) + } else { + if value, ok := value.(float64); ok { + if math.IsNaN(value) || math.IsInf(value, 0) { + event.RootFields.Delete(cpuPctKey) + } else { + event.RootFields.Put(cpuPctKey, value/100) + } + } } return r.PushReporterV2.Event(event) } diff --git a/x-pack/metricbeat/module/cloudfoundry/container/container_test.go b/x-pack/metricbeat/module/cloudfoundry/container/container_test.go index 46b1206f1b3..ceabb62cc01 100644 --- a/x-pack/metricbeat/module/cloudfoundry/container/container_test.go +++ b/x-pack/metricbeat/module/cloudfoundry/container/container_test.go @@ -61,7 +61,7 @@ func TestMetricSet(t *testing.T) { expectedFields := common.MapStr{ "cloudfoundry.app.id": "1234", - "cloudfoundry.container.cpu.pct": float64(12.34), + "cloudfoundry.container.cpu.pct": float64(0.1234), "cloudfoundry.container.disk.bytes": uint64(0), "cloudfoundry.container.disk.quota.bytes": uint64(0), "cloudfoundry.container.instance_index": int32(0), @@ -116,7 +116,7 @@ func TestMetricValuesAreNumbers(t *testing.T) { case "1234": v, err := e.RootFields.GetValue(cpuPctKey) if assert.NoError(t, err, "checking cpu pct") { - assert.Equal(t, 12.34, v.(float64)) + assert.Equal(t, 0.1234, v.(float64)) } default: t.Errorf("unexpected app: %s", app) diff --git a/x-pack/metricbeat/module/cloudfoundry/fields.go b/x-pack/metricbeat/module/cloudfoundry/fields.go index 7aeb261e2e3..668743ca974 100644 --- a/x-pack/metricbeat/module/cloudfoundry/fields.go +++ b/x-pack/metricbeat/module/cloudfoundry/fields.go @@ -19,5 +19,5 @@ func init() { // AssetCloudfoundry returns asset data. // This is the base64 encoded gzipped contents of module/cloudfoundry. func AssetCloudfoundry() string { - return "eJzcVbFyGjEQ7fmKN25oYj7gihRxJjPuXDh1LKQ90KCTLtIKcn+fkeDwHQhDAqTIFcywkt57eqvdfcSKugrSuKhqF63y3QRgzYYqPDwNwg8TQFGQXresna3weQIAeQu+bfegcSoamgCeDIlAFebEYgLUmowKVT7yCCsaOiJNH3ctVVh4F9tdpEA5RhsipuP7YI+2om7jvBrEi5jb73VJ+RhcDVqTZdTeNeNbzkYnXlwIem4Ia2EiBWgrTVSEqXSWhbbkp5/Sn2iZ/BTCKkzz1unsSL9o2yP5QzMuEC/a1mgp0jJ4SWiIvZbQASIEJ7VgUthoXg5vcejnUJNWo/BpV8+I6wU+f03e8ljqsRV7964x5G2P8tYDhnfknTfhTIoPn/JFptnAwkr6oa2iX0UDjbOLP3LvOUH13vUEwxzPKWEGsJsVRck2zlrJRTW1ceJw5Uwyn16+IwaxILTkJVkWCyoTN9Q4383mHVO4jRf4krCSGTGQ6gk+Iv8ZHYv7SBBroY1ILeAjHUqH1R0tCOz8yQRk7n/iwJGM93rODfDKas4Yo1rOkTtWcvq9bQNMiH0Z7/SfyBqZkUxckapErHRdk6fUNebEG6LthDAiMFg3NFS0G35Oyug9qbJAdizMDQVmvO0cRe182aGePG+76jllhP/oMeX7lDMVrS73/b8mTogXEB9mCddMnNee64D4dwAAAP//IdTbYA==" + return "eJzcVbGS2jAQ7fmKnWtocnyAixS5TGauu+JS3wnpGTTIkiOtIP77jAQGG5uDHJAiKphhvXrv6a129UgrNAVJ46IqXbTKNxMi1mxQ0MNTJ/wwIVII0uuatbMFfZ0QEeUU+rHNocqpaDAh8jAQAQXNwWJCVGoYFYq85ZGsqDAgTYubGgUtvIv1LjJC2UfrIqbt+2CLtkKzcV514qOY2/W6RN5GriSsYZlK76r+KWe9HS8uBD03oLUwEYG0lSYq0FQ6y0Jb+OmX9Cdahp+SsIqmOXU6G+gXdT2Q3zXjAvGiro2WIn0mXoIqsNeSdCARgpNaMBRtNC+7pzj2s6tJq174tKtnxLUCn78nb7kvdWjF3r1rDHnfo7y3gOGAvPMmnCnx8VW+yDQbWFiJN20Vfo8aaJxd/JV7zwmq9a4l6NZ4joQZiN1sVJSs46yWPKomSGGg3krjxHHCmZo+vfykGMQCVMNLWBYLjPNXqJxvZvOGEW5jCX1LWMmTGKBago/If0XH4j4SxFpoI9Ik+EiH0mF1RwsCO3+yAJn7nzgwkHFo6zwHr2zqjNFr6Ry5Y0On39vOwYTYdvNO/4mqwfRk0hWlSsRKlyU80vCYgzfA9qEwIjCxrtBVtHsDnZTRe6hxgexYmBsKzHjb55RK58cdaslz2lXXKSP8R5cpn2e8UtHq8fH/aeKEeAHxcZUOzJ94cV5briPiPwEAAP//2qzeKw==" } diff --git a/x-pack/metricbeat/module/cloudfoundry/util.go b/x-pack/metricbeat/module/cloudfoundry/util.go deleted file mode 100644 index 4775ac539be..00000000000 --- a/x-pack/metricbeat/module/cloudfoundry/util.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package cloudfoundry - -import ( - "fmt" - "math" - - "github.com/elastic/beats/v7/libbeat/common" -) - -// HasNonNumericFloat checks if an event has a non-numeric float in the specific key. -// It returns false and an error if the key cannot be found in the event -func HasNonNumericFloat(event common.MapStr, key string) (bool, error) { - v, err := event.GetValue(key) - if err != nil { - return false, fmt.Errorf("getting value for key %s: %w", key, err) - } - - if v, ok := v.(float64); ok && (math.IsNaN(v) || math.IsInf(v, 0)) { - return true, nil - } - - return false, nil -} diff --git a/x-pack/metricbeat/module/cloudfoundry/util_test.go b/x-pack/metricbeat/module/cloudfoundry/util_test.go deleted file mode 100644 index 3570607707f..00000000000 --- a/x-pack/metricbeat/module/cloudfoundry/util_test.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package cloudfoundry - -import ( - "math" - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/v7/libbeat/common" -) - -func TestHasNonNumericFloat(t *testing.T) { - type caseKey struct { - key string - expectedFound bool - expectedErr bool - } - cases := []struct { - title string - event common.MapStr - keys []caseKey - }{ - { - title: "Empty event", - event: common.MapStr{}, - keys: []caseKey{ - {"", false, true}, - {"somekey", false, true}, - }, - }, - { - title: "Event with non-numeric values", - event: common.MapStr{ - "someobject": common.MapStr{ - "inf": math.Inf(1), - "nan": math.NaN(), - "number": int64(42), - "float": float64(42), - }, - }, - keys: []caseKey{ - {"", false, true}, - {"someobject", false, false}, - {"someobject.inf", true, false}, - {"someobject.nan", true, false}, - {"someobject.number", false, false}, - {"someobject.float", false, false}, - {"someobject.notexists", false, true}, - }, - }, - } - - for _, c := range cases { - for _, k := range c.keys { - t.Run(c.title+"/"+k.key, func(t *testing.T) { - found, err := HasNonNumericFloat(c.event, k.key) - assert.Equal(t, k.expectedFound, found, "key has numeric float") - if k.expectedErr { - assert.Error(t, err) - } else { - assert.NoError(t, err) - } - }) - } - } -} diff --git a/x-pack/metricbeat/module/cloudfoundry/value/value.go b/x-pack/metricbeat/module/cloudfoundry/value/value.go index de2b3d60a69..62fb02f5e39 100644 --- a/x-pack/metricbeat/module/cloudfoundry/value/value.go +++ b/x-pack/metricbeat/module/cloudfoundry/value/value.go @@ -6,6 +6,7 @@ package value import ( "fmt" + "math" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/metricbeat/mb" @@ -51,11 +52,11 @@ type valueReporter struct { } func (r *valueReporter) Event(event mb.Event) bool { - found, err := cloudfoundry.HasNonNumericFloat(event.RootFields, "cloudfoundry.value.value") + value, err := event.RootFields.GetValue("cloudfoundry.value.value") if err != nil { r.logger.Debugf("Unexpected failure while checking for non-numeric values: %v", err) } - if found { + if value, ok := value.(float64); ok && (math.IsNaN(value) || math.IsInf(value, 0)) { r.logger.Debugf("Ignored event with float value that is not a number: %+v", event) return true }