From b95db65c5a8f38f987573439578715cf4132b453 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 15 Jul 2021 11:06:30 +0200 Subject: [PATCH 1/5] Use common host parser in vsphere module --- CHANGELOG.next.asciidoc | 1 + .../module/vsphere/datastore/datastore.go | 30 ++-------- metricbeat/module/vsphere/host/host.go | 30 ++-------- metricbeat/module/vsphere/metricset.go | 59 +++++++++++++++++++ metricbeat/module/vsphere/test_vsphere.py | 6 ++ .../vsphere/virtualmachine/virtualmachine.go | 31 ++++------ 6 files changed, 89 insertions(+), 68 deletions(-) create mode 100644 metricbeat/module/vsphere/metricset.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 914324833c5..203b5ff5b95 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -140,6 +140,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added `statsd.mappings` configuration for Statsd module {pull}26220[26220] - Added Airflow lightweight module {pull}26220[26220] - Add state_job metricset to Kubernetes module{pull}26479[26479] +- Recover source.address field in vsphere module {issue}26902[26902] {pull}[] *Packetbeat* diff --git a/metricbeat/module/vsphere/datastore/datastore.go b/metricbeat/module/vsphere/datastore/datastore.go index f794384d453..8f911aa5848 100644 --- a/metricbeat/module/vsphere/datastore/datastore.go +++ b/metricbeat/module/vsphere/datastore/datastore.go @@ -19,12 +19,12 @@ package datastore import ( "context" - "net/url" "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/vsphere" "github.com/vmware/govmomi" "github.com/vmware/govmomi/view" @@ -33,41 +33,23 @@ import ( func init() { mb.Registry.MustAddMetricSet("vsphere", "datastore", New, + mb.WithHostParser(vsphere.HostParser), mb.DefaultMetricSet(), ) } // MetricSet type defines all fields of the MetricSet type MetricSet struct { - mb.BaseMetricSet - HostURL *url.URL - Insecure bool + *vsphere.MetricSet } -// New create a new instance of the MetricSet +// NewMetricSet create a new instance of the MetricSet func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - config := struct { - Username string `config:"username"` - Password string `config:"password"` - Insecure bool `config:"insecure"` - }{} - - if err := base.Module().UnpackConfig(&config); err != nil { - return nil, err - } - - u, err := url.Parse(base.HostData().URI) + ms, err := vsphere.NewMetricSet(base) if err != nil { return nil, err } - - u.User = url.UserPassword(config.Username, config.Password) - - return &MetricSet{ - BaseMetricSet: base, - HostURL: u, - Insecure: config.Insecure, - }, nil + return &MetricSet{ms}, nil } // Fetch methods implements the data gathering and data conversion to the right diff --git a/metricbeat/module/vsphere/host/host.go b/metricbeat/module/vsphere/host/host.go index e8bc5d090bb..c294f7c9c26 100644 --- a/metricbeat/module/vsphere/host/host.go +++ b/metricbeat/module/vsphere/host/host.go @@ -20,13 +20,13 @@ package host import ( "context" "fmt" - "net/url" "strings" "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/vsphere" "github.com/vmware/govmomi" "github.com/vmware/govmomi/property" @@ -38,41 +38,23 @@ import ( func init() { mb.Registry.MustAddMetricSet("vsphere", "host", New, + mb.WithHostParser(vsphere.HostParser), mb.DefaultMetricSet(), ) } // MetricSet type defines all fields of the MetricSet type MetricSet struct { - mb.BaseMetricSet - HostURL *url.URL - Insecure bool + *vsphere.MetricSet } -// New create a new instance of the MetricSet +// NewMetricSet create a new instance of the MetricSet func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - config := struct { - Username string `config:"username"` - Password string `config:"password"` - Insecure bool `config:"insecure"` - }{} - - if err := base.Module().UnpackConfig(&config); err != nil { - return nil, err - } - - u, err := url.Parse(base.HostData().URI) + ms, err := vsphere.NewMetricSet(base) if err != nil { return nil, err } - - u.User = url.UserPassword(config.Username, config.Password) - - return &MetricSet{ - BaseMetricSet: base, - HostURL: u, - Insecure: config.Insecure, - }, nil + return &MetricSet{ms}, nil } // Fetch methods implements the data gathering and data conversion to the right diff --git a/metricbeat/module/vsphere/metricset.go b/metricbeat/module/vsphere/metricset.go new file mode 100644 index 00000000000..b95f1e77a1d --- /dev/null +++ b/metricbeat/module/vsphere/metricset.go @@ -0,0 +1,59 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package vsphere + +import ( + "net/url" + + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/mb/parse" +) + +var HostParser = parse.URLHostParserBuilder{ + DefaultScheme: "https", + DefaultPath: "/sdk", +}.Build() + +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + mb.BaseMetricSet + Insecure bool + HostURL *url.URL +} + +// NewMetricSet create a new instance of the MetricSet +func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { + config := struct { + Insecure bool `config:"insecure"` + }{} + + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + u, err := url.Parse(base.HostData().URI) + if err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + HostURL: u, + Insecure: config.Insecure, + }, nil +} diff --git a/metricbeat/module/vsphere/test_vsphere.py b/metricbeat/module/vsphere/test_vsphere.py index c0c2b0db4bb..edc06a16c9f 100644 --- a/metricbeat/module/vsphere/test_vsphere.py +++ b/metricbeat/module/vsphere/test_vsphere.py @@ -42,6 +42,8 @@ def test_datastore(self): self.assertCountEqual(self.de_dot(VSPHERE_FIELDS), evt.keys(), evt) + self.assertEqual(evt["service"]["address"], self.get_hosts()[0]) + self.assert_fields_are_documented(evt) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") @@ -71,6 +73,8 @@ def test_host(self): self.assertCountEqual(self.de_dot(VSPHERE_FIELDS), evt.keys(), evt) + self.assertEqual(evt["service"]["address"], self.get_hosts()[0]) + self.assert_fields_are_documented(evt) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") @@ -100,4 +104,6 @@ def test_virtualmachine(self): self.assertCountEqual(self.de_dot(VSPHERE_FIELDS), evt.keys(), evt) + self.assertEqual(evt["service"]["address"], self.get_hosts()[0]) + self.assert_fields_are_documented(evt) diff --git a/metricbeat/module/vsphere/virtualmachine/virtualmachine.go b/metricbeat/module/vsphere/virtualmachine/virtualmachine.go index 938bf91c97b..0e9da7fc2d6 100644 --- a/metricbeat/module/vsphere/virtualmachine/virtualmachine.go +++ b/metricbeat/module/vsphere/virtualmachine/virtualmachine.go @@ -20,11 +20,11 @@ package virtualmachine import ( "context" "fmt" - "net/url" "strings" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/vsphere" "github.com/pkg/errors" "github.com/vmware/govmomi" @@ -38,25 +38,26 @@ import ( func init() { mb.Registry.MustAddMetricSet("vsphere", "virtualmachine", New, + mb.WithHostParser(vsphere.HostParser), mb.DefaultMetricSet(), ) } // MetricSet type defines all fields of the MetricSet type MetricSet struct { - mb.BaseMetricSet - HostURL *url.URL - Insecure bool + *vsphere.MetricSet GetCustomFields bool } -// New create a new instance of the MetricSet +// NewMetricSet create a new instance of the MetricSet func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + ms, err := vsphere.NewMetricSet(base) + if err != nil { + return nil, err + } + config := struct { - Username string `config:"username"` - Password string `config:"password"` - Insecure bool `config:"insecure"` - GetCustomFields bool `config:"get_custom_fields"` + GetCustomFields bool `config:"get_custom_fields"` }{ GetCustomFields: false, } @@ -64,18 +65,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } - - u, err := url.Parse(base.HostData().URI) - if err != nil { - return nil, err - } - - u.User = url.UserPassword(config.Username, config.Password) - return &MetricSet{ - BaseMetricSet: base, - HostURL: u, - Insecure: config.Insecure, + MetricSet: ms, GetCustomFields: config.GetCustomFields, }, nil } From 0a79ec53890e13a4d87fdb60993793436bb6bedf Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 15 Jul 2021 11:15:45 +0200 Subject: [PATCH 2/5] Fix changelog --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 203b5ff5b95..73ad5efeeed 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -140,7 +140,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added `statsd.mappings` configuration for Statsd module {pull}26220[26220] - Added Airflow lightweight module {pull}26220[26220] - Add state_job metricset to Kubernetes module{pull}26479[26479] -- Recover source.address field in vsphere module {issue}26902[26902] {pull}[] +- Recover source.address field in vsphere module {issue}26902[26902] {pull}26904[26904] *Packetbeat* From 1864a542c8fae67a0e172f2aed8d2f4783112b35 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 15 Jul 2021 11:16:13 +0200 Subject: [PATCH 3/5] Fix changelog again --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 73ad5efeeed..b4a39b8182f 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -140,7 +140,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added `statsd.mappings` configuration for Statsd module {pull}26220[26220] - Added Airflow lightweight module {pull}26220[26220] - Add state_job metricset to Kubernetes module{pull}26479[26479] -- Recover source.address field in vsphere module {issue}26902[26902] {pull}26904[26904] +- Recover service.address field in vsphere module {issue}26902[26902] {pull}26904[26904] *Packetbeat* From c7ccec553461cefa11bae5cf17666d2510d47173 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 15 Jul 2021 11:17:12 +0200 Subject: [PATCH 4/5] Regenerate data.json --- .../module/vsphere/datastore/_meta/data.json | 10 +++++----- metricbeat/module/vsphere/host/_meta/data.json | 15 +++++++++------ .../module/vsphere/virtualmachine/_meta/data.json | 5 +---- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/metricbeat/module/vsphere/datastore/_meta/data.json b/metricbeat/module/vsphere/datastore/_meta/data.json index fb290513a92..944ecb1189f 100644 --- a/metricbeat/module/vsphere/datastore/_meta/data.json +++ b/metricbeat/module/vsphere/datastore/_meta/data.json @@ -10,21 +10,21 @@ "period": 10000 }, "service": { - "address": "http://127.0.0.1:51027/sdk", + "address": "127.0.0.1:33365", "type": "vsphere" }, "vsphere": { "datastore": { "capacity": { "free": { - "bytes": 267958476800 + "bytes": 37120094208 }, "total": { - "bytes": 484736266240 + "bytes": 74686664704 }, "used": { - "bytes": 216777789440, - "pct": 0.44720769733509097 + "bytes": 37566570496, + "pct": 0.502988996026061 } }, "fstype": "local", diff --git a/metricbeat/module/vsphere/host/_meta/data.json b/metricbeat/module/vsphere/host/_meta/data.json index c1948cdc304..05443939f10 100644 --- a/metricbeat/module/vsphere/host/_meta/data.json +++ b/metricbeat/module/vsphere/host/_meta/data.json @@ -1,14 +1,17 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { - "hostname": "host.example.com", - "name": "host.example.com" + "event": { + "dataset": "vsphere.host", + "duration": 115000, + "module": "vsphere" }, "metricset": { - "host": "http://127.0.0.1:43843/sdk", - "module": "vsphere", "name": "host", - "rtt": 115 + "period": 10000 + }, + "service": { + "address": "127.0.0.1:38517", + "type": "vsphere" }, "vsphere": { "host": { diff --git a/metricbeat/module/vsphere/virtualmachine/_meta/data.json b/metricbeat/module/vsphere/virtualmachine/_meta/data.json index 4dfa302758c..2dea5c91363 100644 --- a/metricbeat/module/vsphere/virtualmachine/_meta/data.json +++ b/metricbeat/module/vsphere/virtualmachine/_meta/data.json @@ -10,7 +10,7 @@ "period": 10000 }, "service": { - "address": "http://127.0.0.1:37231/sdk", + "address": "127.0.0.1:39149", "type": "vsphere" }, "vsphere": { @@ -43,9 +43,6 @@ } }, "name": "ha-host_VM0", - "network_names": [ - "VM Network" - ], "os": "otherGuest" } } From 761236676843a36b1ca4bc953030afc5b20aef14 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 15 Jul 2021 11:30:35 +0200 Subject: [PATCH 5/5] Fix comments --- metricbeat/module/vsphere/datastore/datastore.go | 4 ++-- metricbeat/module/vsphere/host/host.go | 4 ++-- metricbeat/module/vsphere/metricset.go | 4 ++-- metricbeat/module/vsphere/virtualmachine/virtualmachine.go | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/metricbeat/module/vsphere/datastore/datastore.go b/metricbeat/module/vsphere/datastore/datastore.go index 8f911aa5848..f37171c65a3 100644 --- a/metricbeat/module/vsphere/datastore/datastore.go +++ b/metricbeat/module/vsphere/datastore/datastore.go @@ -38,12 +38,12 @@ func init() { ) } -// MetricSet type defines all fields of the MetricSet +// MetricSet type defines all fields of the MetricSet. type MetricSet struct { *vsphere.MetricSet } -// NewMetricSet create a new instance of the MetricSet +// New creates a new instance of the MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { ms, err := vsphere.NewMetricSet(base) if err != nil { diff --git a/metricbeat/module/vsphere/host/host.go b/metricbeat/module/vsphere/host/host.go index c294f7c9c26..6f7d19b5d57 100644 --- a/metricbeat/module/vsphere/host/host.go +++ b/metricbeat/module/vsphere/host/host.go @@ -43,12 +43,12 @@ func init() { ) } -// MetricSet type defines all fields of the MetricSet +// MetricSet type defines all fields of the MetricSet. type MetricSet struct { *vsphere.MetricSet } -// NewMetricSet create a new instance of the MetricSet +// New creates a new instance of the MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { ms, err := vsphere.NewMetricSet(base) if err != nil { diff --git a/metricbeat/module/vsphere/metricset.go b/metricbeat/module/vsphere/metricset.go index b95f1e77a1d..c7089b52ac7 100644 --- a/metricbeat/module/vsphere/metricset.go +++ b/metricbeat/module/vsphere/metricset.go @@ -29,14 +29,14 @@ var HostParser = parse.URLHostParserBuilder{ DefaultPath: "/sdk", }.Build() -// MetricSet type defines all fields of the MetricSet +// MetricSet type defines all fields of the MetricSet. type MetricSet struct { mb.BaseMetricSet Insecure bool HostURL *url.URL } -// NewMetricSet create a new instance of the MetricSet +// NewMetricSet creates a new instance of the MetricSet. func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { config := struct { Insecure bool `config:"insecure"` diff --git a/metricbeat/module/vsphere/virtualmachine/virtualmachine.go b/metricbeat/module/vsphere/virtualmachine/virtualmachine.go index 0e9da7fc2d6..13af0168765 100644 --- a/metricbeat/module/vsphere/virtualmachine/virtualmachine.go +++ b/metricbeat/module/vsphere/virtualmachine/virtualmachine.go @@ -43,13 +43,13 @@ func init() { ) } -// MetricSet type defines all fields of the MetricSet +// MetricSet type defines all fields of the MetricSet. type MetricSet struct { *vsphere.MetricSet GetCustomFields bool } -// NewMetricSet create a new instance of the MetricSet +// New creates a new instance of the MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { ms, err := vsphere.NewMetricSet(base) if err != nil {