Skip to content

Commit

Permalink
[Metricbeat] Migrate couchbase cluster & pool metricset to new data g…
Browse files Browse the repository at this point in the history
…eneration

The data.json is now generated based on example JSON documents. In case they change, it means the code change in some way. This will make it much easier to detect changes.

Also the basic tests for the cluster are now covered by the new testing.

The error handling moved over to the new error interface for simplicity.
  • Loading branch information
ruflin committed Mar 7, 2019
1 parent 8d3911e commit 7843861
Show file tree
Hide file tree
Showing 19 changed files with 380 additions and 332 deletions.
2 changes: 2 additions & 0 deletions metricbeat/mb/testing/data_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ import (
"github.com/elastic/beats/metricbeat/mb"

// TODO: generate include file for these tests automatically moving forward
_ "github.com/elastic/beats/metricbeat/module/couchbase/cluster"
_ "github.com/elastic/beats/metricbeat/module/couchbase/node"
_ "github.com/elastic/beats/metricbeat/module/kibana/status"
_ "github.com/elastic/beats/metricbeat/module/php_fpm/pool"
_ "github.com/elastic/beats/metricbeat/module/php_fpm/process"
Expand Down
24 changes: 10 additions & 14 deletions metricbeat/module/couchbase/cluster/_meta/data.json
Original file line number Diff line number Diff line change
@@ -1,29 +1,25 @@
{
"@timestamp": "2017-10-12T08:05:34.853Z",
"agent": {
"hostname": "host.example.com",
"name": "host.example.com"
},
"@timestamp": "2019-03-01T08:05:34.853Z",
"couchbase": {
"cluster": {
"hdd": {
"free": {
"bytes": 48341763359
"bytes": 46902679716
},
"quota": {
"total": {
"bytes": 185929859072
"bytes": 63381999616
}
},
"total": {
"bytes": 185929859072
"bytes": 63381999616
},
"used": {
"by_data": {
"bytes": 23665027
"bytes": 16369010
},
"value": {
"bytes": 137588095713
"bytes": 16479319900
}
}
},
Expand Down Expand Up @@ -56,14 +52,14 @@
}
},
"total": {
"bytes": 16683503616
"bytes": 8359174144
},
"used": {
"by_data": {
"bytes": 53961776
"bytes": 53962016
},
"value": {
"bytes": 14443474944
"bytes": 8004751360
}
}
}
Expand All @@ -78,7 +74,7 @@
"name": "cluster"
},
"service": {
"address": "localhost:8091",
"address": "127.0.0.1:55555",
"type": "couchbase"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"storageTotals":{"ram":{"total":6246748160,"quotaTotal":314572800,"quotaUsed":104857600,"used":3155247104,"usedByData":53962032,"quotaUsedPerNode":104857600,"quotaTotalPerNode":314572800},"hdd":{"total":62725623808,"quotaTotal":62725623808,"used":5645306142,"usedByData":17923953,"free":57080317666}},"ftsMemoryQuota":512,"indexMemoryQuota":300,"memoryQuota":300,"name":"default","alerts":[],"alertsSilenceURL":"/controller/resetAlerts?token=0&uuid=9179f751ee7308159a00974c5da5229f","nodes":[{"systemStats":{"cpu_utilization_rate":5.670103092783505,"swap_total":1073737728,"swap_used":0,"mem_total":6246748160,"mem_free":5239775232},"interestingStats":{"cmd_get":0,"couch_docs_actual_disk_size":17154348,"couch_docs_data_size":17140736,"couch_spatial_data_size":0,"couch_spatial_disk_size":0,"couch_views_actual_disk_size":769605,"couch_views_data_size":769605,"curr_items":7303,"curr_items_tot":7303,"ep_bg_fetched":0,"get_hits":0,"mem_used":53962032,"ops":0,"vb_replica_curr_items":0},"uptime":"30","memoryTotal":6246748160,"memoryFree":5239775232,"mcdMemoryReserved":4765,"mcdMemoryAllocated":4765,"couchApiBase":"http://172.17.0.2:8092/","couchApiBaseHTTPS":"https://172.17.0.2:18092/","otpCookie":"snyykghcidiksmqs","clusterMembership":"active","recoveryType":"none","status":"healthy","otpNode":"[email protected]","thisNode":true,"hostname":"172.17.0.2:8091","clusterCompatibility":262149,"version":"4.5.1-2844-enterprise","os":"x86_64-unknown-linux-gnu","ports":{"sslProxy":11214,"httpsMgmt":18091,"httpsCAPI":18092,"proxy":11211,"direct":11210},"services":["index","kv","n1ql"]}],"buckets":{"uri":"/pools/default/buckets?v=42234267&uuid=9179f751ee7308159a00974c5da5229f","terseBucketsBase":"/pools/default/b/","terseStreamingBucketsBase":"/pools/default/bs/"},"remoteClusters":{"uri":"/pools/default/remoteClusters?uuid=9179f751ee7308159a00974c5da5229f","validateURI":"/pools/default/remoteClusters?just_validate=1"},"controllers":{"addNode":{"uri":"/controller/addNodeV2?uuid=9179f751ee7308159a00974c5da5229f"},"rebalance":{"uri":"/controller/rebalance?uuid=9179f751ee7308159a00974c5da5229f"},"failOver":{"uri":"/controller/failOver?uuid=9179f751ee7308159a00974c5da5229f"},"startGracefulFailover":{"uri":"/controller/startGracefulFailover?uuid=9179f751ee7308159a00974c5da5229f"},"reAddNode":{"uri":"/controller/reAddNode?uuid=9179f751ee7308159a00974c5da5229f"},"reFailOver":{"uri":"/controller/reFailOver?uuid=9179f751ee7308159a00974c5da5229f"},"ejectNode":{"uri":"/controller/ejectNode?uuid=9179f751ee7308159a00974c5da5229f"},"setRecoveryType":{"uri":"/controller/setRecoveryType?uuid=9179f751ee7308159a00974c5da5229f"},"setAutoCompaction":{"uri":"/controller/setAutoCompaction?uuid=9179f751ee7308159a00974c5da5229f","validateURI":"/controller/setAutoCompaction?just_validate=1"},"clusterLogsCollection":{"startURI":"/controller/startLogsCollection?uuid=9179f751ee7308159a00974c5da5229f","cancelURI":"/controller/cancelLogsCollection?uuid=9179f751ee7308159a00974c5da5229f"},"replication":{"createURI":"/controller/createReplication?uuid=9179f751ee7308159a00974c5da5229f","validateURI":"/controller/createReplication?just_validate=1"}},"rebalanceStatus":"none","rebalanceProgressUri":"/pools/default/rebalanceProgress","stopRebalanceUri":"/controller/stopRebalance?uuid=9179f751ee7308159a00974c5da5229f","nodeStatusesUri":"/nodeStatuses","maxBucketCount":10,"autoCompactionSettings":{"parallelDBAndViewCompaction":false,"databaseFragmentationThreshold":{"percentage":30,"size":"undefined"},"viewFragmentationThreshold":{"percentage":30,"size":"undefined"},"indexCompactionMode":"circular","indexCircularCompaction":{"daysOfWeek":"Sunday,Monday,Tuesday,Wednesday,Thursday,Friday,Saturday","interval":{"fromHour":0,"toHour":0,"fromMinute":0,"toMinute":0,"abortOutside":false}},"indexFragmentationThreshold":{"percentage":30}},"tasks":{"uri":"/pools/default/tasks?v=74113599"},"counters":{},"indexStatusURI":"/indexStatus?v=21137658","checkPermissionsURI":"/pools/default/checkPermissions?v=11181330","serverGroupsUri":"/pools/default/serverGroups?v=52184775"}
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
[
{
"couchbase": {
"cluster": {
"hdd": {
"free": {
"bytes": 57080317666
},
"quota": {
"total": {
"bytes": 62725623808
}
},
"total": {
"bytes": 62725623808
},
"used": {
"by_data": {
"bytes": 17923953
},
"value": {
"bytes": 5645306142
}
}
},
"max_bucket_count": 10,
"quota": {
"index_memory": {
"mb": 300
},
"memory": {
"mb": 300
}
},
"ram": {
"quota": {
"total": {
"per_node": {
"bytes": 314572800
},
"value": {
"bytes": 314572800
}
},
"used": {
"per_node": {
"bytes": 104857600
},
"value": {
"bytes": 104857600
}
}
},
"total": {
"bytes": 6246748160
},
"used": {
"by_data": {
"bytes": 53962032
},
"value": {
"bytes": 3155247104
}
}
}
}
},
"event": {
"dataset": "couchbase.cluster",
"duration": 115000,
"module": "couchbase"
},
"metricset": {
"name": "cluster"
},
"service": {
"address": "127.0.0.1:55555",
"type": "couchbase"
}
}
]
2 changes: 2 additions & 0 deletions metricbeat/module/couchbase/cluster/_meta/testdata/config.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
type: http
url: "/pools/default"
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
[
{
"couchbase": {
"cluster": {
"hdd": {
"free": {
"bytes": 46902679716
},
"quota": {
"total": {
"bytes": 63381999616
}
},
"total": {
"bytes": 63381999616
},
"used": {
"by_data": {
"bytes": 16369010
},
"value": {
"bytes": 16479319900
}
}
},
"max_bucket_count": 10,
"quota": {
"index_memory": {
"mb": 300
},
"memory": {
"mb": 300
}
},
"ram": {
"quota": {
"total": {
"per_node": {
"bytes": 314572800
},
"value": {
"bytes": 314572800
}
},
"used": {
"per_node": {
"bytes": 104857600
},
"value": {
"bytes": 104857600
}
}
},
"total": {
"bytes": 8359174144
},
"used": {
"by_data": {
"bytes": 53962016
},
"value": {
"bytes": 8004751360
}
}
}
}
},
"event": {
"dataset": "couchbase.cluster",
"duration": 115000,
"module": "couchbase"
},
"metricset": {
"name": "cluster"
},
"service": {
"address": "127.0.0.1:55555",
"type": "couchbase"
}
}
]
10 changes: 3 additions & 7 deletions metricbeat/module/couchbase/cluster/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
package cluster

import (
"github.com/elastic/beats/libbeat/logp"
"github.com/elastic/beats/metricbeat/helper"
"github.com/elastic/beats/metricbeat/mb"
"github.com/elastic/beats/metricbeat/mb/parse"
Expand All @@ -36,8 +35,6 @@ var (
}.Build()
)

var logger = logp.NewLogger("couchbase.cluster")

// init registers the MetricSet with the central registry.
// The New method will be called after the setup of the module and before starting to fetch data
func init() {
Expand Down Expand Up @@ -68,15 +65,14 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
// Fetch methods implements the data gathering and data conversion to the right
// format. It publishes the event which is then forwarded to the output. In case
// of an error set the Error field of mb.Event or simply call report.Error().
func (m *MetricSet) Fetch(reporter mb.ReporterV2) {
func (m *MetricSet) Fetch(reporter mb.ReporterV2) error {
content, err := m.http.FetchContent()
if err != nil {
logger.Error(err)
reporter.Error(err)
return
return err
}

reporter.Event(mb.Event{
MetricSetFields: eventMapping(content),
})
return nil
}
15 changes: 0 additions & 15 deletions metricbeat/module/couchbase/cluster/cluster_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,21 +42,6 @@ func TestFetch(t *testing.T) {
t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events[0])
}

func TestData(t *testing.T) {
compose.EnsureUp(t, "couchbase")

f := mbtest.NewReportingMetricSetV2(t, getConfig())
events, errs := mbtest.ReportingFetchV2(f)
if len(errs) > 0 {
t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs)
}

assert.NotEmpty(t, events)
if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil {
t.Fatal("write", err)
}
}

func getConfig() map[string]interface{} {
return map[string]interface{}{
"module": "couchbase",
Expand Down
Loading

0 comments on commit 7843861

Please sign in to comment.