From af4c0f9f8301e8701a31794bb819e660f71a33ff Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Thu, 7 Mar 2019 12:02:38 +0100 Subject: [PATCH] [Metricbeat] Migrate couchbase cluster & pool metricset to new data generation (#11112) The data.json is now generated based on example JSON documents. In case they change, it means the code change in some way. This will make it much easier to detect changes. Also the basic tests for the cluster are now covered by the new testing. The error handling moved over to the new error interface for simplicity. --- metricbeat/mb/testing/data_test.go | 2 + .../module/couchbase/cluster/_meta/data.json | 24 ++-- .../cluster/_meta/testdata/4.5.1.json | 1 + .../_meta/testdata/4.5.1.json-expected.json | 81 +++++++++++ .../cluster/_meta/testdata/config.yml | 2 + .../testdata/docs.json} | 0 .../_meta/testdata/docs.json-expected.json | 81 +++++++++++ .../cluster/cluster_integration_test.go | 15 -- .../module/couchbase/cluster/cluster_test.go | 120 ---------------- .../module/couchbase/node/_meta/data.json | 36 +++-- .../couchbase/node/_meta/testdata/4.5.1.json | 1 + .../_meta/testdata/4.5.1.json-expected.json | 89 ++++++++++++ .../couchbase/node/_meta/testdata/config.yml | 2 + .../testdata/docs.json} | 0 .../_meta/testdata/docs.json-expected.json | 89 ++++++++++++ .../couchbase/node/node_integration_test.go | 15 -- metricbeat/module/couchbase/node/node_test.go | 133 ------------------ 17 files changed, 374 insertions(+), 317 deletions(-) create mode 100644 metricbeat/module/couchbase/cluster/_meta/testdata/4.5.1.json create mode 100644 metricbeat/module/couchbase/cluster/_meta/testdata/4.5.1.json-expected.json create mode 100644 metricbeat/module/couchbase/cluster/_meta/testdata/config.yml rename metricbeat/module/couchbase/cluster/{testdata/sample_response.json => _meta/testdata/docs.json} (100%) create mode 100644 metricbeat/module/couchbase/cluster/_meta/testdata/docs.json-expected.json delete mode 100644 metricbeat/module/couchbase/cluster/cluster_test.go create mode 100644 metricbeat/module/couchbase/node/_meta/testdata/4.5.1.json create mode 100644 metricbeat/module/couchbase/node/_meta/testdata/4.5.1.json-expected.json create mode 100644 metricbeat/module/couchbase/node/_meta/testdata/config.yml rename metricbeat/module/couchbase/node/{testdata/sample_response.json => _meta/testdata/docs.json} (100%) create mode 100644 metricbeat/module/couchbase/node/_meta/testdata/docs.json-expected.json delete mode 100644 metricbeat/module/couchbase/node/node_test.go diff --git a/metricbeat/mb/testing/data_test.go b/metricbeat/mb/testing/data_test.go index 803bfec6d3c3..199dc28b3936 100644 --- a/metricbeat/mb/testing/data_test.go +++ b/metricbeat/mb/testing/data_test.go @@ -36,6 +36,8 @@ import ( "github.com/elastic/beats/metricbeat/mb" // TODO: generate include file for these tests automatically moving forward + _ "github.com/elastic/beats/metricbeat/module/couchbase/cluster" + _ "github.com/elastic/beats/metricbeat/module/couchbase/node" _ "github.com/elastic/beats/metricbeat/module/kibana/status" _ "github.com/elastic/beats/metricbeat/module/php_fpm/pool" _ "github.com/elastic/beats/metricbeat/module/php_fpm/process" diff --git a/metricbeat/module/couchbase/cluster/_meta/data.json b/metricbeat/module/couchbase/cluster/_meta/data.json index 461df930f17d..9ee9137dd7bd 100644 --- a/metricbeat/module/couchbase/cluster/_meta/data.json +++ b/metricbeat/module/couchbase/cluster/_meta/data.json @@ -1,29 +1,25 @@ { - "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, + "@timestamp": "2019-03-01T08:05:34.853Z", "couchbase": { "cluster": { "hdd": { "free": { - "bytes": 48341763359 + "bytes": 46902679716 }, "quota": { "total": { - "bytes": 185929859072 + "bytes": 63381999616 } }, "total": { - "bytes": 185929859072 + "bytes": 63381999616 }, "used": { "by_data": { - "bytes": 23665027 + "bytes": 16369010 }, "value": { - "bytes": 137588095713 + "bytes": 16479319900 } } }, @@ -56,14 +52,14 @@ } }, "total": { - "bytes": 16683503616 + "bytes": 8359174144 }, "used": { "by_data": { - "bytes": 53961776 + "bytes": 53962016 }, "value": { - "bytes": 14443474944 + "bytes": 8004751360 } } } @@ -78,7 +74,7 @@ "name": "cluster" }, "service": { - "address": "localhost:8091", + "address": "127.0.0.1:55555", "type": "couchbase" } } \ No newline at end of file diff --git a/metricbeat/module/couchbase/cluster/_meta/testdata/4.5.1.json b/metricbeat/module/couchbase/cluster/_meta/testdata/4.5.1.json new file mode 100644 index 000000000000..d620439f1382 --- /dev/null +++ b/metricbeat/module/couchbase/cluster/_meta/testdata/4.5.1.json @@ -0,0 +1 @@ +{"storageTotals":{"ram":{"total":6246748160,"quotaTotal":314572800,"quotaUsed":104857600,"used":3155247104,"usedByData":53962032,"quotaUsedPerNode":104857600,"quotaTotalPerNode":314572800},"hdd":{"total":62725623808,"quotaTotal":62725623808,"used":5645306142,"usedByData":17923953,"free":57080317666}},"ftsMemoryQuota":512,"indexMemoryQuota":300,"memoryQuota":300,"name":"default","alerts":[],"alertsSilenceURL":"/controller/resetAlerts?token=0&uuid=9179f751ee7308159a00974c5da5229f","nodes":[{"systemStats":{"cpu_utilization_rate":5.670103092783505,"swap_total":1073737728,"swap_used":0,"mem_total":6246748160,"mem_free":5239775232},"interestingStats":{"cmd_get":0,"couch_docs_actual_disk_size":17154348,"couch_docs_data_size":17140736,"couch_spatial_data_size":0,"couch_spatial_disk_size":0,"couch_views_actual_disk_size":769605,"couch_views_data_size":769605,"curr_items":7303,"curr_items_tot":7303,"ep_bg_fetched":0,"get_hits":0,"mem_used":53962032,"ops":0,"vb_replica_curr_items":0},"uptime":"30","memoryTotal":6246748160,"memoryFree":5239775232,"mcdMemoryReserved":4765,"mcdMemoryAllocated":4765,"couchApiBase":"http://172.17.0.2:8092/","couchApiBaseHTTPS":"https://172.17.0.2:18092/","otpCookie":"snyykghcidiksmqs","clusterMembership":"active","recoveryType":"none","status":"healthy","otpNode":"ns_1@127.0.0.1","thisNode":true,"hostname":"172.17.0.2:8091","clusterCompatibility":262149,"version":"4.5.1-2844-enterprise","os":"x86_64-unknown-linux-gnu","ports":{"sslProxy":11214,"httpsMgmt":18091,"httpsCAPI":18092,"proxy":11211,"direct":11210},"services":["index","kv","n1ql"]}],"buckets":{"uri":"/pools/default/buckets?v=42234267&uuid=9179f751ee7308159a00974c5da5229f","terseBucketsBase":"/pools/default/b/","terseStreamingBucketsBase":"/pools/default/bs/"},"remoteClusters":{"uri":"/pools/default/remoteClusters?uuid=9179f751ee7308159a00974c5da5229f","validateURI":"/pools/default/remoteClusters?just_validate=1"},"controllers":{"addNode":{"uri":"/controller/addNodeV2?uuid=9179f751ee7308159a00974c5da5229f"},"rebalance":{"uri":"/controller/rebalance?uuid=9179f751ee7308159a00974c5da5229f"},"failOver":{"uri":"/controller/failOver?uuid=9179f751ee7308159a00974c5da5229f"},"startGracefulFailover":{"uri":"/controller/startGracefulFailover?uuid=9179f751ee7308159a00974c5da5229f"},"reAddNode":{"uri":"/controller/reAddNode?uuid=9179f751ee7308159a00974c5da5229f"},"reFailOver":{"uri":"/controller/reFailOver?uuid=9179f751ee7308159a00974c5da5229f"},"ejectNode":{"uri":"/controller/ejectNode?uuid=9179f751ee7308159a00974c5da5229f"},"setRecoveryType":{"uri":"/controller/setRecoveryType?uuid=9179f751ee7308159a00974c5da5229f"},"setAutoCompaction":{"uri":"/controller/setAutoCompaction?uuid=9179f751ee7308159a00974c5da5229f","validateURI":"/controller/setAutoCompaction?just_validate=1"},"clusterLogsCollection":{"startURI":"/controller/startLogsCollection?uuid=9179f751ee7308159a00974c5da5229f","cancelURI":"/controller/cancelLogsCollection?uuid=9179f751ee7308159a00974c5da5229f"},"replication":{"createURI":"/controller/createReplication?uuid=9179f751ee7308159a00974c5da5229f","validateURI":"/controller/createReplication?just_validate=1"}},"rebalanceStatus":"none","rebalanceProgressUri":"/pools/default/rebalanceProgress","stopRebalanceUri":"/controller/stopRebalance?uuid=9179f751ee7308159a00974c5da5229f","nodeStatusesUri":"/nodeStatuses","maxBucketCount":10,"autoCompactionSettings":{"parallelDBAndViewCompaction":false,"databaseFragmentationThreshold":{"percentage":30,"size":"undefined"},"viewFragmentationThreshold":{"percentage":30,"size":"undefined"},"indexCompactionMode":"circular","indexCircularCompaction":{"daysOfWeek":"Sunday,Monday,Tuesday,Wednesday,Thursday,Friday,Saturday","interval":{"fromHour":0,"toHour":0,"fromMinute":0,"toMinute":0,"abortOutside":false}},"indexFragmentationThreshold":{"percentage":30}},"tasks":{"uri":"/pools/default/tasks?v=74113599"},"counters":{},"indexStatusURI":"/indexStatus?v=21137658","checkPermissionsURI":"/pools/default/checkPermissions?v=11181330","serverGroupsUri":"/pools/default/serverGroups?v=52184775"} diff --git a/metricbeat/module/couchbase/cluster/_meta/testdata/4.5.1.json-expected.json b/metricbeat/module/couchbase/cluster/_meta/testdata/4.5.1.json-expected.json new file mode 100644 index 000000000000..8dbbb27b4198 --- /dev/null +++ b/metricbeat/module/couchbase/cluster/_meta/testdata/4.5.1.json-expected.json @@ -0,0 +1,81 @@ +[ + { + "couchbase": { + "cluster": { + "hdd": { + "free": { + "bytes": 57080317666 + }, + "quota": { + "total": { + "bytes": 62725623808 + } + }, + "total": { + "bytes": 62725623808 + }, + "used": { + "by_data": { + "bytes": 17923953 + }, + "value": { + "bytes": 5645306142 + } + } + }, + "max_bucket_count": 10, + "quota": { + "index_memory": { + "mb": 300 + }, + "memory": { + "mb": 300 + } + }, + "ram": { + "quota": { + "total": { + "per_node": { + "bytes": 314572800 + }, + "value": { + "bytes": 314572800 + } + }, + "used": { + "per_node": { + "bytes": 104857600 + }, + "value": { + "bytes": 104857600 + } + } + }, + "total": { + "bytes": 6246748160 + }, + "used": { + "by_data": { + "bytes": 53962032 + }, + "value": { + "bytes": 3155247104 + } + } + } + } + }, + "event": { + "dataset": "couchbase.cluster", + "duration": 115000, + "module": "couchbase" + }, + "metricset": { + "name": "cluster" + }, + "service": { + "address": "127.0.0.1:55555", + "type": "couchbase" + } + } +] \ No newline at end of file diff --git a/metricbeat/module/couchbase/cluster/_meta/testdata/config.yml b/metricbeat/module/couchbase/cluster/_meta/testdata/config.yml new file mode 100644 index 000000000000..1351708ffb0c --- /dev/null +++ b/metricbeat/module/couchbase/cluster/_meta/testdata/config.yml @@ -0,0 +1,2 @@ +type: http +url: "/pools/default" diff --git a/metricbeat/module/couchbase/cluster/testdata/sample_response.json b/metricbeat/module/couchbase/cluster/_meta/testdata/docs.json similarity index 100% rename from metricbeat/module/couchbase/cluster/testdata/sample_response.json rename to metricbeat/module/couchbase/cluster/_meta/testdata/docs.json diff --git a/metricbeat/module/couchbase/cluster/_meta/testdata/docs.json-expected.json b/metricbeat/module/couchbase/cluster/_meta/testdata/docs.json-expected.json new file mode 100644 index 000000000000..093aaed8a871 --- /dev/null +++ b/metricbeat/module/couchbase/cluster/_meta/testdata/docs.json-expected.json @@ -0,0 +1,81 @@ +[ + { + "couchbase": { + "cluster": { + "hdd": { + "free": { + "bytes": 46902679716 + }, + "quota": { + "total": { + "bytes": 63381999616 + } + }, + "total": { + "bytes": 63381999616 + }, + "used": { + "by_data": { + "bytes": 16369010 + }, + "value": { + "bytes": 16479319900 + } + } + }, + "max_bucket_count": 10, + "quota": { + "index_memory": { + "mb": 300 + }, + "memory": { + "mb": 300 + } + }, + "ram": { + "quota": { + "total": { + "per_node": { + "bytes": 314572800 + }, + "value": { + "bytes": 314572800 + } + }, + "used": { + "per_node": { + "bytes": 104857600 + }, + "value": { + "bytes": 104857600 + } + } + }, + "total": { + "bytes": 8359174144 + }, + "used": { + "by_data": { + "bytes": 53962016 + }, + "value": { + "bytes": 8004751360 + } + } + } + } + }, + "event": { + "dataset": "couchbase.cluster", + "duration": 115000, + "module": "couchbase" + }, + "metricset": { + "name": "cluster" + }, + "service": { + "address": "127.0.0.1:55555", + "type": "couchbase" + } + } +] \ No newline at end of file diff --git a/metricbeat/module/couchbase/cluster/cluster_integration_test.go b/metricbeat/module/couchbase/cluster/cluster_integration_test.go index 72a94a530d31..dd1812388a58 100644 --- a/metricbeat/module/couchbase/cluster/cluster_integration_test.go +++ b/metricbeat/module/couchbase/cluster/cluster_integration_test.go @@ -42,21 +42,6 @@ func TestFetch(t *testing.T) { t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events[0]) } -func TestData(t *testing.T) { - compose.EnsureUp(t, "couchbase") - - f := mbtest.NewReportingMetricSetV2(t, getConfig()) - events, errs := mbtest.ReportingFetchV2(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - - assert.NotEmpty(t, events) - if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { - t.Fatal("write", err) - } -} - func getConfig() map[string]interface{} { return map[string]interface{}{ "module": "couchbase", diff --git a/metricbeat/module/couchbase/cluster/cluster_test.go b/metricbeat/module/couchbase/cluster/cluster_test.go deleted file mode 100644 index d828f3509a77..000000000000 --- a/metricbeat/module/couchbase/cluster/cluster_test.go +++ /dev/null @@ -1,120 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// +build !integration - -package cluster - -import ( - "io/ioutil" - "net/http" - "net/http/httptest" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/libbeat/common" - mbtest "github.com/elastic/beats/metricbeat/mb/testing" -) - -func TestFetchEventContents(t *testing.T) { - absPath, err := filepath.Abs("./testdata/") - assert.NoError(t, err) - - // response is a raw response from a couchbase - response, err := ioutil.ReadFile(absPath + "/sample_response.json") - assert.NoError(t, err) - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(200) - w.Header().Set("Content-Type", "application/json;") - w.Write([]byte(response)) - })) - defer server.Close() - - config := map[string]interface{}{ - "module": "couchbase", - "metricsets": []string{"cluster"}, - "hosts": []string{server.URL}, - } - - f := mbtest.NewReportingMetricSetV2(t, config) - events, errs := mbtest.ReportingFetchV2(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) - event := events[0].MetricSetFields - - t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event.StringToPrint()) - - hdd := event["hdd"].(common.MapStr) - hdd_free := hdd["free"].(common.MapStr) - assert.EqualValues(t, 46902679716, hdd_free["bytes"]) - - hdd_total := hdd["total"].(common.MapStr) - assert.EqualValues(t, 63381999616, hdd_total["bytes"]) - - hdd_used := hdd["used"].(common.MapStr) - hdd_used_value := hdd_used["value"].(common.MapStr) - assert.EqualValues(t, 16479319900, hdd_used_value["bytes"]) - - hdd_used_by_data := hdd_used["by_data"].(common.MapStr) - assert.EqualValues(t, 16369010, hdd_used_by_data["bytes"]) - - hdd_quota := hdd["quota"].(common.MapStr) - hdd_quota_total := hdd_quota["total"].(common.MapStr) - assert.EqualValues(t, 63381999616, hdd_quota_total["bytes"]) - - assert.EqualValues(t, 10, event["max_bucket_count"]) - - quota := event["quota"].(common.MapStr) - quota_index_memory := quota["index_memory"].(common.MapStr) - assert.EqualValues(t, 300, quota_index_memory["mb"]) - - quota_memory := quota["memory"].(common.MapStr) - assert.EqualValues(t, 300, quota_memory["mb"]) - - ram := event["ram"].(common.MapStr) - - ram_quota := ram["quota"].(common.MapStr) - - ram_quota_total := ram_quota["total"].(common.MapStr) - ram_quota_total_value := ram_quota_total["value"].(common.MapStr) - assert.EqualValues(t, 314572800, ram_quota_total_value["bytes"]) - - ram_quota_total_per_node := ram_quota_total["per_node"].(common.MapStr) - assert.EqualValues(t, 314572800, ram_quota_total_per_node["bytes"]) - - ram_quota_used := ram_quota["used"].(common.MapStr) - ram_quota_used_value := ram_quota_used["value"].(common.MapStr) - assert.EqualValues(t, 104857600, ram_quota_used_value["bytes"]) - - ram_quota_used_per_node := ram_quota_used["per_node"].(common.MapStr) - assert.EqualValues(t, 104857600, ram_quota_used_per_node["bytes"]) - - ram_total := ram["total"].(common.MapStr) - assert.EqualValues(t, 8359174144, ram_total["bytes"]) - - ram_used := ram["used"].(common.MapStr) - ram_used_value := ram_used["value"].(common.MapStr) - assert.EqualValues(t, 8004751360, ram_used_value["bytes"]) - - ram_used_by_data := ram_used["by_data"].(common.MapStr) - assert.EqualValues(t, 53962016, ram_used_by_data["bytes"]) -} diff --git a/metricbeat/module/couchbase/node/_meta/data.json b/metricbeat/module/couchbase/node/_meta/data.json index 750af9accd9a..58cc3b95835a 100644 --- a/metricbeat/module/couchbase/node/_meta/data.json +++ b/metricbeat/module/couchbase/node/_meta/data.json @@ -1,19 +1,15 @@ { - "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, + "@timestamp": "2019-03-01T08:05:34.853Z", "couchbase": { "node": { "cmd_get": 0, "couch": { "docs": { "data_size": { - "bytes": 19151872 + "bytes": 9792512 }, "disk_size": { - "bytes": 22895585 + "bytes": 13563791 } }, "spatial": { @@ -26,15 +22,15 @@ }, "views": { "data_size": { - "bytes": 769446 + "bytes": 2805219 }, "disk_size": { - "bytes": 769446 + "bytes": 2805219 } } }, "cpu_utilization_rate": { - "pct": 19.64735516372796 + "pct": 29.64705882352941 }, "current_items": { "total": 7303, @@ -42,37 +38,37 @@ }, "ep_bg_fetched": 0, "get_hits": 0, - "hostname": "172.21.0.2:8091", + "hostname": "172.17.0.2:8091", "mcd_memory": { "allocated": { - "bytes": 12728 + "bytes": 6377 }, "reserved": { - "bytes": 12728 + "bytes": 6377 } }, "memory": { "free": { - "bytes": 7447130112 + "bytes": 4678324224 }, "total": { - "bytes": 16683503616 + "bytes": 8359174144 }, "used": { - "bytes": 53961776 + "bytes": 53962016 } }, "ops": 0, "swap": { "total": { - "bytes": 2147479552 + "bytes": 4189057024 }, "used": { - "bytes": 0 + "bytes": 135168 } }, "uptime": { - "sec": 10612 + "sec": 7260 }, "vb_replica_curr_items": 0 } @@ -86,7 +82,7 @@ "name": "node" }, "service": { - "address": "localhost:8091", + "address": "127.0.0.1:55555", "type": "couchbase" } } \ No newline at end of file diff --git a/metricbeat/module/couchbase/node/_meta/testdata/4.5.1.json b/metricbeat/module/couchbase/node/_meta/testdata/4.5.1.json new file mode 100644 index 000000000000..d620439f1382 --- /dev/null +++ b/metricbeat/module/couchbase/node/_meta/testdata/4.5.1.json @@ -0,0 +1 @@ +{"storageTotals":{"ram":{"total":6246748160,"quotaTotal":314572800,"quotaUsed":104857600,"used":3155247104,"usedByData":53962032,"quotaUsedPerNode":104857600,"quotaTotalPerNode":314572800},"hdd":{"total":62725623808,"quotaTotal":62725623808,"used":5645306142,"usedByData":17923953,"free":57080317666}},"ftsMemoryQuota":512,"indexMemoryQuota":300,"memoryQuota":300,"name":"default","alerts":[],"alertsSilenceURL":"/controller/resetAlerts?token=0&uuid=9179f751ee7308159a00974c5da5229f","nodes":[{"systemStats":{"cpu_utilization_rate":5.670103092783505,"swap_total":1073737728,"swap_used":0,"mem_total":6246748160,"mem_free":5239775232},"interestingStats":{"cmd_get":0,"couch_docs_actual_disk_size":17154348,"couch_docs_data_size":17140736,"couch_spatial_data_size":0,"couch_spatial_disk_size":0,"couch_views_actual_disk_size":769605,"couch_views_data_size":769605,"curr_items":7303,"curr_items_tot":7303,"ep_bg_fetched":0,"get_hits":0,"mem_used":53962032,"ops":0,"vb_replica_curr_items":0},"uptime":"30","memoryTotal":6246748160,"memoryFree":5239775232,"mcdMemoryReserved":4765,"mcdMemoryAllocated":4765,"couchApiBase":"http://172.17.0.2:8092/","couchApiBaseHTTPS":"https://172.17.0.2:18092/","otpCookie":"snyykghcidiksmqs","clusterMembership":"active","recoveryType":"none","status":"healthy","otpNode":"ns_1@127.0.0.1","thisNode":true,"hostname":"172.17.0.2:8091","clusterCompatibility":262149,"version":"4.5.1-2844-enterprise","os":"x86_64-unknown-linux-gnu","ports":{"sslProxy":11214,"httpsMgmt":18091,"httpsCAPI":18092,"proxy":11211,"direct":11210},"services":["index","kv","n1ql"]}],"buckets":{"uri":"/pools/default/buckets?v=42234267&uuid=9179f751ee7308159a00974c5da5229f","terseBucketsBase":"/pools/default/b/","terseStreamingBucketsBase":"/pools/default/bs/"},"remoteClusters":{"uri":"/pools/default/remoteClusters?uuid=9179f751ee7308159a00974c5da5229f","validateURI":"/pools/default/remoteClusters?just_validate=1"},"controllers":{"addNode":{"uri":"/controller/addNodeV2?uuid=9179f751ee7308159a00974c5da5229f"},"rebalance":{"uri":"/controller/rebalance?uuid=9179f751ee7308159a00974c5da5229f"},"failOver":{"uri":"/controller/failOver?uuid=9179f751ee7308159a00974c5da5229f"},"startGracefulFailover":{"uri":"/controller/startGracefulFailover?uuid=9179f751ee7308159a00974c5da5229f"},"reAddNode":{"uri":"/controller/reAddNode?uuid=9179f751ee7308159a00974c5da5229f"},"reFailOver":{"uri":"/controller/reFailOver?uuid=9179f751ee7308159a00974c5da5229f"},"ejectNode":{"uri":"/controller/ejectNode?uuid=9179f751ee7308159a00974c5da5229f"},"setRecoveryType":{"uri":"/controller/setRecoveryType?uuid=9179f751ee7308159a00974c5da5229f"},"setAutoCompaction":{"uri":"/controller/setAutoCompaction?uuid=9179f751ee7308159a00974c5da5229f","validateURI":"/controller/setAutoCompaction?just_validate=1"},"clusterLogsCollection":{"startURI":"/controller/startLogsCollection?uuid=9179f751ee7308159a00974c5da5229f","cancelURI":"/controller/cancelLogsCollection?uuid=9179f751ee7308159a00974c5da5229f"},"replication":{"createURI":"/controller/createReplication?uuid=9179f751ee7308159a00974c5da5229f","validateURI":"/controller/createReplication?just_validate=1"}},"rebalanceStatus":"none","rebalanceProgressUri":"/pools/default/rebalanceProgress","stopRebalanceUri":"/controller/stopRebalance?uuid=9179f751ee7308159a00974c5da5229f","nodeStatusesUri":"/nodeStatuses","maxBucketCount":10,"autoCompactionSettings":{"parallelDBAndViewCompaction":false,"databaseFragmentationThreshold":{"percentage":30,"size":"undefined"},"viewFragmentationThreshold":{"percentage":30,"size":"undefined"},"indexCompactionMode":"circular","indexCircularCompaction":{"daysOfWeek":"Sunday,Monday,Tuesday,Wednesday,Thursday,Friday,Saturday","interval":{"fromHour":0,"toHour":0,"fromMinute":0,"toMinute":0,"abortOutside":false}},"indexFragmentationThreshold":{"percentage":30}},"tasks":{"uri":"/pools/default/tasks?v=74113599"},"counters":{},"indexStatusURI":"/indexStatus?v=21137658","checkPermissionsURI":"/pools/default/checkPermissions?v=11181330","serverGroupsUri":"/pools/default/serverGroups?v=52184775"} diff --git a/metricbeat/module/couchbase/node/_meta/testdata/4.5.1.json-expected.json b/metricbeat/module/couchbase/node/_meta/testdata/4.5.1.json-expected.json new file mode 100644 index 000000000000..5d4ea32844aa --- /dev/null +++ b/metricbeat/module/couchbase/node/_meta/testdata/4.5.1.json-expected.json @@ -0,0 +1,89 @@ +[ + { + "couchbase": { + "node": { + "cmd_get": 0, + "couch": { + "docs": { + "data_size": { + "bytes": 17140736 + }, + "disk_size": { + "bytes": 17154348 + } + }, + "spatial": { + "data_size": { + "bytes": 0 + }, + "disk_size": { + "bytes": 0 + } + }, + "views": { + "data_size": { + "bytes": 769605 + }, + "disk_size": { + "bytes": 769605 + } + } + }, + "cpu_utilization_rate": { + "pct": 5.670103092783505 + }, + "current_items": { + "total": 7303, + "value": 7303 + }, + "ep_bg_fetched": 0, + "get_hits": 0, + "hostname": "172.17.0.2:8091", + "mcd_memory": { + "allocated": { + "bytes": 4765 + }, + "reserved": { + "bytes": 4765 + } + }, + "memory": { + "free": { + "bytes": 5239775232 + }, + "total": { + "bytes": 6246748160 + }, + "used": { + "bytes": 53962032 + } + }, + "ops": 0, + "swap": { + "total": { + "bytes": 1073737728 + }, + "used": { + "bytes": 0 + } + }, + "uptime": { + "sec": 30 + }, + "vb_replica_curr_items": 0 + } + }, + "event": { + "dataset": "couchbase.node", + "duration": 115000, + "module": "couchbase" + }, + "metricset": { + "name": "node" + }, + "service": { + "address": "127.0.0.1:55555", + "type": "couchbase" + } + } +] \ No newline at end of file diff --git a/metricbeat/module/couchbase/node/_meta/testdata/config.yml b/metricbeat/module/couchbase/node/_meta/testdata/config.yml new file mode 100644 index 000000000000..1351708ffb0c --- /dev/null +++ b/metricbeat/module/couchbase/node/_meta/testdata/config.yml @@ -0,0 +1,2 @@ +type: http +url: "/pools/default" diff --git a/metricbeat/module/couchbase/node/testdata/sample_response.json b/metricbeat/module/couchbase/node/_meta/testdata/docs.json similarity index 100% rename from metricbeat/module/couchbase/node/testdata/sample_response.json rename to metricbeat/module/couchbase/node/_meta/testdata/docs.json diff --git a/metricbeat/module/couchbase/node/_meta/testdata/docs.json-expected.json b/metricbeat/module/couchbase/node/_meta/testdata/docs.json-expected.json new file mode 100644 index 000000000000..c43091bbf5b0 --- /dev/null +++ b/metricbeat/module/couchbase/node/_meta/testdata/docs.json-expected.json @@ -0,0 +1,89 @@ +[ + { + "couchbase": { + "node": { + "cmd_get": 0, + "couch": { + "docs": { + "data_size": { + "bytes": 9792512 + }, + "disk_size": { + "bytes": 13563791 + } + }, + "spatial": { + "data_size": { + "bytes": 0 + }, + "disk_size": { + "bytes": 0 + } + }, + "views": { + "data_size": { + "bytes": 2805219 + }, + "disk_size": { + "bytes": 2805219 + } + } + }, + "cpu_utilization_rate": { + "pct": 29.64705882352941 + }, + "current_items": { + "total": 7303, + "value": 7303 + }, + "ep_bg_fetched": 0, + "get_hits": 0, + "hostname": "172.17.0.2:8091", + "mcd_memory": { + "allocated": { + "bytes": 6377 + }, + "reserved": { + "bytes": 6377 + } + }, + "memory": { + "free": { + "bytes": 4678324224 + }, + "total": { + "bytes": 8359174144 + }, + "used": { + "bytes": 53962016 + } + }, + "ops": 0, + "swap": { + "total": { + "bytes": 4189057024 + }, + "used": { + "bytes": 135168 + } + }, + "uptime": { + "sec": 7260 + }, + "vb_replica_curr_items": 0 + } + }, + "event": { + "dataset": "couchbase.node", + "duration": 115000, + "module": "couchbase" + }, + "metricset": { + "name": "node" + }, + "service": { + "address": "127.0.0.1:55555", + "type": "couchbase" + } + } +] \ No newline at end of file diff --git a/metricbeat/module/couchbase/node/node_integration_test.go b/metricbeat/module/couchbase/node/node_integration_test.go index 9513e91cf7a0..3612e63238a2 100644 --- a/metricbeat/module/couchbase/node/node_integration_test.go +++ b/metricbeat/module/couchbase/node/node_integration_test.go @@ -42,21 +42,6 @@ func TestFetch(t *testing.T) { t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events[0]) } -func TestData(t *testing.T) { - compose.EnsureUp(t, "couchbase") - - f := mbtest.NewReportingMetricSetV2(t, getConfig()) - events, errs := mbtest.ReportingFetchV2(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - - assert.NotEmpty(t, events) - if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { - t.Fatal("write", err) - } -} - func getConfig() map[string]interface{} { return map[string]interface{}{ "module": "couchbase", diff --git a/metricbeat/module/couchbase/node/node_test.go b/metricbeat/module/couchbase/node/node_test.go deleted file mode 100644 index 293195de01ff..000000000000 --- a/metricbeat/module/couchbase/node/node_test.go +++ /dev/null @@ -1,133 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// +build !integration - -package node - -import ( - "io/ioutil" - "net/http" - "net/http/httptest" - "path/filepath" - "testing" - - "github.com/elastic/beats/libbeat/common" - mbtest "github.com/elastic/beats/metricbeat/mb/testing" - - "github.com/stretchr/testify/assert" -) - -func TestFetchEventContents(t *testing.T) { - absPath, err := filepath.Abs("./testdata/") - assert.NoError(t, err) - - // response is a raw response from a couchbase - response, err := ioutil.ReadFile(absPath + "/sample_response.json") - assert.NoError(t, err) - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(200) - w.Header().Set("Content-Type", "application/json;") - w.Write([]byte(response)) - })) - defer server.Close() - - config := map[string]interface{}{ - "module": "couchbase", - "metricsets": []string{"node"}, - "hosts": []string{server.URL}, - } - - f := mbtest.NewReportingMetricSetV2(t, config) - events, errs := mbtest.ReportingFetchV2(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - - assert.NotEmpty(t, events) - event := events[0].MetricSetFields - - t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event.StringToPrint()) - - assert.EqualValues(t, 0, event["cmd_get"]) - - couch := event["couch"].(common.MapStr) - - couch_docs := couch["docs"].(common.MapStr) - couch_docs_disk_size := couch_docs["disk_size"].(common.MapStr) - assert.EqualValues(t, 13563791, couch_docs_disk_size["bytes"]) - - couch_docs_data_size := couch_docs["data_size"].(common.MapStr) - assert.EqualValues(t, 9792512, couch_docs_data_size["bytes"]) - - couch_spatial := couch["spatial"].(common.MapStr) - couch_spatial_data_size := couch_spatial["data_size"].(common.MapStr) - assert.EqualValues(t, 0, couch_spatial_data_size["bytes"]) - - couch_spatial_disk_size := couch_spatial["disk_size"].(common.MapStr) - assert.EqualValues(t, 0, couch_spatial_disk_size["bytes"]) - - couch_views := couch["views"].(common.MapStr) - couch_views_disk_size := couch_views["disk_size"].(common.MapStr) - assert.EqualValues(t, 2805219, couch_views_disk_size["bytes"]) - - couch_views_data_size := couch_views["data_size"].(common.MapStr) - assert.EqualValues(t, 2805219, couch_views_data_size["bytes"]) - - event_cpu_utilization_rate := event["cpu_utilization_rate"].(common.MapStr) - assert.EqualValues(t, 29.64705882352941, event_cpu_utilization_rate["pct"]) - - current_items := event["current_items"].(common.MapStr) - assert.EqualValues(t, 7303, current_items["value"]) - assert.EqualValues(t, 7303, current_items["total"]) - - assert.EqualValues(t, 0, event["ep_bg_fetched"]) - assert.EqualValues(t, 0, event["get_hits"]) - assert.Equal(t, "172.17.0.2:8091", event["hostname"]) - - mcd_memory := event["mcd_memory"].(common.MapStr) - mcd_memory_reserved := mcd_memory["reserved"].(common.MapStr) - assert.EqualValues(t, 6377, mcd_memory_reserved["bytes"]) - - mcd_memory_allocated := mcd_memory["allocated"].(common.MapStr) - assert.EqualValues(t, 6377, mcd_memory_allocated["bytes"]) - - memory := event["memory"].(common.MapStr) - memory_total := memory["total"].(common.MapStr) - assert.EqualValues(t, 8359174144, memory_total["bytes"]) - - memory_free := memory["free"].(common.MapStr) - assert.EqualValues(t, 4678324224, memory_free["bytes"]) - - memory_used := memory["used"].(common.MapStr) - assert.EqualValues(t, 53962016, memory_used["bytes"]) - - assert.EqualValues(t, 0, event["ops"]) - - swap := event["swap"].(common.MapStr) - swap_total := swap["total"].(common.MapStr) - assert.EqualValues(t, 4189057024, swap_total["bytes"]) - - swap_used := swap["used"].(common.MapStr) - assert.EqualValues(t, 135168, swap_used["bytes"]) - - event_uptime := event["uptime"].(common.MapStr) - assert.EqualValues(t, 7260, event_uptime["sec"]) - - assert.EqualValues(t, 0, event["vb_replica_curr_items"]) -}