From 9f1787c545d6832cf20af19e9419f26afc7219f0 Mon Sep 17 00:00:00 2001 From: ruflin Date: Tue, 7 Feb 2017 07:52:17 +0100 Subject: [PATCH] Improve couchbase module * Fix typo spacial to spatial. Change fieldnames couchbase.node.couch.*.actual_disk_size.* to couchbase.node.couch.*.disk_size.* for consistency. * Add system tests to verify docs * Update data.json --- CHANGELOG.asciidoc | 1 + metricbeat/docs/fields.asciidoc | 4 +- metricbeat/metricbeat.template-es2x.json | 8 +- metricbeat/metricbeat.template.json | 8 +- .../module/couchbase/bucket/_meta/data.json | 6 +- .../module/couchbase/cluster/_meta/data.json | 16 ++-- .../module/couchbase/node/_meta/data.json | 38 +++++----- .../module/couchbase/node/_meta/fields.yml | 4 +- metricbeat/module/couchbase/node/data.go | 6 +- metricbeat/module/couchbase/node/node_test.go | 18 ++--- metricbeat/module/couchbase/testing.go | 7 +- metricbeat/tests/system/test_couchbase.py | 75 +++++++++++++++++++ 12 files changed, 136 insertions(+), 55 deletions(-) create mode 100644 metricbeat/tests/system/test_couchbase.py diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 3453fc6086a..524dbbf4ad8 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -20,6 +20,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff] - Linux cgroup metrics are now enabled by default for the system process metricset. The configuration option for the feature was renamed from `cgroups` to `process.cgroups.enabled`. {pull}3519[3519] +- Change fieldnames couchbase.node.couch.*.actual_disk_size.* to couchbase.node.couch.*.disk_size.* {pull}3545[3545] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index c4f1040b32c..b02fc71be99 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -1059,7 +1059,7 @@ Number of get commands [float] -=== couchbase.node.couch.docs.actual_disk_size.bytes +=== couchbase.node.couch.docs.disk_size.bytes type: long @@ -1095,7 +1095,7 @@ couchSpatialDiskSize field [float] -=== couchbase.node.couch.views.actual_disk_size.bytes +=== couchbase.node.couch.views.disk_size.bytes type: long diff --git a/metricbeat/metricbeat.template-es2x.json b/metricbeat/metricbeat.template-es2x.json index 03cdfebdf0d..bf7b50bf72b 100644 --- a/metricbeat/metricbeat.template-es2x.json +++ b/metricbeat/metricbeat.template-es2x.json @@ -589,14 +589,14 @@ "properties": { "docs": { "properties": { - "actual_disk_size": { + "data_size": { "properties": { "bytes": { "type": "long" } } }, - "data_size": { + "disk_size": { "properties": { "bytes": { "type": "long" @@ -625,14 +625,14 @@ }, "views": { "properties": { - "actual_disk_size": { + "data_size": { "properties": { "bytes": { "type": "long" } } }, - "data_size": { + "disk_size": { "properties": { "bytes": { "type": "long" diff --git a/metricbeat/metricbeat.template.json b/metricbeat/metricbeat.template.json index 316dae56668..eaa515f51b9 100644 --- a/metricbeat/metricbeat.template.json +++ b/metricbeat/metricbeat.template.json @@ -584,14 +584,14 @@ "properties": { "docs": { "properties": { - "actual_disk_size": { + "data_size": { "properties": { "bytes": { "type": "long" } } }, - "data_size": { + "disk_size": { "properties": { "bytes": { "type": "long" @@ -620,14 +620,14 @@ }, "views": { "properties": { - "actual_disk_size": { + "data_size": { "properties": { "bytes": { "type": "long" } } }, - "data_size": { + "disk_size": { "properties": { "bytes": { "type": "long" diff --git a/metricbeat/module/couchbase/bucket/_meta/data.json b/metricbeat/module/couchbase/bucket/_meta/data.json index 16f70befbbe..bab5ccb47a7 100644 --- a/metricbeat/module/couchbase/bucket/_meta/data.json +++ b/metricbeat/module/couchbase/bucket/_meta/data.json @@ -8,13 +8,13 @@ "bucket": { "data": { "used": { - "bytes": 12597798 + "bytes": 12585517 } }, "disk": { "fetches": 0, "used": { - "bytes": 16368995 + "bytes": 16356812 } }, "item_count": 7303, @@ -36,7 +36,7 @@ } }, "metricset": { - "host": "couchbase:8091", + "host": "localhost:8091", "module": "couchbase", "name": "bucket", "rtt": 115 diff --git a/metricbeat/module/couchbase/cluster/_meta/data.json b/metricbeat/module/couchbase/cluster/_meta/data.json index f090375706a..764ddff37d7 100644 --- a/metricbeat/module/couchbase/cluster/_meta/data.json +++ b/metricbeat/module/couchbase/cluster/_meta/data.json @@ -8,22 +8,22 @@ "cluster": { "hdd": { "free": { - "bytes": 50705599693 + "bytes": 49181251462 }, "quota": { "total": { - "bytes": 63381999616 + "bytes": 67371577344 } }, "total": { - "bytes": 63381999616 + "bytes": 67371577344 }, "used": { "by_data": { - "bytes": 16368995 + "bytes": 16356812 }, "value": { - "bytes": 12676399923 + "bytes": 18190325882 } } }, @@ -56,21 +56,21 @@ } }, "total": { - "bytes": 8360542208 + "bytes": 2096275456 }, "used": { "by_data": { "bytes": 53962160 }, "value": { - "bytes": 7660535808 + "bytes": 1235513344 } } } } }, "metricset": { - "host": "couchbase:8091", + "host": "localhost:8091", "module": "couchbase", "name": "cluster", "rtt": 115 diff --git a/metricbeat/module/couchbase/node/_meta/data.json b/metricbeat/module/couchbase/node/_meta/data.json index 0511a104e8e..e9158740302 100644 --- a/metricbeat/module/couchbase/node/_meta/data.json +++ b/metricbeat/module/couchbase/node/_meta/data.json @@ -9,14 +9,14 @@ "cmd_get": 0, "couch": { "docs": { - "actual_disk_size": { - "bytes": 13575997 - }, "data_size": { - "bytes": 9804800 + "bytes": 9796608 + }, + "disk_size": { + "bytes": 13567903 } }, - "spacial": { + "spatial": { "data_size": { "bytes": 0 }, @@ -25,16 +25,16 @@ } }, "views": { - "actual_disk_size": { - "bytes": 2792998 - }, "data_size": { - "bytes": 2792998 + "bytes": 2788909 + }, + "disk_size": { + "bytes": 2788909 } } }, "cpu_utilization_rate": { - "pct": 87.08791 + "pct": 4.568527918781726 }, "current_items": { "total": 7303, @@ -42,21 +42,21 @@ }, "ep_bg_fetched": 0, "get_hits": 0, - "hostname": "172.22.0.9:8091", + "hostname": "172.20.0.2:8091", "mcd_memory": { "allocated": { - "bytes": 6378 + "bytes": 1599 }, "reserved": { - "bytes": 6378 + "bytes": 1599 } }, "memory": { "free": { - "bytes": 2273726464 + "bytes": 1435561984 }, "total": { - "bytes": 8360542208 + "bytes": 2096275456 }, "used": { "bytes": 53962160 @@ -65,20 +65,20 @@ "ops": 0, "swap": { "total": { - "bytes": 4189057024 + "bytes": 0 }, "used": { - "bytes": 58044416 + "bytes": 0 } }, "uptime": { - "sec": 50492 + "sec": 891 }, "vb_replica_curr_items": 0 } }, "metricset": { - "host": "couchbase:8091", + "host": "localhost:8091", "module": "couchbase", "name": "node", "rtt": 115 diff --git a/metricbeat/module/couchbase/node/_meta/fields.yml b/metricbeat/module/couchbase/node/_meta/fields.yml index b0e7cd58869..a0e435d6403 100644 --- a/metricbeat/module/couchbase/node/_meta/fields.yml +++ b/metricbeat/module/couchbase/node/_meta/fields.yml @@ -9,7 +9,7 @@ type: long description: > Number of get commands - - name: couch.docs.actual_disk_size.bytes + - name: couch.docs.disk_size.bytes format: bytes type: long description: > @@ -27,7 +27,7 @@ type: long description: > couchSpatialDiskSize field - - name: couch.views.actual_disk_size.bytes + - name: couch.views.disk_size.bytes type: long description: > Amount of disk space occupied by Couch views (bytes) diff --git a/metricbeat/module/couchbase/node/data.go b/metricbeat/module/couchbase/node/data.go index c1d34e165ce..e57744ae8ec 100644 --- a/metricbeat/module/couchbase/node/data.go +++ b/metricbeat/module/couchbase/node/data.go @@ -73,14 +73,14 @@ func eventsMapping(content []byte) []common.MapStr { "cmd_get": NodeItem.InterestingStats.CmdGet, "couch": common.MapStr{ "docs": common.MapStr{ - "actual_disk_size": common.MapStr{ + "disk_size": common.MapStr{ "bytes": NodeItem.InterestingStats.CouchDocsActualDiskSize, }, "data_size": common.MapStr{ "bytes": NodeItem.InterestingStats.CouchDocsDataSize, }, }, - "spacial": common.MapStr{ + "spatial": common.MapStr{ "data_size": common.MapStr{ "bytes": NodeItem.InterestingStats.CouchSpatialDataSize, }, @@ -89,7 +89,7 @@ func eventsMapping(content []byte) []common.MapStr { }, }, "views": common.MapStr{ - "actual_disk_size": common.MapStr{ + "disk_size": common.MapStr{ "bytes": NodeItem.InterestingStats.CouchViewsActualDiskSize, }, "data_size": common.MapStr{ diff --git a/metricbeat/module/couchbase/node/node_test.go b/metricbeat/module/couchbase/node/node_test.go index 22f7ba874d4..21f05922c29 100644 --- a/metricbeat/module/couchbase/node/node_test.go +++ b/metricbeat/module/couchbase/node/node_test.go @@ -46,22 +46,22 @@ func TestFetchEventContents(t *testing.T) { couch := event["couch"].(common.MapStr) couch_docs := couch["docs"].(common.MapStr) - couch_docs_actual_disk_size := couch_docs["actual_disk_size"].(common.MapStr) - assert.EqualValues(t, 13563791, couch_docs_actual_disk_size["bytes"]) + couch_docs_disk_size := couch_docs["disk_size"].(common.MapStr) + assert.EqualValues(t, 13563791, couch_docs_disk_size["bytes"]) couch_docs_data_size := couch_docs["data_size"].(common.MapStr) assert.EqualValues(t, 9792512, couch_docs_data_size["bytes"]) - couch_spacial := couch["spacial"].(common.MapStr) - couch_spacial_data_size := couch_spacial["data_size"].(common.MapStr) - assert.EqualValues(t, 0, couch_spacial_data_size["bytes"]) + couch_spatial := couch["spatial"].(common.MapStr) + couch_spatial_data_size := couch_spatial["data_size"].(common.MapStr) + assert.EqualValues(t, 0, couch_spatial_data_size["bytes"]) - couch_spacial_disk_size := couch_spacial["disk_size"].(common.MapStr) - assert.EqualValues(t, 0, couch_spacial_disk_size["bytes"]) + couch_spatial_disk_size := couch_spatial["disk_size"].(common.MapStr) + assert.EqualValues(t, 0, couch_spatial_disk_size["bytes"]) couch_views := couch["views"].(common.MapStr) - couch_views_actual_disk_size := couch_views["actual_disk_size"].(common.MapStr) - assert.EqualValues(t, 2805219, couch_views_actual_disk_size["bytes"]) + couch_views_disk_size := couch_views["disk_size"].(common.MapStr) + assert.EqualValues(t, 2805219, couch_views_disk_size["bytes"]) couch_views_data_size := couch_views["data_size"].(common.MapStr) assert.EqualValues(t, 2805219, couch_views_data_size["bytes"]) diff --git a/metricbeat/module/couchbase/testing.go b/metricbeat/module/couchbase/testing.go index f91981e6cf8..d3b28ca0945 100644 --- a/metricbeat/module/couchbase/testing.go +++ b/metricbeat/module/couchbase/testing.go @@ -3,5 +3,10 @@ package couchbase import "os" func GetEnvDSN() string { - return os.Getenv("COUCHBASE_DSN") + dsn := os.Getenv("COUCHBASE_DSN") + + if len(dsn) == 0 { + dsn = "http://Administrator:password@localhost:8091" + } + return dsn } diff --git a/metricbeat/tests/system/test_couchbase.py b/metricbeat/tests/system/test_couchbase.py new file mode 100644 index 00000000000..d46bedb2823 --- /dev/null +++ b/metricbeat/tests/system/test_couchbase.py @@ -0,0 +1,75 @@ +import os +import metricbeat +import unittest + + +class Test(metricbeat.BaseTest): + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + def test_bucket(self): + """ + couchbase bucket metricset test + """ + self.render_config_template(modules=[{ + "name": "couchbase", + "metricsets": ["bucket"], + "hosts": self.get_hosts(), + "period": "1s" + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0, max_timeout=20) + proc.check_kill_and_wait() + + output = self.read_output_json() + self.assertTrue(len(output) >= 1) + evt = output[0] + print evt + + self.assert_fields_are_documented(evt) + + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + def test_cluster(self): + """ + couchbase cluster metricset test + """ + self.render_config_template(modules=[{ + "name": "couchbase", + "metricsets": ["cluster"], + "hosts": self.get_hosts(), + "period": "1s" + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0, max_timeout=20) + proc.check_kill_and_wait() + + output = self.read_output_json() + self.assertTrue(len(output) >= 1) + evt = output[0] + print evt + + self.assert_fields_are_documented(evt) + + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + def test_node(self): + """ + couchbase node metricset test + """ + self.render_config_template(modules=[{ + "name": "couchbase", + "metricsets": ["node"], + "hosts": self.get_hosts(), + "period": "1s" + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0, max_timeout=20) + proc.check_kill_and_wait() + + output = self.read_output_json() + self.assertTrue(len(output) >= 1) + evt = output[0] + print evt + + self.assert_fields_are_documented(evt) + + def get_hosts(self): + return [os.getenv('COUCHBASE_HOST', 'localhost') + ':' + + os.getenv('COUCHBASE_PORT', '8091')]