diff --git a/couchbase/assets/configuration/spec.yaml b/couchbase/assets/configuration/spec.yaml index 4b4b84a87927c..22f82cd0e43dc 100644 --- a/couchbase/assets/configuration/spec.yaml +++ b/couchbase/assets/configuration/spec.yaml @@ -28,6 +28,13 @@ files: value: example: http://localhost:4986 type: string + - name: index_stats_url + description: | + The URL to get Index Statistics, available since Couchbase 7.0. + See https://docs.couchbase.com/server/current/rest-api/rest-index-stats.html + value: + example: http://localhost:9102 + type: string - template: instances/http - template: instances/default - template: logs diff --git a/couchbase/datadog_checks/couchbase/config_models/defaults.py b/couchbase/datadog_checks/couchbase/config_models/defaults.py index 9ab7fbe071e7c..f07b20b9e7119 100644 --- a/couchbase/datadog_checks/couchbase/config_models/defaults.py +++ b/couchbase/datadog_checks/couchbase/config_models/defaults.py @@ -64,6 +64,10 @@ def instance_headers(field, value): return get_default_field_value(field, value) +def instance_index_stats_url(field, value): + return 'http://localhost:9102' + + def instance_kerberos_auth(field, value): return 'disabled' diff --git a/couchbase/datadog_checks/couchbase/config_models/instance.py b/couchbase/datadog_checks/couchbase/config_models/instance.py index a44b8575c5c6b..0130ad240f75a 100644 --- a/couchbase/datadog_checks/couchbase/config_models/instance.py +++ b/couchbase/datadog_checks/couchbase/config_models/instance.py @@ -45,6 +45,7 @@ class Config: empty_default_hostname: Optional[bool] extra_headers: Optional[Mapping[str, Any]] headers: Optional[Mapping[str, Any]] + index_stats_url: Optional[str] kerberos_auth: Optional[str] kerberos_cache: Optional[str] kerberos_delegate: Optional[bool] diff --git a/couchbase/datadog_checks/couchbase/couchbase.py b/couchbase/datadog_checks/couchbase/couchbase.py index 36bc3c43616b9..97281b8304eb0 100644 --- a/couchbase/datadog_checks/couchbase/couchbase.py +++ b/couchbase/datadog_checks/couchbase/couchbase.py @@ -10,12 +10,17 @@ import requests from six import string_types +from six.moves.urllib.parse import urljoin from datadog_checks.base import AgentCheck, ConfigurationError from datadog_checks.couchbase.couchbase_consts import ( BUCKET_STATS, COUCHBASE_STATS_PATH, COUCHBASE_VITALS_PATH, + INDEX_STATS_COUNT_METRICS, + INDEX_STATS_METRICS_PATH, + INDEX_STATS_SERVICE_CHECK_NAME, + INDEXER_STATE_MAP, NODE_CLUSTER_SERVICE_CHECK_NAME, NODE_HEALTH_SERVICE_CHECK_NAME, NODE_HEALTH_TRANSLATION, @@ -43,6 +48,7 @@ def __init__(self, name, init_config, instances): super(Couchbase, self).__init__(name, init_config, instances) self._sync_gateway_url = self.instance.get('sync_gateway_url', None) + self._index_stats_url = self.instance.get('index_stats_url') self._server = self.instance.get('server', None) if self._server is None: raise ConfigurationError("The server must be specified") @@ -50,6 +56,7 @@ def __init__(self, name, init_config, instances): self._tags.append('instance:{}'.format(self._server)) self._previous_status = None + self._version = None def _create_metrics(self, data): # Get storage metrics @@ -183,6 +190,12 @@ def check(self, _): self._create_metrics(data) if self._sync_gateway_url: self._collect_sync_gateway_metrics() + try: + # Error handling in case Couchbase changes their versioning format + if self._index_stats_url and self._version and int(self._version.split(".")[0]) >= 7: + self._collect_index_stats_metrics() + except Exception as e: + self.log.debug(str(e)) def _collect_version(self, data): nodes = data['stats']['nodes'] @@ -198,9 +211,9 @@ def _collect_version(self, data): build_separator = version.rindex('-') version = list(version) version[build_separator] = '+' - version = ''.join(version) + self._version = ''.join(version) - self.set_metadata('version', version) + self.set_metadata('version', self._version) def get_data(self): # The dictionary to be returned. @@ -318,7 +331,7 @@ def _collect_sync_gateway_metrics(self): try: data = self._get_stats(url).get('syncgateway', {}) except requests.exceptions.RequestException as e: - msg = "Error accessing the Sync Gateway monitoring endpoint %s: %s," % url, str(e) + msg = "Error accessing the Sync Gateway monitoring endpoint %s: %s," % (url, str(e)) self.log.debug(msg) self.service_check(SG_SERVICE_CHECK_NAME, AgentCheck.CRITICAL, msg, self._tags) return @@ -402,3 +415,73 @@ def extract_seconds_value(self, value): unit = 'us' return float(val) / TO_SECONDS[unit] + + def _collect_index_stats_metrics(self): + url = urljoin(self._index_stats_url, INDEX_STATS_METRICS_PATH) + try: + data = self._get_stats(url) + except requests.exceptions.RequestException as e: + msg = "Error accessing the Index Statistics endpoint: %s: %s" % (url, str(e)) + self.log.warning(msg) + self.service_check(INDEX_STATS_SERVICE_CHECK_NAME, AgentCheck.CRITICAL, self._tags, msg) + return + + self.service_check(INDEX_STATS_SERVICE_CHECK_NAME, AgentCheck.OK, self._tags) + + for keyspace in data: + if keyspace == "indexer": + # The indexer object provides metric about the index node + for mname, mval in data.get(keyspace).items(): + self._submit_index_node_metrics(mname, mval) + else: + index_tags = self._extract_index_tags(keyspace) + self._tags + for mname, mval in data.get(keyspace).items(): + self._submit_per_index_metrics(mname, mval, index_tags) + + def _extract_index_tags(self, keyspace): + # Index Keyspaces can come in different formats: + # partition, bucket:index_name, bucket:collection:index_name, bucket:scope:collection:index_name + # For variations missing the scope and collection, they refer to the default scope and collection respectively + # https://docs.couchbase.com/server/current/n1ql/n1ql-language-reference/createprimaryindex.html#keyspace-ref + tag_arr = keyspace.split(":") + if len(tag_arr) == 2: + bucket, index_name = tag_arr + scope, collection = ["default", "default"] + elif len(tag_arr) == 3: + bucket, collection, index_name = tag_arr + scope = 'default' + elif len(tag_arr) == 4: + bucket, scope, collection, index_name = tag_arr + else: + # Catch all incase the keyspace has either none or 3 or more separators(':') + # There is a documented example of partition-num being a possible keyspace: + # https://docs.couchbase.com/server/current/rest-api/rest-index-stats.html#_get_index_stats + # But we shouldn't encounter this since we don't query the index api with the needed params + # (Version 1.19.0+ of the Couchbase check) + formatted_tags = [] + self.log.debug("Unable to extract tags from keyspace: %s", keyspace) + return formatted_tags + + formatted_tags = [ + 'bucket:{}'.format(bucket), + 'scope:{}'.format(scope), + 'collection:{}'.format(collection), + 'index_name:{}'.format(index_name), + ] + return formatted_tags + + def _submit_index_node_metrics(self, mname, mval): + namespace = 'couchbase.indexer' + f_mname = '.'.join([namespace, mname]) + if mname == "indexer_state": + self.gauge(f_mname, INDEXER_STATE_MAP[mval], self._tags) + else: + self.gauge(f_mname, mval, self._tags) + + def _submit_per_index_metrics(self, mname, mval, tags): + namespace = 'couchbase.index' + f_mname = '.'.join([namespace, mname]) + if mname in INDEX_STATS_COUNT_METRICS: + self.monotonic_count(f_mname, mval, tags) + else: + self.gauge(f_mname, mval, tags) diff --git a/couchbase/datadog_checks/couchbase/couchbase_consts.py b/couchbase/datadog_checks/couchbase/couchbase_consts.py index cb81b574f8560..3a3808477db76 100644 --- a/couchbase/datadog_checks/couchbase/couchbase_consts.py +++ b/couchbase/datadog_checks/couchbase/couchbase_consts.py @@ -6,12 +6,14 @@ COUCHBASE_STATS_PATH = '/pools/default' COUCHBASE_VITALS_PATH = '/admin/vitals' SG_METRICS_PATH = '/_expvar' +INDEX_STATS_METRICS_PATH = '/api/v1/stats' # Service Checks SERVICE_CHECK_NAME = 'couchbase.can_connect' SG_SERVICE_CHECK_NAME = 'couchbase.sync_gateway.can_connect' NODE_CLUSTER_SERVICE_CHECK_NAME = 'couchbase.by_node.cluster_membership' NODE_HEALTH_SERVICE_CHECK_NAME = 'couchbase.by_node.health' +INDEX_STATS_SERVICE_CHECK_NAME = 'couchbase.index_stats.can_connect' NODE_MEMBERSHIP_TRANSLATION = { 'active': AgentCheck.OK, @@ -323,3 +325,18 @@ "import_partitions", "import_processing_time", ] + +INDEX_STATS_COUNT_METRICS = [ + "cache_hits", + "cache_misses", + "items_count", + "num_docs_indexed", + "num_items_flushed", + "num_requests", + "num_rows_returned", + "num_scan_errors", + "num_scan_timeouts", + "scan_bytes_read", +] + +INDEXER_STATE_MAP = {'Active': 0, 'Pause': 1, 'Warmup': 2} diff --git a/couchbase/datadog_checks/couchbase/data/conf.yaml.example b/couchbase/datadog_checks/couchbase/data/conf.yaml.example index 49966eccf87d7..6dd076ee5e4f5 100644 --- a/couchbase/datadog_checks/couchbase/data/conf.yaml.example +++ b/couchbase/datadog_checks/couchbase/data/conf.yaml.example @@ -62,6 +62,12 @@ instances: # # sync_gateway_url: http://localhost:4986 + ## @param index_stats_url - string - optional - default: http://localhost:9102 + ## The URL to get Index Statistics, available since Couchbase 7.0. + ## See https://docs.couchbase.com/server/current/rest-api/rest-index-stats.html + # + # index_stats_url: http://localhost:9102 + ## @param proxy - mapping - optional ## This overrides the `proxy` setting in `init_config`. ## diff --git a/couchbase/metadata.csv b/couchbase/metadata.csv index 9344ba7c66fee..3f26f0880168c 100644 --- a/couchbase/metadata.csv +++ b/couchbase/metadata.csv @@ -10,7 +10,7 @@ couchbase.ram.quota_total,gauge,,byte,,RAM quota,0,couchbase,ram quota couchbase.ram.used_by_data,gauge,,byte,,RAM used for data,0,couchbase,ram data used couchbase.by_bucket.avg_bg_wait_time,gauge,,microsecond,,Average background wait time,-1,couchbase,avg bg wait couchbase.by_bucket.avg_disk_commit_time,gauge,,second,,Average disk commit time,-1,couchbase,avg commit time -couchbase.by_bucket.avg_disk_update_time,gauge,,microsecond,,Average disk update time ,-1,couchbase,avg update time +couchbase.by_bucket.avg_disk_update_time,gauge,,microsecond,,"Average disk update time ",-1,couchbase,avg update time couchbase.by_bucket.bg_wait_total,gauge,,byte,,Bytes read,0,couchbase,read couchbase.by_bucket.bytes_read,gauge,,byte,,Bytes read,0,couchbase,read couchbase.by_bucket.bytes_written,gauge,,byte,,Bytes written,0,couchbase,written @@ -33,7 +33,7 @@ couchbase.by_bucket.couch_views_fragmentation,gauge,,percent,,View fragmentation couchbase.by_bucket.couch_views_ops,gauge,,operation,,View operations,0,couchbase,view ops couchbase.by_bucket.cpu_idle_ms,gauge,,millisecond,,CPU idle milliseconds,0,couchbase,cpu idle ms couchbase.by_bucket.cpu_utilization_rate,gauge,,percent,,CPU utilization percentage,0,couchbase,cpu util -couchbase.by_bucket.curr_connections,gauge,,connection,,Current bucket connections,0,couchbase,conns +couchbase.by_bucket.curr_connections,gauge,,connection,,The current number of bucket connections,0,couchbase,conns couchbase.by_bucket.curr_items_tot,gauge,,item,,Total number of items,0,couchbase,total items couchbase.by_bucket.curr_items,gauge,,item,,Number of active items in memory,0,couchbase,mem items couchbase.by_bucket.decr_hits,gauge,,hit,,Decrement hits,1,couchbase,decr hits @@ -132,7 +132,7 @@ couchbase.by_bucket.replication_docs_rep_queue,gauge,,item,,,0,couchbase,repl do couchbase.by_bucket.replication_meta_latency_aggr,gauge,,second,,,0,couchbase,repl meta latency aggr couchbase.by_bucket.rest_requests,gauge,,request,second,Number of HTTP requests,0,couchbase,rest requests couchbase.by_bucket.swap_total,gauge,,byte,,Total amount of swap available,0,couchbase,swap total -couchbase.by_bucket.swap_used,gauge,,byte,,Amount of swap used ,0,couchbase,swap used +couchbase.by_bucket.swap_used,gauge,,byte,,"Amount of swap used ",0,couchbase,swap used couchbase.by_bucket.vb_active_eject,gauge,,item,second,Number of items per second being ejected to disk from active vBuckets,0,couchbase,vb active eject couchbase.by_bucket.vb_active_itm_memory,gauge,,item,,Amount of active user data cached in RAM in this bucket,0,couchbase,vb active item mem couchbase.by_bucket.vb_active_meta_data_memory,gauge,,item,,Amount of active item metadata consuming RAM in this bucket,0,couchbase,vb active meta mem @@ -313,4 +313,35 @@ couchbase.sync_gateway.shared_bucket_import.import_high_seq,gauge,,,,The highest couchbase.sync_gateway.shared_bucket_import.import_partitions,count,,,,The total number of import partitions.,0,couchbase, couchbase.sync_gateway.shared_bucket_import.import_processing_time,count,,,,The total time taken to process a document import.,0,couchbase, couchbase.sync_gateway.system_memory_total,count,,byte,,The total memory available on the system in bytes.,0,couchbase, -couchbase.sync_gateway.warn_count,count,,byte,,The total number of warnings logged.,0,couchbase, \ No newline at end of file +couchbase.sync_gateway.warn_count,count,,byte,,The total number of warnings logged.,0,couchbase, +couchbase.indexer.indexer_state,gauge,,,,"The current state of the Index service on this node (0 = Active, 1 = Pause, 2 = Warmup)",0,couchbase, +couchbase.indexer.memory_quota,gauge,,byte,,The memory quota assigned to the Index service on this node by user configuration,0,couchbase, +couchbase.indexer.memory_total_storage,gauge,,byte,,The total size allocated in the indexer across all indexes. This also accounts for memory fragmentation,0,couchbase, +couchbase.indexer.memory_used,gauge,,byte,,The amount of memory used by the Index service on this node,0,couchbase, +couchbase.indexer.total_indexer_gc_pause_ns,gauge,,nanosecond,,The total time the indexer has spent in GC pause since the last startup,0,couchbase, +couchbase.index.avg_drain_rate,gauge,,item,second,The average number of items flushed from memory to disk storage per second,0,couchbase, +couchbase.index.avg_item_size,gauge,,byte,,The average size of the keys,0,couchbase, +couchbase.index.avg_scan_latency,gauge,,nanosecond,,The average time to serve a scan request,0,couchbase, +couchbase.index.cache_hit_percent,gauge,,percent,,The percentage of memory accesses that were served from the managed cache,0,couchbase, +couchbase.index.cache_hits,count,,,,The number of accesses to this index data from RAM,0,couchbase, +couchbase.index.cache_misses,count,,,,The number of accesses to this index data from disk,0,couchbase, +couchbase.index.data_size,gauge,,byte,,The size of indexable data that is maintained for the index or partition,0,couchbase, +couchbase.index.disk_size,gauge,,byte,,The total disk file size consumed by the index or partition,0,couchbase, +couchbase.index.frag_percent,gauge,,percent,,The percentage fragmentation of the index,0,couchbase, +couchbase.index.initial_build_progress,gauge,,percent,,"The percentage of the initial build progress for the index. When the initial build is completed, the value is 100. For an index partition, the value is listed as 0",0,couchbase, +couchbase.index.items_count,count,,item,,The number of items currently indexed,0,couchbase, +couchbase.index.last_known_scan_time,gauge,,nanosecond,,"Timestamp of the last scan request received for this index (Unix timestamp in nanoseconds). This may be useful for determining whether this index is currently unused. Note: This statistic is persisted to disk every 15 minutes, so it is preserved when the indexer restarts",0,couchbase, +couchbase.index.num_docs_indexed,count,,document,,The number of documents indexed by the indexer since last startup,0,couchbase, +couchbase.index.num_docs_pending,gauge,,document,,The number of documents pending to be indexed,0,couchbase, +couchbase.index.num_docs_queued,gauge,,document,,The number of documents queued to be indexed,0,couchbase, +couchbase.index.num_items_flushed,count,,item,,The number of items flushed from memory to disk storage,0,couchbase, +couchbase.index.num_pending_requests,gauge,,request,,The number of requests received but not yet served by the indexer,0,couchbase, +couchbase.index.num_requests,count,,request,,The number of requests served by the indexer since last startup,0,couchbase, +couchbase.index.num_rows_returned,count,,row,,The total number of rows returned so far by the indexer,0,couchbase, +couchbase.index.num_scan_errors,count,,request,,The number of requests that failed due to errors other than timeout,0,couchbase, +couchbase.index.num_scan_timeouts,count,,request,,"The number of requests that timed out, either waiting for snapshots or during scan in progress",0,couchbase, +couchbase.index.recs_in_mem,gauge,,record,,"For standard index storage, this is the number of records in this index that are stored in memory. For memory-optimized index storage, this is the same as items_count",0,couchbase, +couchbase.index.recs_on_disk,gauge,,record,,"For standard index storage, this is the number of records in this index that are stored on disk. For memory-optimized index storage, this is 0",0,couchbase, +couchbase.index.resident_percent,gauge,,percent,,The percentage of data held in memory,0,couchbase, +couchbase.index.scan_bytes_read,count,,byte,,The number of bytes read by a scan since last startup,0,couchbase, +couchbase.index.total_scan_duration,gauge,,nanosecond,,The total time spent by the indexer in scanning rows since last startup,0,couchbase, \ No newline at end of file diff --git a/couchbase/tests/common.py b/couchbase/tests/common.py index 8e1cee90acdad..e12d679f318e9 100644 --- a/couchbase/tests/common.py +++ b/couchbase/tests/common.py @@ -12,20 +12,30 @@ PORT = '8091' QUERY_PORT = '8093' SG_PORT = '4985' +INDEX_STATS_PORT = '9102' # Tags and common bucket name CUSTOM_TAGS = ['optional:tag1'] CHECK_TAGS = CUSTOM_TAGS + ['instance:http://{}:{}'.format(HOST, PORT)] BUCKET_NAME = 'cb_bucket' +INDEX_STATS_TAGS = CHECK_TAGS + [ + 'bucket:cb_bucket', + 'collection:default', + 'index_name:gamesim_primary', + 'scope:default', +] URL = 'http://{}:{}'.format(HOST, PORT) QUERY_URL = 'http://{}:{}'.format(HOST, QUERY_PORT) SG_URL = 'http://{}:{}'.format(HOST, SG_PORT) +INDEX_STATS_URL = 'http://{}:{}'.format(HOST, INDEX_STATS_PORT) CB_CONTAINER_NAME = 'couchbase-standalone' USER = 'Administrator' PASSWORD = 'password' -DEFAULT_INSTANCE = {'server': URL, 'user': USER, 'password': PASSWORD, 'timeout': 0.5, 'tags': CUSTOM_TAGS} +COUCHBASE_MAJOR_VERSION = int(os.getenv('COUCHBASE_VERSION').split(".")[0]) + +DEFAULT_INSTANCE = {'server': URL, 'user': USER, 'password': PASSWORD, 'timeout': 1, 'tags': CUSTOM_TAGS} SYNC_GATEWAY_METRICS = [ "couchbase.sync_gateway.admin_net_bytes_recv", @@ -132,3 +142,43 @@ "couchbase.sync_gateway.system_memory_total", "couchbase.sync_gateway.warn_count", ] + +INDEX_STATS_INDEXER_METRICS = [ + 'couchbase.indexer.indexer_state', + 'couchbase.indexer.memory_quota', + 'couchbase.indexer.memory_total_storage', + 'couchbase.indexer.memory_used', + 'couchbase.indexer.total_indexer_gc_pause_ns', +] + +INDEX_STATS_GAUGE_METRICS = [ + 'couchbase.index.avg_drain_rate', + 'couchbase.index.avg_item_size', + 'couchbase.index.avg_scan_latency', + 'couchbase.index.cache_hit_percent', + 'couchbase.index.data_size', + 'couchbase.index.disk_size', + 'couchbase.index.frag_percent', + 'couchbase.index.initial_build_progress', + 'couchbase.index.last_known_scan_time', + 'couchbase.index.num_docs_pending', + 'couchbase.index.num_docs_queued', + 'couchbase.index.num_pending_requests', + 'couchbase.index.recs_in_mem', + 'couchbase.index.recs_on_disk', + 'couchbase.index.resident_percent', + 'couchbase.index.total_scan_duration', +] + +INDEX_STATS_COUNT_METRICS = [ + 'couchbase.index.cache_hits', + 'couchbase.index.cache_misses', + 'couchbase.index.items_count', + 'couchbase.index.num_docs_indexed', + 'couchbase.index.num_items_flushed', + 'couchbase.index.num_requests', + 'couchbase.index.num_rows_returned', + 'couchbase.index.num_scan_errors', + 'couchbase.index.num_scan_timeouts', + 'couchbase.index.scan_bytes_read', +] diff --git a/couchbase/tests/compose/standalone.compose b/couchbase/tests/compose/standalone.compose index f06138a02304f..43203f5350c49 100644 --- a/couchbase/tests/compose/standalone.compose +++ b/couchbase/tests/compose/standalone.compose @@ -5,6 +5,7 @@ services: image: "couchbase/server:${COUCHBASE_VERSION}" ports: - 8091-8094:8091-8094 + - 9102:9102 container_name: ${CB_CONTAINER_NAME} couchbase-sync-gateway: container_name: couchbase-sync-gateway diff --git a/couchbase/tests/conftest.py b/couchbase/tests/conftest.py index 0bf19f018b0de..59a9fb0a1f37d 100644 --- a/couchbase/tests/conftest.py +++ b/couchbase/tests/conftest.py @@ -15,9 +15,11 @@ from .common import ( BUCKET_NAME, CB_CONTAINER_NAME, + COUCHBASE_MAJOR_VERSION, CUSTOM_TAGS, DEFAULT_INSTANCE, HERE, + INDEX_STATS_URL, PASSWORD, PORT, QUERY_URL, @@ -38,7 +40,7 @@ def instance_query(): 'server': URL, 'user': USER, 'password': PASSWORD, - 'timeout': 0.5, + 'timeout': 1, 'tags': CUSTOM_TAGS, 'query_monitoring_url': QUERY_URL, } @@ -50,28 +52,42 @@ def instance_sg(): 'server': URL, 'user': USER, 'password': PASSWORD, - 'timeout': 0.5, + 'timeout': 1, 'tags': CUSTOM_TAGS, 'sync_gateway_url': SG_URL, } +@pytest.fixture +def instance_index_stats(): + return { + 'server': URL, + 'user': USER, + 'password': PASSWORD, + 'timeout': 1, + 'tags': CUSTOM_TAGS, + 'index_stats_url': INDEX_STATS_URL, + } + + @pytest.fixture(scope="session") def dd_environment(): """ Spin up and initialize couchbase """ - + conditions = [ + WaitFor(couchbase_container), + WaitFor(couchbase_init), + WaitFor(couchbase_setup), + WaitFor(node_stats), + WaitFor(bucket_stats), + ] + if COUCHBASE_MAJOR_VERSION >= 7: + conditions.append(WaitFor(load_sample_bucket)) with docker_run( compose_file=os.path.join(HERE, 'compose', 'standalone.compose'), env_vars={'CB_CONTAINER_NAME': CB_CONTAINER_NAME}, - conditions=[ - WaitFor(couchbase_container), - WaitFor(couchbase_init), - WaitFor(couchbase_setup), - WaitFor(node_stats), - WaitFor(bucket_stats), - ], + conditions=conditions, attempts=2, ): yield DEFAULT_INSTANCE @@ -155,7 +171,7 @@ def couchbase_init(): '--services', 'data,index,fts,query', '--cluster-ramsize', - '256', + '512', '--cluster-index-ramsize', '256', '--cluster-fts-ramsize', @@ -167,6 +183,35 @@ def couchbase_init(): return r.status_code == requests.codes.ok +def load_sample_bucket(): + """ + Load sample data bucket + """ + + # Resources used: + # https://docs.couchbase.com/server/current/manage/manage-settings/install-sample-buckets.html + + bucket_loader_args = [ + 'docker', + 'exec', + CB_CONTAINER_NAME, + 'cbdocloader', + '-c', + 'localhost:{}'.format(PORT), + '-u', + USER, + '-p', + PASSWORD, + '-d', + '/opt/couchbase/samples/gamesim-sample.zip', + '-b', + 'cb_bucket', + '-m', + '256', + ] + subprocess.check_call(bucket_loader_args) + + def node_stats(): """ Wait for couchbase to generate node stats diff --git a/couchbase/tests/test_couchbase.py b/couchbase/tests/test_couchbase.py index cdfe16d19deb5..eb27a6a96966c 100644 --- a/couchbase/tests/test_couchbase.py +++ b/couchbase/tests/test_couchbase.py @@ -2,11 +2,14 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +import time + import mock import pytest from datadog_checks.couchbase import Couchbase from datadog_checks.couchbase.couchbase_consts import ( + INDEX_STATS_SERVICE_CHECK_NAME, NODE_CLUSTER_SERVICE_CHECK_NAME, NODE_HEALTH_SERVICE_CHECK_NAME, QUERY_STATS, @@ -14,7 +17,17 @@ SG_SERVICE_CHECK_NAME, ) -from .common import BUCKET_NAME, CHECK_TAGS, PORT, SYNC_GATEWAY_METRICS +from .common import ( + BUCKET_NAME, + CHECK_TAGS, + COUCHBASE_MAJOR_VERSION, + INDEX_STATS_COUNT_METRICS, + INDEX_STATS_GAUGE_METRICS, + INDEX_STATS_INDEXER_METRICS, + INDEX_STATS_TAGS, + PORT, + SYNC_GATEWAY_METRICS, +) NODE_STATS = [ 'cmd_get', @@ -34,6 +47,9 @@ 'vb_replica_curr_items', ] +if COUCHBASE_MAJOR_VERSION == 7: + NODE_STATS += ['index_data_size', 'index_disk_size'] + TOTAL_STATS = [ 'hdd.free', 'hdd.used', @@ -73,28 +89,6 @@ def test_service_check(aggregator, instance, couchbase_container_ip): ) -@pytest.mark.integration -@pytest.mark.usefixtures("dd_environment") -def test_metrics(aggregator, instance, couchbase_container_ip): - """ - Test couchbase metrics not including 'couchbase.query.' - """ - couchbase = Couchbase('couchbase', {}, instances=[instance]) - couchbase.check(None) - - # Assert each type of metric (buckets, nodes, totals) except query - _assert_bucket_metrics(aggregator, BUCKET_TAGS + ['device:{}'.format(BUCKET_NAME)]) - - # Assert 'couchbase.by_node.' metrics - node_tags = CHECK_TAGS + [ - 'node:{}:{}'.format(couchbase_container_ip, PORT), - 'device:{}:{}'.format(couchbase_container_ip, PORT), - ] - _assert_stats(aggregator, node_tags) - - aggregator.assert_all_metrics_covered() - - @pytest.mark.e2e def test_e2e(dd_agent_check, instance, couchbase_container_ip): """ @@ -194,7 +188,51 @@ def _assert_bucket_metrics(aggregator, tags, device=None): def _assert_stats(aggregator, node_tags, device=None): for mname in NODE_STATS: aggregator.assert_metric('couchbase.by_node.{}'.format(mname), tags=node_tags, count=1, device=device) - # Assert 'couchbase.' metrics for mname in TOTAL_STATS: aggregator.assert_metric('couchbase.{}'.format(mname), tags=CHECK_TAGS, count=1) + + +@pytest.mark.skipif(COUCHBASE_MAJOR_VERSION < 7, reason='Index metrics are only available for Couchbase 7+') +@pytest.mark.integration +@pytest.mark.usefixtures("dd_environment") +def test_index_stats_metrics(aggregator, dd_run_check, instance_index_stats, couchbase_container_ip): + """ + Test Index Statistics metrics (prefixed "couchbase.index." and "couchbase.indexer.") + """ + couchbase = Couchbase('couchbase', {}, [instance_index_stats]) + dd_run_check(couchbase) + for mname in INDEX_STATS_INDEXER_METRICS: + aggregator.assert_metric(mname, metric_type=aggregator.GAUGE, tags=CHECK_TAGS) + + for mname in INDEX_STATS_GAUGE_METRICS: + aggregator.assert_metric(mname, metric_type=aggregator.GAUGE, tags=INDEX_STATS_TAGS) + + for mname in INDEX_STATS_COUNT_METRICS: + aggregator.assert_metric(mname, metric_type=aggregator.MONOTONIC_COUNT, tags=INDEX_STATS_TAGS) + + aggregator.assert_service_check(INDEX_STATS_SERVICE_CHECK_NAME, status=Couchbase.OK, tags=CHECK_TAGS) + + +@pytest.mark.integration +@pytest.mark.usefixtures("dd_environment") +def test_metrics(aggregator, dd_run_check, instance, couchbase_container_ip): + """ + Test couchbase metrics not including 'couchbase.query.' + """ + # Few metrics are only available after some time post launch. Sleep to ensure they're present before we validate + time.sleep(15) + couchbase = Couchbase('couchbase', {}, instances=[instance]) + dd_run_check(couchbase) + + # Assert each type of metric (buckets, nodes, totals) except query + _assert_bucket_metrics(aggregator, BUCKET_TAGS + ['device:{}'.format(BUCKET_NAME)]) + + # Assert 'couchbase.by_node.' metrics + node_tags = CHECK_TAGS + [ + 'node:{}:{}'.format(couchbase_container_ip, PORT), + 'device:{}:{}'.format(couchbase_container_ip, PORT), + ] + _assert_stats(aggregator, node_tags) + + aggregator.assert_all_metrics_covered() diff --git a/couchbase/tests/test_unit.py b/couchbase/tests/test_unit.py index 085a6841f4f73..b46743fd508d9 100644 --- a/couchbase/tests/test_unit.py +++ b/couchbase/tests/test_unit.py @@ -91,3 +91,34 @@ def test_config(test_case, dd_run_check, extra_config, expected_http_kwargs, ins ) http_wargs.update(expected_http_kwargs) r.get.assert_called_with('http://localhost:8091/pools/default/tasks', **http_wargs) + + +@pytest.mark.parametrize( + 'test_input, expected_tags', + [ + ('partition', []), + ('bucket:index_name', ['bucket:bucket', 'scope:default', 'collection:default', 'index_name:index_name']), + ( + 'bucket:collection:index_name', + ['bucket:bucket', 'scope:default', 'collection:collection', 'index_name:index_name'], + ), + ( + 'bucket:scope:collection:index_name', + ['bucket:bucket', 'scope:scope', 'collection:collection', 'index_name:index_name'], + ), + ( + 'foo:baz:bar:fiz:buz', + [], + ), + ], +) +def test_extract_index_tags(instance, test_input, expected_tags): + couchbase = Couchbase('couchbase', {}, [instance]) + """ + Test to ensure that tags are extracted properly from keyspaces. Takes into account the different + forms of the keyspace and extract the tags from them accordingly. Docs: + https://docs.couchbase.com/server/current/rest-api/rest-index-stats.html#responses-3 + https://docs.couchbase.com/server/current/n1ql/n1ql-language-reference/createprimaryindex.html#keyspace-ref + """ + test_output = couchbase._extract_index_tags(test_input) + assert eval(str(test_output)) == expected_tags diff --git a/couchbase/tox.ini b/couchbase/tox.ini index 93e81e42c3b81..1beca3dac5939 100644 --- a/couchbase/tox.ini +++ b/couchbase/tox.ini @@ -2,7 +2,7 @@ minversion = 2.0 basepython = py38 envlist = - py{27,38}-{5.5.3} + py{27,38}-{5.5.3,7.0.2} [testenv] ensure_default_envdir = true @@ -17,7 +17,9 @@ platform = linux|darwin|win32 deps = -e../datadog_checks_base[deps] -rrequirements-dev.txt -setenv = COUCHBASE_VERSION=5.5.3 +setenv = + 5.5.3: COUCHBASE_VERSION=5.5.3 + 7.0.2: COUCHBASE_VERSION=7.0.2 passenv = DOCKER* COMPOSE*