Add support for Index Statistics metrics

couchbase/assets/configuration/spec.yaml

@@ -28,6 +28,13 @@ files:
       value:
         example: http://localhost:4986
         type: string
+    - name: index_stats_url
+      description: |
+        Url to get Index Statistics, available since couchbase 7.0.
+        See https://docs.couchbase.com/server/current/rest-api/rest-index-stats.html
+      value:
+        example: http://localhost:9102
+        type: string
     - template: instances/http
     - template: instances/default
   - template: logs

couchbase/datadog_checks/couchbase/config_models/defaults.py

@@ -64,6 +64,10 @@ def instance_headers(field, value):
     return get_default_field_value(field, value)
 
 
+def instance_index_stats_url(field, value):
+    return 'http://localhost:9102'
+
+
 def instance_kerberos_auth(field, value):
     return 'disabled'
 

couchbase/datadog_checks/couchbase/config_models/instance.py

@@ -45,6 +45,7 @@ class Config:
     empty_default_hostname: Optional[bool]
     extra_headers: Optional[Mapping[str, Any]]
     headers: Optional[Mapping[str, Any]]
+    index_stats_url: Optional[str]
     kerberos_auth: Optional[str]
     kerberos_cache: Optional[str]
     kerberos_delegate: Optional[bool]

couchbase/datadog_checks/couchbase/couchbase.py

@@ -16,6 +16,8 @@
     BUCKET_STATS,
     COUCHBASE_STATS_PATH,
     COUCHBASE_VITALS_PATH,
+    INDEX_STATS_METRICS_PATH,
+    INDEX_STATS_SERVICE_CHECK_NAME,
     NODE_CLUSTER_SERVICE_CHECK_NAME,
     NODE_HEALTH_SERVICE_CHECK_NAME,
     NODE_HEALTH_TRANSLATION,
@@ -43,6 +45,7 @@ def __init__(self, name, init_config, instances):
         super(Couchbase, self).__init__(name, init_config, instances)
 
         self._sync_gateway_url = self.instance.get('sync_gateway_url', None)
+        self._index_stats_url = self.instance.get('index_stats_url', None)
         self._server = self.instance.get('server', None)
         if self._server is None:
             raise ConfigurationError("The server must be specified")
@@ -183,6 +186,8 @@ def check(self, _):
         self._create_metrics(data)
         if self._sync_gateway_url:
             self._collect_sync_gateway_metrics()
+        if self._index_stats_url:
+            self._collect_index_stats_metrics()
 
     def _collect_version(self, data):
         nodes = data['stats']['nodes']
@@ -318,7 +323,7 @@ def _collect_sync_gateway_metrics(self):
         try:
             data = self._get_stats(url).get('syncgateway', {})
         except requests.exceptions.RequestException as e:
-            msg = "Error accessing the Sync Gateway monitoring endpoint %s: %s," % url, str(e)
+            msg = "Error accessing the Sync Gateway monitoring endpoint %s: %s," % (url, str(e))
             self.log.debug(msg)
             self.service_check(SG_SERVICE_CHECK_NAME, AgentCheck.CRITICAL, msg, self._tags)
             return
@@ -402,3 +407,66 @@ def extract_seconds_value(self, value):
             unit = 'us'
 
         return float(val) / TO_SECONDS[unit]
+
+    def _collect_index_stats_metrics(self):
+        url = '{}{}'.format(self._index_stats_url, INDEX_STATS_METRICS_PATH)
+        self.log.error(url)
+        try:
+            data = self._get_stats(url)
+        except requests.exceptions.RequestException as e:
+            msg = "Error accessing the Index Statistics endpoint: %s: %s" % (url, str(e))
+            self.log.debug(e)
+            self.service_check(INDEX_STATS_SERVICE_CHECK_NAME, AgentCheck.CRITICAL, self._tags, msg)
+            return
+
+        self.service_check(INDEX_STATS_SERVICE_CHECK_NAME, AgentCheck.OK, self._tags)
+
+        for keyspace in data:
+            if keyspace == "indexer":
+                # The indexer object provides metric about the index node
+                for mname, mval in data.get(keyspace).items():
+                    self._submit_index_node_metrics(mname, mval, self._tags)
+            else:
+                index_tags = self._extract_index_tags(keyspace) + self._tags
+                for mname, mval in data.get(keyspace).items():
+                    self._submit_per_index_metrics(mname, mval, index_tags)
+
+    def _extract_index_tags(self, keyspace):
+        # Index Keyspaces can come in different formats:
+        # partition, bucket:index_name, bucket:collection:index_name, bucket:scope:collection:index_name
+        # For variations missing the scope and collection, they refer to the default scope and collection respectively
+        # https://docs.couchbase.com/server/current/n1ql/n1ql-language-reference/createprimaryindex.html#keyspace-ref
+        tag_arr = keyspace.split(":")
+        if len(tag_arr) == 1:
+            partition = tag_arr[0]
+            formatted_tags = [
+                'partition:{}'.format(partition),
+            ]
+            return formatted_tags
+        elif len(tag_arr) == 2:
+            bucket, index_name = tag_arr
+            scope, collection = ["default", "default"]
+        elif len(tag_arr) == 3:
+            bucket, collection, index_name = tag_arr
+            scope = 'default'
+        else:
+            bucket, scope, collection, index_name = tag_arr
+        formatted_tags = [
+            'bucket:{}'.format(bucket),
+            'scope:{}'.format(scope),
+            'collection:{}'.format(collection),
+            'index_name:{}'.format(index_name),
+        ]
+        return formatted_tags
+
+    def _submit_index_node_metrics(self, mname, mval, tags):
+        index_state_map = {'Active': 0, 'Pause': 1, 'Warmup': 2}
+        namespace = '.'.join(['couchbase', 'indexer'])
+        if mname == "indexer_state":
+            self.gauge('.'.join([namespace, mname]), index_state_map[mval], tags)
+        else:
+            self.gauge('.'.join([namespace, mname]), mval, tags)
+
+    def _submit_per_index_metrics(self, mname, mval, tags):
+        namespace = '.'.join(['couchbase', 'index'])
+        self.gauge('.'.join([namespace, mname]), mval, tags)

couchbase/datadog_checks/couchbase/couchbase_consts.py

@@ -6,12 +6,14 @@
 COUCHBASE_STATS_PATH = '/pools/default'
 COUCHBASE_VITALS_PATH = '/admin/vitals'
 SG_METRICS_PATH = '/_expvar'
+INDEX_STATS_METRICS_PATH = '/api/v1/stats'
 
 # Service Checks
 SERVICE_CHECK_NAME = 'couchbase.can_connect'
 SG_SERVICE_CHECK_NAME = 'couchbase.sync_gateway.can_connect'
 NODE_CLUSTER_SERVICE_CHECK_NAME = 'couchbase.by_node.cluster_membership'
 NODE_HEALTH_SERVICE_CHECK_NAME = 'couchbase.by_node.health'
+INDEX_STATS_SERVICE_CHECK_NAME = 'couchbase.index_stats.can_connect'
 
 NODE_MEMBERSHIP_TRANSLATION = {
     'active': AgentCheck.OK,

couchbase/datadog_checks/couchbase/data/conf.yaml.example

@@ -62,6 +62,12 @@ instances:
     #
     # sync_gateway_url: http://localhost:4986
 
+    ## @param index_stats_url - string - optional - default: http://localhost:9102
+    ## Url to get Index Statistics, available since couchbase 7.0.
+    ## See https://docs.couchbase.com/server/current/rest-api/rest-index-stats.html
+    #
+    # index_stats_url: http://localhost:9102
+
     ## @param proxy - mapping - optional
     ## This overrides the `proxy` setting in `init_config`.
     ##

couchbase/metadata.csv

@@ -313,4 +313,36 @@ couchbase.sync_gateway.shared_bucket_import.import_high_seq,gauge,,,,The highest
 couchbase.sync_gateway.shared_bucket_import.import_partitions,count,,,,The total number of import partitions.,0,couchbase,
 couchbase.sync_gateway.shared_bucket_import.import_processing_time,count,,,,The total time taken to process a document import.,0,couchbase,
 couchbase.sync_gateway.system_memory_total,count,,byte,,The total memory available on the system in bytes.,0,couchbase,
-couchbase.sync_gateway.warn_count,count,,byte,,The total number of warnings logged.,0,couchbase,
+couchbase.sync_gateway.warn_count,count,,byte,,The total number of warnings logged.,0,couchbase,
+couchbase.indexer.indexer_state,gauge,,,,"Current state of the Index service on this node (0 = Active, 1 = Pause, 2 = Warmup)",0,couchbase,
+couchbase.indexer.memory_quota,gauge,,byte,,Memory quota assigned to the Index service on this node by user configuration,0,couchbase,
+couchbase.indexer.memory_total_storage,gauge,,byte,,The total size allocated in the indexer across all indexes. This also accounts for memory fragmentation,0,couchbase,
+couchbase.indexer.memory_used,gauge,,byte,,Amount of memory used by the Index service on this node,0,couchbase,
+couchbase.indexer.total_indexer_gc_pause_ns,gauge,,nanosecond,,The total time the indexer has spent in GC pause since last startup,0,couchbase,
+couchbase.index.avg_drain_rate,gauge,,,,Average number of items flushed from memory to disk storage per second,0,couchbase,
+couchbase.index.avg_item_size,gauge,,,,Average size of the keys,0,couchbase,
+couchbase.index.avg_scan_latency,gauge,,nanosecond,,Average time to serve a scan request,0,couchbase,
+couchbase.index.cache_hit_percent,gauge,,,,Percentage of memory accesses that were served from the managed cache,0,couchbase,
+couchbase.index.cache_hits,gauge,,,,Accesses to this index data from RAM,0,couchbase,
+couchbase.index.cache_misses,gauge,,,,Accesses to this index data from disk,0,couchbase,
+couchbase.index.data_size,gauge,,byte,,The size of indexable data that is maintained for the index or partition,0,couchbase,
+couchbase.index.disk_size,gauge,,byte,,Total disk file size consumed by the index or partition,0,couchbase,
+couchbase.index.frag_percent,gauge,,,,Percentage fragmentation of the index,0,couchbase,
+couchbase.index.initial_build_progress,gauge,,,,"Percentage initial build progress for the index. When initial build is completed, the value is 100. For an index partition, the value is listed as 0",0,couchbase,
+couchbase.index.items_count,gauge,,,,The number of items currently indexed,0,couchbase,
+couchbase.index.last_known_scan_time,gauge,,nanosecond,,"Time of the last scan request received for this index (Unix timestamp in nanoseconds). This may be useful for determining whether this index is currently unused. Note: This statistic is persisted to disk every 15 minutes, so it is preserved when the indexer restarts",0,couchbase,
+couchbase.index.memory_used,gauge,,unit,,description,0,couchbase,
+couchbase.index.num_docs_indexed,gauge,,,,Number of documents indexed by the indexer since last startup,0,couchbase,
+couchbase.index.num_docs_pending,gauge,,,,Number of documents pending to be indexed,0,couchbase,
+couchbase.index.num_docs_queued,gauge,,,,Number of documents queued to be indexed,0,couchbase,
+couchbase.index.num_items_flushed,gauge,,,,Number of items flushed from memory to disk storage,0,couchbase,
+couchbase.index.num_pending_requests,gauge,,,,Number of requests received but not yet served by the indexer,0,couchbase,
+couchbase.index.num_requests,gauge,,,,Number of requests served by the indexer since last startup,0,couchbase,
+couchbase.index.num_rows_returned,gauge,,,,Total number of rows returned so far by the indexer,0,couchbase,
+couchbase.index.num_scan_errors,gauge,,,,Number of requests that failed due to errors other than timeout,0,couchbase,
+couchbase.index.num_scan_timeouts,gauge,,,,"Number of requests that timed out, either waiting for snapshots or during scan in progress",0,couchbase,
+couchbase.index.recs_in_mem,gauge,,,,"For standard index storage, this is the number of records in this index that are stored in memory. For memory-optimized index storage, this is the same as items_count",0,couchbase,
+couchbase.index.recs_on_disk,gauge,,,,"For standard index storage, this is the number of records in this index that are stored on disk. For memory-optimized index storage, this is 0",0,couchbase,
+couchbase.index.resident_percent,gauge,,,,Percentage of the data held in memory,0,couchbase,
+couchbase.index.scan_bytes_read,gauge,,,,Number of bytes read by a scan since last startup,0,couchbase,
+couchbase.index.total_scan_duration,gauge,,,,Total time spent by the indexer in scanning rows since last startup,0,couchbase,

couchbase/tests/common.py

@@ -12,6 +12,7 @@
 PORT = '8091'
 QUERY_PORT = '8093'
 SG_PORT = '4985'
+INDEX_STATS_PORT = '9102'
 
 # Tags and common bucket name
 CUSTOM_TAGS = ['optional:tag1']
@@ -21,11 +22,14 @@
 URL = 'http://{}:{}'.format(HOST, PORT)
 QUERY_URL = 'http://{}:{}'.format(HOST, QUERY_PORT)
 SG_URL = 'http://{}:{}'.format(HOST, SG_PORT)
+INDEX_STATS_URL = 'http://{}:{}'.format(HOST, INDEX_STATS_PORT)
 CB_CONTAINER_NAME = 'couchbase-standalone'
 USER = 'Administrator'
 PASSWORD = 'password'
 
-DEFAULT_INSTANCE = {'server': URL, 'user': USER, 'password': PASSWORD, 'timeout': 0.5, 'tags': CUSTOM_TAGS}
+COUCHBASE_MAJOR_VERSION = int(os.getenv('COUCHBASE_VERSION').split(".")[0])
+
+DEFAULT_INSTANCE = {'server': URL, 'user': USER, 'password': PASSWORD, 'timeout': 1, 'tags': CUSTOM_TAGS}
 
 SYNC_GATEWAY_METRICS = [
     "couchbase.sync_gateway.admin_net_bytes_recv",
@@ -132,3 +136,37 @@
     "couchbase.sync_gateway.system_memory_total",
     "couchbase.sync_gateway.warn_count",
 ]
+
+INDEX_STATS_METRICS = [
+    'couchbase.indexer.indexer_state',
+    'couchbase.indexer.memory_quota',
+    'couchbase.indexer.memory_total_storage',
+    'couchbase.indexer.memory_used',
+    'couchbase.indexer.total_indexer_gc_pause_ns',
+    'couchbase.index.avg_drain_rate',
+    'couchbase.index.avg_item_size',
+    'couchbase.index.avg_scan_latency',
+    'couchbase.index.cache_hit_percent',
+    'couchbase.index.cache_hits',
+    'couchbase.index.cache_misses',
+    'couchbase.index.data_size',
+    'couchbase.index.disk_size',
+    'couchbase.index.frag_percent',
+    'couchbase.index.initial_build_progress',
+    'couchbase.index.items_count',
+    'couchbase.index.last_known_scan_time',
+    'couchbase.index.num_docs_indexed',
+    'couchbase.index.num_docs_pending',
+    'couchbase.index.num_docs_queued',
+    'couchbase.index.num_items_flushed',
+    'couchbase.index.num_pending_requests',
+    'couchbase.index.num_requests',
+    'couchbase.index.num_rows_returned',
+    'couchbase.index.num_scan_errors',
+    'couchbase.index.num_scan_timeouts',
+    'couchbase.index.recs_in_mem',
+    'couchbase.index.recs_on_disk',
+    'couchbase.index.resident_percent',
+    'couchbase.index.scan_bytes_read',
+    'couchbase.index.total_scan_duration',
+]

couchbase/tests/compose/standalone.compose

@@ -5,6 +5,7 @@ services:
     image: "couchbase/server:${COUCHBASE_VERSION}"
     ports:
       - 8091-8094:8091-8094
+      - 9102:9102
     container_name: ${CB_CONTAINER_NAME}
   couchbase-sync-gateway:
     container_name: couchbase-sync-gateway

couchbase/tests/conftest.py

@@ -18,6 +18,7 @@
     CUSTOM_TAGS,
     DEFAULT_INSTANCE,
     HERE,
+    INDEX_STATS_URL,
     PASSWORD,
     PORT,
     QUERY_URL,
@@ -38,7 +39,7 @@ def instance_query():
         'server': URL,
         'user': USER,
         'password': PASSWORD,
-        'timeout': 0.5,
+        'timeout': 1,
         'tags': CUSTOM_TAGS,
         'query_monitoring_url': QUERY_URL,
     }
@@ -50,28 +51,43 @@ def instance_sg():
         'server': URL,
         'user': USER,
         'password': PASSWORD,
-        'timeout': 0.5,
+        'timeout': 1,
         'tags': CUSTOM_TAGS,
         'sync_gateway_url': SG_URL,
     }
 
 
+@pytest.fixture
+def instance_index_stats():
+    return {
+        'server': URL,
+        'user': USER,
+        'password': PASSWORD,
+        'timeout': 1,
+        'tags': CUSTOM_TAGS,
+        'index_stats_url': INDEX_STATS_URL,
+    }
+
+
 @pytest.fixture(scope="session")
 def dd_environment():
     """
     Spin up and initialize couchbase
     """
-
+    couchdb_version = os.environ["COUCHBASE_VERSION"][0]
+    conditions = [
+        WaitFor(couchbase_container),
+        WaitFor(couchbase_init),
+        WaitFor(couchbase_setup),
+        WaitFor(node_stats),
+        WaitFor(bucket_stats),
+    ]
+    if int(couchdb_version) >= 7:
+        conditions.append(WaitFor(load_sample_bucket))
     with docker_run(
         compose_file=os.path.join(HERE, 'compose', 'standalone.compose'),
         env_vars={'CB_CONTAINER_NAME': CB_CONTAINER_NAME},
-        conditions=[
-            WaitFor(couchbase_container),
-            WaitFor(couchbase_init),
-            WaitFor(couchbase_setup),
-            WaitFor(node_stats),
-            WaitFor(bucket_stats),
-        ],
+        conditions=conditions,
         attempts=2,
     ):
         yield DEFAULT_INSTANCE
@@ -155,7 +171,7 @@ def couchbase_init():
         '--services',
         'data,index,fts,query',
         '--cluster-ramsize',
-        '256',
+        '512',
         '--cluster-index-ramsize',
         '256',
         '--cluster-fts-ramsize',
@@ -167,6 +183,35 @@ def couchbase_init():
     return r.status_code == requests.codes.ok
 
 
+def load_sample_bucket():
+    """
+    Load sample data bucket
+    """
+
+    # Resources used:
+    # https://docs.couchbase.com/server/current/manage/manage-settings/install-sample-buckets.html
+
+    bucket_loader_args = [
+        'docker',
+        'exec',
+        CB_CONTAINER_NAME,
+        'cbdocloader',
+        '-c',
+        'localhost:{}'.format(PORT),
+        '-u',
+        USER,
+        '-p',
+        PASSWORD,
+        '-d',
+        '/opt/couchbase/samples/gamesim-sample.zip',
+        '-b',
+        'cb_bucket',
+        '-m',
+        '256',
+    ]
+    subprocess.check_call(bucket_loader_args)
+
+
 def node_stats():
     """
     Wait for couchbase to generate node stats