[PGBouncer] Adds support for 1.8 new metrics

pgbouncer/CHANGELOG.md

@@ -1,5 +1,12 @@
 # CHANGELOG - pgbouncer
 
+1.1.0 / Unreleased
+==================
+
+### Changes
+
+* [FEATURE] Added new metrics for PGBouncer 1.8 `SHOW STATS` and `SHOW POOLS` See [#1011][]
+
 1.0.2 / [2017-11-21]
 ==================
 
@@ -24,3 +31,4 @@
 <!--- The following link definition list is generated by PimpMyChangelog --->
 [#295]: https://github.com/DataDog/integrations-core/issues/295
 [#735]: https://github.com/DataDog/integrations-core/issues/735
+[#1011]: https://github.com/DataDog/integrations-core/issues/1011

pgbouncer/README.md

@@ -65,6 +65,8 @@ The PgBouncer check is compatible with all major platforms.
 ### Metrics
 See [metadata.csv](https://github.com/DataDog/integrations-core/blob/master/pgbouncer/metadata.csv) for a list of metrics provided by this check.
 
+Note: Not all metrics are available with all versions of PGBouncer. 
+
 ### Events
 The PGboucer check does not include any event at this time.
 

pgbouncer/datadog_checks/pgbouncer/__init__.py

@@ -2,6 +2,6 @@
 
 PgBouncer = pgbouncer.PgBouncer
 
-__version__ = "1.0.2"
+__version__ = "1.1.0"
 
 __all__ = ['pgbouncer']

pgbouncer/datadog_checks/pgbouncer/pgbouncer.py

@@ -33,14 +33,24 @@ class PgBouncer(AgentCheck):
             ('database', 'db'),
         ],
         'metrics': [
-            ('total_requests',       ('pgbouncer.stats.requests_per_second', RATE)),
-            ('total_received',       ('pgbouncer.stats.bytes_received_per_second', RATE)),
-            ('total_sent',           ('pgbouncer.stats.bytes_sent_per_second', RATE)),
-            ('total_query_time',     ('pgbouncer.stats.total_query_time', GAUGE)),
-            ('avg_req',              ('pgbouncer.stats.avg_req', GAUGE)),
-            ('avg_recv',             ('pgbouncer.stats.avg_recv', GAUGE)),
-            ('avg_sent',             ('pgbouncer.stats.avg_sent', GAUGE)),
-            ('avg_query',            ('pgbouncer.stats.avg_query', GAUGE)),
+            ('total_requests',       ('pgbouncer.stats.requests_per_second', RATE)),       # < 1.8
+            ('total_received',       ('pgbouncer.stats.bytes_received_per_second', RATE)), # All versions
+            ('total_sent',           ('pgbouncer.stats.bytes_sent_per_second', RATE)),     # All versions
+            ('total_query_time',     ('pgbouncer.stats.total_query_time', GAUGE)),         # All versions
+            ('avg_req',              ('pgbouncer.stats.avg_req', GAUGE)),                  # < 1.8
+            ('avg_recv',             ('pgbouncer.stats.avg_recv', GAUGE)),                 # All versions
+            ('avg_sent',             ('pgbouncer.stats.avg_sent', GAUGE)),                 # All versions
+            ('avg_query',            ('pgbouncer.stats.avg_query', GAUGE)),                # < 1.8
+
+            ('avg_query_time',       ('pgbouncer.stats.avg_query', GAUGE)),                 # >= 1.8
+            ('total_xact_count',     ('pgbouncer.stats.transactions_per_second', RATE)),   # >= 1.8
+            ('total_query_count',    ('pgbouncer.stats.queries_per_second', RATE)),        # >= 1.8
+            ('total_xact_time',      ('pgbouncer.stats.total_transaction_time', GAUGE)),   # >= 1.8
+            ('total_wait_time',      ('pgbouncer.stats.total_client_wait_time', GAUGE)),   # >= 1.8
+            ('avg_xact_count',       ('pgbouncer.stats.avg_transactions_per_second', RATE)), # >= 1.8
+            ('avg_query_count',      ('pgbouncer.stats.avg_queries_per_second', RATE)),              # >= 1.8
+            ('avg_xact_time',        ('pgbouncer.stats.avg_transactions_time', GAUGE)),    # >= 1.8
+            ('avg_wait_time',        ('pgbouncer.stats.avg_wait_time', GAUGE))             # >= 1.8
         ],
         'query': """SHOW STATS""",
     }
@@ -84,22 +94,22 @@ def _collect_stats(self, db, instance_tags):
         """Query pgbouncer for various metrics
         """
 
-        metric_scope = [self.STATS_METRICS, self.POOLS_METRICS]
+        results = None
+        metrics_scope = [self.STATS_METRICS, self.POOLS_METRICS]
 
         try:
             cursor = db.cursor()
-            results = None
-
-            for scope in metric_scope:
 
+            for scope in metrics_scope:
+                column_names = []
+                query = scope['query']
                 metrics = scope['metrics']
-                cols = [m[0] for m in metrics]
-
+                descriptors = scope['descriptors']
                 try:
-                    query = scope['query']
-                    self.log.debug("Running query: %s" % query)
                     cursor.execute(query)
 
+                    ## Get list of columns in the Query output
+                    [column_names.append(value.name) for value in cursor.description]
                     results = cursor.fetchall()
 
                 except pg.Error as e:
@@ -110,26 +120,25 @@ def _collect_stats(self, db, instance_tags):
                     if row[0] == self.DB_NAME:
                         continue
 
-                    desc = scope['descriptors']
+                    query_output = dict(zip(column_names, row))
+                    self.log.debug("Ran query: %s and returned: %s", query, query_output)
 
-                    # Some versions of pgbouncer have extra fields at the end of SHOW POOLS
-                    if len(row) == len(cols) + len(desc) + 1:
-                        row = row[:-1]
-                    elif len(row) == len(cols) + len(desc) + 2:
-                        row = row[:-2]
+                    tags = list(instance_tags)
 
-                    assert len(row) == len(cols) + len(desc)
+                    for descriptor_name, descriptor_tag in descriptors:
+                        if descriptor_name in column_names:
+                            tags.append("{}:{}".format(descriptor_tag, query_output[descriptor_name]))
 
-                    tags = list(instance_tags)
-                    tags += ["%s:%s" % (d[0][1], d[1]) for d in zip(desc, row[:len(desc)])]
                     for i, (key_name, (mname, mtype)) in enumerate(metrics):
-                        value = row[i + len(desc)]
-                        mtype(self, mname, value, tags)
+                        if key_name in column_names:
+                            value = query_output[key_name]
+                            mtype(self, mname, value, tags)
 
             if not results:
-                self.warning('No results were found for query: "%s"' % query)
+                self.warning("No results were found for query: '%s'" % query)
 
             cursor.close()
+
         except pg.Error as e:
             self.log.error("Connection error: %s" % str(e))
             raise ShouldRestartException
@@ -216,11 +225,6 @@ def check(self, instance):
         else:
             key = '%s:%s' % (host, port)
 
-        if tags is None:
-            tags = []
-        else:
-            tags = list(set(tags))
-
         try:
             db = self._get_connection(key, host, port, user, password,
                                       database_url=database_url)

pgbouncer/manifest.json

@@ -11,7 +11,7 @@
     "mac_os"
   ],
   "package_deps": false,
-  "version": "1.0.2",
+  "version": "1.1.0",
   "guid": "51386802-4502-4991-b592-27eff1ca111c",
   "public_title": "Datadog-PGBouncer Integration",
   "categories":["data store"],

pgbouncer/metadata.csv

@@ -1,17 +1,25 @@
 metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name
-pgbouncer.stats.requests_per_second,gauge,,request,second,The request rate,0,pgbouncer,requests
-pgbouncer.stats.bytes_received_per_second,gauge,,byte,second,The total network traffic received,0,pgbouncer,bytes received
-pgbouncer.stats.bytes_sent_per_second,gauge,,byte,second,The total network traffic sent,0,pgbouncer,bytes sent
+pgbouncer.stats.requests_per_second,rate,,request,second,The request rate,0,pgbouncer,requests
+pgbouncer.stats.transactions_per_second,rate,,request,second, The transaction rate,0,pgbouncer,transaction_requests
+pgbouncer.stats.queries_per_second,rate,,request,second, The query rate,0,pgbouncer,query_requests
+pgbouncer.stats.bytes_received_per_second,rate,,byte,second,The total network traffic received,0,pgbouncer,bytes received
+pgbouncer.stats.bytes_sent_per_second,rate,,byte,second,The total network traffic sent,0,pgbouncer,bytes sent
 pgbouncer.stats.total_query_time,gauge,,microsecond,,Time spent by pgbouncer actively querying PostgreSQL,0,pgbouncer,query time
+pgbouncer.stats.total_transaction_time,gauge,,microsecond,,Time spent by pgbouncer in a transaction,0,pgbouncer,total transaction time
+pgbouncer.stats.total_client_wait_time,gauge,,microsecond,,Time spent by clients waiting for server,0,pgbouncer,total wait time
 pgbouncer.stats.avg_req,gauge,,request,second,The average number of requests per second in last stat period,0,pgbouncer,requests per second
 pgbouncer.stats.avg_recv,gauge,,byte,second,The client network traffic received,0,pgbouncer,client bytes received
 pgbouncer.stats.avg_sent,gauge,,byte,second,The client network traffic sent,0,pgbouncer,client bytes sent
 pgbouncer.stats.avg_query,gauge,,microsecond,,The average query duration,-1,pgbouncer,query duration
+pgbouncer.stats.avg_transactions_per_second,rate,,request,second,The number of transactions per second,0,pgbouncer,avg transaction count
+pgbouncer.stats.avg_queries_per_second,rate,,request,second,The number of queries per second,0,pgbouncer,avg query count
+pgbouncer.stats.avg_transactions_time,gauge,,microsecond,,The average transaction duration,0,pgbouncer,avg transaction time
+pgbouncer.stats.avg_wait_time,gauge,,microsecond,,The time spend by clients waiting for server averaged per second,0,pgbouncer,avg wait time
 pgbouncer.pools.cl_active,gauge,,connection,,Client connections linked to server connection and able to process queries,0,pgbouncer,active conns
 pgbouncer.pools.cl_waiting,gauge,,connection,,Client connections waiting on a server connection,-1,pgbouncer,waiting conns
 pgbouncer.pools.sv_active,gauge,,connection,,Server connections linked to a client connection,0,pgbouncer,server conns
 pgbouncer.pools.sv_idle,gauge,,connection,,Server connections idle and ready for a client query,0,pgbouncer,server idle conns
 pgbouncer.pools.sv_used,gauge,,connection,,"Server connections idle more than server_check_delay, needing server_check_query",-1,pgbouncer,server used conns
 pgbouncer.pools.sv_tested,gauge,,connection,,Server connections currently running either server_reset_query or server_check_query,0,pgbouncer,server tested conns
 pgbouncer.pools.sv_login,gauge,,connection,,Server connections currently in the process of logging in,0,pgbouncer,server login conns
-pgbouncer.pools.maxwait,gauge,,second,,Age of oldest unserved client connection,-1,pgbouncer,max client wait
+pgbouncer.pools.maxwait,gauge,,second,,Age of oldest unserved client connection,-1,pgbouncer,max client wait

pgbouncer/test/test_pgbouncer.py

@@ -61,10 +61,10 @@ def test_checks(self):
         self.assertMetric('pgbouncer.pools.maxwait')
 
         self.assertMetric('pgbouncer.stats.total_query_time')
-        self.assertMetric('pgbouncer.stats.avg_req')
+        # self.assertMetric('pgbouncer.stats.avg_req')
         self.assertMetric('pgbouncer.stats.avg_recv')
         self.assertMetric('pgbouncer.stats.avg_sent')
-        self.assertMetric('pgbouncer.stats.avg_query')
+        # self.assertMetric('pgbouncer.stats.avg_query')
         # Rate metrics, need 2 collection rounds
         try:
             connection = pg.connect(
@@ -80,7 +80,7 @@ def test_checks(self):
             pass
         time.sleep(1)
         self.run_check(config)
-        self.assertMetric('pgbouncer.stats.requests_per_second')
+        # self.assertMetric('pgbouncer.stats.requests_per_second')
         self.assertMetric('pgbouncer.stats.bytes_received_per_second')
         self.assertMetric('pgbouncer.stats.bytes_sent_per_second')