PoC of docker-engine Prometheus metrics collector

Refactored Test Suite for docker_engine

.gitignore

@@ -1,3 +1,4 @@
+.idea/
 debian/files
 debian/files
 debian/tcollector.debhelper.log
@@ -10,3 +11,16 @@ debian/tcollector/
 *.pid
 *.log
 .DS_Store
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/dictionaries
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml

.travis.yml

@@ -2,7 +2,7 @@ language: python
 python:
   - "2.7"
 install:
-  - pip install pylint ordereddict mysqlclient
+  - pip install pylint ordereddict mysqlclient requests feedparser prometheus_client
 script:
   - ./pylint-runner.py -s
   - ./tests.py

collectors/0/docker.py

@@ -203,6 +203,7 @@ def main():
             for level1 in os.listdir(CGROUP_PATH + "/lxc"):
                 if os.path.isdir(CGROUP_PATH + "/lxc/"+level1):
                     readdockerstats(CGROUP_PATH + "/lxc/"+level1, level1)
+        print("sleeping")
         time.sleep(COLLECTION_INTERVAL)
 
 if __name__ == "__main__":

collectors/0/docker_engine.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+# This file is part of tcollector.
+# Copyright (C) 2010-2013  The tcollector Authors.
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.  This program is distributed in the hope that it
+# will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser
+# General Public License for more details.  You should have received a copy
+# of the GNU Lesser General Public License along with this program.  If not,
+# see <http://www.gnu.org/licenses/>.
+"""Imports Docker stats from the docker-api"""
+
+import sys
+
+from collectors.etc import docker_engine_conf
+from collectors.lib.docker_engine.docker_metrics import DockerMetrics
+
+CONFIG = docker_engine_conf.get_config()
+ENABLED = docker_engine_conf.enabled()
+METRICS_PATH = CONFIG['metrics_path']
+
+
+def main():
+    if not ENABLED:
+        sys.stderr.write("Docker-engine collector is not enabled")
+        sys.exit(13)
+
+    """docker_cpu main loop"""
+    cli = DockerMetrics(METRICS_PATH)
+
+    for m in cli.get_endpoint():
+        print m.get_metric_lines()
+
+
+if __name__ == "__main__":
+    sys.exit(main())

collectors/etc/docker_conf.py

@@ -13,7 +13,7 @@
 # see <http://www.gnu.org/licenses/>.
 
 def enabled():
-  return False
+  return True
 
 def get_config():
   """Configuration for the Docker collector

collectors/etc/docker_engine_conf.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+# This file is part of tcollector.
+# Copyright (C) 2015  The tcollector Authors.
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.  This program is distributed in the hope that it
+# will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser
+# General Public License for more details.  You should have received a copy
+# of the GNU Lesser General Public License along with this program.  If not,
+# see <http://www.gnu.org/licenses/>.
+
+def enabled():
+  return True
+
+def get_config():
+  """Configuration for the Docker engine (Prometeus) collector """
+  config = {
+    'interval': 15,
+    'default_dims': '',
+    'metrics_path': 'http://localhost:3376/metrics'
+  }
+  return config

collectors/lib/docker_engine/docker_metrics.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# This file is part of tcollector.
+# Copyright (C) 2010-2013  The tcollector Authors.
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.  This program is distributed in the hope that it
+# will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser
+# General Public License for more details.  You should have received a copy
+# of the GNU Lesser General Public License along with this program.  If not,
+# see <http://www.gnu.org/licenses/>.
+
+import time
+
+import requests
+from prometheus_client.parser import text_string_to_metric_families
+
+from collectors.lib.docker_engine.metric import Metric
+
+
+class DockerMetrics(object):
+    def __init__(self, url):
+        self._url = url
+        self.event_time = time.gmtime()
+
+    def get_endpoint(self):
+        """ Fetches the endpoint """
+        ret = []
+        r = requests.get(self._url)
+        if r.status_code != 200:
+            print "Error %s: %s" % (r.status_code, r.text)
+        else:
+            for line in r.iter_lines():
+                if not line.startswith("#"):
+                    ret.extend(self.eval_prometheus_line(self.event_time, line))
+        return ret
+
+    @staticmethod
+    def eval_prometheus_line(etime, line):
+        ret = []
+        for family in text_string_to_metric_families(line):
+            for sample in family.samples:
+                dims = []
+                for kv in sample[1].items():
+                    dims.append("%s=%s" % kv)
+                m = Metric("docker.{0}".format(*sample), etime, sample[2], dims)
+                ret.append(m)
+        return ret

collectors/lib/docker_engine/metric.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+# This file is part of tcollector.
+# Copyright (C) 2010-2013  The tcollector Authors.
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.  This program is distributed in the hope that it
+# will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser
+# General Public License for more details.  You should have received a copy
+# of the GNU Lesser General Public License along with this program.  If not,
+# see <http://www.gnu.org/licenses/>.
+
+import time
+
+from collectors.etc import docker_engine_conf
+
+CONFIG = docker_engine_conf.get_config()
+DEFAULT_DIMS = CONFIG['default_dims']
+
+
+class Metric(object):
+    def __init__(self, name, etime, value, tags=None):
+        self.name = name
+        self.value = value
+        self.event_time = etime
+        if tags is None:
+            self.dims = set([])
+        else:
+            self.dims = set(tags)
+        self.dims.update(set(DEFAULT_DIMS))
+
+    def add_dims(self, dims):
+        self.dims.update(set(dims))
+
+    def get_metric_lines(self):
+        """ return in OpenTSDB format
+        <name> <time_epoch> <value> [key=val] [key1=val1]...
+        """
+        m = "%s %s %s" % (self.name, int(time.mktime(self.event_time)), self.value)
+        return "%s %s" % (m, " ".join(sorted(list(self.dims))))

collectors/lib/docker_engine/stats.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+# This file is part of tcollector.
+# Copyright (C) 2010-2013  The tcollector Authors.
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.  This program is distributed in the hope that it
+# will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser
+# General Public License for more details.  You should have received a copy
+# of the GNU Lesser General Public License along with this program.  If not,
+# see <http://www.gnu.org/licenses/>.
+
+class Stats(object):
+    def __init__(self, container, mtime):
+        self.dims = [
+            "container_name=%s" % self.trim_container_name(container),
+            "container_id=%s" % container['Id'],
+            "image_name=%s" % container['Image'],
+            "image_id=%s" % container['ImageID']
+        ]
+        self.event_time = mtime
+
+    @staticmethod
+    def trim_container_name(container):
+        return container["Names"][0].strip("/")

collectors/test/docker_engine/test_docker_metrics.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# This file is part of tcollector.
+# Copyright (C) 2010-2013  The tcollector Authors.
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.  This program is distributed in the hope that it
+# will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser
+# General Public License for more details.  You should have received a copy
+# of the GNU Lesser General Public License along with this program.  If not,
+# see <http://www.gnu.org/licenses/>.
+
+import unittest
+
+import time
+
+from collectors.lib.docker_engine.docker_metrics import DockerMetrics
+from collectors.lib.docker_engine.metric import Metric
+
+
+class TestDockerMetrics(unittest.TestCase):
+    def setUp(self):
+        self.now = time.gmtime()
+        self.line = 'engine_daemon_network_actions_seconds_count{action="connect"} 2'
+
+    def test_eval_prometheus_line(self):
+        expected = Metric("docker.engine_daemon_network_actions_seconds_count", self.now, 2.0)
+        expected.add_dims(["action=connect"])
+        expected_line = expected.get_metric_lines()
+        provided = DockerMetrics.eval_prometheus_line(self.now, self.line)[0]
+        provided_line = provided.get_metric_lines()
+        self.assertEqual(expected_line, provided_line)
+
+if __name__ == '__main__':
+    unittest.main()

collectors/test/docker_engine/test_metric.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+# This file is part of tcollector.
+# Copyright (C) 2010-2013  The tcollector Authors.
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.  This program is distributed in the hope that it
+# will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser
+# General Public License for more details.  You should have received a copy
+# of the GNU Lesser General Public License along with this program.  If not,
+# see <http://www.gnu.org/licenses/>.
+
+import unittest
+
+import time
+from unittest import TestCase
+
+from collectors.lib.docker_engine.metric import Metric
+
+
+class TestMetric(TestCase):
+
+    def setUp(self):
+        self.now = time.gmtime()
+        self.metric = Metric("metric", self.now, 10)
+
+    def test_metric_name(self):
+        self.assertEqual(self.metric.name, "metric")
+
+    def test_metric_value(self):
+        self.assertEqual(self.metric.value, 10)
+
+    def test_metric_event_time(self):
+        self.assertEqual(self.metric.event_time, self.now)
+
+    def test_metric_dims(self):
+        self.assertEqual(self.metric.dims, set([]))
+
+    def test_add_dims(self):
+        self.metric.add_dims(["hello=world"])
+        self.assertEqual(self.metric.dims, {"hello=world"})
+
+    def test_get_metric_lines(self):
+        expected = "metric %s 10 " % int(time.mktime(self.now))
+        self.assertEqual(expected, self.metric.get_metric_lines())
+
+    def test_get_metric_lines_with_single_dim(self):
+        self.metric.add_dims(["hello=world"])
+        expected = "metric %s 10 hello=world" % int(time.mktime(self.now))
+        self.assertEqual(expected, self.metric.get_metric_lines())
+
+    def test_get_metric_lines_with_multiple_dim(self):
+        self.metric.add_dims(["hello=world", "foo=bar"])
+        expected = "metric %s 10 foo=bar hello=world" % int(time.mktime(self.now))
+        self.assertEqual(expected, self.metric.get_metric_lines())
+
+
+if __name__ == '__main__':
+    unittest.main()

collectors/test/docker_engine/test_stats.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+# This file is part of tcollector.
+# Copyright (C) 2010-2013  The tcollector Authors.
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.  This program is distributed in the hope that it
+# will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser
+# General Public License for more details.  You should have received a copy
+# of the GNU Lesser General Public License along with this program.  If not,
+# see <http://www.gnu.org/licenses/>.
+
+import unittest
+
+import time
+
+from collectors.lib.docker_engine.stats import Stats
+
+
+class TestStats(unittest.TestCase):
+    def setUp(self):
+        self.now = time.gmtime()
+        self.container = {
+            "Names": ["/name"],
+            "Id": "cntHASH",
+            "Image": "qnib/test",
+            "ImageID": "sha256:123"
+        }
+
+    def test_stats_dims(self):
+        expected_dims = [
+            "container_name=name",
+            "container_id=cntHASH",
+            "image_name=qnib/test",
+            "image_id=sha256:123"
+        ]
+
+        s = Stats(self.container, self.now)
+
+        self.assertEqual(expected_dims, s.dims)
+
+    def test_stats_etime(self):
+        s = Stats(self.container, self.now)
+        self.assertEqual(self.now, s.event_time)
+
+    def test_trim_container_name(self):
+        # cnt = {"Names": ["/name"]}
+        expected = "name"
+        self.assertEqual(expected, Stats.trim_container_name(self.container))
+
+
+if __name__ == '__main__':
+    unittest.main()