Handled deprecated ES stats (#61)

* fixed logic for removing deprecated metrics

* string to int for version numbers

* added unit test

* add dependency

* added dependency

* added dependency

* fixed test errors

* mocked module collectd

* fixed docker build error

* edited README file

* halting execution on unit test failure

* halting execution on unit test failure

* halting execution on unit test failure

* halting execution on unit test failure

* halting execution on unit test failure

* halting execution on unit test failure

* halting execution on unit test failure

* halting execution on unit test failure

.circleci/config.yml

@@ -12,7 +12,7 @@ jobs:
             set -x
             VER="17.03.0-ce"
             apt-get update -q
-            apt-get install -yq curl python
+            apt-get install -yq curl python-pip
             curl -L -o /tmp/docker-$VER.tgz https://get.docker.com/builds/Linux/x86_64/docker-$VER.tgz
             tar -xz -C /tmp -f /tmp/docker-$VER.tgz
             mv /tmp/docker/* /usr/bin
@@ -27,8 +27,10 @@ jobs:
           name: Run basic tests
           working_directory: ~/code/tests
           command: |
+            pip install pytest
+            pip install mock
             bash run_tests.sh | tee /tmp/test.log || true  # The test command always exits non-zero
-            grep -v -E 'FAILED|ERROR' /tmp/test.log || exit 1
+            ! grep -E 'FAILED|ERROR' /tmp/test.log > /dev/null || exit 1
       - run:
           name: Run integration tests
           working_directory: ~/code/tests/integration

.gitignore

@@ -1,2 +1,7 @@
 .idea/
-*.tmp
+*.tmp
+.Python
+bin/
+include/
+lib/
+__pycache__/

.travis.yml

@@ -4,5 +4,6 @@ python:
   - "2.7"
 install:
   pip install flake8
+  pip install pytest
 before_script:
   flake8 *.py

elasticsearch_collectd.py

@@ -755,14 +755,14 @@ def remove_deprecated_elements(deprecated, elements, version):
     # Attempt to parse the major, minor, and revision
     (major, minor, revision) = version.split('.')
 
-    # Sanitize alphas and betas from revision number
-    revision = revision.split('-')[0]
+    # Strings to int and sanitize alphas and betas from revision number
+    (major, minor, revision) = (int(major), int(minor), int(revision.split('-')[0]))
 
     # Iterate over deprecation lists and remove any keys that were deprecated
     # prior to the current version
     for dep in deprecated:
-        if (major >= dep['major']) \
-                or (major == dep['major'] and minor >= dep['minor']) \
+        if (major > dep['major']) \
+                or (major == dep['major'] and minor > dep['minor']) \
                 or (major == dep['major'] and minor == dep['minor']
                     and revision >= dep['revision']):
             if type(elements) is list:

elasticsearch_collectd_test.py

@@ -0,0 +1,30 @@
+import sys
+
+import mock
+import pytest
+
+
+class MockCollectd(mock.MagicMock):
+    pass
+
+
+sys.modules['collectd'] = MockCollectd()
+
+
+from elasticsearch_collectd import remove_deprecated_elements
+
+
+@pytest.mark.parametrize("deprecated_elements, input_elements, version, expected_elements", [
+    (
+            [{'major': 1, 'minor': 3, 'revision': 100, 'keys': ['element1', 'element3']}],
+            ['element2', 'element3'],
+            '1.03.00104',
+            ['element2']
+    )
+])
+def test_remove_deprecated_elements(deprecated_elements, input_elements, version, expected_elements):
+    print deprecated_elements, input_elements, version, expected_elements
+    elements = remove_deprecated_elements(deprecated_elements, input_elements, version)
+    assert len(elements) == len(expected_elements)
+    for i in range(len(elements)):
+        assert elements[i] == expected_elements[i]

tests/integration/test.py

@@ -42,7 +42,6 @@ def wait_for_metrics_from_each_cluster():
             print 'Waiting for metric: %s from cluster %s...' % (metric, c)
             eventually_true(lambda: any([metric in str(m.get('type_instance')) for m in get_metric_data()]),
                             TIMEOUT_SECS - (time() - start))
-
             print 'metric: %s Found! from cluster: %s' % (metric, c)
 
 

tests/run_tests.sh

@@ -8,6 +8,13 @@ PYTHON_VERSION=$(${PYTHON} -V 2>&1)
 
 echo "Interpreter version: ${PYTHON_VERSION}"
 
+echo "Running unit tests"
+pytest ../elasticsearch_collectd_test.py
+status=$?
+if [[ "$status" != 0 ]]; then
+    exit ${status}
+fi
+
 tmpfile=$(mktemp /tmp/run_tests.sh.XXXXXX)
 trap 'rm -f $tmpfile' 1 2 3 15
 for scenario in `ls data`; do