Merge pull request #5 from signalfx/tls

Add support for ssl_cert_validation

.circleci/config.yml

@@ -1,26 +1,26 @@
-version: '2'
+version: 2.1
 jobs:
   build:
     docker:
-      - image: ubuntu:yakkety
+      - image: ubuntu:18.04
     working_directory: ~/code
     steps:
       - setup_remote_docker
       - run:
           name: Install Docker client
           command: |
             set -x
-            VER="17.03.0-ce"
+            VER="18.09.6"
             apt-get update -q
-            apt-get install -yq curl python-pip
-            curl -L -o /tmp/docker-$VER.tgz https://get.docker.com/builds/Linux/x86_64/docker-$VER.tgz
+            apt-get install -yq curl python-pip git
+            curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$VER.tgz
             tar -xz -C /tmp -f /tmp/docker-$VER.tgz
             mv /tmp/docker/* /usr/bin
       - run:
           name: Install docker-compose
           command: |
             set -x
-            curl -L https://github.com/docker/compose/releases/download/1.11.2/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose
+            curl -L "https://github.com/docker/compose/releases/download/1.24.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
             chmod +x /usr/local/bin/docker-compose
       - checkout
       - run:
@@ -32,4 +32,4 @@ jobs:
       - run:
           name: Run integration tests
           working_directory: ~/code/integration-test
-          command: bash run.sh
+          command: ./run.sh

README.md

@@ -12,7 +12,7 @@ An etcd [collectd](http://www.collectd.org/) plugin which users can use to send
 ## Requirements
 
 * collectd 4.9 or later (for the Python plugin)
-* Python 2.6 or later
+* Python 2.7 or later
 * etcd 2.0.8 or later
 
 ## Configuration
@@ -33,6 +33,7 @@ Optional configurations keys include:
 Specify path to keyfile and certificate if certificate based authentication of clients is enabled on your etcd server
 * ssl_keyfile - path to file
 * ssl_certificate - path to file
+* ssl_cert_validation - if False, don't require SSL certificate validation (default True to validate certificates)
 
 Provide a custom file that lists trusted CA certificates, required when keyfile and certificate are provided
 * ssl_ca_certs - path to file

etcd_plugin.py

@@ -1,11 +1,13 @@
 #!/usr/bin/env python
-import urllib2
-import urllib_ssl_handler
-import json
 import collections
+
 import collectd
+import requests
 import six
+import urllib3
 
+# Prevents spamming when not validating certificates.
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 LEADER = "StateLeader"
 FOLLOWER = "StateFollower"
@@ -108,7 +110,9 @@ def read_config(conf):
     http_timeout = DEFAULT_API_TIMEOUT
 
     required_keys = frozenset(('Host', 'Port'))
-    ssl_keys = {}
+    ssl_keys = {
+        'ssl_cert_validation': True
+    }
     testing = False
 
     for val in conf.children:
@@ -136,6 +140,11 @@ def read_config(conf):
             ssl_keys['ssl_certificate'] = val.values[0]
         elif val.key == 'ssl_ca_certs' and val.values[0]:
             ssl_keys['ssl_ca_certs'] = val.values[0]
+        elif val.key == "ssl_cert_validation" and val.values[0]:
+            # Doesn't use str_to_bool because the function defaults to
+            # false and we want to default to true.
+            if val.values[0].strip().lower() == 'false':
+                ssl_keys['ssl_cert_validation'] = False
         elif val.key == 'Testing' and str_to_bool(val.values[0]):
             testing = True
 
@@ -436,10 +445,9 @@ def get_json(data, url):
     response = make_api_call(data, url)
     try:
         if response:
-            return json.load(response)
-    except ValueError, e:
+            return response.json()
+    except ValueError as e:
         collectd.error("ERROR: JSON parsing failed: (%s) %s" % (e, url))
-        return
 
 
 def get_text(data, url):
@@ -448,39 +456,52 @@ def get_text(data, url):
     '''
     response = make_api_call(data, url)
     if response:
-        return response.read()
+        return response.text
 
 
 def make_api_call(data, url):
-    collectd.debug("GETTING THIS  URL %s" % url)
+    collectd.debug("GETTING THIS URL %s" % url)
     try:
-        key_file, cert_file, ca_certs = get_ssl_params(data)
-        opener = urllib2.build_opener(urllib_ssl_handler.HTTPSHandler(
-            key_file=key_file, cert_file=cert_file, ca_certs=ca_certs))
+        key_file, cert_file, ca_certs, cert_validation = get_ssl_params(data)
+
+        args = {
+            "url": url,
+            "verify": True,
+        }
+
+        if key_file and cert_file:
+            args["cert"] = (cert_file, key_file)
+
+        if ca_certs:
+            args["verify"] = ca_certs
 
-        response = opener.open(url)
-        return response
-    except (urllib2.HTTPError, urllib2.URLError), e:
+        if not cert_validation:
+            args["verify"] = False
+
+        return requests.get(**args)
+    except requests.RequestException as e:
         collectd.error("ERROR: API call failed: (%s) %s" % (e, url))
 
 
 def get_ssl_params(data):
     '''
     Helper method to prepare auth tuple
     '''
+    ssl_keys = data['ssl_keys']
+
     key_file = None
     cert_file = None
     ca_certs = None
+    cert_validation = ssl_keys['ssl_cert_validation']
 
-    ssl_keys = data['ssl_keys']
     if 'ssl_certificate' in ssl_keys and 'ssl_keyfile' in ssl_keys:
         key_file = ssl_keys['ssl_keyfile']
         cert_file = ssl_keys['ssl_certificate']
 
     if 'ssl_ca_certs' in ssl_keys:
         ca_certs = ssl_keys['ssl_ca_certs']
 
-    return (key_file, cert_file, ca_certs)
+    return (key_file, cert_file, ca_certs, cert_validation)
 
 
 def prepare_and_dispatch_metric(name, value, _type, dimensions):

integration-test/20-etcd-test.conf

@@ -1,34 +1,50 @@
 LoadPlugin python
 
 <Plugin python>
-  ModulePath "/opt/collectd-etcd"
+    ModulePath "/opt/collectd-etcd"
 
-  Import etcd_plugin
-  <Module etcd_plugin>
-  	Host "etcd238"
-  	Port "2379"
-    Cluster 1
-    Dimension foo bar
-  	Interval 10
-  </Module>
-  <Module etcd_plugin>
-  	Host "etcd310"
-  	Port "2379"
-    Cluster 1
-  	Interval 10
-  </Module>
-  <Module etcd_plugin>
-  	Host "etcd324"
-  	Port "2379"
-    Cluster 2
-  	Interval 10
-    EnhancedMetrics True
-  </Module>
-  <Module etcd_plugin>
-  	Host "etcd208"
-  	Port "2379"
-    Cluster 2
-  	Interval 10
-    EnhancedMetrics True
-  </Module>
+    Import etcd_plugin
+
+    <Module etcd_plugin>
+        Host "etcd208"
+        Port "2379"
+        Cluster 1
+        Interval 10
+        EnhancedMetrics True
+    </Module>
+    <Module etcd_plugin>
+        Host "etcd238"
+        Port "2379"
+        Cluster 2
+        Dimension foo bar
+        Interval 10
+    </Module>
+    <Module etcd_plugin>
+        Host "etcd310"
+        Port "2379"
+        Cluster 3
+        Interval 10
+    </Module>
+    <Module etcd_plugin>
+        Host "etcd324"
+        Port "2379"
+        Cluster 4
+        Interval 10
+        EnhancedMetrics True
+        ssl_keyfile "/opt/testing/certs/client.key"
+        ssl_certificate "/opt/testing/certs/client.crt"
+        ssl_ca_certs "/opt/testing/certs/server.crt"
+        ssl_cert_validation True
+    </Module>
+    <Module etcd_plugin>
+        Host "etcd324-tls-unverified"
+        Port "2379"
+        Cluster 5
+        Interval 10
+        EnhancedMetrics True
+        ssl_keyfile "/opt/testing/certs/client.key"
+        ssl_certificate "/opt/testing/certs/client.crt"
+        ssl_ca_certs "/opt/testing/certs/server.crt"
+        ssl_cert_validation False
+    </Module>
 </Plugin>

integration-test/Dockerfile.collectd

@@ -7,10 +7,7 @@ ENV COLLECTD_INTERVAL=3 COLLECTD_HOSTNAME=etcd-test DISABLE_AGGREGATION=true DIS
 RUN apt-get update &&\
     apt-get install -yq netcat curl
 
-CMD /.docker/setup_etcd
-ADD integration-test/setup_etcd /.docker/setup_etcd
-
 ## The context of the image build should be the root dir of this repo!!
 ADD etcd_plugin.py /opt/collectd-etcd/
-ADD urllib_ssl_handler.py /opt/collectd-etcd/
 ADD integration-test/20-etcd-test.conf /etc/collectd/managed_config/
+ADD integration-test/certs/* /opt/testing/certs/

integration-test/Dockerfile.etcd32

@@ -0,0 +1,2 @@
+FROM quay.io/coreos/etcd:v3.2
+COPY certs/* /opt/testing/certs/

integration-test/certs/client.crt

@@ -0,0 +1,27 @@
+-----BEGIN CERTIFICATE-----
+MIIElzCCAn8CAQEwDQYJKoZIhvcNAQEFBQAwEjEQMA4GA1UEAwwHZXRjZDMyNDAg
+Fw0xOTA1MDkxOTM0MzVaGA8yMjkzMDIyMDE5MzQzNVowDzENMAsGA1UEAwwEdXNl
+cjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALY16iHyTYoJfG4WFLP7
+nnPq4lLcr1+ij+HM8BZBMHalJ5L7sKVK3RU3rxJy0QQtvfQBr94ly/8XU8l0fX7E
+IbAYnr0NSI3RneP9dwcKdYfekqwYcxsY8ixpyLD4vdfBd/PqTLOa+f0ymlnvuXpU
+n64b2bquY4R45MD27lr5YWvByqD5mQkI1qn4QHyW86KMP3mGGT+1c0oWCYmJcqj0
+ty5WfBIYfdhaqr6W2U6fM9NIBpz4TNP5OP1Ba1HmrRBLNoS6vyoQOLuW+B3zbCKm
+G9ni3PtX+yDcB3v+9gOQmcPXeTRmTbQfbJoFlD9lf8kWUrMqFsA/f+dAdraWCYAJ
+i39ReR71S1DPrpn89vOgnPuVquScydEW90Sl1NuftJ0uh4WkBiX6gcz7EmuCrJXR
+2FUhjLdWc7WC7hS/7C0j0/rvba0IukgD1OaN4kreaVVBP4WT7Y4VWsCTTkZ2xRkX
+PbTTs1Iqj1Q4yRzrMAZ7Rd2Seb5o0j7pTIKHqr7B9++2GJ/OA3TcLPOaN0vepca7
+VYavvFy+5XxBezjgiknFKS9GXjiB5+pKNZou5wrGckfkSJo8Ndf/l8rKrCjiD71c
+zk776x2xAiHOG8jpBdUPZIwR0mv2tqXt/NN219CF2rV0VCgslTbXA/FZooRipQac
++0d0bBT0bIq6GDTAeLKQ8LuXAgMBAAEwDQYJKoZIhvcNAQEFBQADggIBALJjluL1
+g6+Wdr/B6unkHW9BSo5TFeIHLxxFggjP8pFhROcikPXti1vvx9tSqSbBngGfW/YL
+LU5ZEhELcQVQUU0mSdQigMRW249Vp5CL+tD0UkbzD68dmfNgN6Vt0NRQQH5Ut1Su
+8bBIRZj5bO5czEZoy+n+VJdTUrs9IquNDLRyrA9OzQ2q4EmCApP0OEgWvMH70Osv
+lbjFce97aKU/8slNUbJztROzKYOh4Kv1shAYmxPWye3Aeiod/g4fpRM0Xq39gGSO
+ZHTAh/7IXe6BFjl6DmT/2hVD9g5keuN0doOtEU4PQeYU8f+39nDZEsRGIIyZ/9j3
+08Hq6UyUuwt42nz+XpGQn5rGT4CbSdYiVMgeEvuz+LmaBSUGw30A4Y4uNQfl2Ra3
+OIGK3wbTemoe8AUW0+UYuRr+/rMggJ9+qrVTUIRSuiX5SF0pF5sbtNlJdJ6bSd5q
+UUT/D/TF22ZXOOICTDbBEYEIMR4Mre1RIkcoToQplTOzUGsK+4L+jWpcnM/FTZMw
+FydFUo105Jq2aUIVzRBEbT7McijEFSVQNO3IkxYk/79GKVjkie9v2hP8u9ShhM2E
+wrpycc6s3LyQ86TWE6hD/6wD5/WC6Bsez5zij5NIaNwZaDVnHCXdwHe1Ww92owDj
+zOx2DX7xWkDDZDiZWmbqnQWue7YVO46YicuQ
+-----END CERTIFICATE-----