From 1d6e2db49bc79991b3d62c31e7aca8b081db6085 Mon Sep 17 00:00:00 2001 From: Jay Camp Date: Thu, 9 May 2019 18:04:06 -0400 Subject: [PATCH 1/3] Add support for ssl_cert_validation --- README.md | 3 +- etcd_plugin.py | 53 +++++-- integration-test/20-etcd-test.conf | 74 +++++---- integration-test/Dockerfile.collectd | 4 - .../Dockerfile.collectd.amazon2017.03 | 58 ------- integration-test/certs/client.crt | 27 ++++ integration-test/certs/client.key | 52 +++++++ integration-test/certs/client_csr.pem | 26 ++++ integration-test/certs/server.crt | 27 ++++ integration-test/certs/server.key | 52 +++++++ integration-test/docker-compose.yml | 141 +++++++++--------- integration-test/run.sh | 13 +- integration-test/setup_etcd | 16 -- integration-test/test.py | 10 +- requirements.txt | 1 + urllib_ssl_handler.py | 109 -------------- 16 files changed, 343 insertions(+), 323 deletions(-) delete mode 100644 integration-test/Dockerfile.collectd.amazon2017.03 create mode 100644 integration-test/certs/client.crt create mode 100644 integration-test/certs/client.key create mode 100644 integration-test/certs/client_csr.pem create mode 100644 integration-test/certs/server.crt create mode 100644 integration-test/certs/server.key mode change 100644 => 100755 integration-test/run.sh delete mode 100755 integration-test/setup_etcd delete mode 100644 urllib_ssl_handler.py diff --git a/README.md b/README.md index 97bc689..55df0fa 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ An etcd [collectd](http://www.collectd.org/) plugin which users can use to send ## Requirements * collectd 4.9 or later (for the Python plugin) -* Python 2.6 or later +* Python 2.7 or later * etcd 2.0.8 or later ## Configuration @@ -33,6 +33,7 @@ Optional configurations keys include: Specify path to keyfile and certificate if certificate based authentication of clients is enabled on your etcd server * ssl_keyfile - path to file * ssl_certificate - path to file +* ssl_cert_validation - if False, don't require SSL certificate validation (default True to validate certificates) Provide a custom file that lists trusted CA certificates, required when keyfile and certificate are provided * ssl_ca_certs - path to file diff --git a/etcd_plugin.py b/etcd_plugin.py index 6970252..0891343 100644 --- a/etcd_plugin.py +++ b/etcd_plugin.py @@ -1,11 +1,13 @@ #!/usr/bin/env python -import urllib2 -import urllib_ssl_handler import json import collections import collectd +import requests import six +import urllib3 +# Prevents spamming when not validating certificates. +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) LEADER = "StateLeader" FOLLOWER = "StateFollower" @@ -108,7 +110,9 @@ def read_config(conf): http_timeout = DEFAULT_API_TIMEOUT required_keys = frozenset(('Host', 'Port')) - ssl_keys = {} + ssl_keys = { + 'ssl_cert_validation': True + } testing = False for val in conf.children: @@ -136,6 +140,11 @@ def read_config(conf): ssl_keys['ssl_certificate'] = val.values[0] elif val.key == 'ssl_ca_certs' and val.values[0]: ssl_keys['ssl_ca_certs'] = val.values[0] + elif val.key == "ssl_cert_validation" and val.values[0]: + # Doesn't use str_to_bool because the function defaults to false and + # we want to default to true. + if val.values[0].strip().lower() == 'false': + ssl_keys['ssl_cert_validation'] = False elif val.key == 'Testing' and str_to_bool(val.values[0]): testing = True @@ -436,10 +445,9 @@ def get_json(data, url): response = make_api_call(data, url) try: if response: - return json.load(response) - except ValueError, e: + return response.json() + except ValueError as e: collectd.error("ERROR: JSON parsing failed: (%s) %s" % (e, url)) - return def get_text(data, url): @@ -448,19 +456,30 @@ def get_text(data, url): ''' response = make_api_call(data, url) if response: - return response.read() + return response.text def make_api_call(data, url): - collectd.debug("GETTING THIS URL %s" % url) + collectd.debug("GETTING THIS URL %s" % url) try: - key_file, cert_file, ca_certs = get_ssl_params(data) - opener = urllib2.build_opener(urllib_ssl_handler.HTTPSHandler( - key_file=key_file, cert_file=cert_file, ca_certs=ca_certs)) + key_file, cert_file, ca_certs, cert_validation = get_ssl_params(data) - response = opener.open(url) - return response - except (urllib2.HTTPError, urllib2.URLError), e: + args = { + "url": url, + "verify": True, + } + + if key_file and cert_file: + args["cert"] = (cert_file, key_file) + + if ca_certs: + args["verify"] = ca_certs + + if not cert_validation: + args["verify"] = False + + return requests.get(**args) + except requests.RequestException as e: collectd.error("ERROR: API call failed: (%s) %s" % (e, url)) @@ -468,11 +487,13 @@ def get_ssl_params(data): ''' Helper method to prepare auth tuple ''' + ssl_keys = data['ssl_keys'] + key_file = None cert_file = None ca_certs = None + cert_validation = ssl_keys['ssl_cert_validation'] - ssl_keys = data['ssl_keys'] if 'ssl_certificate' in ssl_keys and 'ssl_keyfile' in ssl_keys: key_file = ssl_keys['ssl_keyfile'] cert_file = ssl_keys['ssl_certificate'] @@ -480,7 +501,7 @@ def get_ssl_params(data): if 'ssl_ca_certs' in ssl_keys: ca_certs = ssl_keys['ssl_ca_certs'] - return (key_file, cert_file, ca_certs) + return (key_file, cert_file, ca_certs, cert_validation) def prepare_and_dispatch_metric(name, value, _type, dimensions): diff --git a/integration-test/20-etcd-test.conf b/integration-test/20-etcd-test.conf index 0d149cf..e75e0be 100644 --- a/integration-test/20-etcd-test.conf +++ b/integration-test/20-etcd-test.conf @@ -1,34 +1,50 @@ LoadPlugin python - ModulePath "/opt/collectd-etcd" + ModulePath "/opt/collectd-etcd" - Import etcd_plugin - - Host "etcd238" - Port "2379" - Cluster 1 - Dimension foo bar - Interval 10 - - - Host "etcd310" - Port "2379" - Cluster 1 - Interval 10 - - - Host "etcd324" - Port "2379" - Cluster 2 - Interval 10 - EnhancedMetrics True - - - Host "etcd208" - Port "2379" - Cluster 2 - Interval 10 - EnhancedMetrics True - + Import etcd_plugin + + + Host "etcd208" + Port "2379" + Cluster 1 + Interval 10 + EnhancedMetrics True + + + Host "etcd238" + Port "2379" + Cluster 2 + Dimension foo bar + Interval 10 + + + Host "etcd310" + Port "2379" + Cluster 3 + Interval 10 + + + Host "etcd324" + Port "2379" + Cluster 4 + Interval 10 + EnhancedMetrics True + ssl_keyfile "/opt/testing/certs/client.key" + ssl_certificate "/opt/testing/certs/client.crt" + ssl_ca_certs "/opt/testing/certs/server.crt" + ssl_cert_validation True + + + Host "etcd324-tls-invalid" + Port "2379" + Cluster 5 + Interval 10 + EnhancedMetrics True + ssl_keyfile "/opt/testing/certs/client.key" + ssl_certificate "/opt/testing/certs/client.crt" + ssl_ca_certs "/opt/testing/certs/server.crt" + ssl_cert_validation False + diff --git a/integration-test/Dockerfile.collectd b/integration-test/Dockerfile.collectd index ec3c882..5fd6be2 100644 --- a/integration-test/Dockerfile.collectd +++ b/integration-test/Dockerfile.collectd @@ -7,10 +7,6 @@ ENV COLLECTD_INTERVAL=3 COLLECTD_HOSTNAME=etcd-test DISABLE_AGGREGATION=true DIS RUN apt-get update &&\ apt-get install -yq netcat curl -CMD /.docker/setup_etcd -ADD integration-test/setup_etcd /.docker/setup_etcd - ## The context of the image build should be the root dir of this repo!! ADD etcd_plugin.py /opt/collectd-etcd/ -ADD urllib_ssl_handler.py /opt/collectd-etcd/ ADD integration-test/20-etcd-test.conf /etc/collectd/managed_config/ diff --git a/integration-test/Dockerfile.collectd.amazon2017.03 b/integration-test/Dockerfile.collectd.amazon2017.03 deleted file mode 100644 index 964af58..0000000 --- a/integration-test/Dockerfile.collectd.amazon2017.03 +++ /dev/null @@ -1,58 +0,0 @@ -FROM amazonlinux:2017.03 - -ENV needed_deps="tar curl" -ENV stage="release" -ENV aws_linux_rpm="SignalFx-collectd-RPMs-AWS_EC2_Linux-${stage}-latest.noarch.rpm" -ENV aws_linux="https://dl.signalfx.com/rpms/SignalFx-rpms/${stage}/${aws_linux_rpm}" -ENV needed_rpm=$aws_linux -ENV needed_rpm_name=$aws_linux_rpm -ENV aws_linux_plugin_rpm="SignalFx-collectd_plugin-RPMs-AWS_EC2_Linux-${stage}-latest.noarch.rpm" -ENV aws_linux_plugin="https://dl.signalfx.com/rpms/SignalFx-rpms/${stage}/${aws_linux_plugin_rpm}" -ENV needed_plugin_rpm=$aws_linux_plugin -ENV needed_plugin_rpm_name=$aws_linux_plugin_rpm -ENV yum_flags="-y" - -RUN yum $yum_flags install $needed_deps -RUN printf "Downloading SignalFx RPM %s\n" "$needed_rpm" \ - && curl -sSL $insecure $needed_rpm -o "$needed_rpm_name" -RUN printf "Installing SignalFx RPM\n" \ - && yum $yum_flags install $needed_rpm_name -RUN rm -f $needed_rpm_name -# RUN type setsebool > /dev/null 2>&1 && setsebool -P collectd_tcp_network_connect on -RUN export EPEL_REPO=$(yum repolist enabled | grep "epel/x86_64") -RUN if [ -n "$EPEL_REPO" ]; then yum_flags="$yum_flags --disablerepo=epel"; fi -RUN printf "Installing collectd\n" \ - && yum $yum_flags install collectd -RUN printf "Installing base-plugins\n" \ - && yum $yum_flags install collectd-disk collectd-write_http - -RUN printf "Downloading SignalFx plugin RPM\n" \ - && curl -sSL $insecure $needed_plugin_rpm -o $needed_plugin_rpm_name -RUN printf "Installing SignalFx plugin RPM\n" \ - && yum -y install $needed_plugin_rpm_name -RUN rm -f $needed_plugin_rpm_name -RUN printf "Installing signalfx-collectd-plugin\n" \ - && yum -y install signalfx-collectd-plugin - -# add config -ADD integration-test/collectd.conf /etc/collectd.conf - -# Debian is super minimalistic -RUN yum install -y nc -RUN yum -y install python-pip -RUN easy_install-2.6 pip -RUN pip2.6 install requests -RUN pip2.6 install six - - -CMD /.docker/setup_etcd -ADD integration-test/setup_etcd /.docker/setup_etcd - -## The context of the image build should be the root dir of this repo!! -ADD etcd_plugin.py /opt/collectd-etcd/ -ADD urllib_ssl_handler.py /opt/collectd-etcd/ -ADD integration-test/20-etcd-test.conf /etc/collectd.d/managed_config/ - -## Mock up a run.sh file -RUN echo $'#! /bin/bash \nexec collectd -C /etc/collectd.conf -f' > /.docker/run.sh -RUN chmod +x /.docker/run.sh diff --git a/integration-test/certs/client.crt b/integration-test/certs/client.crt new file mode 100644 index 0000000..ab7b2b3 --- /dev/null +++ b/integration-test/certs/client.crt @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE----- +MIIElzCCAn8CAQEwDQYJKoZIhvcNAQEFBQAwEjEQMA4GA1UEAwwHZXRjZDMyNDAg +Fw0xOTA1MDkxOTM0MzVaGA8yMjkzMDIyMDE5MzQzNVowDzENMAsGA1UEAwwEdXNl +cjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALY16iHyTYoJfG4WFLP7 +nnPq4lLcr1+ij+HM8BZBMHalJ5L7sKVK3RU3rxJy0QQtvfQBr94ly/8XU8l0fX7E +IbAYnr0NSI3RneP9dwcKdYfekqwYcxsY8ixpyLD4vdfBd/PqTLOa+f0ymlnvuXpU +n64b2bquY4R45MD27lr5YWvByqD5mQkI1qn4QHyW86KMP3mGGT+1c0oWCYmJcqj0 +ty5WfBIYfdhaqr6W2U6fM9NIBpz4TNP5OP1Ba1HmrRBLNoS6vyoQOLuW+B3zbCKm +G9ni3PtX+yDcB3v+9gOQmcPXeTRmTbQfbJoFlD9lf8kWUrMqFsA/f+dAdraWCYAJ +i39ReR71S1DPrpn89vOgnPuVquScydEW90Sl1NuftJ0uh4WkBiX6gcz7EmuCrJXR +2FUhjLdWc7WC7hS/7C0j0/rvba0IukgD1OaN4kreaVVBP4WT7Y4VWsCTTkZ2xRkX +PbTTs1Iqj1Q4yRzrMAZ7Rd2Seb5o0j7pTIKHqr7B9++2GJ/OA3TcLPOaN0vepca7 +VYavvFy+5XxBezjgiknFKS9GXjiB5+pKNZou5wrGckfkSJo8Ndf/l8rKrCjiD71c +zk776x2xAiHOG8jpBdUPZIwR0mv2tqXt/NN219CF2rV0VCgslTbXA/FZooRipQac ++0d0bBT0bIq6GDTAeLKQ8LuXAgMBAAEwDQYJKoZIhvcNAQEFBQADggIBALJjluL1 +g6+Wdr/B6unkHW9BSo5TFeIHLxxFggjP8pFhROcikPXti1vvx9tSqSbBngGfW/YL +LU5ZEhELcQVQUU0mSdQigMRW249Vp5CL+tD0UkbzD68dmfNgN6Vt0NRQQH5Ut1Su +8bBIRZj5bO5czEZoy+n+VJdTUrs9IquNDLRyrA9OzQ2q4EmCApP0OEgWvMH70Osv +lbjFce97aKU/8slNUbJztROzKYOh4Kv1shAYmxPWye3Aeiod/g4fpRM0Xq39gGSO +ZHTAh/7IXe6BFjl6DmT/2hVD9g5keuN0doOtEU4PQeYU8f+39nDZEsRGIIyZ/9j3 +08Hq6UyUuwt42nz+XpGQn5rGT4CbSdYiVMgeEvuz+LmaBSUGw30A4Y4uNQfl2Ra3 +OIGK3wbTemoe8AUW0+UYuRr+/rMggJ9+qrVTUIRSuiX5SF0pF5sbtNlJdJ6bSd5q +UUT/D/TF22ZXOOICTDbBEYEIMR4Mre1RIkcoToQplTOzUGsK+4L+jWpcnM/FTZMw +FydFUo105Jq2aUIVzRBEbT7McijEFSVQNO3IkxYk/79GKVjkie9v2hP8u9ShhM2E +wrpycc6s3LyQ86TWE6hD/6wD5/WC6Bsez5zij5NIaNwZaDVnHCXdwHe1Ww92owDj +zOx2DX7xWkDDZDiZWmbqnQWue7YVO46YicuQ +-----END CERTIFICATE----- diff --git a/integration-test/certs/client.key b/integration-test/certs/client.key new file mode 100644 index 0000000..e6fbf12 --- /dev/null +++ b/integration-test/certs/client.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQC2Neoh8k2KCXxu +FhSz+55z6uJS3K9foo/hzPAWQTB2pSeS+7ClSt0VN68SctEELb30Aa/eJcv/F1PJ +dH1+xCGwGJ69DUiN0Z3j/XcHCnWH3pKsGHMbGPIsaciw+L3XwXfz6kyzmvn9MppZ +77l6VJ+uG9m6rmOEeOTA9u5a+WFrwcqg+ZkJCNap+EB8lvOijD95hhk/tXNKFgmJ +iXKo9LcuVnwSGH3YWqq+ltlOnzPTSAac+EzT+Tj9QWtR5q0QSzaEur8qEDi7lvgd +82wiphvZ4tz7V/sg3Ad7/vYDkJnD13k0Zk20H2yaBZQ/ZX/JFlKzKhbAP3/nQHa2 +lgmACYt/UXke9UtQz66Z/PbzoJz7larknMnRFvdEpdTbn7SdLoeFpAYl+oHM+xJr +gqyV0dhVIYy3VnO1gu4Uv+wtI9P6722tCLpIA9TmjeJK3mlVQT+Fk+2OFVrAk05G +dsUZFz2007NSKo9UOMkc6zAGe0Xdknm+aNI+6UyCh6q+wffvthifzgN03CzzmjdL +3qXGu1WGr7xcvuV8QXs44IpJxSkvRl44gefqSjWaLucKxnJH5EiaPDXX/5fKyqwo +4g+9XM5O++sdsQIhzhvI6QXVD2SMEdJr9ral7fzTdtfQhdq1dFQoLJU21wPxWaKE +YqUGnPtHdGwU9GyKuhg0wHiykPC7lwIDAQABAoICAH9tmy1ccF/OHIg8xwPMP9df +eCJtFfTzEl7e3TUjKAqBm/iOLs6aWZ1CERz80TJz7vlbyKu6kTz4Cm/EGRPzhH/L +6A/c31XXN+2/UGPLK8D2pmrWuoFD4/GtlV42x0XBluste7DL/HLX1dx566Qi1H2G +GrIUzaJSjeBpkO5so3Yh1FzZ0e3gkYfTBv5a705JeQa7AqNseZFhchAO4PJKGmW8 +tA5PtIr/dZPOmXEDo9Hp+D4E5G4GSJXND4n6oiPYBeuzUofPSm3euezwVTqyIp4x +q0vYDte/KTjNaS27Dd5Jrgk4VrVWmLPqnO/tQv3Vr8Z+8+RFRGTsz1kH+uLhSf10 +JXCmTvU+eKcmsEBuruBMm4qy1XaU6kZdAvj4oFlLwCOJ/v8sbFHLDA379kHQv+n4 +y1li1BN9FmhwdcJa9kjEoBumLa35qqlxcSq4FFhAtzhLaL5rg15guTcQEQGCXEJ/ +KHhQ/a7Dv2ozOhuEcTIJTQwj9qIUxO9gij41Hjp6cQd65Kijrt3kvmGOdBIpptRN +MW58ZJB0hCAIgmLaT3v5jSQLuY2OyjwLJEjzsUxZFDpebHgaZ0MhpVaRMG6Q59y0 +6UVIdijBG+0t3FhGd0Sq1310AV/DTKl0oV6KPETjsMohaptlqMjHNY8kwrUM0uaF +ujg80GdFLsJ0Oewm+XRxAoIBAQDsQybOQJoEehhC0yTkRk4zcoPmbTTxoQrcqyVj +VcX2jxisjUFELg3eEuNpvqE+onO9fXbdbgF2dpV0TspwEmJxB8E33ZQimkh06cUI +FfwPeGpLdfZ3vMNpltrnkmhSn/zqcIdlp61z+Aiyz/7QktQXWXTQNCwApHD0MH/X +HU41RHAdR+H8HsH6SAo3vh/brRZqOjRp1DZ+fc/nk9mdM8BIqhrR4dQNLVnQTsh7 +1KOfvx2vKofdrLefmkGvsoOvSevcinz+ypDWu9KCJsZ0i9GW18nWkpuDzxv1dtVd +F+DbgUC3Yy0OB7IiEHRzw3iUOF/eyQzYeQSOPPg6vV6UsqwJAoIBAQDFbsfSl2Mh +v1HQFUG1OI91oAgc9ffz2DRIUJqP2uaLcAgl0sHq9kteXEjAj18w4RxlzOrhLwAt +lt+lWamW2QRlFGvyDgy1pyjuFmjuTzDiMtV+j/2a+2J0lMZHBCprBqpCBSuUeRrM +qz497OXGylkk5CVfyUy9IPAdxskESbY9ClrYJhYN4x7xhThApGNL2slWc2K9pPh5 +iCO247U6X0nMCWXGVaDaCqpRlCcw1ltHqtRVpH90GegBCVVyJRqGjlCV6EmCEU9c ++KEVp8EaAi7oz7OYHkFiKTjL+syRNfxk0x4MnTa4CyQZesepHnuCf1t4B5Ie+Zsy +isSpz9u1NVKfAoIBAQCvwIRticEI84p4q3kWFva1bsqP700jMhlSbj4x1uycrXY9 +ZhbXEStft81ULGrmnRJcM6jor5yG80/aZYJSPFremlYxqLM5ymt8DSp1CWWoXHs2 +B1qBV/pF4eKVwWO5kyohe/Q8uq9dRXWzDucPYSpE9FbhSu4wzCL6g9mVvEchePet +0HtbgcXq2QxAyz4HvIqgGwxCbYrcLdsq+eE3//daFrA80TFdXA8j1ndtJpFRlGoQ +lCLd1E4EfEvA15r+Go9KT/sLzsRP+QhvqUiYCiqve3a6F/V35+l0gCHYTpsGru/l +e/FYLoFktbjrJM3d9NHceNMFdYDeDpzZlWV2WDOBAoIBAGrhwS2hpRoE7+59Q3gu +6YMdIqdDBXut5QftEu+j6sE74j1Prolb0ot/3CbatwbIJ055NAK6JheIIQTr2GKA +k9+Awbtlnb3Ng9v4p8+zcVxTdr6whvOa+b7xN7IEcE5Gam+ZSvFy6m44mlZ95BrN +Ops9I3qKiI+uwC7uia5XlV/QNkPjwna8KOlJXJ2qINNU20CA8MiIAbgRWnxNI7Q5 +F8NspvtLKexAb9oAIYNP2FShB5hhmForGC1Pd0F3ErgbXUb6XKkcY+GJ+/K0QVaq +Lej9YFWOs4jHlibfXFWUa1HN9OuRRKk35tAgZuyIQwbvmX4OYy1/pWQkOblQfLnJ +5zMCggEBAIUYtaoU+0UKjTlgSy4BIuIN5NB18mcuhYnqz7CPYubUXf9gSGMhMC88 +Q2WrTDN2siF+VxUVMHRRDZ8RzGfuTaHop4US5PjmEpaNLnVgfqaqElLcTPZuXdtt +Sg2hvIYNHqJO8BNx3WJHSZlx4PGlEOVoOI0JsrEPqYkUdsXKv5BASPOYVaPWLpmt +SbOIZTAzUXY7Iy6KKY8UZkDtrV3+1S4DYZ6Ve83i59tNj8LPfrJ0xqObVKD/HgLW +d0EPq/M0wFCXvfi7A2FFHM60kd7PLScZdTULsw7ypd/blOYBFwl5i1JgxHhg2nC7 +a0f8QV4fZzKHFBoNXts8ivDxPGkRnGk= +-----END PRIVATE KEY----- diff --git a/integration-test/certs/client_csr.pem b/integration-test/certs/client_csr.pem new file mode 100644 index 0000000..50bb274 --- /dev/null +++ b/integration-test/certs/client_csr.pem @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIEVDCCAjwCAQAwDzENMAsGA1UEAwwEdXNlcjCCAiIwDQYJKoZIhvcNAQEBBQAD +ggIPADCCAgoCggIBALY16iHyTYoJfG4WFLP7nnPq4lLcr1+ij+HM8BZBMHalJ5L7 +sKVK3RU3rxJy0QQtvfQBr94ly/8XU8l0fX7EIbAYnr0NSI3RneP9dwcKdYfekqwY +cxsY8ixpyLD4vdfBd/PqTLOa+f0ymlnvuXpUn64b2bquY4R45MD27lr5YWvByqD5 +mQkI1qn4QHyW86KMP3mGGT+1c0oWCYmJcqj0ty5WfBIYfdhaqr6W2U6fM9NIBpz4 +TNP5OP1Ba1HmrRBLNoS6vyoQOLuW+B3zbCKmG9ni3PtX+yDcB3v+9gOQmcPXeTRm +TbQfbJoFlD9lf8kWUrMqFsA/f+dAdraWCYAJi39ReR71S1DPrpn89vOgnPuVquSc +ydEW90Sl1NuftJ0uh4WkBiX6gcz7EmuCrJXR2FUhjLdWc7WC7hS/7C0j0/rvba0I +ukgD1OaN4kreaVVBP4WT7Y4VWsCTTkZ2xRkXPbTTs1Iqj1Q4yRzrMAZ7Rd2Seb5o +0j7pTIKHqr7B9++2GJ/OA3TcLPOaN0vepca7VYavvFy+5XxBezjgiknFKS9GXjiB +5+pKNZou5wrGckfkSJo8Ndf/l8rKrCjiD71czk776x2xAiHOG8jpBdUPZIwR0mv2 +tqXt/NN219CF2rV0VCgslTbXA/FZooRipQac+0d0bBT0bIq6GDTAeLKQ8LuXAgMB +AAGgADANBgkqhkiG9w0BAQsFAAOCAgEAJwdtsskJAj3PSnAAMhwjQSmtGeEnu0CU +ulPlXbib9+nPlVR2iX6CiEjiSNryDPq2DwiRBS29En0WlzJGZLH6EAD617e2A2p3 +dt+n3XcN6q5azfwyHSF3IDgFuytT8smSFyr+jBhgn/JYcViVWwzAMRHR/cfZzLnY +QFhSpU4NFByoHySsOJi1LwRhHb5KZd7WE3g14ypDiQ2/8p+qzAzkFYrBVnKQlNjy +YFn6SuL3a1jUkBFC+pwMPSXQf+bEQ9eHXYR9K7EcqKg0wcaVMghrU9obcTqVLn5b +fft8GZ9EDOMLmuuW99hkE8/4Z1ydSS+3bE2ZFQVKYtBERAc+ZpAdjweysMm6m5yU +KpseDfeMDT3S16/V3zYf73snyJu7JUd4ti/SB77IRIe/oDiHjmJW2ekn6M47ZsZa +r0def9OzDw1NScOoy7ijPyQXByl0vFKZGI2eVNhByWxpJJuODLOA9JyUTSB7fk7v +gPMKuATniLE+INp8GvZxcqPSB9U+4E2v33XuB7t354/hGJ3zOT4i0jP1JrhRVGae +nZxYMpooE67/8s/HtlJAPnncZ7GqsDRtI0DHU+AJPpi8Qqhx6Py+0rHwvEHEOZnI +mB9OHlDG+DuOCqcqzi/scSpOLApZRnkca4iSzJONb0wgGBIEzpSWTR9JOL8W0poW +R3++SLsludY= +-----END CERTIFICATE REQUEST----- diff --git a/integration-test/certs/server.crt b/integration-test/certs/server.crt new file mode 100644 index 0000000..a039dd6 --- /dev/null +++ b/integration-test/certs/server.crt @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE----- +MIIEojCCAooCCQCeLWyvcmUysTANBgkqhkiG9w0BAQsFADASMRAwDgYDVQQDDAdl +dGNkMzI0MCAXDTE5MDUwOTE5MTY1M1oYDzIyOTMwMjIwMTkxNjUzWjASMRAwDgYD +VQQDDAdldGNkMzI0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAzlUX +IowVlrtKm4a7mrkcPOnSEoOFUJG5A8jRTnMfrk9UqHEkntBtyMX6bNRdTs6UwCVn +vpwqXDyePRtefan9f7anbOhC6GFg+WtuArttR4NYASTAncxtye5O+NQt4tjf3P4r +QM4l7vEMSgga0ZX2BUUttYtFAtXiPWXKh/M+u4SyGZtXiIupQ75+hb/FHaTy5jpE +nL5dkLJ/jcisiTuJmrbLYiIccyeyJvtNBoUQeLxGfqkqoMe/L+W3v200QlwqVQza +NNkDX+YW8yqYMKe1oHVgxD5RklLfyzROw45ZLZba7Pcg0JKn855syJeFhSERrM9Z +B6KfyR9VK+Z4rlCFJb7D+EP0o8s6+xsRtHqSNLibeULSMn9FELjODLCywAq2efRv +lxmF3zPrwoSh+OJza/w2J8SASxi1ymuMiTjMttPhaNb/LUuNUFxr++Oa5N1/UEat +AeWAkoDwE8pQxMU2uRZaRM2iX28KD/Kw/BWCWOQqLR5ppARq99dlnNfzB0zGteXB +/1sMEnnaWHuZLZBjuBAlxhuTZH23Ad1gJK4gsN0guF3e/PF3WPJOt0pJbIbnk9Bb +paor+gyThefDl7cmrf1icjIzN24o/4heidjPF2ps7NaHMY/jwY0R29Xf/FYJYHl9 +K1dpgsgtnAQIe4LdDVRsx3DoZlV9hB+NxyItnt8CAwEAATANBgkqhkiG9w0BAQsF +AAOCAgEAh8rZANRbL4kMjAE1fyQPSq0z2POk5ZgkaaWZhq+BvfHjsDBXoVGsBrKw +umpAXDJ6wLMJ8Ol6bYMrREI9B/hRdVN9n7taBUyPCmLADOlSj6NHlwafhYc0Tamd +bv43dE0QG2N8/pF4q20x0QZ+iJKmNKWFX/9Z00I5uV2huMC+fiMXPjetF2ADEgWM +iTO8c9YqIyJxRNvmCbpqjzolZQFkDbQfOcZpZGy6b2w53+TwGPFWOBDjxuel6eKH +pKCirm0H2izHMszHSsIBBVi6BxkaPrNX+3Q/jy/7l+Alhub4Tg19jEH0tQN4AhBh +3uHxn4vVfjzsHOy6Va0pe6UvZoc8AY/vjEw6bpSYKvuOWWvyBNtO2vNyGZh2eQBF +GkkWVCtIyfeMjogO5uijnPuFjYovE7/EqwJYnjnQKbz3q1ZDgP2KRKbIaYh1lcg8 +AzSzbwERE3tNx70hKTnI10DVo4MnAs+BQ8TDZEf1FQM97E/fNXzKoadfc6EqsVoS +U8VRmBLZfqUwwjYD3RXrWb0GBRoWxNoC78fHKmORL2OJG7K/GeLXZqHkEdx7Phag +4l3KM0b7yrpvqcH94IRBDM0nz2Gb9ETacKfyFwgl29PsV/8BW9cnUSu7Pxy5wv4K +MbALAggCdkk42zYOlpPIQCdCo8csHd9iQVZlOhigC4VRcLtcUpg= +-----END CERTIFICATE----- diff --git a/integration-test/certs/server.key b/integration-test/certs/server.key new file mode 100644 index 0000000..dac1928 --- /dev/null +++ b/integration-test/certs/server.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDOVRcijBWWu0qb +hruauRw86dISg4VQkbkDyNFOcx+uT1SocSSe0G3Ixfps1F1OzpTAJWe+nCpcPJ49 +G159qf1/tqds6ELoYWD5a24Cu21Hg1gBJMCdzG3J7k741C3i2N/c/itAziXu8QxK +CBrRlfYFRS21i0UC1eI9ZcqH8z67hLIZm1eIi6lDvn6Fv8UdpPLmOkScvl2Qsn+N +yKyJO4matstiIhxzJ7Im+00GhRB4vEZ+qSqgx78v5be/bTRCXCpVDNo02QNf5hbz +Kpgwp7WgdWDEPlGSUt/LNE7Djlktltrs9yDQkqfznmzIl4WFIRGsz1kHop/JH1Ur +5niuUIUlvsP4Q/Sjyzr7GxG0epI0uJt5QtIyf0UQuM4MsLLACrZ59G+XGYXfM+vC +hKH44nNr/DYnxIBLGLXKa4yJOMy20+Fo1v8tS41QXGv745rk3X9QRq0B5YCSgPAT +ylDExTa5FlpEzaJfbwoP8rD8FYJY5CotHmmkBGr312Wc1/MHTMa15cH/WwwSedpY +e5ktkGO4ECXGG5NkfbcB3WAkriCw3SC4Xd788XdY8k63SklshueT0Fulqiv6DJOF +58OXtyat/WJyMjM3bij/iF6J2M8Xamzs1ocxj+PBjRHb1d/8VglgeX0rV2mCyC2c +BAh7gt0NVGzHcOhmVX2EH43HIi2e3wIDAQABAoICACJkdJLBnuedGHaMIsFLkbVl +lCqZ2aLwZ1KGKr1U2zcG6imZkm0WePY6ad+yFkuIi6rrkTtodHcjc4zj/wAbUbH0 +FDxHCsAJW0y354J1E2YUMpseMS0cYkCChI9dbridRdNkWnlW42PMsiu05iOBR5zh +Rxk3ua0bUbbCPNwkY5SGLdmu1wxfmzs7iYNpz8V9NS/DDqTB5n60Aib0CC5nyiwL +xCL7ZJqdR0jQIqqvi2tUlruOHxjlsqydMd+qqEdXbYb3uXeNpM5bX//wXaXejpvM +dPDBIn+taLhKY8ac96htuNg+6S1bBLvK8H8PIqZqOLQBa+ml5dR0cJH8vKjxqv+u +6DoH7Jf19n28W+G0/2cYMuKZ2J+LNSI9Q6wTGY2tk/du6V7i8l2Ho1A+IGLdntUg +BsBc24l/wfla42M/pfLBajQlezieb/nMw3sMHToIxy4tSqCRpLZkmzoPYUCjy9eJ +9su5JJZGhUq26EOGTuYNT2TF9bTvTUiCBIzfsDX50vczZnSUZY7+pekB9S4hP3V/ +2Hdida2jdaCQWWLLDJY4sVVaAHYvdwxkwDXoJbomFQn85zOQElSW3PYdwcuC1WbA +PxZyBGTXDX7EWYXYT3CQiUOgzW/pRekIJ5RwBW61NDv9Qxaobm9zoOT54sDXBuzu +3hx36fpjx1895vcvp5LhAoIBAQDzw4wRpy9K7q/Wgcpd6x+VCDLWuJyxq6dY3oQo +JR+Hp41D3hdF+Ixs37IQfPVBfMepFf9GqhUR/OjaKaoDF6vCBfMQelIsDKzEGiOJ +Xzvrjw/oQPg3Iy6Kd3LnNPswN9fcFrLMQfcLyGTShUjisa+RXaIf1VRiW0GJtdUa +iP7Ujqzl3jVWd8aZRvEGSIOzHZzzEA6+lJx7JuT4LfVQklwo2z+4pab7LJlwFjrE +LumVQY6QRXWW92GrSDGE8odh7IAk5FTc6CEShu/544HCI5riEBvaY0lRLRrCqOBR +aXhq2Akjr2KBTHPT7i89c/kq1pU6oV5V5gaKhWr2AqOkrMFVAoIBAQDYsIkQfsOm +fGOSs8SYjI1qNTCQZRsacvHlR+W9Ajgzh4raW166rpbg5jJacgtEr1YUatvSlCnz +VKYGEsTkUGfKx+0ErrU2vFhOPHonm5kOaBLlAwHM/L7DuH7kh94WD/Su0etu8x9K +5QacHfUJU2sTfdj0BcK4abSh7sxr1hMbrUcvwFs+7aS8jxmoie+MhxiZVgn424iF +uGt+4a8G9MHhLr0XEuYKOrjbohFHTquJhiXTanKlfKopk8Q/ArcsPfw4KBfe1cDh +n9y1ABbDNmLxTpBxwdcG/Xa/hacRfuNIZMaRWXTznZOjt91iYUOXTeYbmRapbz6R +d1MQFrCZX29jAoIBADZQzUYltZEBzGaEtx3kRT5E/GcB7dBdCkXwFwyG8RaeDAki +VJc/wbKEyb2IynFUJ30SrLYHMfi3HP93py1XOetavgVe7C5qiJiNpnt2n/hmAUpk +J6rJiRZLdWv1Tz84Qj4FRMPksvKKLYUHM+1nYFKIxZZIR6w+30SbIqft/LJKNFZo +BASHXdZLi2s1/zqLD6zNWCu5G4EJcp2mR/IzGpvz9qfjKWjwUCQ8n82VEnEykVyu +TXxdFQNAa+BO1kHIm+kcg5hRlQT20kQhIImsu1dB9aLc2rpezYawcINTeAR+rNKp +ws5CxdBkU1xjGhMki03sJQQI5E47WlEE14fbAp0CggEACOaXern6R2hsnqYLuUhs +tqQgro24o0l9bjGZ2j2xGUzEA1wlc/2f7PUHtHdixxt8qbIJ/YhggMJFX5J9y/MR +kF7N+ZdDklACQJRzYBUgIFteC1IoTMdE/r0hWTnLgZR+NIAi75fE/UqTDu6cFhn7 +F/3fX4atDO4qoUwx0WwltMlVVzzLgrY4QhaOs0GXOWqDNCk4l6RM41FD+R5EFkEP +hbRpx/zavijTVxI9K3SrzyZXf2ur8ehBBRwwv3XZGFQjfBsJf/m/xoTOiBDM3o/p +WIglnsabkonuyscuqaVQqLSbA3VtPGSIn/z5dwUPxbNJdHDBYYxUQZ1pKAwGjQZC +iQKCAQEAsrKm4zqOln72czIxDoXipO3N8kHluHrIhkuMroZwf+RHymfNYqYw1EBZ +r7JcQBPHp1nCUcLlxjbQkLd+IeEkikDEXlfQwvD/SXQO/YgnJNd9+yUlJQ6xvh75 +W2vh953KkGZ9u8BttjFk0/FXAXgSyNXzcg/erSVJWe5wUTk0M9BHwnG/S0Pjfx8j +1swEl73a/7Ed1aYDwoZ1jk0HucsImJKVTNkT5A6oVOwrPHSSixOORKbkHygQBHCW +msup/1Xc+Gv9F6gORUmO7HWnAb/MiA7LutPkQudSQ9r+oqIG7b+aOyY6kBFMCYF+ +dkxEKrb7ue/MR76uN4LO4Okp1bCkQA== +-----END PRIVATE KEY----- diff --git a/integration-test/docker-compose.yml b/integration-test/docker-compose.yml index 696e73f..8f70f81 100644 --- a/integration-test/docker-compose.yml +++ b/integration-test/docker-compose.yml @@ -1,4 +1,11 @@ version: '2' + +x-common: &x-common + ports: + - 2379 + volumes: + - .:/opt/testing + services: collectd: build: @@ -7,97 +14,90 @@ services: environment: SF_API_TOKEN: testing SF_INGEST_HOST: fake_sfx + volumes: + - .:/opt/testing depends_on: - fake_sfx - - etcd238 - - etcd310 - - etcd324 - etcd208 - - collectdamazon2017: - build: - context: .. - dockerfile: integration-test/Dockerfile.collectd.amazon2017.03 - depends_on: - - fake_sfx - etcd238 - etcd310 - etcd324 - - etcd208 + - etcd324-tls-invalid etcd208: + <<: *x-common image: quay.io/coreos/etcd:v2.0.8 - ports: - - 2379 + hostname: etcd208 command: > - --name - etcd208 - -advertise-client-urls - http://etcd208:2379 - -listen-client-urls - http://0.0.0.0:2379 - -initial-advertise-peer-urls - http://etcd208:2380 - -listen-peer-urls - http://0.0.0.0:2380 - -initial-cluster - etcd208=http://etcd208:2380 + --name etcd208 + --advertise-client-urls http://etcd208:2379 + --listen-client-urls http://0.0.0.0:2379 + --initial-advertise-peer-urls http://etcd208:2380 + --listen-peer-urls http://0.0.0.0:2380 + --initial-cluster etcd208=http://etcd208:2380 etcd238: + <<: *x-common image: quay.io/coreos/etcd:v2.3.8 - ports: - - 2379 + hostname: etcd238 command: > - --name - etcd238 - -advertise-client-urls - http://etcd238:2379 - -listen-client-urls - http://0.0.0.0:2379 - -initial-advertise-peer-urls - http://etcd238:2380 - -listen-peer-urls - http://0.0.0.0:2380 - -initial-cluster - etcd238=http://etcd238:2380 + --name etcd238 + --advertise-client-urls http://etcd238:2379 + --listen-client-urls http://0.0.0.0:2379 + --initial-advertise-peer-urls http://etcd238:2380 + --listen-peer-urls http://0.0.0.0:2380 + --initial-cluster etcd238=http://etcd238:2380 etcd310: + <<: *x-common + hostname: etcd310 image: quay.io/coreos/etcd:v3.1.0 - ports: - - 2379 command: > - etcd --name - etcd310 - -advertise-client-urls - http://etcd310:2379 - -listen-client-urls - http://0.0.0.0:2379 - -initial-advertise-peer-urls - http://etcd310:2380 - -listen-peer-urls - http://0.0.0.0:2380 - -initial-cluster - etcd310=http://etcd310:2380 + etcd --name etcd310 + --advertise-client-urls http://etcd310:2379 + --listen-client-urls http://0.0.0.0:2379 + --initial-advertise-peer-urls http://etcd310:2380 + --listen-peer-urls http://0.0.0.0:2380 + --initial-cluster etcd310=http://etcd310:2380 etcd324: + <<: *x-common + hostname: etcd324 image: quay.io/coreos/etcd:v3.2 - ports: - - 2379 + # Note: certs are valid for this container, has CN=etcd324. command: > - etcd --name - etcd324 - -advertise-client-urls - http://etcd324:2379 - -listen-client-urls - http://0.0.0.0:2379 - -initial-advertise-peer-urls - http://etcd324:2380 - -listen-peer-urls - http://0.0.0.0:2380 - -initial-cluster - etcd324=http://etcd324:2380 + etcd + --name etcd324 + --trusted-ca-file /opt/testing/certs/server.crt + --cert-file /opt/testing/certs/server.crt + --key-file /opt/testing/certs/server.key + --client-cert-auth + --advertise-client-urls https://etcd324:2379 + --listen-client-urls https://0.0.0.0:2379 + --initial-advertise-peer-urls http://etcd324:2380 + --listen-peer-urls http://0.0.0.0:2380 + --initial-cluster etcd324=http://etcd324:2380 + + etcd324-tls-invalid: + <<: *x-common + hostname: etcd324-tls-invalid + image: quay.io/coreos/etcd:v3.2 + command: > + etcd + --name etcd324-tls-invalid + --trusted-ca-file /opt/testing/certs/server.crt + --cert-file /opt/testing/certs/server.crt + --key-file /opt/testing/certs/server.key + --client-cert-auth + --advertise-client-urls https://etcd324-tls-invalid:2379 + --listen-client-urls https://0.0.0.0:2379 + --initial-advertise-peer-urls http://etcd324-tls-invalid:2380 + --listen-peer-urls http://0.0.0.0:2380 + --initial-cluster etcd324-tls-invalid=http://etcd324-tls-invalid:2380 fake_sfx: + hostname: fake_sfx + stop_grace_period: 0s build: context: . dockerfile: Dockerfile.sink @@ -108,10 +108,3 @@ services: dockerfile: Dockerfile.test depends_on: - collectd - - test26: # this is to test that python2.6 works (because collectd is linked to 2.6 on amazon) - build: - context: . - dockerfile: Dockerfile.test - depends_on: - - collectdamazon2017 diff --git a/integration-test/run.sh b/integration-test/run.sh old mode 100644 new mode 100755 index b185466..494611c --- a/integration-test/run.sh +++ b/integration-test/run.sh @@ -5,21 +5,10 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd $DIR printf "Running tests against python2.7\n" +docker-compose build || exit 1 docker-compose run --rm test status=$? docker-compose down printf $status - -if [ "$status" != "0" ]; then exit $status; fi - -printf "\nRunning tests against python2.6" -docker-compose run --rm test26 -status=$? - -docker-compose down - -printf $status - -exit $status diff --git a/integration-test/setup_etcd b/integration-test/setup_etcd deleted file mode 100755 index b402498..0000000 --- a/integration-test/setup_etcd +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -wait_for () { - host=$1 - while ! nc -z $host 2379 - do - sleep 0.2 - done -} - -for host in etcd208 etcd238 etcd310 etcd324 -do - wait_for $host -done - -exec /.docker/run.sh diff --git a/integration-test/test.py b/integration-test/test.py index 118464c..d1c0e36 100644 --- a/integration-test/test.py +++ b/integration-test/test.py @@ -1,8 +1,9 @@ #!/usr/bin/env python +from __future__ import print_function + import httplib import json from time import time, sleep -from subprocess import call # Quick and dirty integration test for multiple broker support for etcd for # one collectd instance. This test script is intended to be run with @@ -12,10 +13,11 @@ # integration tests if so desired. ETCD_HOSTS = [ + 'etcd208', 'etcd238', 'etcd310', 'etcd324', - 'etcd208', + 'etcd324-tls-invalid', ] TIMEOUT_SECS = 60 @@ -32,10 +34,10 @@ def get_metric_data(): def wait_for_metrics_from_each_member(): start = time() for member in ETCD_HOSTS: - print 'Waiting for metrics from member %s...' % (member,) + print('Waiting for metrics from member %s...' % (member,)) eventually_true(lambda: any([member in m.get('plugin_instance').split(':')[0] for m in get_metric_data()]), TIMEOUT_SECS - (time() - start)) - print 'Found!' + print('Found!') def eventually_true(f, timeout_secs): diff --git a/requirements.txt b/requirements.txt index ffe2fce..cbe6385 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ six +requests >= 2.0 diff --git a/urllib_ssl_handler.py b/urllib_ssl_handler.py deleted file mode 100644 index c5cb3a6..0000000 --- a/urllib_ssl_handler.py +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env python -__all__ = ['match_hostname', 'CertificateError'] -import socket -import ssl -import re -from six.moves import http_client -from six.moves import urllib - - -class CertificateError(ValueError): - pass - - -def match_hostname(cert, hostname): - if not cert: - raise ValueError("empty or no certificate") - dnsnames = [] - san = cert.get('subjectAltName', ()) - for key, value in san: - if key == 'DNS': - if _dnsname_to_pat(value).match(hostname): - return - dnsnames.append(value) - if not dnsnames: - for sub in cert.get('subject', ()): - for key, value in sub: - if key == 'commonName': - if _dnsname_to_pat(value).match(hostname): - return - dnsnames.append(value) - if len(dnsnames) > 1: - raise CertificateError("hostname %r doesn't match either of %s" % ( - hostname, - ', '.join(map(repr, dnsnames)))) - elif len(dnsnames) == 1: - raise CertificateError("hostname %r doesn't match %r" % ( - hostname, - dnsnames[0])) - else: - raise CertificateError("no appropriate commonName or \ - subjectAltName fields were found") - - -def _dnsname_to_pat(dn): - pats = [] - for frag in dn.split(r'.'): - if frag == '*': - pats.append('[^.]+') - else: - frag = re.escape(frag) - pats.append(frag.replace(r'\*', '[^.]*')) - return re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) - - -class HTTPSConnection(http_client.HTTPSConnection): - def __init__(self, host, **kwargs): - self.ca_certs = kwargs.pop('ca_certs', None) - self.checker = kwargs.pop('checker', match_hostname) - http_client.HTTPSConnection.__init__(self, host, **kwargs) - - def connect(self): - args = [(self.host, self.port), self.timeout, ] - if hasattr(self, 'source_address'): - args.append(self.source_address) - sock = socket.create_connection(*args) - - if getattr(self, '_tunnel_host', None): - self.sock = sock - self._tunnel() - kwargs = {} - if self.ca_certs is not None: - kwargs.update( - cert_reqs=ssl.CERT_REQUIRED, - ca_certs=self.ca_certs) - self.sock = ssl.wrap_socket(sock, - keyfile=self.key_file, - certfile=self.cert_file, - **kwargs) - if self.checker is not None: - try: - self.checker(self.sock.getpeercert(), self.host) - except CertificateError: - self.sock.shutdown(socket.SHUT_RDWR) - self.sock.close() - raise - - -class HTTPSHandler(urllib.request.HTTPSHandler): - def __init__(self, key_file=None, cert_file=None, ca_certs=None, - checker=match_hostname): - urllib.request.HTTPSHandler.__init__(self) - self.key_file = key_file - self.cert_file = cert_file - self.ca_certs = ca_certs - self.checker = checker - - def https_open(self, req): - return self.do_open(self.getConnection, req) - - def getConnection(self, host, **kwargs): - d = dict(cert_file=self.cert_file, - key_file=self.key_file, - ca_certs=self.ca_certs, - checker=self.checker) - d.update(kwargs) - return HTTPSConnection(host, **d) - - -__all__.append('HTTPSHandler') From 7d12c308c4951f8f4f72710e2c45e091ab6854f5 Mon Sep 17 00:00:00 2001 From: Jay Camp Date: Fri, 10 May 2019 11:17:34 -0400 Subject: [PATCH 2/3] circleci --- .circleci/config.yml | 15 ++++++++------- etcd_plugin.py | 6 +++--- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f745a50..cfb4178 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,8 +1,9 @@ -version: '2' +--- +version: 2.1 jobs: build: docker: - - image: ubuntu:yakkety + - image: ubuntu:18.04 working_directory: ~/code steps: - setup_remote_docker @@ -10,17 +11,17 @@ jobs: name: Install Docker client command: | set -x - VER="17.03.0-ce" + VER="18.09.6" apt-get update -q - apt-get install -yq curl python-pip - curl -L -o /tmp/docker-$VER.tgz https://get.docker.com/builds/Linux/x86_64/docker-$VER.tgz + apt-get install -yq curl python-pip git + curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$VER.tgz tar -xz -C /tmp -f /tmp/docker-$VER.tgz mv /tmp/docker/* /usr/bin - run: name: Install docker-compose command: | set -x - curl -L https://github.com/docker/compose/releases/download/1.11.2/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose + curl -L "https://github.com/docker/compose/releases/download/1.24.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose - checkout - run: @@ -32,4 +33,4 @@ jobs: - run: name: Run integration tests working_directory: ~/code/integration-test - command: bash run.sh + command: ./run.sh diff --git a/etcd_plugin.py b/etcd_plugin.py index 0891343..ad85b88 100644 --- a/etcd_plugin.py +++ b/etcd_plugin.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -import json import collections + import collectd import requests import six @@ -141,8 +141,8 @@ def read_config(conf): elif val.key == 'ssl_ca_certs' and val.values[0]: ssl_keys['ssl_ca_certs'] = val.values[0] elif val.key == "ssl_cert_validation" and val.values[0]: - # Doesn't use str_to_bool because the function defaults to false and - # we want to default to true. + # Doesn't use str_to_bool because the function defaults to + # false and we want to default to true. if val.values[0].strip().lower() == 'false': ssl_keys['ssl_cert_validation'] = False elif val.key == 'Testing' and str_to_bool(val.values[0]): From b36058498364f141457b239944c0854c72d5ef22 Mon Sep 17 00:00:00 2001 From: Jay Camp Date: Fri, 10 May 2019 12:51:50 -0400 Subject: [PATCH 3/3] remove volumes because doesn't work with circleci --- .circleci/config.yml | 1 - integration-test/20-etcd-test.conf | 2 +- integration-test/Dockerfile.collectd | 1 + integration-test/Dockerfile.etcd32 | 2 ++ integration-test/docker-compose.yml | 26 +++++++++++++------------- integration-test/run.sh | 1 + integration-test/test.py | 2 +- 7 files changed, 19 insertions(+), 16 deletions(-) create mode 100644 integration-test/Dockerfile.etcd32 diff --git a/.circleci/config.yml b/.circleci/config.yml index cfb4178..e246ba4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,4 +1,3 @@ ---- version: 2.1 jobs: build: diff --git a/integration-test/20-etcd-test.conf b/integration-test/20-etcd-test.conf index e75e0be..5beccf2 100644 --- a/integration-test/20-etcd-test.conf +++ b/integration-test/20-etcd-test.conf @@ -37,7 +37,7 @@ LoadPlugin python ssl_cert_validation True - Host "etcd324-tls-invalid" + Host "etcd324-tls-unverified" Port "2379" Cluster 5 Interval 10 diff --git a/integration-test/Dockerfile.collectd b/integration-test/Dockerfile.collectd index 5fd6be2..4829755 100644 --- a/integration-test/Dockerfile.collectd +++ b/integration-test/Dockerfile.collectd @@ -10,3 +10,4 @@ RUN apt-get update &&\ ## The context of the image build should be the root dir of this repo!! ADD etcd_plugin.py /opt/collectd-etcd/ ADD integration-test/20-etcd-test.conf /etc/collectd/managed_config/ +ADD integration-test/certs/* /opt/testing/certs/ \ No newline at end of file diff --git a/integration-test/Dockerfile.etcd32 b/integration-test/Dockerfile.etcd32 new file mode 100644 index 0000000..429ef3a --- /dev/null +++ b/integration-test/Dockerfile.etcd32 @@ -0,0 +1,2 @@ +FROM quay.io/coreos/etcd:v3.2 +COPY certs/* /opt/testing/certs/ \ No newline at end of file diff --git a/integration-test/docker-compose.yml b/integration-test/docker-compose.yml index 8f70f81..746dce6 100644 --- a/integration-test/docker-compose.yml +++ b/integration-test/docker-compose.yml @@ -3,8 +3,6 @@ version: '2' x-common: &x-common ports: - 2379 - volumes: - - .:/opt/testing services: collectd: @@ -14,15 +12,13 @@ services: environment: SF_API_TOKEN: testing SF_INGEST_HOST: fake_sfx - volumes: - - .:/opt/testing depends_on: - fake_sfx - etcd208 - etcd238 - etcd310 - etcd324 - - etcd324-tls-invalid + - etcd324-tls-unverified etcd208: <<: *x-common @@ -62,8 +58,10 @@ services: etcd324: <<: *x-common + build: + context: . + dockerfile: Dockerfile.etcd32 hostname: etcd324 - image: quay.io/coreos/etcd:v3.2 # Note: certs are valid for this container, has CN=etcd324. command: > etcd @@ -78,22 +76,24 @@ services: --listen-peer-urls http://0.0.0.0:2380 --initial-cluster etcd324=http://etcd324:2380 - etcd324-tls-invalid: + etcd324-tls-unverified: <<: *x-common - hostname: etcd324-tls-invalid - image: quay.io/coreos/etcd:v3.2 + build: + context: . + dockerfile: Dockerfile.etcd32 + hostname: etcd324-tls-unverified command: > etcd - --name etcd324-tls-invalid + --name etcd324-tls-unverified --trusted-ca-file /opt/testing/certs/server.crt --cert-file /opt/testing/certs/server.crt --key-file /opt/testing/certs/server.key --client-cert-auth - --advertise-client-urls https://etcd324-tls-invalid:2379 + --advertise-client-urls https://etcd324-tls-unverified:2379 --listen-client-urls https://0.0.0.0:2379 - --initial-advertise-peer-urls http://etcd324-tls-invalid:2380 + --initial-advertise-peer-urls http://etcd324-tls-unverified:2380 --listen-peer-urls http://0.0.0.0:2380 - --initial-cluster etcd324-tls-invalid=http://etcd324-tls-invalid:2380 + --initial-cluster etcd324-tls-unverified=http://etcd324-tls-unverified:2380 fake_sfx: hostname: fake_sfx diff --git a/integration-test/run.sh b/integration-test/run.sh index 494611c..fe6c613 100755 --- a/integration-test/run.sh +++ b/integration-test/run.sh @@ -12,3 +12,4 @@ status=$? docker-compose down printf $status +exit $status diff --git a/integration-test/test.py b/integration-test/test.py index d1c0e36..9de4958 100644 --- a/integration-test/test.py +++ b/integration-test/test.py @@ -17,7 +17,7 @@ 'etcd238', 'etcd310', 'etcd324', - 'etcd324-tls-invalid', + 'etcd324-tls-unverified', ] TIMEOUT_SECS = 60