Make OpenMetrics use the RequestsWrapper

crio/datadog_checks/crio/crio.py

@@ -11,11 +11,10 @@ class CrioCheck(OpenMetricsBaseCheck):
 
     DEFAULT_METRIC_LIMIT = 0
 
-    def __init__(self, name, init_config, agentConfig, instances=None):
+    def __init__(self, name, init_config, instances):
         super(CrioCheck, self).__init__(
             name,
             init_config,
-            agentConfig,
             instances,
             default_instances={
                 "crio": {

crio/tests/test_crio.py

@@ -33,7 +33,7 @@ def test_crio(aggregator, mock_data):
     Testing crio.
     """
 
-    c = CrioCheck(CHECK_NAME, None, {}, [instance])
+    c = CrioCheck(CHECK_NAME, {}, [instance])
     c.check(instance)
     aggregator.assert_metric(NAMESPACE + '.operations.count')
     aggregator.assert_metric(NAMESPACE + '.operations.latency.count')

datadog_checks_base/datadog_checks/base/checks/openmetrics/base_check.py

@@ -34,6 +34,14 @@ class OpenMetricsBaseCheck(OpenMetricsScraperMixin, AgentCheck):
 
     DEFAULT_METRIC_LIMIT = 2000
 
+    HTTP_CONFIG_REMAPPER = {
+        'ssl_verify': {'name': 'tls_verify'},
+        'ssl_cert': {'name': 'tls_cert'},
+        'ssl_private_key': {'name': 'tls_private_key'},
+        'ssl_ca_cert': {'name': 'tls_ca_cert'},
+        'prometheus_timeout': {'name': 'timeout'},
+    }
+
     def __init__(self, *args, **kwargs):
         args = list(args)
         default_instances = kwargs.pop('default_instances', None) or {}
@@ -49,6 +57,7 @@ def __init__(self, *args, **kwargs):
 
         super(OpenMetricsBaseCheck, self).__init__(*args, **kwargs)
         self.config_map = {}
+        self.http_handlers = {}
         self.default_instances = default_instances
         self.default_namespace = default_namespace
 

datadog_checks_base/datadog_checks/base/checks/openmetrics/mixins.py

@@ -10,12 +10,11 @@
 import requests
 from prometheus_client.parser import text_fd_to_metric_families
 from six import PY3, iteritems, itervalues, string_types
-from urllib3.exceptions import InsecureRequestWarning
 
 from ...config import is_affirmative
 from ...errors import CheckException
 from ...utils.common import to_string
-from ...utils.warnings_util import disable_warnings_ctx
+from ...utils.http import RequestsWrapper
 from .. import AgentCheck
 
 if PY3:
@@ -288,8 +287,29 @@ def create_scraper_configuration(self, instance=None):
         if config['metadata_metric_name'] and config['metadata_label_map']:
             config['_default_metric_transformers'][config['metadata_metric_name']] = self.transform_metadata
 
+        # Set up the HTTP wrapper for this endpoint
+        self._set_up_http_handler(endpoint, config)
+
         return config
 
+    def _set_up_http_handler(self, endpoint, scraper_config):
+        # TODO: Deprecate this behavior in Agent 8
+        if scraper_config['ssl_verify'] is False:
+            scraper_config.setdefault('tls_ignore_warning', True)
+
+        http_handler = self.http_handlers[endpoint] = RequestsWrapper(
+            scraper_config, self.init_config, self.HTTP_CONFIG_REMAPPER, self.log
+        )
+
+        headers = http_handler.options['headers']
+
+        bearer_token = scraper_config['_bearer_token']
+        if bearer_token is not None:
+            headers['Authorization'] = 'Bearer {}'.format(bearer_token)
+
+        # TODO: Determine if we really need this
+        headers.setdefault('accept-encoding', 'gzip')
+
     def parse_metric_family(self, response, scraper_config):
         """
         Parse the MetricFamily from a valid requests.Response object to provide a MetricFamily object (see [0])
@@ -556,53 +576,11 @@ def poll(self, scraper_config, headers=None):
             raise
 
     def send_request(self, endpoint, scraper_config, headers=None):
-        # Determine the headers
-        if headers is None:
-            headers = {}
-        if 'accept-encoding' not in headers:
-            headers['accept-encoding'] = 'gzip'
-        headers.update(scraper_config['extra_headers'])
-
-        # Add the bearer token to headers
-        bearer_token = scraper_config['_bearer_token']
-        if bearer_token is not None:
-            auth_header = {'Authorization': 'Bearer {}'.format(bearer_token)}
-            headers.update(auth_header)
-
-        # Determine the SSL verification settings
-        cert = None
-        if isinstance(scraper_config['ssl_cert'], string_types):
-            if isinstance(scraper_config['ssl_private_key'], string_types):
-                cert = (scraper_config['ssl_cert'], scraper_config['ssl_private_key'])
-            else:
-                cert = scraper_config['ssl_cert']
-
-        verify = scraper_config['ssl_verify']
-        # TODO: deprecate use as `ssl_verify` boolean
-        if scraper_config['ssl_ca_cert'] is False:
-            verify = False
-
-        disable_insecure_warnings = False
-        if isinstance(scraper_config['ssl_ca_cert'], string_types):
-            verify = scraper_config['ssl_ca_cert']
-        elif verify is False:
-            disable_insecure_warnings = True
-
-        # Determine the authentication settings
-        username = scraper_config['username']
-        password = scraper_config['password']
-        auth = (username, password) if username is not None and password is not None else None
-
-        with disable_warnings_ctx(InsecureRequestWarning, disable=disable_insecure_warnings):
-            return requests.get(
-                endpoint,
-                headers=headers,
-                stream=True,
-                timeout=scraper_config['prometheus_timeout'],
-                cert=cert,
-                verify=verify,
-                auth=auth,
-            )
+        kwargs = {}
+        if headers:
+            kwargs['headers'] = headers
+
+        return self.http_handlers[scraper_config['prometheus_url']].get(endpoint, stream=True, **kwargs)
 
     def get_hostname_for_sample(self, sample, scraper_config):
         """

kube_controller_manager/datadog_checks/kube_controller_manager/kube_controller_manager.py

@@ -81,7 +81,7 @@ class KubeControllerManagerCheck(KubeLeaderElectionMixin, OpenMetricsBaseCheck):
         "record_namespace": "kube-system",
     }
 
-    def __init__(self, name, init_config, agentConfig, instances=None):
+    def __init__(self, name, init_config, instances):
         self.QUEUE_METRICS_TRANSFORMERS = {
             '_adds': self.queue_adds,
             '_depth': self.queue_depth,
@@ -103,7 +103,6 @@ def __init__(self, name, init_config, agentConfig, instances=None):
         super(KubeControllerManagerCheck, self).__init__(
             name,
             init_config,
-            agentConfig,
             instances,
             default_instances={
                 "kube_controller_manager": {

kube_controller_manager/tests/test_kube_controller_manager.py

@@ -57,14 +57,14 @@ def mock_leader():
 
 
 def test_check_metrics_with_deprecated(aggregator, mock_metrics, mock_leader):
-    c = KubeControllerManagerCheck(CHECK_NAME, None, {}, [instance])
+    c = KubeControllerManagerCheck(CHECK_NAME, {}, [instance])
     c.check(instance)
 
     generic_check_metrics(aggregator, True)
 
 
 def test_check_metrics_without_deprecated(aggregator, mock_metrics, mock_leader):
-    c = KubeControllerManagerCheck(CHECK_NAME, None, {}, [instance])
+    c = KubeControllerManagerCheck(CHECK_NAME, {}, [instance])
     c.check(instance2)
 
     generic_check_metrics(aggregator, False)

kube_metrics_server/datadog_checks/kube_metrics_server/kube_metrics_server.py

@@ -82,11 +82,10 @@ class KubeMetricsServerCheck(OpenMetricsBaseCheck):
 
     KUBE_METRICS_SERVER_NAMESPACE = "kube_metrics_server"
 
-    def __init__(self, name, init_config, agentConfig, instances=None):
+    def __init__(self, name, init_config, instances):
         super(KubeMetricsServerCheck, self).__init__(
             name,
             init_config,
-            agentConfig,
             instances,
             default_instances={
                 "kube_metrics_server": {

kube_metrics_server/tests/test_kube_metrics_server.py

@@ -35,7 +35,7 @@ def mock_metrics():
 
 
 def test_check_metrics(aggregator, mock_metrics):
-    c = KubeMetricsServerCheck(CHECK_NAME, None, {}, [instance])
+    c = KubeMetricsServerCheck(CHECK_NAME, {}, [instance])
     c.check(instance)
 
     def assert_metric(name, **kwargs):

kube_proxy/datadog_checks/kube_proxy/kube_proxy.py

@@ -7,11 +7,10 @@
 class KubeProxyCheck(OpenMetricsBaseCheck):
     DEFAULT_METRIC_LIMIT = 0
 
-    def __init__(self, name, init_config, agentConfig, instances=None):
+    def __init__(self, name, init_config, instances):
         super(KubeProxyCheck, self).__init__(
             name,
             init_config,
-            agentConfig,
             instances,
             default_instances={
                 "kubeproxy": {

kube_proxy/tests/test_kube_proxy.py

@@ -51,7 +51,7 @@ def test_check_iptables(aggregator, mock_iptables):
     Testing Kube_proxy in iptables mode.
     """
 
-    c = KubeProxyCheck(CHECK_NAME, None, {}, [instance])
+    c = KubeProxyCheck(CHECK_NAME, {}, [instance])
     c.check(instance)
     aggregator.assert_metric(NAMESPACE + '.cpu.time')
     aggregator.assert_metric(NAMESPACE + '.mem.resident')
@@ -74,7 +74,7 @@ def test_check_userspace(aggregator, mock_userspace):
     """
     Testing Kube_proxy in userspace mode.
     """
-    c = KubeProxyCheck(CHECK_NAME, None, {}, [instance])
+    c = KubeProxyCheck(CHECK_NAME, {}, [instance])
     c.check(instance)
     aggregator.assert_metric(NAMESPACE + '.cpu.time')
     aggregator.assert_metric(NAMESPACE + '.mem.resident')

kube_scheduler/datadog_checks/kube_scheduler/kube_scheduler.py

@@ -103,11 +103,10 @@ class KubeSchedulerCheck(KubeLeaderElectionMixin, OpenMetricsBaseCheck):
         "record_namespace": "kube-system",
     }
 
-    def __init__(self, name, init_config, agentConfig, instances=None):
+    def __init__(self, name, init_config, instances):
         super(KubeSchedulerCheck, self).__init__(
             name,
             init_config,
-            agentConfig,
             instances,
             default_instances={
                 "kube_scheduler": {

kube_scheduler/tests/test_kube_scheduler_1_13.py

@@ -46,7 +46,7 @@ def mock_leader():
 
 
 def test_check_metrics_1_13(aggregator, mock_metrics, mock_leader):
-    c = KubeSchedulerCheck(CHECK_NAME, None, {}, [instance])
+    c = KubeSchedulerCheck(CHECK_NAME, {}, [instance])
     c.check(instance)
 
     def assert_metric(name, **kwargs):

kube_scheduler/tests/test_kube_scheduler_1_14.py

@@ -46,7 +46,7 @@ def mock_leader():
 
 
 def test_check_metrics_1_14(aggregator, mock_metrics, mock_leader):
-    c = KubeSchedulerCheck(CHECK_NAME, None, {}, [instance])
+    c = KubeSchedulerCheck(CHECK_NAME, {}, [instance])
     c.check(instance)
 
     def assert_metric(name, **kwargs):

kubelet/datadog_checks/kubelet/kubelet.py

@@ -163,7 +163,7 @@ class KubeletCheck(CadvisorPrometheusScraperMixin, OpenMetricsBaseCheck, Cadviso
 
     COUNTER_METRICS = {'kubelet_evictions': 'kubelet.evictions'}
 
-    def __init__(self, name, init_config, agentConfig, instances=None):
+    def __init__(self, name, init_config, instances):
         self.NAMESPACE = 'kubernetes'
         if instances is not None and len(instances) > 1:
             raise Exception('Kubelet check only supports one configured instance.')
@@ -172,7 +172,7 @@ def __init__(self, name, init_config, agentConfig, instances=None):
         cadvisor_instance = self._create_cadvisor_prometheus_instance(inst)
         kubelet_instance = self._create_kubelet_prometheus_instance(inst)
         generic_instances = [cadvisor_instance, kubelet_instance]
-        super(KubeletCheck, self).__init__(name, init_config, agentConfig, generic_instances)
+        super(KubeletCheck, self).__init__(name, init_config, generic_instances)
 
         self.cadvisor_legacy_port = inst.get('cadvisor_port', CADVISOR_DEFAULT_PORT)
         self.cadvisor_legacy_url = None

kubelet/tests/test_cadvisor.py

@@ -57,7 +57,7 @@ def test_detect_cadvisor_port_zero():
 def test_kubelet_check_cadvisor(monkeypatch, aggregator, tagger):
     instance_with_tag = {"tags": ["instance:tag"], "cadvisor_port": 4194}
     cadvisor_url = "http://valid:port/url"
-    check = KubeletCheck('kubelet', None, {}, [instance_with_tag])
+    check = KubeletCheck('kubelet', {}, [instance_with_tag])
     monkeypatch.setattr(
         check, 'retrieve_pod_list', mock.Mock(return_value=json.loads(mock_from_file('pods_list_1.2.json')))
     )

kubelet/tests/test_common.py

@@ -120,9 +120,9 @@ def test_credentials_empty():
     assert creds.cert_pair() is None
     assert creds.headers("https://dummy") is None
 
-    scraper = OpenMetricsBaseCheck('prometheus', {}, {})
-    scraper_config = scraper.create_scraper_configuration()
-    scraper_config['prometheus_url'] = "https://dummy"
+    instance = {'prometheus_url': 'https://dummy', 'namespace': 'foo'}
+    scraper = OpenMetricsBaseCheck('prometheus', {}, [instance])
+    scraper_config = scraper.create_scraper_configuration(instance)
     creds.configure_scraper(scraper_config)
     assert scraper_config['ssl_ca_cert'] is None
     assert scraper_config['ssl_cert'] is None
@@ -138,9 +138,9 @@ def test_credentials_certificates():
     assert creds.cert_pair() == ("crt", "key")
     assert creds.headers("https://dummy") is None
 
-    scraper = OpenMetricsBaseCheck('prometheus', {}, {})
-    scraper_config = scraper.create_scraper_configuration({})
-    scraper_config['prometheus_url'] = "https://dummy"
+    instance = {'prometheus_url': 'https://dummy', 'namespace': 'foo'}
+    scraper = OpenMetricsBaseCheck('prometheus', {}, [instance])
+    scraper_config = scraper.create_scraper_configuration(instance)
     creds.configure_scraper(scraper_config)
     assert scraper_config['ssl_ca_cert'] == "ca_cert"
     assert scraper_config['ssl_cert'] == "crt"
@@ -159,9 +159,9 @@ def test_credentials_token_noverify():
     # Make sure we don't leak the token over http
     assert creds.headers("http://dummy") is None
 
-    scraper = OpenMetricsBaseCheck('prometheus', {}, {})
-    scraper_config = scraper.create_scraper_configuration()
-    scraper_config['prometheus_url'] = 'https://dummy'
+    instance = {'prometheus_url': 'https://dummy', 'namespace': 'foo'}
+    scraper = OpenMetricsBaseCheck('prometheus', {}, [instance])
+    scraper_config = scraper.create_scraper_configuration(instance)
     creds.configure_scraper(scraper_config)
     assert scraper_config['ssl_ca_cert'] is False
     assert scraper_config['ssl_cert'] is None