Add request wrappers to php_fpm (#4264)

* Add request wrappers to php_fpm

* Add standard configs

* fix line

php_fpm/datadog_checks/php_fpm/data/conf.yaml.example

@@ -1,4 +1,30 @@
 init_config:
+    ## @param proxy - object - optional
+    ## Set HTTP or HTTPS proxies for all instances. Use the `no_proxy` list
+    ## to specify hosts that must bypass proxies.
+    ##
+    ## The SOCKS protocol is also supported:
+    ##
+    ## socks5://user:pass@host:port
+    ##
+    ## Using the scheme `socks5` causes the DNS resolution to happen on the
+    ## client, rather than on the proxy server. This is in line with `curl`,
+    ## which uses the scheme to decide whether to do the DNS resolution on
+    ## the client or proxy. If you want to resolve the domains on the proxy
+    ## server, use `socks5h` as the scheme.
+    #
+    # proxy:
+    #   http: http://<PROXY_SERVER_FOR_HTTP>:<PORT>
+    #   https: https://<PROXY_SERVER_FOR_HTTPS>:<PORT>
+    #   no_proxy:
+    #     - <HOSTNAME_1>
+    #     - <HOSTNAME_2>
+
+    ## @param skip_proxy - boolean - optional - default: false
+    ## If set to `true`, this makes the check bypass any proxy
+    ## settings enabled and attempt to reach services directly.
+    #
+    # skip_proxy: false
 
 instances:
 
@@ -36,36 +62,12 @@ instances:
     #
     ping_reply: pong
 
-    ## @param user - string - optional
-    ## Use the `user` parameter to set your username
-    ## if you have basic authentication on your ping and status pages.
-    #
-    # user: <USERNAME>
-
-    ## @param password - string - optional
-    ## Use the `password` parameter to set your password
-    ## if you have basic authentication on your ping and status pages.
-    #
-    # password: <PASSWORD>
-
     ## @param http_host - string - optional
     ## If your FPM pool is only accessible via a specific HTTP vhost, you can
     ## pass in a custom Host header like so
     #
     # http_host: <HOST_ENDPOINT>
 
-    ## @param timeout - integer - optional - default: 20
-    ## If you need to specify a custom timeout in seconds.
-    #
-    # timeout: 20
-
-    ## @param disable_ssl_validation - boolean - optional - default: false
-    ## The disable_ssl_validation instructs the check to skip the validation
-    ## of the SSL certificate of the URL being tested.
-    ## Set to true if you want to disable SSL certificate validation.
-    #
-    # disable_ssl_validation: false
-
     ## @param tags - list of key:value elements - optional
     ## List of tags to attach to every metric, event, and service check emitted by this Integration.
     ##
@@ -74,3 +76,165 @@ instances:
     # tags:
     #   - <KEY_1>:<VALUE_1>
     #   - <KEY_2>:<VALUE_2>
+
+    ## @param username - string - optional
+    ## If the API endpoint is behind basic auth, enter here the required username.
+    #
+    # username: <USERNAME>
+
+    ## @param password - string - optional
+    ## If the API endpoint is behind basic auth, enter here the required password.
+    #
+    # password: <PASSWORD>
+
+    ## @param ntlm_domain - string - optional
+    ## If your services uses NTLM authentication, you can
+    ## specify a domain that is used in the check. For NTLM Auth,
+    ## append the username to domain, not as the `username` parameter.
+    ## Example: <NTLM_DOMAIN>/<USERNAME>
+    #
+    # ntlm_domain: <DOMAIN>
+
+    ## @param connect_timeout - integer - optional
+    ## Overrides the default connection timeout value,
+    ## and fails the check if the time to establish the (TCP) connection
+    ## exceeds the connect_timeout value (in seconds)
+    #
+    # connect_timeout: <VALUE_IN_SECOND>
+
+    ## @param read_timeout - integer - optional
+    ## Overrides the default received timeout value, and fails the check if the time to receive
+    ## the server status from the Apache server exceeds the receive_timeout value (in seconds)
+    #
+    # read_timeout: <VALUE_IN_SECOND>
+
+    ## @param proxy - object - optional
+    ## This overrides the `proxy` setting in `init_config`.
+    ##
+    ## Set HTTP or HTTPS proxies. Use the `no_proxy` list
+    ## to specify hosts that must bypass proxies.
+    ##
+    ## The SOCKS protocol is also supported:
+    ##
+    ## socks5://user:pass@host:port
+    ##
+    ## Using the scheme `socks5` causes the DNS resolution to happen on the
+    ## client, rather than on the proxy server. This is in line with `curl`,
+    ## which uses the scheme to decide whether to do the DNS resolution on
+    ## the client or proxy. If you want to resolve the domains on the proxy
+    ## server, use `socks5h` as the scheme.
+    #
+    # proxy:
+    #   http: http://<PROXY_SERVER_FOR_HTTP>:<PORT>
+    #   https: https://<PROXY_SERVER_FOR_HTTPS>:<PORT>
+    #   no_proxy:
+    #     - <HOSTNAME_1>
+    #     - <HOSTNAME_2>
+
+    ## @param skip_proxy - boolean - optional - default: false
+    ## This overrides the `skip_proxy` setting in `init_config`.
+    ##
+    ## If set to `true`, this makes the check bypass any proxy
+    ## settings enabled and attempt to reach services directly.
+    #
+    # skip_proxy: false
+
+    ## @param kerberos_auth - string - optional - default: disabled
+    ## If your service uses Kerberos authentication, you can specify the Kerberos
+    ## strategy to use between:
+    ##  * required
+    ##  * optional
+    ##  * disabled
+    ##
+    ## See https://github.com/requests/requests-kerberos#mutual-authentication
+    #
+    # kerberos_auth: disabled
+
+    ## @param kerberos_delegate - boolean - optional - default: false
+    ## Set to `true` to enable kerberos delegation of credentials to a server that requests delegation.
+    ## See https://github.com/requests/requests-kerberos#delegation
+    #
+    # kerberos_delegate: false
+
+    ## @param kerberos_force_initiate - boolean - optional - default: false
+    ## Set to `true` to preemptively initiate the Kerberos GSS exchange and present a Kerberos ticket on the initial
+    ## request (and all subsequent requests).
+    ## See https://github.com/requests/requests-kerberos#preemptive-authentication
+    #
+    # kerberos_force_initiate: false
+
+    ## @param kerberos_hostname - string - optional
+    ## Override the hostname used for the Kerberos GSS exchange if its DNS name doesn't match its kerberos
+    ## hostname (e.g., behind a content switch or load balancer).
+    ## See https://github.com/requests/requests-kerberos#hostname-override
+    #
+    # kerberos_hostname: null
+
+    ## @param kerberos_principal - string - optional
+    ## Set an explicit principal, to force Kerberos to look for a matching credential cache for the named user.
+    ## See https://github.com/requests/requests-kerberos#explicit-principal
+    #
+    # kerberos_principal: null
+
+    ## @param kerberos_keytab - string - optional
+    ## Set the path to your Kerberos key tab file.
+    #
+    # kerberos_keytab: <KEYTAB_FILE_PATH>
+
+    ## @param tls_verify - boolean - optional - default: true
+    ## Instructs the check to validate the TLS certificate of services.
+    #
+    # tls_verify: true
+
+    ## @param tls_ignore_warning - boolean - optional - default: false
+    ## If `tls_verify` is disabled, security warnings are logged by the check.
+    ## Disable these by setting `tls_ignore_warning` to true.
+    #
+    # tls_ignore_warning: false
+
+    ## @param tls_cert - string - optional
+    ## The path to a single file in PEM format containing a certificate as well as any
+    ## number of CA certificates needed to establish the certificate’s authenticity for
+    ## use when connecting to services. It may also contain an unencrypted private key to use.
+    #
+    # tls_cert: <CERT_PATH>
+
+    ## @param tls_private_key - string - optional
+    ## The unencrypted private key to use for `tls_cert` when connecting to services. This is
+    ## required if `tls_cert` is set and it does not already contain a private key.
+    #
+    # tls_private_key: <PRIVATE_KEY_PATH>
+
+    ## @param tls_ca_cert - string - optional
+    ## The path to a file of concatenated CA certificates in PEM format or a directory
+    ## containing several CA certificates in PEM format. If a directory, the directory
+    ## must have been processed using the c_rehash utility supplied with OpenSSL. See:
+    ## https://www.openssl.org/docs/manmaster/man3/SSL_CTX_load_verify_locations.html
+    #
+    # tls_ca_cert: <CA_CERT_PATH>
+
+    ## @param headers - list of key:value elements - optional
+    ## The headers parameter allows you to send specific headers with every request.
+    ## You can use it for explicitly specifying the host header or adding headers for
+    ## authorization purposes.
+    ##
+    ## This overrides any default headers.
+    #
+    # headers:
+    #   Host: <ALTERNATIVE_HOSTNAME>
+    #   X-Auth-Token: <AUTH_TOKEN>
+
+    ## @param timeout - integer - optional - default: 10
+    ## The timeout for connecting to services.
+    #
+    # timeout: 10
+
+    ## @param log_requests - boolean - optional - default: false
+    ## Whether or not to debug log the HTTP(S) requests made, including the method and URL.
+    #
+    # log_requests: false
+
+    ## @param persist_connections - boolean - optional - default: false
+    ## Whether or not to persist cookies and use connection pooling for increased performance.
+    #
+    # persist_connections: false

php_fpm/datadog_checks/php_fpm/php_fpm.py

@@ -5,14 +5,12 @@
 import random
 import time
 
-import requests
 from flup.client.fcgi_app import FCGIApp
 from six import PY3, StringIO, iteritems, string_types
 from six.moves.urllib.parse import urlparse
 
 from datadog_checks.checks import AgentCheck
 from datadog_checks.config import is_affirmative
-from datadog_checks.utils.headers import headers
 
 # Relax param filtering
 FCGIApp._environPrefixes.extend(('DOCUMENT_', 'SCRIPT_'))
@@ -67,44 +65,39 @@ class PHPFPMCheck(AgentCheck):
         'slow requests': 'php_fpm.requests.slow',
     }
 
+    HTTP_CONFIG_REMAPPER = {
+        'user': {'name': 'username'},
+        'disable_ssl_validation': {'name': 'tls_verify', 'invert': True, 'default': False},
+    }
+
+    def __init__(self, name, init_config, instances):
+        super(PHPFPMCheck, self).__init__(name, init_config, instances)
+        if 'http_host' in self.instance:
+            self.http.options['headers']['Host'] = self.instance['http_host']
+
     def check(self, instance):
         status_url = instance.get('status_url')
         ping_url = instance.get('ping_url')
         use_fastcgi = is_affirmative(instance.get('use_fastcgi', False))
         ping_reply = instance.get('ping_reply')
 
-        auth = None
-        user = instance.get('user')
-        password = instance.get('password')
-
         tags = instance.get('tags', [])
         http_host = instance.get('http_host')
 
-        timeout = instance.get('timeout', DEFAULT_TIMEOUT)
-
-        disable_ssl_validation = is_affirmative(instance.get('disable_ssl_validation', False))
-
-        if user and password:
-            auth = (user, password)
-
         if status_url is None and ping_url is None:
             raise BadConfigError("No status_url or ping_url specified for this instance")
 
         pool = None
         if status_url is not None:
             try:
-                pool = self._process_status(
-                    status_url, auth, tags, http_host, timeout, disable_ssl_validation, use_fastcgi
-                )
+                pool = self._process_status(status_url, tags, http_host, use_fastcgi)
             except Exception as e:
                 self.log.error("Error running php_fpm check: {}".format(e))
 
         if ping_url is not None:
-            self._process_ping(
-                ping_url, ping_reply, auth, tags, pool, http_host, timeout, disable_ssl_validation, use_fastcgi
-            )
+            self._process_ping(ping_url, ping_reply, tags, pool, http_host, use_fastcgi)
 
-    def _process_status(self, status_url, auth, tags, http_host, timeout, disable_ssl_validation, use_fastcgi):
+    def _process_status(self, status_url, tags, http_host, use_fastcgi):
         data = {}
         try:
             if use_fastcgi:
@@ -114,14 +107,7 @@ def _process_status(self, status_url, auth, tags, http_host, timeout, disable_ss
                 # informations, which could be nice to parse and output as metrics
                 max_attempts = 3
                 for i in range(max_attempts):
-                    resp = requests.get(
-                        status_url,
-                        auth=auth,
-                        timeout=timeout,
-                        headers=headers(self.agentConfig, http_host=http_host),
-                        verify=not disable_ssl_validation,
-                        params={'json': True},
-                    )
+                    resp = self.http.get(status_url, params={'json': True})
 
                     # Exponential backoff, wait at most (max_attempts - 1) times in case we get a 503.
                     # Delay in seconds is (2^i + random amount of seconds between 0 and 1)
@@ -160,9 +146,7 @@ def _process_status(self, status_url, auth, tags, http_host, timeout, disable_ss
         # return pool, to tag the service check with it if we have one
         return pool_name
 
-    def _process_ping(
-        self, ping_url, ping_reply, auth, tags, pool_name, http_host, timeout, disable_ssl_validation, use_fastcgi
-    ):
+    def _process_ping(self, ping_url, ping_reply, tags, pool_name, http_host, use_fastcgi):
         if ping_reply is None:
             ping_reply = 'pong'
 
@@ -176,13 +160,7 @@ def _process_ping(
             if use_fastcgi:
                 response = self.request_fastcgi(ping_url).decode('utf-8')
             else:
-                resp = requests.get(
-                    ping_url,
-                    auth=auth,
-                    timeout=timeout,
-                    headers=headers(self.agentConfig, http_host=http_host),
-                    verify=not disable_ssl_validation,
-                )
+                resp = self.http.get(ping_url)
                 resp.raise_for_status()
                 response = resp.text
 

php_fpm/tests/conftest.py

@@ -27,7 +27,7 @@
 
 @pytest.fixture
 def check():
-    return PHPFPMCheck('php_fpm', {}, {})
+    return PHPFPMCheck('php_fpm', {}, [DEFAULT_INSTANCE])
 
 
 @pytest.fixture