diff --git a/spark/datadog_checks/spark/data/conf.yaml.example b/spark/datadog_checks/spark/data/conf.yaml.example index f5751961fcae9..d0d2da75f3b3f 100644 --- a/spark/datadog_checks/spark/data/conf.yaml.example +++ b/spark/datadog_checks/spark/data/conf.yaml.example @@ -1,5 +1,32 @@ init_config: + ## @param proxy - object - optional + ## Set HTTP or HTTPS proxies for all instances. Use the `no_proxy` list + ## to specify hosts that must bypass proxies. + ## + ## The SOCKS protocol is also supported like so: + ## + ## socks5://user:pass@host:port + ## + ## Using the scheme `socks5` causes the DNS resolution to happen on the + ## client, rather than on the proxy server. This is in line with `curl`, + ## which uses the scheme to decide whether to do the DNS resolution on + ## the client or proxy. If you want to resolve the domains on the proxy + ## server, use `socks5h` as the scheme. + # + # proxy: + # http: http://: + # https: https://: + # no_proxy: + # - + # - + + ## @param skip_proxy - boolean - optional - default: false + ## If set to true, this makes the check bypass any proxy + ## settings enabled and attempt to reach services directly. + # + # skip_proxy: false + instances: ## @param spark_url - string - required @@ -56,6 +83,11 @@ instances: # # spark_proxy_enabled: false + ## @param streaming_metrics - boolean - optional - default: true + ## Enable collection of streaming statistics. + # + # streaming_metrics: true + ## @param tags - list of key:value elements - optional ## List of tags to attach to every metric, event, and service check emitted by this Integration. ## @@ -65,37 +97,56 @@ instances: # - : # - : - ## @param username - string - optional - ## If your service uses basic HTTP authentication, specify a username to be used in the check. - # - # username: - - ## @param password - string - optional - ## If your service uses basic HTTP authentication, specify a password to be used in the check. - # - # password: - - ## @param streaming_metrics - boolean - optional - default: true - ## Enable collection of streaming statistics. + ## @param proxy - object - optional + ## This overrides the `proxy` setting in `init_config`. + ## + ## Set HTTP or HTTPS proxies. Use the `no_proxy` list + ## to specify hosts that must bypass proxies. + ## + ## The SOCKS protocol is also supported like so: + ## + ## socks5://user:pass@host:port + ## + ## Using the scheme `socks5` causes the DNS resolution to happen on the + ## client, rather than on the proxy server. This is in line with `curl`, + ## which uses the scheme to decide whether to do the DNS resolution on + ## the client or proxy. If you want to resolve the domains on the proxy + ## server, use `socks5h` as the scheme. + # + # proxy: + # http: http://: + # https: https://: + # no_proxy: + # - + # - + + ## @param skip_proxy - boolean - optional - default: false + ## This overrides the `skip_proxy` setting in `init_config`. + ## + ## If set to true, this makes the check bypass any proxy + ## settings enabled and attempt to reach services directly. # - # streaming_metrics: true + # skip_proxy: false - ## @param ssl_verify - boolean - optional - default: false - ## Instruct the check to validate SSL certificates when connecting to `spark_url`. + ## @param username - string - optional + ## The username to use if services are behind basic auth. # - # ssl_verify: false + # username: - ## @param ssl_cert - boolean - optional - ## Path to the certificate file used to identify the connection to `spark_url`. + ## @param ntlm_domain - string - optional + ## If your services uses NTLM authentication, you can optionally + ## specify a domain that will be used in the check. For NTLM Auth, + ## append the username to domain, not as the `username` parameter. + ## Example: example_ntlm_domain\example_username # - # ssl_cert: + # ntlm_domain: - ## @param ssl_key - boolean - optional - ## Path to the private keyfile used to identify the connection to `spark_url`. + ## @param password - string - optional + ## The password to use if services are behind basic or NTLM auth. # - # ssl_key: + # password: - ## @param kerberos - string - optional - default: disabled + ## @param kerberos_auth - string - optional - default: disabled ## If your service uses Kerberos authentication, you can specify the Kerberos ## strategy to use between: ## * required @@ -104,7 +155,7 @@ instances: ## ## See https://github.com/requests/requests-kerberos#mutual-authentication # - # kerberos: disabled + # kerberos_auth: disabled ## @param kerberos_delegate - boolean - optional - default: false ## Set to true to enable kerberos delegation of credentials to a server that requests delegation. @@ -136,3 +187,61 @@ instances: ## Set the path to your Kerberos key tab file. # # kerberos_keytab: + + ## @param tls_verify - boolean - optional - default: true + ## Instructs the check to validate the TLS certificate of services. + # + # tls_verify: true + + ## @param tls_ignore_warning - boolean - optional - default: false + ## If you disable `tls_verify` you will receive security warnings in logs. + ## Disable those by setting `tls_ignore_warning` to true. + # + # tls_ignore_warning: false + + ## @param tls_cert - string - optional + ## The path to a single file in PEM format containing a certificate as well as any + ## number of CA certificates needed to establish the certificate’s authenticity for + ## use when connecting to services. It may also contain an unencrypted private key to use. + # + # tls_cert: + + ## @param tls_private_key - string - optional + ## The unencrypted private key to use for `tls_cert` when connecting to services. This is + ## required if `tls_cert` is set and it does not already contain a private key. + # + # tls_private_key: + + ## @param tls_ca_cert - string - optional + ## The path to a file of concatenated CA certificates in PEM format or a directory + ## containing several CA certificates in PEM format. If a directory, the directory + ## must have been processed using the c_rehash utility supplied with OpenSSL. See: + ## https://www.openssl.org/docs/manmaster/man3/SSL_CTX_load_verify_locations.html + # + # tls_ca_cert: + + ## @param headers - list of key:value elements - optional + ## The headers parameter allows you to send specific headers with every request. + ## This is useful for explicitly specifying the host header or adding headers for + ## authorization purposes. + ## + ## This overrides any default headers. + # + # headers: + # Host: alternative.host.example.com + # X-Auth-Token: + + ## @param timeout - integer - optional - default: 10 + ## The timeout for connecting to services. + # + # timeout: 10 + + ## @param log_requests - boolean - optional - default: false + ## Whether or not to debug log the HTTP(S) requests made, including the method and URL. + # + # log_requests: false + + ## @param persist_connections - boolean - optional - default: false + ## Whether or not to persist cookies and use connection pooling for increased performance. + # + # persist_connections: false diff --git a/spark/datadog_checks/spark/spark.py b/spark/datadog_checks/spark/spark.py index a967e8b3766d2..ae0d6bb5d7631 100644 --- a/spark/datadog_checks/spark/spark.py +++ b/spark/datadog_checks/spark/spark.py @@ -1,11 +1,8 @@ # (C) Datadog, Inc. 2018 # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -import os from collections import namedtuple -import requests -import requests_kerberos from bs4 import BeautifulSoup from requests.exceptions import ConnectionError, HTTPError, InvalidURL, Timeout from simplejson import JSONDecodeError @@ -14,12 +11,6 @@ from datadog_checks.base import AgentCheck, ConfigurationError, is_affirmative -KERBEROS_STRATEGIES = { - 'required': requests_kerberos.REQUIRED, - 'optional': requests_kerberos.OPTIONAL, - 'disabled': requests_kerberos.DISABLED, -} - # Identifier for cluster master address in `spark.yaml` MASTER_ADDRESS = 'spark_url' DEPRECATED_MASTER_ADDRESS = 'resourcemanager_uri' @@ -154,6 +145,13 @@ class SparkCheck(AgentCheck): + HTTP_CONFIG_REMAPPER = { + 'ssl_verify': {'name': 'tls_verify'}, + 'ssl_cert': {'name': 'tls_cert'}, + 'ssl_key': {'name': 'tls_private_key'}, + 'verify': {'name': 'tls_ca_cert'}, + } + def check(self, instance): # Get additional tags from the conf file tags = instance.get('tags', []) @@ -162,29 +160,28 @@ def check(self, instance): else: tags = list(set(tags)) - requests_config = self._get_requests_config(instance) cluster_name = instance.get('cluster_name') if not cluster_name: raise ConfigurationError('The cluster_name must be specified in the instance configuration') tags.append('cluster_name:%s' % cluster_name) - spark_apps = self._get_running_apps(instance, requests_config) + spark_apps = self._get_running_apps(instance) # Get the job metrics - self._spark_job_metrics(instance, spark_apps, tags, requests_config) + self._spark_job_metrics(instance, spark_apps, tags) # Get the stage metrics - self._spark_stage_metrics(instance, spark_apps, tags, requests_config) + self._spark_stage_metrics(instance, spark_apps, tags) # Get the executor metrics - self._spark_executor_metrics(instance, spark_apps, tags, requests_config) + self._spark_executor_metrics(instance, spark_apps, tags) # Get the rdd metrics - self._spark_rdd_metrics(instance, spark_apps, tags, requests_config) + self._spark_rdd_metrics(instance, spark_apps, tags) # Get the streaming statistics metrics if is_affirmative(instance.get('streaming_metrics', True)): - self._spark_streaming_statistics_metrics(instance, spark_apps, tags, requests_config) + self._spark_streaming_statistics_metrics(instance, spark_apps, tags) # Report success after gathering all metrics from the ApplicationMaster if spark_apps: @@ -199,36 +196,6 @@ def check(self, instance): message='Connection to ApplicationMaster "%s" was successful' % am_address, ) - @classmethod - def _get_requests_config(cls, instance): - auth = None - - # Authenticate our connection to JMX endpoint if required - kerberos = instance.get('kerberos') - username = instance.get('username') - password = instance.get('password') - if username is not None and password is not None: - auth = (username, password) - elif kerberos is not None: - if kerberos not in KERBEROS_STRATEGIES: - raise Exception('Invalid Kerberos strategy `{}`'.format(kerberos)) - - auth = requests_kerberos.HTTPKerberosAuth( - mutual_authentication=KERBEROS_STRATEGIES[kerberos], - delegate=is_affirmative(instance.get('kerberos_delegate', False)), - force_preemptive=is_affirmative(instance.get('kerberos_force_initiate', False)), - hostname_override=instance.get('kerberos_hostname'), - principal=instance.get('kerberos_principal'), - ) - - return RequestsConfig( - auth=auth, - ssl_verify=instance.get('ssl_verify'), - ssl_cert=instance.get('ssl_cert'), - ssl_key=instance.get('ssl_key'), - kerberos_keytab=instance.get('kerberos_keytab'), - ) - def _get_master_address(self, instance): """ Get the master address from the instance configuration @@ -262,7 +229,7 @@ def _get_request_url(self, instance, url): self.log.debug('Request URL returned: %s', _url) return _url - def _get_running_apps(self, instance, requests_config): + def _get_running_apps(self, instance): """ Determine what mode was specified """ @@ -288,25 +255,25 @@ def _get_running_apps(self, instance, requests_config): if cluster_mode == SPARK_STANDALONE_MODE: # check for PRE-20 pre20 = is_affirmative(instance.get(SPARK_PRE_20_MODE, False)) - return self._standalone_init(master_address, pre20, requests_config, tags) + return self._standalone_init(master_address, pre20, tags) elif cluster_mode == SPARK_MESOS_MODE: - running_apps = self._mesos_init(instance, master_address, requests_config, tags) - return self._get_spark_app_ids(running_apps, requests_config, tags) + running_apps = self._mesos_init(instance, master_address, tags) + return self._get_spark_app_ids(running_apps, tags) elif cluster_mode == SPARK_YARN_MODE: - running_apps = self._yarn_init(master_address, requests_config, tags) - return self._get_spark_app_ids(running_apps, requests_config, tags) + running_apps = self._yarn_init(master_address, tags) + return self._get_spark_app_ids(running_apps, tags) else: raise Exception('Invalid setting for %s. Received %s.' % (SPARK_CLUSTER_MODE, cluster_mode)) - def _standalone_init(self, spark_master_address, pre_20_mode, requests_config, tags): + def _standalone_init(self, spark_master_address, pre_20_mode, tags): """ Return a dictionary of {app_id: (app_name, tracking_url)} for the running Spark applications """ metrics_json = self._rest_request_to_json( - spark_master_address, SPARK_MASTER_STATE_PATH, SPARK_STANDALONE_SERVICE_CHECK, requests_config, tags + spark_master_address, SPARK_MASTER_STATE_PATH, SPARK_STANDALONE_SERVICE_CHECK, tags ) running_apps = {} @@ -318,13 +285,13 @@ def _standalone_init(self, spark_master_address, pre_20_mode, requests_config, t # Parse through the HTML to grab the application driver's link try: - app_url = self._get_standalone_app_url(app_id, spark_master_address, requests_config, tags) + app_url = self._get_standalone_app_url(app_id, spark_master_address, tags) if app_id and app_name and app_url: if pre_20_mode: self.log.debug('Getting application list in pre-20 mode') applist = self._rest_request_to_json( - app_url, SPARK_APPS_PATH, SPARK_STANDALONE_SERVICE_CHECK, requests_config, tags + app_url, SPARK_APPS_PATH, SPARK_STANDALONE_SERVICE_CHECK, tags ) for appl in applist: aid = appl.get('id') @@ -347,15 +314,13 @@ def _standalone_init(self, spark_master_address, pre_20_mode, requests_config, t self.log.info("Returning running apps %s" % running_apps) return running_apps - def _mesos_init(self, instance, master_address, requests_config, tags): + def _mesos_init(self, instance, master_address, tags): """ Return a dictionary of {app_id: (app_name, tracking_url)} for running Spark applications. """ running_apps = {} - metrics_json = self._rest_request_to_json( - master_address, MESOS_MASTER_APP_PATH, MESOS_SERVICE_CHECK, requests_config, tags - ) + metrics_json = self._rest_request_to_json(master_address, MESOS_MASTER_APP_PATH, MESOS_SERVICE_CHECK, tags) if metrics_json.get('frameworks'): for app_json in metrics_json.get('frameworks'): @@ -384,11 +349,11 @@ def _mesos_init(self, instance, master_address, requests_config, tags): return running_apps - def _yarn_init(self, rm_address, requests_config, tags): + def _yarn_init(self, rm_address, tags): """ Return a dictionary of {app_id: (app_name, tracking_url)} for running Spark applications. """ - running_apps = self._yarn_get_running_spark_apps(rm_address, requests_config, tags) + running_apps = self._yarn_get_running_spark_apps(rm_address, tags) # Report success after gathering all metrics from ResourceManager self.service_check( @@ -400,19 +365,14 @@ def _yarn_init(self, rm_address, requests_config, tags): return running_apps - def _get_standalone_app_url(self, app_id, spark_master_address, requests_config, tags): + def _get_standalone_app_url(self, app_id, spark_master_address, tags): """ Return the application URL from the app info page on the Spark master. Due to a bug, we need to parse the HTML manually because we cannot fetch JSON data from HTTP interface. """ app_page = self._rest_request( - spark_master_address, - SPARK_MASTER_APP_PATH, - SPARK_STANDALONE_SERVICE_CHECK, - requests_config, - tags, - appId=app_id, + spark_master_address, SPARK_MASTER_APP_PATH, SPARK_STANDALONE_SERVICE_CHECK, tags, appId=app_id ) dom = BeautifulSoup(app_page.text, 'html.parser') @@ -421,7 +381,7 @@ def _get_standalone_app_url(self, app_id, spark_master_address, requests_config, if app_detail_ui_links and len(app_detail_ui_links) == 1: return app_detail_ui_links[0].attrs['href'] - def _yarn_get_running_spark_apps(self, rm_address, requests_config, tags): + def _yarn_get_running_spark_apps(self, rm_address, tags): """ Return a dictionary of {app_id: (app_name, tracking_url)} for running Spark applications. @@ -432,7 +392,6 @@ def _yarn_get_running_spark_apps(self, rm_address, requests_config, tags): rm_address, YARN_APPS_PATH, YARN_SERVICE_CHECK, - requests_config, tags, states=APPLICATION_STATES, applicationTypes=YARN_APPLICATION_TYPES, @@ -453,7 +412,7 @@ def _yarn_get_running_spark_apps(self, rm_address, requests_config, tags): return running_apps - def _get_spark_app_ids(self, running_apps, requests_config, tags): + def _get_spark_app_ids(self, running_apps, tags): """ Traverses the Spark application master in YARN to get a Spark application ID. @@ -461,9 +420,7 @@ def _get_spark_app_ids(self, running_apps, requests_config, tags): """ spark_apps = {} for app_id, (app_name, tracking_url) in iteritems(running_apps): - response = self._rest_request_to_json( - tracking_url, SPARK_APPS_PATH, SPARK_SERVICE_CHECK, requests_config, tags - ) + response = self._rest_request_to_json(tracking_url, SPARK_APPS_PATH, SPARK_SERVICE_CHECK, tags) for app in response: app_id = app.get('id') @@ -474,7 +431,7 @@ def _get_spark_app_ids(self, running_apps, requests_config, tags): return spark_apps - def _spark_job_metrics(self, instance, running_apps, addl_tags, requests_config): + def _spark_job_metrics(self, instance, running_apps, addl_tags): """ Get metrics for each Spark job. """ @@ -482,7 +439,7 @@ def _spark_job_metrics(self, instance, running_apps, addl_tags, requests_config) base_url = self._get_request_url(instance, tracking_url) response = self._rest_request_to_json( - base_url, SPARK_APPS_PATH, SPARK_SERVICE_CHECK, requests_config, addl_tags, app_id, 'jobs' + base_url, SPARK_APPS_PATH, SPARK_SERVICE_CHECK, addl_tags, app_id, 'jobs' ) for job in response: @@ -496,7 +453,7 @@ def _spark_job_metrics(self, instance, running_apps, addl_tags, requests_config) self._set_metrics_from_json(tags, job, SPARK_JOB_METRICS) self._set_metric('spark.job.count', COUNT, 1, tags) - def _spark_stage_metrics(self, instance, running_apps, addl_tags, requests_config): + def _spark_stage_metrics(self, instance, running_apps, addl_tags): """ Get metrics for each Spark stage. """ @@ -504,7 +461,7 @@ def _spark_stage_metrics(self, instance, running_apps, addl_tags, requests_confi base_url = self._get_request_url(instance, tracking_url) response = self._rest_request_to_json( - base_url, SPARK_APPS_PATH, SPARK_SERVICE_CHECK, requests_config, addl_tags, app_id, 'stages' + base_url, SPARK_APPS_PATH, SPARK_SERVICE_CHECK, addl_tags, app_id, 'stages' ) for stage in response: @@ -518,7 +475,7 @@ def _spark_stage_metrics(self, instance, running_apps, addl_tags, requests_confi self._set_metrics_from_json(tags, stage, SPARK_STAGE_METRICS) self._set_metric('spark.stage.count', COUNT, 1, tags) - def _spark_executor_metrics(self, instance, running_apps, addl_tags, requests_config): + def _spark_executor_metrics(self, instance, running_apps, addl_tags): """ Get metrics for each Spark executor. """ @@ -526,7 +483,7 @@ def _spark_executor_metrics(self, instance, running_apps, addl_tags, requests_co base_url = self._get_request_url(instance, tracking_url) response = self._rest_request_to_json( - base_url, SPARK_APPS_PATH, SPARK_SERVICE_CHECK, requests_config, addl_tags, app_id, 'executors' + base_url, SPARK_APPS_PATH, SPARK_SERVICE_CHECK, addl_tags, app_id, 'executors' ) tags = ['app_name:%s' % str(app_name)] @@ -541,7 +498,7 @@ def _spark_executor_metrics(self, instance, running_apps, addl_tags, requests_co if len(response): self._set_metric('spark.executor.count', COUNT, len(response), tags) - def _spark_rdd_metrics(self, instance, running_apps, addl_tags, requests_config): + def _spark_rdd_metrics(self, instance, running_apps, addl_tags): """ Get metrics for each Spark RDD. """ @@ -549,7 +506,7 @@ def _spark_rdd_metrics(self, instance, running_apps, addl_tags, requests_config) base_url = self._get_request_url(instance, tracking_url) response = self._rest_request_to_json( - base_url, SPARK_APPS_PATH, SPARK_SERVICE_CHECK, requests_config, addl_tags, app_id, 'storage/rdd' + base_url, SPARK_APPS_PATH, SPARK_SERVICE_CHECK, addl_tags, app_id, 'storage/rdd' ) tags = ['app_name:%s' % str(app_name)] @@ -561,7 +518,7 @@ def _spark_rdd_metrics(self, instance, running_apps, addl_tags, requests_config) if len(response): self._set_metric('spark.rdd.count', COUNT, len(response), tags) - def _spark_streaming_statistics_metrics(self, instance, running_apps, addl_tags, requests_config): + def _spark_streaming_statistics_metrics(self, instance, running_apps, addl_tags): """ Get metrics for each application streaming statistics. """ @@ -569,13 +526,7 @@ def _spark_streaming_statistics_metrics(self, instance, running_apps, addl_tags, try: base_url = self._get_request_url(instance, tracking_url) response = self._rest_request_to_json( - base_url, - SPARK_APPS_PATH, - SPARK_SERVICE_CHECK, - requests_config, - addl_tags, - app_id, - 'streaming/statistics', + base_url, SPARK_APPS_PATH, SPARK_SERVICE_CHECK, addl_tags, app_id, 'streaming/statistics' ) self.log.debug('streaming/statistics: %s', response) tags = ['app_name:%s' % str(app_name)] @@ -617,7 +568,7 @@ def _set_metric(self, metric_name, metric_type, value, tags=None): else: self.log.error('Metric type "{}" unknown'.format(metric_type)) - def _rest_request(self, url, object_path, service_name, requests_config, tags, *args, **kwargs): + def _rest_request(self, url, object_path, service_name, tags, *args, **kwargs): """ Query the given URL and return the response """ @@ -626,20 +577,6 @@ def _rest_request(self, url, object_path, service_name, requests_config, tags, * if object_path: url = self._join_url_dir(url, object_path) - # Load SSL configuration, if available. - # ssl_verify can be a bool or a string - # (http://docs.python-requests.org/en/latest/user/advanced/#ssl-cert-verification) - if isinstance(requests_config.ssl_verify, bool) or isinstance(requests_config.ssl_verify, str): - verify = requests_config.ssl_verify - else: - verify = None - if requests_config.ssl_cert and requests_config.ssl_key: - cert = (requests_config.ssl_cert, requests_config.ssl_key) - elif requests_config.ssl_cert: - cert = requests_config.ssl_cert - else: - cert = None - # Add args to the url if args: for directory in args: @@ -650,14 +587,9 @@ def _rest_request(self, url, object_path, service_name, requests_config, tags, * query = '&'.join(['{0}={1}'.format(key, value) for key, value in iteritems(kwargs)]) url = urljoin(url, '?' + query) - old_keytab_path = None - if requests_config.kerberos_keytab: - old_keytab_path = os.getenv('KRB5_CLIENT_KTNAME') - os.environ['KRB5_CLIENT_KTNAME'] = requests_config.kerberos_keytab - try: self.log.debug('Spark check URL: %s' % url) - response = requests.get(url, auth=requests_config.auth, verify=verify, cert=cert) + response = self.http.get(url) response.raise_for_status() @@ -686,15 +618,11 @@ def _rest_request(self, url, object_path, service_name, requests_config, tags, * else: return response - finally: - if old_keytab_path is not None: - os.environ['KRB5_CLIENT_KTNAME'] = old_keytab_path - - def _rest_request_to_json(self, address, object_path, service_name, requests_config, tags, *args, **kwargs): + def _rest_request_to_json(self, address, object_path, service_name, tags, *args, **kwargs): """ Query the given URL and return the JSON response """ - response = self._rest_request(address, object_path, service_name, requests_config, tags, *args, **kwargs) + response = self._rest_request(address, object_path, service_name, tags, *args, **kwargs) try: response_json = response.json() diff --git a/spark/requirements.in b/spark/requirements.in index 5657043c29440..8f63371ca71da 100644 --- a/spark/requirements.in +++ b/spark/requirements.in @@ -1,2 +1 @@ beautifulsoup4==4.5.1 -requests-kerberos==0.12.0 diff --git a/spark/tests/test_spark.py b/spark/tests/test_spark.py index 4ee6a31cb0666..4c9a9912ec3a0 100644 --- a/spark/tests/test_spark.py +++ b/spark/tests/test_spark.py @@ -565,7 +565,7 @@ def raise_for_status(self): def test_yarn(aggregator): with mock.patch('requests.get', yarn_requests_get_mock): - c = SparkCheck('spark', None, {}, [YARN_CONFIG]) + c = SparkCheck('spark', {}, [YARN_CONFIG]) c.check(YARN_CONFIG) # Check the succeeded job metrics @@ -614,7 +614,7 @@ def test_yarn(aggregator): def test_auth_yarn(aggregator): with mock.patch('requests.get', yarn_requests_auth_mock): - c = SparkCheck('spark', None, {}, [YARN_AUTH_CONFIG]) + c = SparkCheck('spark', {}, [YARN_AUTH_CONFIG]) c.check(YARN_AUTH_CONFIG) tags = ['url:http://localhost:8088', 'cluster_name:SparkCluster'] + CUSTOM_TAGS @@ -633,7 +633,7 @@ def test_auth_yarn(aggregator): def test_mesos(aggregator): with mock.patch('requests.get', mesos_requests_get_mock): - c = SparkCheck('spark', None, {}, [MESOS_CONFIG]) + c = SparkCheck('spark', {}, [MESOS_CONFIG]) c.check(MESOS_CONFIG) # Check the running job metrics @@ -689,7 +689,7 @@ def test_mesos(aggregator): def test_mesos_filter(aggregator): with mock.patch('requests.get', mesos_requests_get_mock): - c = SparkCheck('spark', None, {}, [MESOS_FILTERED_CONFIG]) + c = SparkCheck('spark', {}, [MESOS_FILTERED_CONFIG]) c.check(MESOS_FILTERED_CONFIG) for sc in aggregator.service_checks(MESOS_SERVICE_CHECK): @@ -701,7 +701,7 @@ def test_mesos_filter(aggregator): def test_standalone_unit(aggregator): with mock.patch('requests.get', standalone_requests_get_mock): - c = SparkCheck('spark', None, {}, [STANDALONE_CONFIG]) + c = SparkCheck('spark', {}, [STANDALONE_CONFIG]) c.check(STANDALONE_CONFIG) # Check the running job metrics @@ -754,7 +754,7 @@ def test_standalone_unit(aggregator): def test_standalone_pre20(aggregator): with mock.patch('requests.get', standalone_requests_pre20_get_mock): - c = SparkCheck('spark', None, {}, [STANDALONE_CONFIG_PRE_20]) + c = SparkCheck('spark', {}, [STANDALONE_CONFIG_PRE_20]) c.check(STANDALONE_CONFIG_PRE_20) # Check the running job metrics @@ -807,7 +807,7 @@ def test_standalone_pre20(aggregator): def test_ssl(): run_ssl_server() - c = SparkCheck('spark', None, {}, [SSL_CONFIG]) + c = SparkCheck('spark', {}, [SSL_CONFIG]) with pytest.raises(requests.exceptions.SSLError): c.check(SSL_CONFIG) @@ -817,7 +817,7 @@ def test_ssl_no_verify(): # Disable ssl warning for self signed cert/no verify urllib3.disable_warnings() run_ssl_server() - c = SparkCheck('spark', None, {}, [SSL_NO_VERIFY_CONFIG]) + c = SparkCheck('spark', {}, [SSL_NO_VERIFY_CONFIG]) c.check(SSL_NO_VERIFY_CONFIG) @@ -826,7 +826,7 @@ def test_ssl_cert(): # Disable ssl warning for self signed cert/no verify urllib3.disable_warnings() run_ssl_server() - c = SparkCheck('spark', None, {}, [SSL_CERT_CONFIG]) + c = SparkCheck('spark', {}, [SSL_CERT_CONFIG]) c.check(SSL_CERT_CONFIG) @@ -855,7 +855,7 @@ def run_ssl_server(): @pytest.mark.usefixtures('dd_environment') @pytest.mark.integration def test_standalone(aggregator, instance_standalone): - c = SparkCheck('spark', None, {}, [instance_standalone]) + c = SparkCheck('spark', {}, [instance_standalone]) c.check(instance_standalone) # Check the running job metrics