Added no_proxy autoconfiguration from rhsm conf and tests (#3507)

Resolves: rhbz#2065233

Signed-off-by: ahitacat <[email protected]>
(cherry picked from commit 894484a)

insights/client/auto_config.py

@@ -55,21 +55,24 @@ def verify_connectivity(config):
         return False
 
 
-def set_auto_configuration(config, hostname, ca_cert, proxy, is_satellite, is_stage):
+def set_auto_configuration(config, hostname, ca_cert, proxy, is_satellite, is_stage, rhsm_no_proxy=None):
     """
     Set config based on discovered data
     """
     logger.debug("Attempting to auto configure!")
     logger.debug("Attempting to auto configure hostname: %s", hostname)
     logger.debug("Attempting to auto configure CA cert: %s", ca_cert)
     logger.debug("Attempting to auto configure proxy: %s", proxy)
+    logger.debug("Attempting to auto configure no_proxy: %s", rhsm_no_proxy)
     saved_base_url = config.base_url
     if ca_cert is not None:
         saved_cert_verify = config.cert_verify
         config.cert_verify = ca_cert
     if proxy is not None:
         saved_proxy = config.proxy
         config.proxy = proxy
+    if rhsm_no_proxy and rhsm_no_proxy != '':
+        config.no_proxy = rhsm_no_proxy
     if is_satellite:
         # satellite
         config.base_url = hostname + '/r/insights'
@@ -128,6 +131,9 @@ def _try_satellite6_configuration(config):
         rhsm_proxy_port = rhsm_config.get('server', 'proxy_port').strip()
         rhsm_proxy_user = rhsm_config.get('server', 'proxy_user').strip()
         rhsm_proxy_pass = rhsm_config.get('server', 'proxy_password').strip()
+        rhsm_no_proxy = rhsm_config.get('server', 'no_proxy').strip()
+        if rhsm_no_proxy.lower() == 'none' or rhsm_no_proxy == '':
+            rhsm_no_proxy = None
 
         proxy = None
 
@@ -170,7 +176,7 @@ def _try_satellite6_configuration(config):
             is_satellite = True
 
         logger.debug("Trying to set auto_configuration")
-        set_auto_configuration(config, rhsm_hostname, rhsm_ca, proxy, is_satellite, is_stage)
+        set_auto_configuration(config, rhsm_hostname, rhsm_ca, proxy, is_satellite, is_stage, rhsm_no_proxy=rhsm_no_proxy)
         return True
     except Exception as e:
         logger.debug(e)

insights/client/config.py

@@ -243,6 +243,10 @@ def _core_collect_default():
         # non-CLI
         'default': False,  # legacy
     },
+    'no_proxy': {
+        # non-CLI
+        'default': None
+    },
     'no_upload': {
         'default': False,
         'opt': ['--no-upload'],

insights/client/connection.py

@@ -257,79 +257,72 @@ def user_agent(self):
 
         return ua
 
+    def get_proxy(self, proxy_info, no_proxy_info, environment):
+        proxies = None
+        proxy_auth = None
+        if '@' in proxy_info:
+            scheme = proxy_info.split(':')[0] + '://'
+            logger.debug("Proxy Scheme: %s", scheme)
+            location = proxy_info.split('@')[1]
+            logger.debug("Proxy Location: %s", location)
+            username = proxy_info.split(
+                '@')[0].split(':')[1].replace('/', '')
+            logger.debug("Proxy User: %s", username)
+            password = proxy_info.split('@')[0].split(':')[2]
+            proxy_auth = requests.auth._basic_auth_str(username, password)
+            proxy_info = scheme + location
+        logger.debug("%s Proxy: %s", environment, proxy_info)
+        proxies = {"https": proxy_info}
+        if no_proxy_info:
+            insights_service_host = urlparse(self.base_url).hostname
+            logger.debug('Found NO_PROXY set. Checking NO_PROXY %s against base URL %s.', no_proxy_info, insights_service_host)
+            for no_proxy_host in no_proxy_info.split(', '):
+                logger.debug('Checking %s against %s', no_proxy_host, insights_service_host)
+                if no_proxy_host == '*':
+                    proxies = None
+                    proxy_auth = None
+                    logger.debug('Found NO_PROXY asterisk(*) wildcard, disabling all proxies.')
+                    break
+                elif no_proxy_host.startswith('.') or no_proxy_host.startswith('*'):
+                    if insights_service_host.endswith(no_proxy_host.replace('*', '')):
+                        proxies = None
+                        proxy_auth = None
+                        logger.debug('Found NO_PROXY range %s matching %s', no_proxy_host, insights_service_host)
+                        break
+                elif no_proxy_host == insights_service_host:
+                    proxies = None
+                    proxy_auth = None
+                    logger.debug('Found NO_PROXY %s exactly matching %s', no_proxy_host, insights_service_host)
+                    break
+        return proxies, proxy_auth
+
     def get_proxies(self):
         """
         Determine proxy configuration
         """
         # Get proxy from ENV or Config
         proxies = None
         proxy_auth = None
-        no_proxy = os.environ.get('NO_PROXY')
-        logger.debug("NO PROXY: %s", no_proxy)
 
         # CONF PROXY TAKES PRECEDENCE OVER ENV PROXY
         conf_proxy = self.config.proxy
-        if ((conf_proxy is not None and
-             conf_proxy.lower() != 'None'.lower() and
-             conf_proxy != "")):
-            if '@' in conf_proxy:
-                scheme = conf_proxy.split(':')[0] + '://'
-                logger.debug("Proxy Scheme: %s", scheme)
-                location = conf_proxy.split('@')[1]
-                logger.debug("Proxy Location: %s", location)
-                username = conf_proxy.split(
-                    '@')[0].split(':')[1].replace('/', '')
-                logger.debug("Proxy User: %s", username)
-                password = conf_proxy.split('@')[0].split(':')[2]
-                proxy_auth = requests.auth._basic_auth_str(username, password)
-                conf_proxy = scheme + location
-            logger.debug("CONF Proxy: %s", conf_proxy)
-            proxies = {"https": conf_proxy}
+        conf_no_proxy = self.config.no_proxy
+
+        if conf_proxy:
+            proxies, proxy_auth = self.get_proxy(conf_proxy, conf_no_proxy, "CONF")
 
         # HANDLE NO PROXY CONF PROXY EXCEPTION VERBIAGE
+        no_proxy = os.environ.get('NO_PROXY')
         if no_proxy and conf_proxy:
             logger.debug("You have environment variable NO_PROXY set "
                          "as well as 'proxy' set in your configuration file. "
                          "NO_PROXY environment variable will be ignored.")
 
-        # IF NO CONF PROXY, GET ENV PROXY AND NO PROXY
-        if proxies is None:
+        # IF NO CONF PROXY and NO_PROX none in conf, GET ENV PROXY AND NO PROXY
+        if proxies is None and conf_no_proxy is None:
             env_proxy = os.environ.get('HTTPS_PROXY')
             if env_proxy:
-                if '@' in env_proxy:
-                    scheme = env_proxy.split(':')[0] + '://'
-                    logger.debug("Proxy Scheme: %s", scheme)
-                    location = env_proxy.split('@')[1]
-                    logger.debug("Proxy Location: %s", location)
-                    username = env_proxy.split('@')[0].split(':')[1].replace('/', '')
-                    logger.debug("Proxy User: %s", username)
-                    password = env_proxy.split('@')[0].split(':')[2]
-                    proxy_auth = requests.auth._basic_auth_str(username, password)
-                    env_proxy = scheme + location
-                logger.debug("ENV Proxy: %s", env_proxy)
-                proxies = {"https": env_proxy}
-            if no_proxy:
-                insights_service_host = urlparse(self.base_url).hostname
-                logger.debug('Found NO_PROXY set. Checking NO_PROXY %s against base URL %s.', no_proxy, insights_service_host)
-                for no_proxy_host in no_proxy.split(','):
-                    logger.debug('Checking %s against %s', no_proxy_host, insights_service_host)
-                    if no_proxy_host == '*':
-                        proxies = None
-                        proxy_auth = None
-                        logger.debug('Found NO_PROXY asterisk(*) wildcard, disabling all proxies.')
-                        break
-                    elif no_proxy_host.startswith('.') or no_proxy_host.startswith('*'):
-                        if insights_service_host.endswith(no_proxy_host.replace('*', '')):
-                            proxies = None
-                            proxy_auth = None
-                            logger.debug('Found NO_PROXY range %s matching %s', no_proxy_host, insights_service_host)
-                            break
-                    elif no_proxy_host == insights_service_host:
-                        proxies = None
-                        proxy_auth = None
-                        logger.debug('Found NO_PROXY %s exactly matching %s', no_proxy_host, insights_service_host)
-                        break
-
+                proxies, proxy_auth = self.get_proxy(env_proxy, no_proxy, "ENV")
         self.proxies = proxies
         self.proxy_auth = proxy_auth
 

insights/tests/client/apps/test_compliance.py

@@ -274,42 +274,42 @@ def test_get_profiles_matching_os(config, os_release_info_mock):
 
 
 @patch("insights.client.apps.compliance.os_release_info", return_value=(None, '6.5'))
-@patch("insights.client.config.InsightsConfig")
+@patch("insights.client.config.InsightsConfig", base_url='localhost.com/app')
 def test_os_release(config, os_release_info_mock):
     compliance_client = ComplianceClient(config)
     assert compliance_client.os_release() == '6.5'
 
 
 @patch("insights.client.apps.compliance.os_release_info", return_value=(None, '6.5'))
-@patch("insights.client.config.InsightsConfig")
+@patch("insights.client.config.InsightsConfig", base_url='localhost.com/tests')
 def test_os_minor_version(config, os_release_info_mock):
     compliance_client = ComplianceClient(config)
     assert compliance_client.os_minor_version() == '5'
 
 
 @patch("insights.client.apps.compliance.os_release_info", return_value=(None, '6.5'))
-@patch("insights.client.config.InsightsConfig")
+@patch("insights.client.config.InsightsConfig", base_url='localhost.com/app')
 def test_os_major_version(config, os_release_info_mock):
     compliance_client = ComplianceClient(config)
     assert compliance_client.os_major_version() == '6'
 
 
-@patch("insights.client.config.InsightsConfig")
+@patch("insights.client.config.InsightsConfig", base_url='localhost.com/app')
 def test_profile_files(config):
     compliance_client = ComplianceClient(config)
     compliance_client.os_release = lambda: '7'
     assert compliance_client.profile_files() == []
 
 
 @patch("insights.client.apps.compliance.call", return_value=(0, PATH))
-@patch("insights.client.config.InsightsConfig")
+@patch("insights.client.config.InsightsConfig", base_url='localhost.com/app')
 def test_find_scap_policy(config, call):
     compliance_client = ComplianceClient(config)
     compliance_client.profile_files = lambda: ['/something']
     assert compliance_client.find_scap_policy('ref_id') == PATH
 
 
-@patch("insights.client.config.InsightsConfig")
+@patch("insights.client.config.InsightsConfig", base_url='localhost.com/app')
 def test_find_scap_policy_with_one_datastream_file(config, tmpdir):
     compliance_client = ComplianceClient(config)
     dir1 = tmpdir.mkdir('scap')
@@ -323,7 +323,7 @@ def test_find_scap_policy_with_one_datastream_file(config, tmpdir):
         assert compliance_client.find_scap_policy('content_profile_anssi_bp28_high') == file
 
 
-@patch("insights.client.config.InsightsConfig")
+@patch("insights.client.config.InsightsConfig", base_url='localhost.com/app')
 def test_find_scap_policy_with_two_datastream_file(config, tmpdir):
     compliance_client = ComplianceClient(config)
     dir1 = tmpdir.mkdir('scap')
@@ -343,15 +343,15 @@ def test_find_scap_policy_with_two_datastream_file(config, tmpdir):
 
 
 @patch("insights.client.apps.compliance.call", return_value=(1, 'bad things happened'.encode('utf-8')))
-@patch("insights.client.config.InsightsConfig")
+@patch("insights.client.config.InsightsConfig", base_url='localhost.com/app')
 def test_find_scap_policy_not_found(config, call):
     compliance_client = ComplianceClient(config)
     compliance_client.profile_files = lambda: ['/something']
     assert compliance_client.find_scap_policy('ref_id') is None
 
 
 @patch("insights.client.apps.compliance.call", return_value=(0, ''.encode('utf-8')))
-@patch("insights.client.config.InsightsConfig")
+@patch("insights.client.config.InsightsConfig", base_url='localhost.com/app')
 def test_run_scan(config, call):
     compliance_client = ComplianceClient(config)
     output_path = '/tmp/oscap_results-ref_id.xml'
@@ -365,7 +365,7 @@ def test_run_scan(config, call):
 
 
 @patch("insights.client.apps.compliance.call", return_value=(1, 'bad things happened'.encode('utf-8')))
-@patch("insights.client.config.InsightsConfig")
+@patch("insights.client.config.InsightsConfig", base_url='localhost.com/app')
 def test_run_scan_fail(config, call):
     compliance_client = ComplianceClient(config)
     output_path = '/tmp/oscap_results-ref_id.xml'
@@ -380,7 +380,7 @@ def test_run_scan_fail(config, call):
 
 
 @patch("insights.client.apps.compliance.call", return_value=(0, ''.encode('utf-8')))
-@patch("insights.client.config.InsightsConfig")
+@patch("insights.client.config.InsightsConfig", base_url='localhost.com/app')
 def test_run_scan_missing_profile(config, call):
     compliance_client = ComplianceClient(config)
     output_path = '/tmp/oscap_results-ref_id.xml'
@@ -390,20 +390,20 @@ def test_run_scan_missing_profile(config, call):
     call.assert_not_called()
 
 
-@patch("insights.client.config.InsightsConfig")
+@patch("insights.client.config.InsightsConfig", base_url='localhost.com/app')
 def test_tailored_file_is_not_downloaded_if_not_needed(config):
     compliance_client = ComplianceClient(config)
     assert compliance_client.download_tailoring_file({'attributes': {'tailored': False}}) is None
 
 
-@patch("insights.client.config.InsightsConfig")
+@patch("insights.client.config.InsightsConfig", base_url='localhost.com/app')
 def test_tailored_file_is_not_downloaded_if_tailored_is_missing(config):
     compliance_client = ComplianceClient(config)
     assert compliance_client.download_tailoring_file({'id': 'foo', 'attributes': {'ref_id': 'aaaaa'}}) is None
 
 
 @patch("insights.client.apps.compliance.open", new_callable=mock_open)
-@patch("insights.client.config.InsightsConfig")
+@patch("insights.client.config.InsightsConfig", base_url='localhost.com/app')
 def test_tailored_file_is_downloaded_from_initial_profile_if_os_minor_version_is_missing(config, call):
     compliance_client = ComplianceClient(config)
     compliance_client.conn.session.get = Mock(return_value=Mock(status_code=200, json=Mock(return_value={'data': [{'attributes': 'data'}]})))
@@ -412,7 +412,7 @@ def test_tailored_file_is_downloaded_from_initial_profile_if_os_minor_version_is
 
 
 @patch("insights.client.apps.compliance.os_release_info", return_value=(None, '6.5'))
-@patch("insights.client.config.InsightsConfig")
+@patch("insights.client.config.InsightsConfig", base_url='localhost.com/app')
 def test_tailored_file_is_not_downloaded_if_os_minor_version_mismatches(config, os_release_info_mock):
     compliance_client = ComplianceClient(config)
     compliance_client.conn.session.get = Mock(return_value=Mock(status_code=200, json=Mock(return_value={'data': [{'attributes': 'data'}]})))
@@ -422,29 +422,29 @@ def test_tailored_file_is_not_downloaded_if_os_minor_version_mismatches(config,
 
 @patch("insights.client.apps.compliance.os_release_info", return_value=(None, '6.5'))
 @patch("insights.client.apps.compliance.open", new_callable=mock_open)
-@patch("insights.client.config.InsightsConfig")
+@patch("insights.client.config.InsightsConfig", base_url='localhost.com/app')
 def test_tailored_file_is_downloaded_if_needed(config, call, os_release_info_mock):
     compliance_client = ComplianceClient(config)
     compliance_client.conn.session.get = Mock(return_value=Mock(status_code=200, json=Mock(return_value={'data': [{'attributes': 'data'}]})))
     assert 'oscap_tailoring_file-aaaaa' in compliance_client.download_tailoring_file({'id': 'foo', 'attributes': {'tailored': True, 'ref_id': 'aaaaa', 'os_minor_version': '5'}})
     assert compliance_client.download_tailoring_file({'id': 'foo', 'attributes': {'tailored': False, 'ref_id': 'aaaaa', 'os_minor_version': '5'}}) is None
 
 
-@patch("insights.client.config.InsightsConfig")
+@patch("insights.client.config.InsightsConfig", base_url='localhost.com/app')
 def test_build_oscap_command_does_not_append_tailoring_path(config):
     compliance_client = ComplianceClient(config)
     expected_command = 'oscap xccdf eval --profile aaaaa --results output_path xml_sample'
     assert expected_command == compliance_client.build_oscap_command('aaaaa', 'xml_sample', 'output_path', None)
 
 
-@patch("insights.client.config.InsightsConfig")
+@patch("insights.client.config.InsightsConfig", base_url='localhost.com/app')
 def test_build_oscap_command_append_tailoring_path(config):
     compliance_client = ComplianceClient(config)
     expected_command = 'oscap xccdf eval --profile aaaaa --tailoring-file tailoring_path --results output_path xml_sample'
     assert expected_command == compliance_client.build_oscap_command('aaaaa', 'xml_sample', 'output_path', 'tailoring_path')
 
 
-@patch("insights.client.config.InsightsConfig")
+@patch("insights.client.config.InsightsConfig", base_url='localhost.com/app')
 def test__get_inventory_id(config):
     compliance_client = ComplianceClient(config)
     compliance_client.conn._fetch_system_by_machine_id = lambda: []