Merge pull request #6903 from cjerdonek/move-trusted-hosts

Move PackageFinder's --trusted-host logic to PipSession

src/pip/_internal/cli/req_command.py

@@ -62,7 +62,7 @@ def _build_session(self, options, retries=None, timeout=None):
                 if options.cache_dir else None
             ),
             retries=retries if retries is not None else options.retries,
-            insecure_hosts=options.trusted_hosts,
+            trusted_hosts=options.trusted_hosts,
             index_urls=self._get_index_urls(options),
         )
 
@@ -276,7 +276,6 @@ def _build_package_finder(
         return PackageFinder.create(
             search_scope=search_scope,
             selection_prefs=selection_prefs,
-            trusted_hosts=options.trusted_hosts,
             session=session,
             target_python=target_python,
         )

src/pip/_internal/commands/list.py

@@ -126,7 +126,6 @@ def _build_package_finder(self, options, session):
         return PackageFinder.create(
             search_scope=search_scope,
             selection_prefs=selection_prefs,
-            trusted_hosts=options.trusted_hosts,
             session=session,
         )
 

src/pip/_internal/download.py

@@ -12,7 +12,7 @@
 import sys
 from contextlib import contextmanager
 
-from pip._vendor import requests, urllib3
+from pip._vendor import requests, six, urllib3
 from pip._vendor.cachecontrol import CacheControlAdapter
 from pip._vendor.cachecontrol.caches import FileCache
 from pip._vendor.lockfile import LockError
@@ -32,7 +32,7 @@
 from pip._internal.exceptions import HashMismatch, InstallationError
 from pip._internal.models.index import PyPI
 # Import ssl from compat so the initial import occurs in only one place.
-from pip._internal.utils.compat import HAS_TLS, ssl
+from pip._internal.utils.compat import HAS_TLS, ipaddress, ssl
 from pip._internal.utils.encoding import auto_decode
 from pip._internal.utils.filesystem import check_path_owner, copy2_fixed
 from pip._internal.utils.glibc import libc_ver
@@ -66,7 +66,7 @@
 
 if MYPY_CHECK_RUNNING:
     from typing import (
-        Callable, Dict, List, IO, Optional, Text, Tuple, Union
+        IO, Callable, Dict, Iterator, List, Optional, Text, Tuple, Union,
     )
     from optparse import Values
 
@@ -77,6 +77,7 @@
     from pip._internal.vcs.versioncontrol import AuthInfo, VersionControl
 
     Credentials = Tuple[str, str, str]
+    SecureOrigin = Tuple[str, str, Optional[str]]
 
     if PY2:
         CopytreeKwargs = TypedDict(
@@ -120,6 +121,20 @@
                    str(exc))
     keyring = None
 
+
+SECURE_ORIGINS = [
+    # protocol, hostname, port
+    # Taken from Chrome's list of secure origins (See: http://bit.ly/1qrySKC)
+    ("https", "*", "*"),
+    ("*", "localhost", "*"),
+    ("*", "127.0.0.0/8", "*"),
+    ("*", "::1/128", "*"),
+    ("file", "*", None),
+    # ssh is always secure.
+    ("ssh", "*", "*"),
+]  # type: List[SecureOrigin]
+
+
 # These are environment variables present when running under various
 # CI systems.  For each variable, some CI systems that use the variable
 # are indicated.  The collection was chosen so that for each of a number
@@ -558,13 +573,21 @@ class PipSession(requests.Session):
     timeout = None  # type: Optional[int]
 
     def __init__(self, *args, **kwargs):
+        """
+        :param trusted_hosts: Domains not to emit warnings for when not using
+            HTTPS.
+        """
         retries = kwargs.pop("retries", 0)
         cache = kwargs.pop("cache", None)
-        insecure_hosts = kwargs.pop("insecure_hosts", [])
+        trusted_hosts = kwargs.pop("trusted_hosts", [])  # type: List[str]
         index_urls = kwargs.pop("index_urls", None)
 
         super(PipSession, self).__init__(*args, **kwargs)
 
+        # Namespace the attribute with "pip_" just in case to prevent
+        # possible conflicts with the base class.
+        self.pip_trusted_hosts = []  # type: List[str]
+
         # Attach our User Agent to the request
         self.headers["User-Agent"] = user_agent()
 
@@ -630,13 +653,26 @@ def __init__(self, *args, **kwargs):
         # Enable file:// urls
         self.mount("file://", LocalFSAdapter())
 
-        # We want to use a non-validating adapter for any requests which are
-        # deemed insecure.
-        for host in insecure_hosts:
-            self.add_insecure_host(host)
+        for host in trusted_hosts:
+            self.add_trusted_host(host, suppress_logging=True)
+
+    def add_trusted_host(self, host, source=None, suppress_logging=False):
+        # type: (str, Optional[str], bool) -> None
+        """
+        :param host: It is okay to provide a host that has previously been
+            added.
+        :param source: An optional source string, for logging where the host
+            string came from.
+        """
+        if not suppress_logging:
+            msg = 'adding trusted host: {!r}'.format(host)
+            if source is not None:
+                msg += ' (from {})'.format(source)
+            logger.info(msg)
+
+        if host not in self.pip_trusted_hosts:
+            self.pip_trusted_hosts.append(host)
 
-    def add_insecure_host(self, host):
-        # type: (str) -> None
         self.mount(build_url_from_netloc(host) + '/', self._insecure_adapter)
         if not netloc_has_port(host):
             # Mount wildcard ports for the same host.
@@ -645,6 +681,90 @@ def add_insecure_host(self, host):
                 self._insecure_adapter
             )
 
+    def iter_secure_origins(self):
+        # type: () -> Iterator[SecureOrigin]
+        for secure_origin in SECURE_ORIGINS:
+            yield secure_origin
+        for host in self.pip_trusted_hosts:
+            yield ('*', host, '*')
+
+    def is_secure_origin(self, location):
+        # type: (Link) -> bool
+        # Determine if this url used a secure transport mechanism
+        parsed = urllib_parse.urlparse(str(location))
+        origin_protocol, origin_host, origin_port = (
+            parsed.scheme, parsed.hostname, parsed.port,
+        )
+
+        # The protocol to use to see if the protocol matches.
+        # Don't count the repository type as part of the protocol: in
+        # cases such as "git+ssh", only use "ssh". (I.e., Only verify against
+        # the last scheme.)
+        origin_protocol = origin_protocol.rsplit('+', 1)[-1]
+
+        # Determine if our origin is a secure origin by looking through our
+        # hardcoded list of secure origins, as well as any additional ones
+        # configured on this PackageFinder instance.
+        for secure_origin in self.iter_secure_origins():
+            secure_protocol, secure_host, secure_port = secure_origin
+            if origin_protocol != secure_protocol and secure_protocol != "*":
+                continue
+
+            try:
+                # We need to do this decode dance to ensure that we have a
+                # unicode object, even on Python 2.x.
+                addr = ipaddress.ip_address(
+                    origin_host
+                    if (
+                        isinstance(origin_host, six.text_type) or
+                        origin_host is None
+                    )
+                    else origin_host.decode("utf8")
+                )
+                network = ipaddress.ip_network(
+                    secure_host
+                    if isinstance(secure_host, six.text_type)
+                    # setting secure_host to proper Union[bytes, str]
+                    # creates problems in other places
+                    else secure_host.decode("utf8")  # type: ignore
+                )
+            except ValueError:
+                # We don't have both a valid address or a valid network, so
+                # we'll check this origin against hostnames.
+                if (origin_host and
+                        origin_host.lower() != secure_host.lower() and
+                        secure_host != "*"):
+                    continue
+            else:
+                # We have a valid address and network, so see if the address
+                # is contained within the network.
+                if addr not in network:
+                    continue
+
+            # Check to see if the port matches.
+            if (origin_port != secure_port and
+                    secure_port != "*" and
+                    secure_port is not None):
+                continue
+
+            # If we've gotten here, then this origin matches the current
+            # secure origin and we should return True
+            return True
+
+        # If we've gotten to this point, then the origin isn't secure and we
+        # will not accept it as a valid location to search. We will however
+        # log a warning that we are ignoring it.
+        logger.warning(
+            "The repository located at %s is not a trusted or secure host and "
+            "is being ignored. If this repository is available via HTTPS we "
+            "recommend you use HTTPS instead, otherwise you may silence "
+            "this warning and allow it anyway with '--trusted-host %s'.",
+            origin_host,
+            origin_host,
+        )
+
+        return False
+
     def request(self, method, url, *args, **kwargs):
         # Allow setting a default timeout on a session
         kwargs.setdefault("timeout", self.timeout)