Allow PEP508 url dependencies in install_requires

news/4187.feature

@@ -0,0 +1,5 @@
+Allow PEP 508 URL requirements to be used as dependencies.
+
+As a security measure, pip will raise an exception when installing packages from
+PyPI if those packages depend on packages not also hosted on PyPI.
+In the future, PyPI will block uploading packages with such external URL dependencies directly.

src/pip/_internal/models/index.py

@@ -2,14 +2,20 @@
 
 
 class Index(object):
-    def __init__(self, url):
+    def __init__(self, url, file_storage_domain):
         self.url = url
         self.netloc = urllib_parse.urlsplit(url).netloc
         self.simple_url = self.url_to_path('simple')
         self.pypi_url = self.url_to_path('pypi')
 
+        # This is part of a temporary hack used to block installs of PyPI
+        # packages which depend on external urls only necessary until PyPI can
+        # block such packages themselves
+        self.file_storage_domain = file_storage_domain
+
     def url_to_path(self, path):
         return urllib_parse.urljoin(self.url, path)
 
 
-PyPI = Index('https://pypi.org/')
+PyPI = Index('https://pypi.org/', 'files.pythonhosted.org')
+TestPyPI = Index('https://test.pypi.org/', 'test-files.pythonhosted.org')

src/pip/_internal/req/req_install.py

@@ -32,6 +32,7 @@
 from pip._internal.locations import (
     PIP_DELETE_MARKER_FILENAME, running_under_virtualenv,
 )
+from pip._internal.models.index import PyPI, TestPyPI
 from pip._internal.req.req_uninstall import UninstallPathSet
 from pip._internal.utils.deprecation import (
     RemovedInPip11Warning, RemovedInPip12Warning,
@@ -169,11 +170,19 @@ def from_req(cls, req, comes_from=None, isolated=False, wheel_cache=None):
             req = Requirement(req)
         except InvalidRequirement:
             raise InstallationError("Invalid requirement: '%s'" % req)
-        if req.url:
+
+        domains_not_allowed = [
+            PyPI.file_storage_domain,
+            TestPyPI.file_storage_domain,
+        ]
+        if req.url and comes_from.link.netloc in domains_not_allowed:
+            # Explicitly disallow pypi packages that depend on external urls
             raise InstallationError(
-                "Direct url requirement (like %s) are not allowed for "
-                "dependencies" % req
+                "Packages installed from PyPI cannot depend on packages "
+                "which are not also hosted on PyPI.\n"
+                "%s depends on %s " % (comes_from.name, req)
             )
+
         return cls(req, comes_from, isolated=isolated, wheel_cache=wheel_cache)
 
     @classmethod

tests/functional/test_install.py

@@ -8,6 +8,7 @@
 import pytest
 
 from pip._internal import pep425tags
+from pip._internal.models.index import PyPI, TestPyPI
 from pip._internal.status_codes import ERROR
 from pip._internal.utils.misc import rmtree
 from tests.lib import (
@@ -1267,9 +1268,31 @@ def test_install_pep508_with_url_in_install_requires(script):
             'ce1a869fe039fbf7e217df36c4653d1dbe657778b2d41709593a0003584405f4'
         ],
     )
-    res = script.pip('install', pkga_path, expect_error=True)
-    assert "Direct url requirement " in res.stderr, str(res)
-    assert "are not allowed for dependencies" in res.stderr, str(res)
+    res = script.pip('install', pkga_path)
+    assert "Successfully installed packaging-15.3" in str(res), str(res)
+
+
+@pytest.mark.network
+@pytest.mark.parametrize('index', (PyPI.simple_url, TestPyPI.simple_url))
+def test_install_from_test_pypi_with_ext_url_dep_is_blocked(script, index):
+    res = script.pip(
+        'install',
+        '--index-url',
+        index,
+        'pep-508-url-deps',
+        expect_error=True,
+    )
+    error_message = (
+        "Packages installed from PyPI cannot depend on packages "
+        "which are not also hosted on PyPI."
+    )
+    error_cause = (
+        "pep-508-url-deps depends on sampleproject@ "
+        "https://github.com/pypa/sampleproject/archive/master.zip"
+    )
+    assert res.returncode == 1
+    assert error_message in res.stderr, str(res)
+    assert error_cause in res.stderr, str(res)
 
 
 def test_installing_scripts_outside_path_prints_warning(script):