Issue 4371 incorrect dependencies when install dev packages

pipenv/core.py

@@ -32,6 +32,7 @@
     is_required_version,
     is_star,
     pep423_name,
+    prepare_default_constraint_file,
     python_version,
 )
 from pipenv.utils.indexes import get_source_list, parse_indexes, prepare_pip_source_args
@@ -41,6 +42,7 @@
     cmd_list_to_shell,
     find_python,
     is_python_command,
+    normalize_path,
     project_python,
     subprocess_run,
     system_which,
@@ -788,6 +790,7 @@ def batch_install(
                 trusted_hosts=trusted_hosts,
                 extra_indexes=extra_indexes,
                 use_pep517=use_pep517,
+                use_constraint=False,  # no need to use constraints, it's written in lockfile
             )
             c.dep = dep
 
@@ -1095,8 +1098,9 @@ def do_lock(
         for k, v in lockfile[section].copy().items():
             if not hasattr(v, "keys"):
                 del lockfile[section][k]
-    # Resolve dev-package dependencies followed by packages dependencies.
-    for is_dev in [True, False]:
+
+    # Resolve package to generate constraints before resolving dev-packages
+    for is_dev in [False, True]:
         pipfile_section = "dev-packages" if is_dev else "packages"
         if project.pipfile_exists:
             packages = project.parsed_pipfile.get(pipfile_section, {})
@@ -1452,12 +1456,14 @@ def pip_install(
     block=True,
     index=None,
     pre=False,
+    dev=False,
     selective_upgrade=False,
     requirements_dir=None,
     extra_indexes=None,
     pypi_mirror=None,
     trusted_hosts=None,
     use_pep517=True,
+    use_constraint=False,
 ):
     piplogger = logging.getLogger("pipenv.patched.pip._internal.commands.install")
     if not trusted_hosts:
@@ -1538,9 +1544,15 @@ def pip_install(
     )
     pip_command.extend(pip_args)
     if r:
-        pip_command.extend(["-r", vistir.path.normalize_path(r)])
+        pip_command.extend(["-r", normalize_path(r)])
     elif line:
         pip_command.extend(line)
+    if dev and use_constraint:
+        constraint_filename = prepare_default_constraint_file(
+            project,
+            dir=requirements_dir,
+        )
+        pip_command.extend(["-c", normalize_path(constraint_filename)])
     pip_command.extend(prepare_pip_source_args(sources))
     if project.s.is_verbose():
         click.echo(f"$ {cmd_list_to_shell(pip_command)}", err=True)
@@ -2127,9 +2139,11 @@ def do_install(
                         selective_upgrade=selective_upgrade,
                         no_deps=False,
                         pre=pre,
+                        dev=dev,
                         requirements_dir=requirements_directory,
                         index=index_url,
                         pypi_mirror=pypi_mirror,
+                        use_constraint=True,
                     )
                     if c.returncode:
                         sp.write_err(

pipenv/resolver.py

@@ -745,12 +745,15 @@ def resolve_packages(pre, clear, verbose, system, write, requirements_dir, packa
         else None
     )
 
-    def resolve(packages, pre, project, sources, clear, system, requirements_dir=None):
+    def resolve(
+        packages, pre, project, sources, clear, system, dev, requirements_dir=None
+    ):
         return resolve_deps(
             packages,
             which,
             project=project,
             pre=pre,
+            dev=dev,
             sources=sources,
             clear=clear,
             allow_global=system,
@@ -769,6 +772,7 @@ def resolve(packages, pre, project, sources, clear, system, requirements_dir=Non
     results, resolver = resolve(
         packages,
         pre=pre,
+        dev=dev,
         project=project,
         sources=sources,
         clear=clear,

pipenv/utils/dependencies.py

@@ -280,6 +280,49 @@ def convert_deps_to_pip(
     return f.name
 
 
+def get_constraints_from_deps(deps):
+    """Get contraints from Pipfile-formatted dependency"""
+    from pipenv.vendor.requirementslib.models.requirements import Requirement
+
+    def is_constraint(dep):
+        # https://pip.pypa.io/en/stable/user_guide/#constraints-files
+        # constraints must have a name, they cannot be editable, and they cannot specify extras.
+        return dep.name and not dep.editable and not dep.extras
+
+    constraints = []
+    for dep_name, dep in deps.items():
+        new_dep = Requirement.from_pipfile(dep_name, dep)
+        if is_constraint(new_dep):
+            c = new_dep.as_line().strip()
+            constraints.append(c)
+    return constraints
+
+
+def prepare_default_constraint_file(
+    project,
+    dir=None,
+):
+    from pipenv.vendor.vistir.path import (
+        create_tracked_tempdir,
+        create_tracked_tempfile,
+    )
+
+    if not dir:
+        dir = create_tracked_tempdir(suffix="-requirements", prefix="pipenv-")
+
+    default_constraints_file = create_tracked_tempfile(
+        mode="w",
+        prefix="pipenv-",
+        suffix="-default-constraints.txt",
+        dir=dir,
+        delete=False,
+    )
+    default_constraints = get_constraints_from_deps(project.packages)
+    default_constraints_file.write("\n".join([c for c in default_constraints]))
+    default_constraints_file.close()
+    return default_constraints_file.name
+
+
 def is_required_version(version, specified_version):
     """Check to see if there's a hard requirement for version
     number provided in the Pipfile.

pipenv/utils/resolver.py

@@ -17,6 +17,9 @@
 from pipenv.patched.pip._internal.operations.build.build_tracker import (
     get_build_tracker,
 )
+from pipenv.patched.pip._internal.req.constructors import (
+    install_req_from_parsed_requirement,
+)
 from pipenv.patched.pip._internal.req.req_file import parse_requirements
 from pipenv.patched.pip._internal.utils.hashes import FAVORITE_HASH
 from pipenv.patched.pip._internal.utils.temp_dir import global_tempdir_manager
@@ -35,6 +38,7 @@
     get_vcs_deps,
     is_pinned_requirement,
     pep423_name,
+    prepare_default_constraint_file,
     translate_markers,
 )
 from .indexes import parse_indexes, prepare_pip_source_args
@@ -117,6 +121,7 @@ def __init__(
         skipped=None,
         clear=False,
         pre=False,
+        dev=False,
     ):
         self.initial_constraints = constraints
         self.req_dir = req_dir
@@ -126,6 +131,7 @@ def __init__(
         self.hashes = {}
         self.clear = clear
         self.pre = pre
+        self.dev = dev
         self.results = None
         self.markers_lookup = markers_lookup if markers_lookup is not None else {}
         self.index_lookup = index_lookup if index_lookup is not None else {}
@@ -134,12 +140,15 @@ def __init__(
         self.requires_python_markers = {}
         self._pip_args = None
         self._constraints = None
+        self._default_constraints = None
         self._parsed_constraints = None
+        self._parsed_default_constraints = None
         self._resolver = None
         self._finder = None
         self._ignore_compatibility_finder = None
         self._session = None
         self._constraint_file = None
+        self._default_constraint_file = None
         self._pip_options = None
         self._pip_command = None
         self._retry_attempts = 0
@@ -420,6 +429,7 @@ def create(
         req_dir: str = None,
         clear: bool = False,
         pre: bool = False,
+        dev: bool = False,
     ) -> "Resolver":
 
         if not req_dir:
@@ -450,6 +460,7 @@ def create(
             skipped=skipped,
             clear=clear,
             pre=pre,
+            dev=dev,
         )
 
     @classmethod
@@ -552,6 +563,14 @@ def constraint_file(self):
             self._constraint_file = self.prepare_constraint_file()
         return self._constraint_file
 
+    @property
+    def default_constraint_file(self):
+        if self._default_constraint_file is None:
+            self._default_constraint_file = prepare_default_constraint_file(
+                self.project, dir=self.req_dir
+            )
+        return self._default_constraint_file
+
     @property
     def pip_options(self):
         if self._pip_options is None:
@@ -631,11 +650,34 @@ def parsed_constraints(self):
         return self._parsed_constraints
 
     @property
-    def constraints(self):
-        from pipenv.patched.pip._internal.req.constructors import (
-            install_req_from_parsed_requirement,
-        )
+    def parsed_default_constraints(self):
+        pip_options = self.pip_options
+        pip_options.extra_index_urls = []
+        if self._parsed_default_constraints is None:
+            self._parsed_default_constraints = parse_requirements(
+                self.default_constraint_file,
+                constraint=True,
+                finder=self.finder,
+                session=self.session,
+                options=pip_options,
+            )
+        return self._parsed_default_constraints
 
+    @property
+    def default_constraints(self):
+        if self._default_constraints is None:
+            self._default_constraints = [
+                install_req_from_parsed_requirement(
+                    c,
+                    isolated=self.pip_options.build_isolation,
+                    user_supplied=False,
+                )
+                for c in self.parsed_default_constraints
+            ]
+        return self._default_constraints
+
+    @property
+    def constraints(self):
         if self._constraints is None:
             self._constraints = [
                 install_req_from_parsed_requirement(
@@ -646,6 +688,9 @@ def constraints(self):
                 )
                 for c in self.parsed_constraints
             ]
+            # Only use default_constraints when installing dev-packages
+            if self.dev:
+                self._constraints += self.default_constraints
         return self._constraints
 
     @contextlib.contextmanager
@@ -870,6 +915,7 @@ def actually_resolve_deps(
     sources,
     clear,
     pre,
+    dev,
     req_dir=None,
 ):
     if not req_dir:
@@ -878,7 +924,7 @@ def actually_resolve_deps(
 
     with warnings.catch_warnings(record=True) as warning_list:
         resolver = Resolver.create(
-            deps, project, index_lookup, markers_lookup, sources, req_dir, clear, pre
+            deps, project, index_lookup, markers_lookup, sources, req_dir, clear, pre, dev
         )
         resolver.resolve()
         hashes = resolver.resolve_hashes()
@@ -1064,6 +1110,7 @@ def resolve_deps(
     python=False,
     clear=False,
     pre=False,
+    dev=False,
     allow_global=False,
     req_dir=None,
 ):
@@ -1094,6 +1141,7 @@ def resolve_deps(
                 sources,
                 clear,
                 pre,
+                dev,
                 req_dir=req_dir,
             )
         except RuntimeError:
@@ -1122,6 +1170,7 @@ def resolve_deps(
                     sources,
                     clear,
                     pre,
+                    dev,
                     req_dir=req_dir,
                 )
             except RuntimeError:

tests/integration/test_install_basic.py

@@ -509,3 +509,29 @@ def test_install_with_unnamed_source(PipenvInstance):
             f.write(contents)
         c = p.pipenv("install")
         assert c.returncode == 0
+
+@pytest.mark.dev
+@pytest.mark.install
+def test_install_dev_use_default_constraints(PipenvInstance):
+    # See https://github.com/pypa/pipenv/issues/4371
+    # See https://github.com/pypa/pipenv/issues/2987
+    with PipenvInstance(chdir=True) as p:
+
+        c = p.pipenv("install requests==2.14.0")
+        assert c.returncode == 0
+        assert "requests" in p.lockfile["default"]
+        assert p.lockfile["default"]["requests"]["version"] == "==2.14.0"
+
+        c = p.pipenv("install --dev requests")
+        assert c.returncode == 0
+        assert "requests" in p.lockfile["develop"]
+        assert p.lockfile["develop"]["requests"]["version"] == "==2.14.0"
+
+        # requests 2.14.0 doesn't require these packages
+        assert "idna" not in p.lockfile["develop"]
+        assert "certifi" not in p.lockfile["develop"]
+        assert "urllib3" not in p.lockfile["develop"]
+        assert "chardet" not in p.lockfile["develop"]
+
+        c = p.pipenv("run python -c 'import urllib3'")
+        assert c.returncode != 0

tests/integration/test_lock.py

@@ -791,3 +791,38 @@ def test_pipenv_respects_package_index_restrictions(PipenvInstance):
                                       'sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a'],
                            'index': 'local', 'version': '==2.19.1'}
         assert p.lockfile['default']['requests'] == expected_result
+
+
+@pytest.mark.dev
+@pytest.mark.lock
+@pytest.mark.install
+def test_dev_lock_use_default_packages_as_constraint(PipenvInstance):
+    # See https://github.com/pypa/pipenv/issues/4371
+    # See https://github.com/pypa/pipenv/issues/2987
+    with PipenvInstance(chdir=True) as p:
+        with open(p.pipfile_path, 'w') as f:
+            contents = """
+[packages]
+requests = "<=2.14.0"
+
+[dev-packages]
+requests = "*"
+                """.strip()
+            f.write(contents)
+
+        c = p.pipenv("lock --dev")
+        assert c.returncode == 0
+        assert "requests" in p.lockfile["default"]
+        assert p.lockfile["default"]["requests"]["version"] == "==2.14.0"
+        assert "requests" in p.lockfile["develop"]
+        assert p.lockfile["develop"]["requests"]["version"] == "==2.14.0"
+
+        # requests 2.14.0 doesn't require these packages
+        assert "idna" not in p.lockfile["develop"]
+        assert "certifi" not in p.lockfile["develop"]
+        assert "urllib3" not in p.lockfile["develop"]
+        assert "chardet" not in p.lockfile["develop"]
+
+        c = p.pipenv("install --dev")
+        c = p.pipenv("run python -c 'import urllib3'")
+        assert c.returncode != 0