From 3626530b4ac5f255ba8e0fb151a096c62f0335f3 Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Mon, 22 Jul 2019 19:17:12 -0700 Subject: [PATCH 01/14] exclude '.tox', '.nox', '.git', '.hg', '.bzr', '.svn' from pip install . to speed installation up --- src/pip/_internal/download.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/download.py b/src/pip/_internal/download.py index 8715eb5b19d..6c2aafdf19d 100644 --- a/src/pip/_internal/download.py +++ b/src/pip/_internal/download.py @@ -939,7 +939,19 @@ def unpack_file_url( if is_dir_url(link): if os.path.isdir(location): rmtree(location) - shutil.copytree(link_path, location, symlinks=True) + shutil.copytree(link_path, + location, + symlinks=True, + ignore=shutil.ignore_patterns( + # Pulling in those directories can potentially be very slow. + # Excludin them speeds things up substantially in some cases. + # see dicsussion at: + # https://github.com/pypa/pip/issues/2195 + # https://github.com/pypa/pip/pull/2196 + '.tox', '.nox', '.git', '.hg', '.bzr', '.svn' + ) + ) + if download_dir: logger.info('Link is a directory, ignoring download_dir') return From 82e89a94c6ec4c6303331710e3be7ad0131b4a3b Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Mon, 22 Jul 2019 19:24:34 -0700 Subject: [PATCH 02/14] news --- news/2195.bugfix | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 news/2195.bugfix diff --git a/news/2195.bugfix b/news/2195.bugfix new file mode 100644 index 00000000000..9016516db76 --- /dev/null +++ b/news/2195.bugfix @@ -0,0 +1,4 @@ +pip install . is very slow in the presence of large directories in the source tree. +Typical cases are source control directories (.git, .svn etc) and test automation directries (.tox, .nox). +This diff excludes the common culprits from the copy to a temporary directory, speeding up pip install . +significantly in such cases. From f285407d7acb6521e2fde077e7eb1a41723908dc Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Mon, 22 Jul 2019 19:53:43 -0700 Subject: [PATCH 03/14] lint --- src/pip/_internal/download.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/pip/_internal/download.py b/src/pip/_internal/download.py index 6c2aafdf19d..a0a3a4a2030 100644 --- a/src/pip/_internal/download.py +++ b/src/pip/_internal/download.py @@ -942,14 +942,12 @@ def unpack_file_url( shutil.copytree(link_path, location, symlinks=True, - ignore=shutil.ignore_patterns( - # Pulling in those directories can potentially be very slow. - # Excludin them speeds things up substantially in some cases. - # see dicsussion at: - # https://github.com/pypa/pip/issues/2195 - # https://github.com/pypa/pip/pull/2196 - '.tox', '.nox', '.git', '.hg', '.bzr', '.svn' - ) + # Pulling in those directories can potentially be very slow. + # Excludin them speeds things up substantially in some cases. + # see dicsussion at: + # https://github.com/pypa/pip/issues/2195 + # https://github.com/pypa/pip/pull/2196 + ignore=shutil.ignore_patterns('.tox', '.nox', '.git', '.hg', '.bzr', '.svn') ) if download_dir: From e8330528bfb0e2dac63e02cd6d1f45fd8f11ce92 Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Mon, 22 Jul 2019 20:22:07 -0700 Subject: [PATCH 04/14] lint take 2 (not sure how to test locally) --- src/pip/_internal/download.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pip/_internal/download.py b/src/pip/_internal/download.py index a0a3a4a2030..c73c639ac2a 100644 --- a/src/pip/_internal/download.py +++ b/src/pip/_internal/download.py @@ -947,8 +947,7 @@ def unpack_file_url( # see dicsussion at: # https://github.com/pypa/pip/issues/2195 # https://github.com/pypa/pip/pull/2196 - ignore=shutil.ignore_patterns('.tox', '.nox', '.git', '.hg', '.bzr', '.svn') - ) + ignore=shutil.ignore_patterns('.tox', '.nox', '.git', '.hg', '.bzr', '.svn')) if download_dir: logger.info('Link is a directory, ignoring download_dir') From 67ce4427dd3500d2f886c1c04cd729cb4b55b242 Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Tue, 23 Jul 2019 11:29:09 -0700 Subject: [PATCH 05/14] fixed lint, excludes only .tox and .nox --- src/pip/_internal/download.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/download.py b/src/pip/_internal/download.py index c73c639ac2a..fbaaffaf6f9 100644 --- a/src/pip/_internal/download.py +++ b/src/pip/_internal/download.py @@ -942,12 +942,12 @@ def unpack_file_url( shutil.copytree(link_path, location, symlinks=True, - # Pulling in those directories can potentially be very slow. - # Excludin them speeds things up substantially in some cases. + # Pulling in those directories can potentially + # be very slow. # see dicsussion at: # https://github.com/pypa/pip/issues/2195 # https://github.com/pypa/pip/pull/2196 - ignore=shutil.ignore_patterns('.tox', '.nox', '.git', '.hg', '.bzr', '.svn')) + ignore=shutil.ignore_patterns('.tox', '.nox')) if download_dir: logger.info('Link is a directory, ignoring download_dir') From 29a8b6ed38dd945d8c88de5571bd3aff3acd63a4 Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Tue, 23 Jul 2019 11:35:41 -0700 Subject: [PATCH 06/14] updated news --- news/2195.bugfix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/news/2195.bugfix b/news/2195.bugfix index 9016516db76..3df54f10fcf 100644 --- a/news/2195.bugfix +++ b/news/2195.bugfix @@ -1,4 +1,4 @@ -pip install . is very slow in the presence of large directories in the source tree. -Typical cases are source control directories (.git, .svn etc) and test automation directries (.tox, .nox). -This diff excludes the common culprits from the copy to a temporary directory, speeding up pip install . -significantly in such cases. +'pip install .' is very slow in the presence of large directories in the source tree. +Specifialy test automation directries (.tox, .nox). +#6770 excludes .tox and .nox from being copied when pip install . is being exectued, +significantly speeding up the build in the presence of large .tox and .nox directories. From 459b4dc238e52351291bf40b78ce35289822b63c Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Tue, 23 Jul 2019 18:23:59 -0700 Subject: [PATCH 07/14] renamed news file to match PR --- news/{2195.bugfix => 6770.bugfix} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename news/{2195.bugfix => 6770.bugfix} (100%) diff --git a/news/2195.bugfix b/news/6770.bugfix similarity index 100% rename from news/2195.bugfix rename to news/6770.bugfix From 6b1aa6f6c00236a1f7952da1c1f944779f2a0757 Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Tue, 23 Jul 2019 18:26:06 -0700 Subject: [PATCH 08/14] simplifies new language --- news/6770.bugfix | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/news/6770.bugfix b/news/6770.bugfix index 3df54f10fcf..174b8f62c8a 100644 --- a/news/6770.bugfix +++ b/news/6770.bugfix @@ -1,4 +1,3 @@ 'pip install .' is very slow in the presence of large directories in the source tree. -Specifialy test automation directries (.tox, .nox). -#6770 excludes .tox and .nox from being copied when pip install . is being exectued, -significantly speeding up the build in the presence of large .tox and .nox directories. +#6770 excludes .tox and .nox from being copied by 'pip install .', significantly speeding +up the builds when those directories are large. From d4171f28d18cd7e6af7b5b0712fa3c6b632435b1 Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Tue, 30 Jul 2019 17:52:12 -0700 Subject: [PATCH 09/14] minor updates --- news/6770.bugfix | 4 +--- src/pip/_internal/download.py | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/news/6770.bugfix b/news/6770.bugfix index 174b8f62c8a..c0ab57ee109 100644 --- a/news/6770.bugfix +++ b/news/6770.bugfix @@ -1,3 +1 @@ -'pip install .' is very slow in the presence of large directories in the source tree. -#6770 excludes .tox and .nox from being copied by 'pip install .', significantly speeding -up the builds when those directories are large. +Skip copying .tox and .nox directories to temporary build directories diff --git a/src/pip/_internal/download.py b/src/pip/_internal/download.py index fbaaffaf6f9..f58ba475c06 100644 --- a/src/pip/_internal/download.py +++ b/src/pip/_internal/download.py @@ -934,7 +934,6 @@ def unpack_file_url( of the link file inside download_dir. """ link_path = url_to_path(link.url_without_fragment) - # If it's a url to a local directory if is_dir_url(link): if os.path.isdir(location): @@ -944,7 +943,8 @@ def unpack_file_url( symlinks=True, # Pulling in those directories can potentially # be very slow. - # see dicsussion at: + # see discussion at: + # https://github.com/pypa/pip/pull/6770 # https://github.com/pypa/pip/issues/2195 # https://github.com/pypa/pip/pull/2196 ignore=shutil.ignore_patterns('.tox', '.nox')) From 77c1504137f92b1b91e26ae13a7d21c776536b61 Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Fri, 2 Aug 2019 18:17:53 -0700 Subject: [PATCH 10/14] unit test --- src/pip/_internal/download.py | 16 +++++++------ tests/unit/test_download.py | 44 +++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/src/pip/_internal/download.py b/src/pip/_internal/download.py index f58ba475c06..07e08af6eee 100644 --- a/src/pip/_internal/download.py +++ b/src/pip/_internal/download.py @@ -936,18 +936,20 @@ def unpack_file_url( link_path = url_to_path(link.url_without_fragment) # If it's a url to a local directory if is_dir_url(link): + + def ignore(d, names): + # Pulling in those directories can potentially + # be very slow. + # see discussion at: + # https://github.com/pypa/pip/pull/6770 + return ['.tox', '.nox'] if d == link_path else [] + if os.path.isdir(location): rmtree(location) shutil.copytree(link_path, location, symlinks=True, - # Pulling in those directories can potentially - # be very slow. - # see discussion at: - # https://github.com/pypa/pip/pull/6770 - # https://github.com/pypa/pip/issues/2195 - # https://github.com/pypa/pip/pull/2196 - ignore=shutil.ignore_patterns('.tox', '.nox')) + ignore=ignore) if download_dir: logger.info('Link is a directory, ignoring download_dir') diff --git a/tests/unit/test_download.py b/tests/unit/test_download.py index 7b421a7d7d5..86493cba27d 100644 --- a/tests/unit/test_download.py +++ b/tests/unit/test_download.py @@ -413,6 +413,50 @@ def test_unpack_file_url_thats_a_dir(self, tmpdir, data): assert os.path.isdir(os.path.join(self.build_dir, 'fspkg')) +@pytest.mark.parametrize('exclude_dir', [ + '.nox', + '.tox' +]) +def test_unpack_file_url_with_excluded_dirs(exclude_dir): + + def touch(path): + with open(path, 'a'): + os.utime(path, None) + + src_dir = mkdtemp() + src_included_file = os.path.join(src_dir, 'file.txt') + src_excluded_dir = os.path.join(src_dir, exclude_dir) + src_excluded_file = os.path.join(src_dir, exclude_dir, 'file.txt') + src_included_dir = os.path.join(src_dir, 'subdir', exclude_dir) + + # set up source directory + os.makedirs(src_excluded_dir, exist_ok=True) + os.makedirs(src_included_dir, exist_ok=True) + touch(src_included_file) + touch(src_excluded_file) + + dst_dir = mkdtemp() + dst_included_file = os.path.join(dst_dir, 'file.txt') + dst_excluded_dir = os.path.join(dst_dir, exclude_dir) + dst_excluded_file = os.path.join(dst_dir, exclude_dir, 'file.txt') + dst_included_dir = os.path.join(dst_dir, 'subdir', exclude_dir) + + try: + src_link = Link(path_to_url(src_dir)) + unpack_file_url( + src_link, + dst_dir, + download_dir=None + ) + assert not os.path.isdir(dst_excluded_dir) + assert not os.path.isfile(dst_excluded_file) + assert os.path.isfile(dst_included_file) + assert os.path.isdir(dst_included_dir) + finally: + rmtree(src_dir) + rmtree(dst_dir) + + class TestSafeFileCache: """ The no_perms test are useless on Windows since SafeFileCache uses From e67f066169b4b4bde68e8ea4a0dd57daf8acaee9 Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Fri, 2 Aug 2019 18:38:13 -0700 Subject: [PATCH 11/14] fixed on python 2.7 --- tests/unit/test_download.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_download.py b/tests/unit/test_download.py index 86493cba27d..0b00644a80a 100644 --- a/tests/unit/test_download.py +++ b/tests/unit/test_download.py @@ -18,7 +18,7 @@ from pip._internal.exceptions import HashMismatch from pip._internal.models.link import Link from pip._internal.utils.hashes import Hashes -from pip._internal.utils.misc import path_to_url +from pip._internal.utils.misc import ensure_dir, path_to_url from tests.lib import create_file @@ -430,8 +430,8 @@ def touch(path): src_included_dir = os.path.join(src_dir, 'subdir', exclude_dir) # set up source directory - os.makedirs(src_excluded_dir, exist_ok=True) - os.makedirs(src_included_dir, exist_ok=True) + ensure_dir(src_excluded_dir) + ensure_dir(src_included_dir) touch(src_included_file) touch(src_excluded_file) From 8e87980d17631ffc55e386ec8042ca10f586d7b1 Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Fri, 2 Aug 2019 23:14:41 -0700 Subject: [PATCH 12/14] responding to comments --- src/pip/_internal/download.py | 8 ++--- tests/unit/test_download.py | 67 +++++++++++++++-------------------- 2 files changed, 33 insertions(+), 42 deletions(-) diff --git a/src/pip/_internal/download.py b/src/pip/_internal/download.py index 07e08af6eee..95963a1d2d5 100644 --- a/src/pip/_internal/download.py +++ b/src/pip/_internal/download.py @@ -938,10 +938,10 @@ def unpack_file_url( if is_dir_url(link): def ignore(d, names): - # Pulling in those directories can potentially - # be very slow. - # see discussion at: - # https://github.com/pypa/pip/pull/6770 + # Pulling in those directories can potentially be very slow, + # exclude the following directories if they appear in the top + # level dir (and only it). + # See discussion at https://github.com/pypa/pip/pull/6770 return ['.tox', '.nox'] if d == link_path else [] if os.path.isdir(location): diff --git a/tests/unit/test_download.py b/tests/unit/test_download.py index 0b00644a80a..068c45dd5a1 100644 --- a/tests/unit/test_download.py +++ b/tests/unit/test_download.py @@ -18,8 +18,8 @@ from pip._internal.exceptions import HashMismatch from pip._internal.models.link import Link from pip._internal.utils.hashes import Hashes -from pip._internal.utils.misc import ensure_dir, path_to_url -from tests.lib import create_file +from pip._internal.utils.misc import path_to_url +from tests.lib import Path, create_file @pytest.fixture(scope="function") @@ -417,44 +417,35 @@ def test_unpack_file_url_thats_a_dir(self, tmpdir, data): '.nox', '.tox' ]) -def test_unpack_file_url_with_excluded_dirs(exclude_dir): - - def touch(path): - with open(path, 'a'): - os.utime(path, None) - - src_dir = mkdtemp() - src_included_file = os.path.join(src_dir, 'file.txt') - src_excluded_dir = os.path.join(src_dir, exclude_dir) - src_excluded_file = os.path.join(src_dir, exclude_dir, 'file.txt') - src_included_dir = os.path.join(src_dir, 'subdir', exclude_dir) +def test_unpack_file_url_excludes_expected_dirs(tmpdir, exclude_dir): + src_dir = tmpdir / 'src' + dst_dir = tmpdir / 'dst' + src_included_file = Path.joinpath(src_dir, 'file.txt') + src_excluded_dir = Path.joinpath(src_dir, exclude_dir) + src_excluded_file = Path.joinpath(src_dir, exclude_dir, 'file.txt') + src_included_dir = Path.joinpath(src_dir, 'subdir', exclude_dir) # set up source directory - ensure_dir(src_excluded_dir) - ensure_dir(src_included_dir) - touch(src_included_file) - touch(src_excluded_file) - - dst_dir = mkdtemp() - dst_included_file = os.path.join(dst_dir, 'file.txt') - dst_excluded_dir = os.path.join(dst_dir, exclude_dir) - dst_excluded_file = os.path.join(dst_dir, exclude_dir, 'file.txt') - dst_included_dir = os.path.join(dst_dir, 'subdir', exclude_dir) - - try: - src_link = Link(path_to_url(src_dir)) - unpack_file_url( - src_link, - dst_dir, - download_dir=None - ) - assert not os.path.isdir(dst_excluded_dir) - assert not os.path.isfile(dst_excluded_file) - assert os.path.isfile(dst_included_file) - assert os.path.isdir(dst_included_dir) - finally: - rmtree(src_dir) - rmtree(dst_dir) + src_excluded_dir.mkdir(parents=True) + src_included_dir.mkdir(parents=True) + Path.touch(src_included_file) + Path.touch(src_excluded_file) + + dst_included_file = Path.joinpath(dst_dir, 'file.txt') + dst_excluded_dir = Path.joinpath(dst_dir, exclude_dir) + dst_excluded_file = Path.joinpath(dst_dir, exclude_dir, 'file.txt') + dst_included_dir = Path.joinpath(dst_dir, 'subdir', exclude_dir) + + src_link = Link(path_to_url(src_dir)) + unpack_file_url( + src_link, + dst_dir, + download_dir=None + ) + assert not os.path.isdir(dst_excluded_dir) + assert not os.path.isfile(dst_excluded_file) + assert os.path.isfile(dst_included_file) + assert os.path.isdir(dst_included_dir) class TestSafeFileCache: From ec47c866c85a6677a1500d376df5897d8524b075 Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Sat, 3 Aug 2019 10:31:42 -0700 Subject: [PATCH 13/14] updated docs --- docs/html/reference/pip_install.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/html/reference/pip_install.rst b/docs/html/reference/pip_install.rst index 96fd1c018ba..e919693b3fe 100644 --- a/docs/html/reference/pip_install.rst +++ b/docs/html/reference/pip_install.rst @@ -694,10 +694,21 @@ does not satisfy the ``--require-hashes`` demand that every package have a local hash. +Local project installs +++++++++++++++++++++++ +pip supports installing local project in both regular mode and editable mode. +You can install local projects by specifying the project path to pip:: + +$ pip install path/to/SomeProject + +During the installation, pip will copy the entire project directory to a temporary location and install from there. +The exception is that pip will exclude .tox and .nox directories present in the top level of the project from being copied. + + .. _`editable-installs`: "Editable" Installs -+++++++++++++++++++ +~~~~~~~~~~~~~~~~~~~ "Editable" installs are fundamentally `"setuptools develop mode" `_ From 6dd5727773ab1d99d57fa7f77b611801e68ce6d3 Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Sat, 3 Aug 2019 11:14:36 -0700 Subject: [PATCH 14/14] responded to comments --- docs/html/reference/pip_install.rst | 2 +- tests/unit/test_download.py | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/html/reference/pip_install.rst b/docs/html/reference/pip_install.rst index e919693b3fe..f56dc9a0e9f 100644 --- a/docs/html/reference/pip_install.rst +++ b/docs/html/reference/pip_install.rst @@ -701,7 +701,7 @@ You can install local projects by specifying the project path to pip:: $ pip install path/to/SomeProject -During the installation, pip will copy the entire project directory to a temporary location and install from there. +During regular installation, pip will copy the entire project directory to a temporary location and install from there. The exception is that pip will exclude .tox and .nox directories present in the top level of the project from being copied. diff --git a/tests/unit/test_download.py b/tests/unit/test_download.py index 068c45dd5a1..afc32e29dca 100644 --- a/tests/unit/test_download.py +++ b/tests/unit/test_download.py @@ -19,7 +19,7 @@ from pip._internal.models.link import Link from pip._internal.utils.hashes import Hashes from pip._internal.utils.misc import path_to_url -from tests.lib import Path, create_file +from tests.lib import create_file @pytest.fixture(scope="function") @@ -420,21 +420,21 @@ def test_unpack_file_url_thats_a_dir(self, tmpdir, data): def test_unpack_file_url_excludes_expected_dirs(tmpdir, exclude_dir): src_dir = tmpdir / 'src' dst_dir = tmpdir / 'dst' - src_included_file = Path.joinpath(src_dir, 'file.txt') - src_excluded_dir = Path.joinpath(src_dir, exclude_dir) - src_excluded_file = Path.joinpath(src_dir, exclude_dir, 'file.txt') - src_included_dir = Path.joinpath(src_dir, 'subdir', exclude_dir) + src_included_file = src_dir.joinpath('file.txt') + src_excluded_dir = src_dir.joinpath(exclude_dir) + src_excluded_file = src_dir.joinpath(exclude_dir, 'file.txt') + src_included_dir = src_dir.joinpath('subdir', exclude_dir) # set up source directory src_excluded_dir.mkdir(parents=True) src_included_dir.mkdir(parents=True) - Path.touch(src_included_file) - Path.touch(src_excluded_file) + src_included_file.touch() + src_excluded_file.touch() - dst_included_file = Path.joinpath(dst_dir, 'file.txt') - dst_excluded_dir = Path.joinpath(dst_dir, exclude_dir) - dst_excluded_file = Path.joinpath(dst_dir, exclude_dir, 'file.txt') - dst_included_dir = Path.joinpath(dst_dir, 'subdir', exclude_dir) + dst_included_file = dst_dir.joinpath('file.txt') + dst_excluded_dir = dst_dir.joinpath(exclude_dir) + dst_excluded_file = dst_dir.joinpath(exclude_dir, 'file.txt') + dst_included_dir = dst_dir.joinpath('subdir', exclude_dir) src_link = Link(path_to_url(src_dir)) unpack_file_url(