From 9de1c8c551089074507d09046335fdb4c2434033 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Fri, 1 Dec 2017 12:22:18 +0100 Subject: [PATCH 1/3] implement an opt-in pipignore mechanism ``` (ve) thinkie merzky ~/projects/pip [master *] $ rm -rf data/ (ve) thinkie merzky ~/projects/pip [master *] $ unset PIP_IGNORE (ve) thinkie merzky ~/projects/pip [master *] $ rm -f .pipignore (ve) thinkie merzky ~/projects/pip [master *] $ time pip install --no-deps --upgrade . Looking in indexes: https://pypi.python.org/simple, https://packagecloud.io/github/git-lfs/pypi/simple Processing /home/merzky/projects/pip Installing collected packages: pip Found existing installation: pip 10.0.0.dev0 Uninstalling pip-10.0.0.dev0: Successfully uninstalled pip-10.0.0.dev0 Running setup.py install for pip ... done Successfully installed pip-10.0.0.dev0 r:0m3.421s u:0m2.828s s:0m0.604s (ve) thinkie merzky ~/projects/pip [master *] $ mkdir data (ve) thinkie merzky ~/projects/pip [master *] $ dd if=/dev/urandom of=data/test.dat count=$((2048*1024)) 2097152+0 records in 2097152+0 records out 1073741824 bytes (1.1 GB, 1.0 GiB) copied, 15.17 s, 70.8 MB/s (ve) thinkie merzky ~/projects/pip [master *] $ time pip install --no-deps --upgrade . Looking in indexes: https://pypi.python.org/simple, https://packagecloud.io/github/git-lfs/pypi/simple Processing /home/merzky/projects/pip Installing collected packages: pip Found existing installation: pip 10.0.0.dev0 Uninstalling pip-10.0.0.dev0: Successfully uninstalled pip-10.0.0.dev0 Running setup.py install for pip ... done Successfully installed pip-10.0.0.dev0 r:0m41.687s u:0m3.420s s:0m3.320s (ve) thinkie merzky ~/projects/pip [master *] $ export PIP_IGNORE=.pipignore (ve) thinkie merzky ~/projects/pip [master *] $ cat > .pipignore # and this data .pipignore # because, why not (ve) thinkie merzky ~/projects/pip [master *] $ time pip install --no-deps --upgrade . Looking in indexes: https://pypi.python.org/simple, https://packagecloud.io/github/git-lfs/pypi/simple Processing /home/merzky/projects/pip Installing collected packages: pip Found existing installation: pip 10.0.0.dev0 Uninstalling pip-10.0.0.dev0: Successfully uninstalled pip-10.0.0.dev0 Running setup.py install for pip ... done Successfully installed pip-10.0.0.dev0 r:0m3.495s u:0m2.780s s:0m0.664s ``` --- src/pip/_internal/download.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/download.py b/src/pip/_internal/download.py index d56a1bb3bb2..64b8b3f288b 100644 --- a/src/pip/_internal/download.py +++ b/src/pip/_internal/download.py @@ -690,11 +690,20 @@ def unpack_file_url(link, location, download_dir=None, hashes=None): """ link_path = url_to_path(link.url_without_fragment) + # if a .pipignore' file location is set and exists, '#' style comments are stripped + ignored = list() + pipignore = os.environ.get('PIP_IGNORE') + if pipignore and os.path.isfile(pipignore): + with open(pipignore, 'r') as fin: + for line in fin.readlines(): + ignored.extend(line.split('#')[0].strip().split()) + # If it's a url to a local directory if is_dir_url(link): if os.path.isdir(location): rmtree(location) - shutil.copytree(link_path, location, symlinks=True) + shutil.copytree(link_path, location, symlinks=True, + ignore=shutil.ignore_patterns(*ignored)) if download_dir: logger.info('Link is a directory, ignoring download_dir') return From 33e4d977fcdd4d0fc6c4885232a75dd234a2d8a0 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Fri, 1 Dec 2017 12:24:27 +0100 Subject: [PATCH 2/3] expand documentation --- src/pip/_internal/download.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/download.py b/src/pip/_internal/download.py index 64b8b3f288b..4671857135b 100644 --- a/src/pip/_internal/download.py +++ b/src/pip/_internal/download.py @@ -690,7 +690,10 @@ def unpack_file_url(link, location, download_dir=None, hashes=None): """ link_path = url_to_path(link.url_without_fragment) - # if a .pipignore' file location is set and exists, '#' style comments are stripped + # if a .pipignore' file location is set and exists, '#' style + # comments are stripped. All other strings (space limited) are interpreted + # as glob-style pattersn to be passed to `shutil.copytree`, and are thus + # excluded from the file cloning operation. ignored = list() pipignore = os.environ.get('PIP_IGNORE') if pipignore and os.path.isfile(pipignore): From 15980b37079313e721a9bb9f170b5a40e9ab6981 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Fri, 1 Dec 2017 12:30:28 +0100 Subject: [PATCH 3/3] add news fragment --- news/2195.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/2195.feature diff --git a/news/2195.feature b/news/2195.feature new file mode 100644 index 00000000000..947d6cd2f36 --- /dev/null +++ b/news/2195.feature @@ -0,0 +1 @@ +Add supporte for a `PIP_IGNORE` environment setting, which, when pointing to a valid file, causes `pip install` to ignore all file patterns listed in there. This can speed up installations from large source trees.