diff --git a/piptools/repositories/base.py b/piptools/repositories/base.py index bb617d809..54849cb7f 100644 --- a/piptools/repositories/base.py +++ b/piptools/repositories/base.py @@ -12,6 +12,8 @@ class BaseRepository(object): def clear_caches(self): """Should clear any caches used by the implementation.""" + @abstractmethod + @contextmanager def freshen_build_caches(self): """Should start with fresh build/source caches.""" diff --git a/piptools/repositories/local.py b/piptools/repositories/local.py index fccd916de..f185f35c3 100644 --- a/piptools/repositories/local.py +++ b/piptools/repositories/local.py @@ -56,8 +56,10 @@ def DEFAULT_INDEX_URL(self): def clear_caches(self): self.repository.clear_caches() + @contextmanager def freshen_build_caches(self): - self.repository.freshen_build_caches() + with self.repository.freshen_build_caches(): + yield def find_best_match(self, ireq, prereleases=None): key = key_from_ireq(ireq) diff --git a/piptools/repositories/pypi.py b/piptools/repositories/pypi.py index e1c4e913c..6a9d07ae8 100644 --- a/piptools/repositories/pypi.py +++ b/piptools/repositories/pypi.py @@ -87,7 +87,8 @@ def __init__(self, pip_args, cache_dir): self._dependencies_cache = {} # Setup file paths - self.freshen_build_caches() + self._build_dir = None + self._source_dir = None self._cache_dir = normalize_path(cache_dir) self._download_dir = fs_str(os.path.join(self._cache_dir, "pkgs")) if PIP_VERSION[:2] <= (20, 2): @@ -95,6 +96,7 @@ def __init__(self, pip_args, cache_dir): self._setup_logging() + @contextmanager def freshen_build_caches(self): """ Start with fresh build/source caches. Will remove any old build @@ -102,14 +104,21 @@ def freshen_build_caches(self): """ self._build_dir = TemporaryDirectory(fs_str("build")) self._source_dir = TemporaryDirectory(fs_str("source")) + try: + yield + finally: + self._build_dir.cleanup() + self._build_dir = None + self._source_dir.cleanup() + self._source_dir = None @property def build_dir(self): - return self._build_dir.name + return self._build_dir.name if self._build_dir else None @property def source_dir(self): - return self._source_dir.name + return self._source_dir.name if self._source_dir else None def clear_caches(self): rmtree(self._download_dir, ignore_errors=True) diff --git a/piptools/resolver.py b/piptools/resolver.py index 954f751ab..40b465b8e 100644 --- a/piptools/resolver.py +++ b/piptools/resolver.py @@ -170,7 +170,13 @@ def resolve(self, max_rounds=10): log.debug("") log.debug(magenta("{:^60}".format("ROUND {}".format(current_round)))) - has_changed, best_matches = self._resolve_one_round() + # If a package version (foo==2.0) was built in a previous round, + # and in this round a different version of foo needs to be built + # (i.e. foo==1.0), the directory will exist already, which will + # cause a pip build failure. The trick is to start with a new + # build cache dir for every round, so this can never happen. + with self.repository.freshen_build_caches(): + has_changed, best_matches = self._resolve_one_round() log.debug("-" * 60) log.debug( "Result of round {}: {}".format( @@ -180,13 +186,6 @@ def resolve(self, max_rounds=10): if not has_changed: break - # If a package version (foo==2.0) was built in a previous round, - # and in this round a different version of foo needs to be built - # (i.e. foo==1.0), the directory will exist already, which will - # cause a pip build failure. The trick is to start with a new - # build cache dir for every round, so this can never happen. - self.repository.freshen_build_caches() - del os.environ["PIP_EXISTS_ACTION"] # Only include hard requirements and not pip constraints diff --git a/setup.cfg b/setup.cfg index 932317dda..a755d81d7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -56,6 +56,7 @@ universal = 1 norecursedirs = .* build dist venv test_data piptools/_compat/* testpaths = tests piptools filterwarnings = + always ignore::PendingDeprecationWarning:pip\._vendor.+ ignore::DeprecationWarning:pip\._vendor.+ markers = diff --git a/tests/conftest.py b/tests/conftest.py index 57abf8a8e..2ee8fc525 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -41,6 +41,10 @@ def __init__(self): with open("tests/test_data/fake-editables.json", "r") as f: self.editables = json.load(f) + @contextmanager + def freshen_build_caches(self): + yield + def get_hashes(self, ireq): # Some fake hashes return { diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index 121b6b0bf..9a9894a8f 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -196,7 +196,7 @@ def test_redacted_urls_in_verbose_output(runner, option): cli, [ "--no-header", - "--no-index", + "--no-emit-index-url", "--no-emit-find-links", "--verbose", option, diff --git a/tests/test_repository_pypi.py b/tests/test_repository_pypi.py index 5e9ec117b..686ea60c5 100644 --- a/tests/test_repository_pypi.py +++ b/tests/test_repository_pypi.py @@ -109,23 +109,30 @@ def test_open_local_or_remote_file__remote_file( response_file_path.write_bytes(content) mock_response = mock.Mock() - mock_response.raw = response_file_path.open("rb") - mock_response.headers = {"content-length": content_length} + with response_file_path.open("rb") as fp: + mock_response.raw = fp + mock_response.headers = {"content-length": content_length} - with mock.patch.object(session, "get", return_value=mock_response): - with open_local_or_remote_file(link, session) as file_stream: - assert file_stream.stream.read() == content - assert file_stream.size == expected_content_length + with mock.patch.object(session, "get", return_value=mock_response): + with open_local_or_remote_file(link, session) as file_stream: + assert file_stream.stream.read() == content + assert file_stream.size == expected_content_length - mock_response.close.assert_called_once() + mock_response.close.assert_called_once() def test_pypirepo_build_dir_is_str(pypi_repository): - assert isinstance(pypi_repository.build_dir, str) + assert pypi_repository.build_dir is None + with pypi_repository.freshen_build_caches(): + assert isinstance(pypi_repository.build_dir, str) + assert pypi_repository.build_dir is None def test_pypirepo_source_dir_is_str(pypi_repository): - assert isinstance(pypi_repository.source_dir, str) + assert pypi_repository.source_dir is None + with pypi_repository.freshen_build_caches(): + assert isinstance(pypi_repository.source_dir, str) + assert pypi_repository.source_dir is None def test_relative_path_cache_dir_is_normalized(from_line):