From da9a432576eb94e460658c408e3524ffabaed3a3 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 3 Nov 2019 18:37:30 +0530 Subject: [PATCH 1/4] Move WheelBuilder and friends to a dedicated module --- src/pip/_internal/commands/install.py | 4 +- src/pip/_internal/commands/wheel.py | 2 +- src/pip/_internal/wheel.py | 515 +------------------------ src/pip/_internal/wheel_builder.py | 530 ++++++++++++++++++++++++++ tests/unit/test_wheel.py | 28 +- 5 files changed, 552 insertions(+), 527 deletions(-) create mode 100644 src/pip/_internal/wheel_builder.py diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 1e1d273fe9a..7dd1dc7b7a0 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -42,7 +42,7 @@ from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.virtualenv import virtualenv_no_global -from pip._internal.wheel import WheelBuilder +from pip._internal.wheel_builder import WheelBuilder if MYPY_CHECK_RUNNING: from optparse import Values @@ -50,7 +50,7 @@ from pip._internal.models.format_control import FormatControl from pip._internal.req.req_install import InstallRequirement - from pip._internal.wheel import BinaryAllowedPredicate + from pip._internal.wheel_builder import BinaryAllowedPredicate logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/commands/wheel.py b/src/pip/_internal/commands/wheel.py index a6912717fc2..e1d59ee4c2b 100644 --- a/src/pip/_internal/commands/wheel.py +++ b/src/pip/_internal/commands/wheel.py @@ -16,7 +16,7 @@ from pip._internal.req.req_tracker import RequirementTracker from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.typing import MYPY_CHECK_RUNNING -from pip._internal.wheel import WheelBuilder +from pip._internal.wheel_builder import WheelBuilder if MYPY_CHECK_RUNNING: from optparse import Values diff --git a/src/pip/_internal/wheel.py b/src/pip/_internal/wheel.py index 0af58056d38..274d9aa7362 100644 --- a/src/pip/_internal/wheel.py +++ b/src/pip/_internal/wheel.py @@ -9,7 +9,6 @@ import collections import compileall import csv -import hashlib import logging import os.path import re @@ -26,50 +25,25 @@ from pip._vendor.packaging.utils import canonicalize_name from pip._vendor.six import StringIO -from pip._internal import pep425tags from pip._internal.exceptions import ( InstallationError, InvalidWheelFilename, UnsupportedWheel, ) from pip._internal.locations import get_major_minor_version -from pip._internal.models.link import Link -from pip._internal.utils.logging import indent_log -from pip._internal.utils.marker_files import has_delete_marker_file -from pip._internal.utils.misc import captured_stdout, ensure_dir, read_chunks -from pip._internal.utils.setuptools_build import ( - make_setuptools_bdist_wheel_args, - make_setuptools_clean_args, -) -from pip._internal.utils.subprocess import ( - LOG_DIVIDER, - call_subprocess, - format_command_args, - runner_with_spinner_message, -) -from pip._internal.utils.temp_dir import TempDirectory +from pip._internal.utils.misc import captured_stdout, ensure_dir from pip._internal.utils.typing import MYPY_CHECK_RUNNING -from pip._internal.utils.ui import open_spinner -from pip._internal.utils.unpacking import unpack_file -from pip._internal.utils.urls import path_to_url -from pip._internal.vcs import vcs +from pip._internal.wheel_builder import hash_file if MYPY_CHECK_RUNNING: from typing import ( Dict, List, Optional, Sequence, Mapping, Tuple, IO, Text, Any, - Iterable, Callable, Set, Pattern, Union, - ) - from pip._internal.req.req_install import InstallRequirement - from pip._internal.operations.prepare import ( - RequirementPreparer + Iterable, Callable, Set, ) - from pip._internal.cache import WheelCache from pip._internal.pep425tags import Pep425Tag InstalledCSVRow = Tuple[str, ...] - BinaryAllowedPredicate = Callable[[InstallRequirement], bool] - VERSION_COMPATIBLE = (1, 0) @@ -82,18 +56,6 @@ def normpath(src, p): return os.path.relpath(src, p).replace(os.path.sep, '/') -def hash_file(path, blocksize=1 << 20): - # type: (str, int) -> Tuple[Any, int] - """Return (hash, length) for path using hashlib.sha256()""" - h = hashlib.sha256() - length = 0 - with open(path, 'rb') as f: - for block in read_chunks(f, size=blocksize): - length += len(block) - h.update(block) - return (h, length) # type: ignore - - def rehash(path, blocksize=1 << 20): # type: (str, int) -> Tuple[str, str] """Return (encoded_digest, length) for path using hashlib.sha256()""" @@ -116,14 +78,6 @@ def open_for_csv(name, mode): return open(name, mode + bin, **nl) -def replace_python_tag(wheelname, new_tag): - # type: (str, str) -> str - """Replace the Python tag in a wheel file name with a new value.""" - parts = wheelname.split('-') - parts[-3] = new_tag - return '-'.join(parts) - - def fix_script(path): # type: (str) -> Optional[bool] """Replace #!python with #!/path/to/python @@ -775,466 +729,3 @@ def supported(self, tags): :param tags: the PEP 425 tags to check the wheel against. """ return not self.file_tags.isdisjoint(tags) - - -def _contains_egg_info( - s, _egg_info_re=re.compile(r'([a-z0-9_.]+)-([a-z0-9_.!+-]+)', re.I)): - # type: (str, Pattern) -> bool - """Determine whether the string looks like an egg_info. - - :param s: The string to parse. E.g. foo-2.1 - """ - return bool(_egg_info_re.search(s)) - - -def should_build( - req, # type: InstallRequirement - need_wheel, # type: bool - check_binary_allowed, # type: BinaryAllowedPredicate -): - # type: (...) -> Optional[bool] - """Return whether an InstallRequirement should be built into a wheel.""" - if req.constraint: - # never build requirements that are merely constraints - return False - if req.is_wheel: - if need_wheel: - logger.info( - 'Skipping %s, due to already being wheel.', req.name, - ) - return False - - if need_wheel: - # i.e. pip wheel, not pip install - return True - - if req.editable or not req.source_dir: - return False - - if not check_binary_allowed(req): - logger.info( - "Skipping wheel build for %s, due to binaries " - "being disabled for it.", req.name, - ) - return False - - return True - - -def should_cache( - req, # type: InstallRequirement - check_binary_allowed, # type: BinaryAllowedPredicate -): - # type: (...) -> Optional[bool] - """ - Return whether a built InstallRequirement can be stored in the persistent - wheel cache, assuming the wheel cache is available, and should_build() - has determined a wheel needs to be built. - """ - if not should_build( - req, need_wheel=False, check_binary_allowed=check_binary_allowed - ): - # never cache if pip install (need_wheel=False) would not have built - # (editable mode, etc) - return False - - if req.link and req.link.is_vcs: - # VCS checkout. Build wheel just for this run - # unless it points to an immutable commit hash in which - # case it can be cached. - assert not req.editable - assert req.source_dir - vcs_backend = vcs.get_backend_for_scheme(req.link.scheme) - assert vcs_backend - if vcs_backend.is_immutable_rev_checkout(req.link.url, req.source_dir): - return True - return False - - link = req.link - base, ext = link.splitext() - if _contains_egg_info(base): - return True - - # Otherwise, build the wheel just for this run using the ephemeral - # cache since we are either in the case of e.g. a local directory, or - # no cache directory is available to use. - return False - - -def format_command_result( - command_args, # type: List[str] - command_output, # type: Text -): - # type: (...) -> str - """Format command information for logging.""" - command_desc = format_command_args(command_args) - text = 'Command arguments: {}\n'.format(command_desc) - - if not command_output: - text += 'Command output: None' - elif logger.getEffectiveLevel() > logging.DEBUG: - text += 'Command output: [use --verbose to show]' - else: - if not command_output.endswith('\n'): - command_output += '\n' - text += 'Command output:\n{}{}'.format(command_output, LOG_DIVIDER) - - return text - - -def get_legacy_build_wheel_path( - names, # type: List[str] - temp_dir, # type: str - req, # type: InstallRequirement - command_args, # type: List[str] - command_output, # type: Text -): - # type: (...) -> Optional[str] - """Return the path to the wheel in the temporary build directory.""" - # Sort for determinism. - names = sorted(names) - if not names: - msg = ( - 'Legacy build of wheel for {!r} created no files.\n' - ).format(req.name) - msg += format_command_result(command_args, command_output) - logger.warning(msg) - return None - - if len(names) > 1: - msg = ( - 'Legacy build of wheel for {!r} created more than one file.\n' - 'Filenames (choosing first): {}\n' - ).format(req.name, names) - msg += format_command_result(command_args, command_output) - logger.warning(msg) - - return os.path.join(temp_dir, names[0]) - - -def _always_true(_): - # type: (Any) -> bool - return True - - -class WheelBuilder(object): - """Build wheels from a RequirementSet.""" - - def __init__( - self, - preparer, # type: RequirementPreparer - wheel_cache, # type: WheelCache - build_options=None, # type: Optional[List[str]] - global_options=None, # type: Optional[List[str]] - check_binary_allowed=None, # type: Optional[BinaryAllowedPredicate] - no_clean=False, # type: bool - path_to_wheelnames=None, # type: Optional[Union[bytes, Text]] - ): - # type: (...) -> None - if check_binary_allowed is None: - # Binaries allowed by default. - check_binary_allowed = _always_true - - self.preparer = preparer - self.wheel_cache = wheel_cache - - self._wheel_dir = preparer.wheel_download_dir - - self.build_options = build_options or [] - self.global_options = global_options or [] - self.check_binary_allowed = check_binary_allowed - self.no_clean = no_clean - # path where to save built names of built wheels - self.path_to_wheelnames = path_to_wheelnames - # file names of built wheel names - self.wheel_filenames = [] # type: List[Union[bytes, Text]] - - def _build_one( - self, - req, # type: InstallRequirement - output_dir, # type: str - python_tag=None, # type: Optional[str] - ): - # type: (...) -> Optional[str] - """Build one wheel. - - :return: The filename of the built wheel, or None if the build failed. - """ - # Install build deps into temporary directory (PEP 518) - with req.build_env: - return self._build_one_inside_env(req, output_dir, - python_tag=python_tag) - - def _build_one_inside_env( - self, - req, # type: InstallRequirement - output_dir, # type: str - python_tag=None, # type: Optional[str] - ): - # type: (...) -> Optional[str] - with TempDirectory(kind="wheel") as temp_dir: - if req.use_pep517: - builder = self._build_one_pep517 - else: - builder = self._build_one_legacy - wheel_path = builder(req, temp_dir.path, python_tag=python_tag) - if wheel_path is not None: - wheel_name = os.path.basename(wheel_path) - dest_path = os.path.join(output_dir, wheel_name) - try: - wheel_hash, length = hash_file(wheel_path) - shutil.move(wheel_path, dest_path) - logger.info('Created wheel for %s: ' - 'filename=%s size=%d sha256=%s', - req.name, wheel_name, length, - wheel_hash.hexdigest()) - logger.info('Stored in directory: %s', output_dir) - return dest_path - except Exception: - pass - # Ignore return, we can't do anything else useful. - self._clean_one(req) - return None - - def _build_one_pep517( - self, - req, # type: InstallRequirement - tempd, # type: str - python_tag=None, # type: Optional[str] - ): - # type: (...) -> Optional[str] - """Build one InstallRequirement using the PEP 517 build process. - - Returns path to wheel if successfully built. Otherwise, returns None. - """ - assert req.metadata_directory is not None - if self.build_options: - # PEP 517 does not support --build-options - logger.error('Cannot build wheel for %s using PEP 517 when ' - '--build-options is present' % (req.name,)) - return None - try: - logger.debug('Destination directory: %s', tempd) - - runner = runner_with_spinner_message( - 'Building wheel for {} (PEP 517)'.format(req.name) - ) - backend = req.pep517_backend - with backend.subprocess_runner(runner): - wheel_name = backend.build_wheel( - tempd, - metadata_directory=req.metadata_directory, - ) - if python_tag: - # General PEP 517 backends don't necessarily support - # a "--python-tag" option, so we rename the wheel - # file directly. - new_name = replace_python_tag(wheel_name, python_tag) - os.rename( - os.path.join(tempd, wheel_name), - os.path.join(tempd, new_name) - ) - # Reassign to simplify the return at the end of function - wheel_name = new_name - except Exception: - logger.error('Failed building wheel for %s', req.name) - return None - return os.path.join(tempd, wheel_name) - - def _build_one_legacy( - self, - req, # type: InstallRequirement - tempd, # type: str - python_tag=None, # type: Optional[str] - ): - # type: (...) -> Optional[str] - """Build one InstallRequirement using the "legacy" build process. - - Returns path to wheel if successfully built. Otherwise, returns None. - """ - wheel_args = make_setuptools_bdist_wheel_args( - req.setup_py_path, - global_options=self.global_options, - build_options=self.build_options, - destination_dir=tempd, - python_tag=python_tag, - ) - - spin_message = 'Building wheel for %s (setup.py)' % (req.name,) - with open_spinner(spin_message) as spinner: - logger.debug('Destination directory: %s', tempd) - - try: - output = call_subprocess( - wheel_args, - cwd=req.unpacked_source_directory, - spinner=spinner, - ) - except Exception: - spinner.finish("error") - logger.error('Failed building wheel for %s', req.name) - return None - - names = os.listdir(tempd) - wheel_path = get_legacy_build_wheel_path( - names=names, - temp_dir=tempd, - req=req, - command_args=wheel_args, - command_output=output, - ) - return wheel_path - - def _clean_one(self, req): - # type: (InstallRequirement) -> bool - clean_args = make_setuptools_clean_args( - req.setup_py_path, - global_options=self.global_options, - ) - - logger.info('Running setup.py clean for %s', req.name) - try: - call_subprocess(clean_args, cwd=req.source_dir) - return True - except Exception: - logger.error('Failed cleaning build dir for %s', req.name) - return False - - def build( - self, - requirements, # type: Iterable[InstallRequirement] - should_unpack=False # type: bool - ): - # type: (...) -> List[InstallRequirement] - """Build wheels. - - :param should_unpack: If True, after building the wheel, unpack it - and replace the sdist with the unpacked version in preparation - for installation. - :return: The list of InstallRequirement that failed to build. - """ - # pip install uses should_unpack=True. - # pip install never provides a _wheel_dir. - # pip wheel uses should_unpack=False. - # pip wheel always provides a _wheel_dir (via the preparer). - assert ( - (should_unpack and not self._wheel_dir) or - (not should_unpack and self._wheel_dir) - ) - - buildset = [] - cache_available = bool(self.wheel_cache.cache_dir) - - for req in requirements: - if not should_build( - req, - need_wheel=not should_unpack, - check_binary_allowed=self.check_binary_allowed, - ): - continue - - if ( - cache_available and - should_cache(req, self.check_binary_allowed) - ): - output_dir = self.wheel_cache.get_path_for_link(req.link) - else: - output_dir = self.wheel_cache.get_ephem_path_for_link( - req.link - ) - - buildset.append((req, output_dir)) - - if not buildset: - return [] - - # TODO by @pradyunsg - # Should break up this method into 2 separate methods. - - # Build the wheels. - logger.info( - 'Building wheels for collected packages: %s', - ', '.join([req.name for (req, _) in buildset]), - ) - - python_tag = None - if should_unpack: - python_tag = pep425tags.implementation_tag - - with indent_log(): - build_success, build_failure = [], [] - for req, output_dir in buildset: - try: - ensure_dir(output_dir) - except OSError as e: - logger.warning( - "Building wheel for %s failed: %s", - req.name, e, - ) - build_failure.append(req) - continue - - wheel_file = self._build_one( - req, output_dir, - python_tag=python_tag, - ) - if wheel_file: - if should_unpack: - # XXX: This is mildly duplicative with prepare_files, - # but not close enough to pull out to a single common - # method. - # The code below assumes temporary source dirs - - # prevent it doing bad things. - if ( - req.source_dir and - not has_delete_marker_file(req.source_dir) - ): - raise AssertionError( - "bad source dir - missing marker") - # Delete the source we built the wheel from - req.remove_temporary_source() - # set the build directory again - name is known from - # the work prepare_files did. - req.source_dir = req.ensure_build_location( - self.preparer.build_dir - ) - # Update the link for this. - req.link = Link(path_to_url(wheel_file)) - assert req.link.is_wheel - # extract the wheel into the dir - unpack_file(req.link.file_path, req.source_dir) - else: - # copy from cache to target directory - try: - ensure_dir(self._wheel_dir) - shutil.copy( - os.path.join(output_dir, wheel_file), - self._wheel_dir, - ) - except OSError as e: - logger.warning( - "Building wheel for %s failed: %s", - req.name, e, - ) - build_failure.append(req) - continue - self.wheel_filenames.append( - os.path.relpath(wheel_file, output_dir) - ) - build_success.append(req) - else: - build_failure.append(req) - - # notify success/failure - if build_success: - logger.info( - 'Successfully built %s', - ' '.join([req.name for req in build_success]), - ) - if build_failure: - logger.info( - 'Failed to build %s', - ' '.join([req.name for req in build_failure]), - ) - # Return a list of requirements that failed to build - return build_failure diff --git a/src/pip/_internal/wheel_builder.py b/src/pip/_internal/wheel_builder.py new file mode 100644 index 00000000000..188f97fadc9 --- /dev/null +++ b/src/pip/_internal/wheel_builder.py @@ -0,0 +1,530 @@ +"""Orchestrator for building wheels from InstallRequirements. +""" + +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + +import logging +import os.path +import re +import shutil + +from pip._internal import pep425tags +from pip._internal.models.link import Link +from pip._internal.utils.logging import indent_log +from pip._internal.utils.marker_files import has_delete_marker_file +from pip._internal.utils.misc import ensure_dir, read_chunks +from pip._internal.utils.setuptools_build import ( + make_setuptools_bdist_wheel_args, + make_setuptools_clean_args, +) +from pip._internal.utils.subprocess import ( + LOG_DIVIDER, + call_subprocess, + format_command_args, + runner_with_spinner_message, +) +from pip._internal.utils.temp_dir import TempDirectory +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.ui import open_spinner +from pip._internal.utils.unpacking import unpack_file +from pip._internal.utils.urls import path_to_url +from pip._internal.vcs import vcs + +if MYPY_CHECK_RUNNING: + from typing import ( + Any, Callable, Iterable, List, Optional, Pattern, Text, Tuple, Union, + ) + + from pip._internal.cache import WheelCache + from pip._internal.operations.prepare import ( + RequirementPreparer + ) + from pip._internal.req.req_install import InstallRequirement + + BinaryAllowedPredicate = Callable[[InstallRequirement], bool] + +logger = logging.getLogger(__name__) + + +def hash_file(path, blocksize=1 << 20): + # type: (str, int) -> Tuple[Any, int] + """Return (hash, length) for path using hashlib.sha256()""" + h = hashlib.sha256() + length = 0 + with open(path, 'rb') as f: + for block in read_chunks(f, size=blocksize): + length += len(block) + h.update(block) + return (h, length) # type: ignore + + +def replace_python_tag(wheelname, new_tag): + # type: (str, str) -> str + """Replace the Python tag in a wheel file name with a new value.""" + parts = wheelname.split('-') + parts[-3] = new_tag + return '-'.join(parts) + + +def _contains_egg_info( + s, _egg_info_re=re.compile(r'([a-z0-9_.]+)-([a-z0-9_.!+-]+)', re.I)): + # type: (str, Pattern) -> bool + """Determine whether the string looks like an egg_info. + + :param s: The string to parse. E.g. foo-2.1 + """ + return bool(_egg_info_re.search(s)) + + +def should_build( + req, # type: InstallRequirement + need_wheel, # type: bool + check_binary_allowed, # type: BinaryAllowedPredicate +): + # type: (...) -> Optional[bool] + """Return whether an InstallRequirement should be built into a wheel.""" + if req.constraint: + # never build requirements that are merely constraints + return False + if req.is_wheel: + if need_wheel: + logger.info( + 'Skipping %s, due to already being wheel.', req.name, + ) + return False + + if need_wheel: + # i.e. pip wheel, not pip install + return True + + if req.editable or not req.source_dir: + return False + + if not check_binary_allowed(req): + logger.info( + "Skipping wheel build for %s, due to binaries " + "being disabled for it.", req.name, + ) + return False + + return True + + +def should_cache( + req, # type: InstallRequirement + check_binary_allowed, # type: BinaryAllowedPredicate +): + # type: (...) -> Optional[bool] + """ + Return whether a built InstallRequirement can be stored in the persistent + wheel cache, assuming the wheel cache is available, and should_build() + has determined a wheel needs to be built. + """ + if not should_build( + req, need_wheel=False, check_binary_allowed=check_binary_allowed + ): + # never cache if pip install (need_wheel=False) would not have built + # (editable mode, etc) + return False + + if req.link and req.link.is_vcs: + # VCS checkout. Build wheel just for this run + # unless it points to an immutable commit hash in which + # case it can be cached. + assert not req.editable + assert req.source_dir + vcs_backend = vcs.get_backend_for_scheme(req.link.scheme) + assert vcs_backend + if vcs_backend.is_immutable_rev_checkout(req.link.url, req.source_dir): + return True + return False + + link = req.link + base, ext = link.splitext() + if _contains_egg_info(base): + return True + + # Otherwise, build the wheel just for this run using the ephemeral + # cache since we are either in the case of e.g. a local directory, or + # no cache directory is available to use. + return False + + +def format_command_result( + command_args, # type: List[str] + command_output, # type: Text +): + # type: (...) -> str + """Format command information for logging.""" + command_desc = format_command_args(command_args) + text = 'Command arguments: {}\n'.format(command_desc) + + if not command_output: + text += 'Command output: None' + elif logger.getEffectiveLevel() > logging.DEBUG: + text += 'Command output: [use --verbose to show]' + else: + if not command_output.endswith('\n'): + command_output += '\n' + text += 'Command output:\n{}{}'.format(command_output, LOG_DIVIDER) + + return text + + +def get_legacy_build_wheel_path( + names, # type: List[str] + temp_dir, # type: str + req, # type: InstallRequirement + command_args, # type: List[str] + command_output, # type: Text +): + # type: (...) -> Optional[str] + """Return the path to the wheel in the temporary build directory.""" + # Sort for determinism. + names = sorted(names) + if not names: + msg = ( + 'Legacy build of wheel for {!r} created no files.\n' + ).format(req.name) + msg += format_command_result(command_args, command_output) + logger.warning(msg) + return None + + if len(names) > 1: + msg = ( + 'Legacy build of wheel for {!r} created more than one file.\n' + 'Filenames (choosing first): {}\n' + ).format(req.name, names) + msg += format_command_result(command_args, command_output) + logger.warning(msg) + + return os.path.join(temp_dir, names[0]) + + +def _always_true(_): + # type: (Any) -> bool + return True + + +class WheelBuilder(object): + """Build wheels from a RequirementSet.""" + + def __init__( + self, + preparer, # type: RequirementPreparer + wheel_cache, # type: WheelCache + build_options=None, # type: Optional[List[str]] + global_options=None, # type: Optional[List[str]] + check_binary_allowed=None, # type: Optional[BinaryAllowedPredicate] + no_clean=False, # type: bool + path_to_wheelnames=None, # type: Optional[Union[bytes, Text]] + ): + # type: (...) -> None + if check_binary_allowed is None: + # Binaries allowed by default. + check_binary_allowed = _always_true + + self.preparer = preparer + self.wheel_cache = wheel_cache + + self._wheel_dir = preparer.wheel_download_dir + + self.build_options = build_options or [] + self.global_options = global_options or [] + self.check_binary_allowed = check_binary_allowed + self.no_clean = no_clean + # path where to save built names of built wheels + self.path_to_wheelnames = path_to_wheelnames + # file names of built wheel names + self.wheel_filenames = [] # type: List[Union[bytes, Text]] + + def _build_one( + self, + req, # type: InstallRequirement + output_dir, # type: str + python_tag=None, # type: Optional[str] + ): + # type: (...) -> Optional[str] + """Build one wheel. + + :return: The filename of the built wheel, or None if the build failed. + """ + # Install build deps into temporary directory (PEP 518) + with req.build_env: + return self._build_one_inside_env(req, output_dir, + python_tag=python_tag) + + def _build_one_inside_env( + self, + req, # type: InstallRequirement + output_dir, # type: str + python_tag=None, # type: Optional[str] + ): + # type: (...) -> Optional[str] + with TempDirectory(kind="wheel") as temp_dir: + if req.use_pep517: + builder = self._build_one_pep517 + else: + builder = self._build_one_legacy + wheel_path = builder(req, temp_dir.path, python_tag=python_tag) + if wheel_path is not None: + wheel_name = os.path.basename(wheel_path) + dest_path = os.path.join(output_dir, wheel_name) + try: + wheel_hash, length = hash_file(wheel_path) + shutil.move(wheel_path, dest_path) + logger.info('Created wheel for %s: ' + 'filename=%s size=%d sha256=%s', + req.name, wheel_name, length, + wheel_hash.hexdigest()) + logger.info('Stored in directory: %s', output_dir) + return dest_path + except Exception: + pass + # Ignore return, we can't do anything else useful. + self._clean_one(req) + return None + + def _build_one_pep517( + self, + req, # type: InstallRequirement + tempd, # type: str + python_tag=None, # type: Optional[str] + ): + # type: (...) -> Optional[str] + """Build one InstallRequirement using the PEP 517 build process. + + Returns path to wheel if successfully built. Otherwise, returns None. + """ + assert req.metadata_directory is not None + if self.build_options: + # PEP 517 does not support --build-options + logger.error('Cannot build wheel for %s using PEP 517 when ' + '--build-options is present' % (req.name,)) + return None + try: + logger.debug('Destination directory: %s', tempd) + + runner = runner_with_spinner_message( + 'Building wheel for {} (PEP 517)'.format(req.name) + ) + backend = req.pep517_backend + with backend.subprocess_runner(runner): + wheel_name = backend.build_wheel( + tempd, + metadata_directory=req.metadata_directory, + ) + if python_tag: + # General PEP 517 backends don't necessarily support + # a "--python-tag" option, so we rename the wheel + # file directly. + new_name = replace_python_tag(wheel_name, python_tag) + os.rename( + os.path.join(tempd, wheel_name), + os.path.join(tempd, new_name) + ) + # Reassign to simplify the return at the end of function + wheel_name = new_name + except Exception: + logger.error('Failed building wheel for %s', req.name) + return None + return os.path.join(tempd, wheel_name) + + def _build_one_legacy( + self, + req, # type: InstallRequirement + tempd, # type: str + python_tag=None, # type: Optional[str] + ): + # type: (...) -> Optional[str] + """Build one InstallRequirement using the "legacy" build process. + + Returns path to wheel if successfully built. Otherwise, returns None. + """ + wheel_args = make_setuptools_bdist_wheel_args( + req.setup_py_path, + global_options=self.global_options, + build_options=self.build_options, + destination_dir=tempd, + python_tag=python_tag, + ) + + spin_message = 'Building wheel for %s (setup.py)' % (req.name,) + with open_spinner(spin_message) as spinner: + logger.debug('Destination directory: %s', tempd) + + try: + output = call_subprocess( + wheel_args, + cwd=req.unpacked_source_directory, + spinner=spinner, + ) + except Exception: + spinner.finish("error") + logger.error('Failed building wheel for %s', req.name) + return None + + names = os.listdir(tempd) + wheel_path = get_legacy_build_wheel_path( + names=names, + temp_dir=tempd, + req=req, + command_args=wheel_args, + command_output=output, + ) + return wheel_path + + def _clean_one(self, req): + # type: (InstallRequirement) -> bool + clean_args = make_setuptools_clean_args( + req.setup_py_path, + global_options=self.global_options, + ) + + logger.info('Running setup.py clean for %s', req.name) + try: + call_subprocess(clean_args, cwd=req.source_dir) + return True + except Exception: + logger.error('Failed cleaning build dir for %s', req.name) + return False + + def build( + self, + requirements, # type: Iterable[InstallRequirement] + should_unpack=False # type: bool + ): + # type: (...) -> List[InstallRequirement] + """Build wheels. + + :param should_unpack: If True, after building the wheel, unpack it + and replace the sdist with the unpacked version in preparation + for installation. + :return: The list of InstallRequirement that failed to build. + """ + # pip install uses should_unpack=True. + # pip install never provides a _wheel_dir. + # pip wheel uses should_unpack=False. + # pip wheel always provides a _wheel_dir (via the preparer). + assert ( + (should_unpack and not self._wheel_dir) or + (not should_unpack and self._wheel_dir) + ) + + buildset = [] + cache_available = bool(self.wheel_cache.cache_dir) + + for req in requirements: + if not should_build( + req, + need_wheel=not should_unpack, + check_binary_allowed=self.check_binary_allowed, + ): + continue + + if ( + cache_available and + should_cache(req, self.check_binary_allowed) + ): + output_dir = self.wheel_cache.get_path_for_link(req.link) + else: + output_dir = self.wheel_cache.get_ephem_path_for_link( + req.link + ) + + buildset.append((req, output_dir)) + + if not buildset: + return [] + + # TODO by @pradyunsg + # Should break up this method into 2 separate methods. + + # Build the wheels. + logger.info( + 'Building wheels for collected packages: %s', + ', '.join([req.name for (req, _) in buildset]), + ) + + python_tag = None + if should_unpack: + python_tag = pep425tags.implementation_tag + + with indent_log(): + build_success, build_failure = [], [] + for req, output_dir in buildset: + try: + ensure_dir(output_dir) + except OSError as e: + logger.warning( + "Building wheel for %s failed: %s", + req.name, e, + ) + build_failure.append(req) + continue + + wheel_file = self._build_one( + req, output_dir, + python_tag=python_tag, + ) + if wheel_file: + if should_unpack: + # XXX: This is mildly duplicative with prepare_files, + # but not close enough to pull out to a single common + # method. + # The code below assumes temporary source dirs - + # prevent it doing bad things. + if ( + req.source_dir and + not has_delete_marker_file(req.source_dir) + ): + raise AssertionError( + "bad source dir - missing marker") + # Delete the source we built the wheel from + req.remove_temporary_source() + # set the build directory again - name is known from + # the work prepare_files did. + req.source_dir = req.ensure_build_location( + self.preparer.build_dir + ) + # Update the link for this. + req.link = Link(path_to_url(wheel_file)) + assert req.link.is_wheel + # extract the wheel into the dir + unpack_file(req.link.file_path, req.source_dir) + else: + # copy from cache to target directory + try: + ensure_dir(self._wheel_dir) + shutil.copy( + os.path.join(output_dir, wheel_file), + self._wheel_dir, + ) + except OSError as e: + logger.warning( + "Building wheel for %s failed: %s", + req.name, e, + ) + build_failure.append(req) + continue + self.wheel_filenames.append( + os.path.relpath(wheel_file, output_dir) + ) + build_success.append(req) + else: + build_failure.append(req) + + # notify success/failure + if build_success: + logger.info( + 'Successfully built %s', + ' '.join([req.name for req in build_success]), + ) + if build_failure: + logger.info( + 'Failed to build %s', + ' '.join([req.name for req in build_failure]), + ) + # Return a list of requirements that failed to build + return build_failure diff --git a/tests/unit/test_wheel.py b/tests/unit/test_wheel.py index fc3a7ddb382..1d0c8e209ae 100644 --- a/tests/unit/test_wheel.py +++ b/tests/unit/test_wheel.py @@ -8,6 +8,7 @@ from mock import Mock, patch from pip._vendor.packaging.requirements import Requirement +import pip._internal.wheel_builder from pip._internal import pep425tags, wheel from pip._internal.commands.wheel import WheelCommand from pip._internal.exceptions import InvalidWheelFilename, UnsupportedWheel @@ -58,7 +59,7 @@ def __init__( ], ) def test_contains_egg_info(s, expected): - result = wheel._contains_egg_info(s) + result = pip._internal.wheel_builder._contains_egg_info(s) assert result == expected @@ -128,7 +129,7 @@ def test_format_tag(file_tag, expected): ], ) def test_should_build(req, need_wheel, disallow_binaries, expected): - should_build = wheel.should_build( + should_build = pip._internal.wheel_builder.should_build( req, need_wheel, check_binary_allowed=lambda req: not disallow_binaries, @@ -157,8 +158,10 @@ def test_should_cache( def check_binary_allowed(req): return not disallow_binaries - should_cache = wheel.should_cache(req, check_binary_allowed) - if not wheel.should_build( + should_cache = pip._internal.wheel_builder.should_cache( + req, check_binary_allowed + ) + if not pip._internal.wheel_builder.should_build( req, need_wheel=False, check_binary_allowed=check_binary_allowed ): # never cache if pip install (need_wheel=False) would not have built) @@ -183,7 +186,7 @@ def test_should_cache_git_sha(script, tmpdir): def test_format_command_result__INFO(caplog): caplog.set_level(logging.INFO) - actual = wheel.format_command_result( + actual = pip._internal.wheel_builder.format_command_result( # Include an argument with a space to test argument quoting. command_args=['arg1', 'second arg'], command_output='output line 1\noutput line 2\n', @@ -202,7 +205,7 @@ def test_format_command_result__INFO(caplog): ]) def test_format_command_result__DEBUG(caplog, command_output): caplog.set_level(logging.DEBUG) - actual = wheel.format_command_result( + actual = pip._internal.wheel_builder.format_command_result( command_args=['arg1', 'arg2'], command_output=command_output, ) @@ -218,7 +221,7 @@ def test_format_command_result__DEBUG(caplog, command_output): @pytest.mark.parametrize('log_level', ['DEBUG', 'INFO']) def test_format_command_result__empty_output(caplog, log_level): caplog.set_level(log_level) - actual = wheel.format_command_result( + actual = pip._internal.wheel_builder.format_command_result( command_args=['arg1', 'arg2'], command_output='', ) @@ -230,7 +233,7 @@ def test_format_command_result__empty_output(caplog, log_level): def call_get_legacy_build_wheel_path(caplog, names): req = make_test_install_req() - wheel_path = wheel.get_legacy_build_wheel_path( + wheel_path = pip._internal.wheel_builder.get_legacy_build_wheel_path( names=names, temp_dir='/tmp/abcd', req=req, @@ -416,8 +419,9 @@ def test_python_tag(): 'simplewheel-1.0-py37-none-any.whl', 'simplewheel-2.0-1-py37-none-any.whl', ] - for name, new in zip(wheelnames, newnames): - assert wheel.replace_python_tag(name, 'py37') == new + for name, expected in zip(wheelnames, newnames): + result = pip._internal.wheel_builder.replace_python_tag(name, 'py37') + assert result == expected def test_check_compatibility(): @@ -745,7 +749,7 @@ def test_skip_building_wheels(self, caplog): with patch('pip._internal.wheel.WheelBuilder._build_one') \ as mock_build_one: wheel_req = Mock(is_wheel=True, editable=False, constraint=False) - wb = wheel.WheelBuilder( + wb = pip._internal.wheel_builder.WheelBuilder( preparer=Mock(), wheel_cache=Mock(cache_dir=None), ) @@ -883,7 +887,7 @@ def prep(self, tmpdir): def test_hash_file(self, tmpdir): self.prep(tmpdir) - h, length = wheel.hash_file(self.test_file) + h, length = pip._internal.wheel_builder.hash_file(self.test_file) assert length == self.test_file_len assert h.hexdigest() == self.test_file_hash From 647d30ec77d550bc021ea2e510c7fac6e019528e Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 3 Nov 2019 18:54:11 +0530 Subject: [PATCH 2/4] Move hash_file to utils.misc Why: Allows for better code reuse, without introducing dependency between wheel->wheel_builder. --- src/pip/_internal/utils/misc.py | 15 +++++++++++++++ src/pip/_internal/wheel.py | 3 +-- src/pip/_internal/wheel_builder.py | 16 ++-------------- tests/unit/test_wheel.py | 3 ++- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/src/pip/_internal/utils/misc.py b/src/pip/_internal/utils/misc.py index b84826350bc..c84177cbd2c 100644 --- a/src/pip/_internal/utils/misc.py +++ b/src/pip/_internal/utils/misc.py @@ -7,6 +7,7 @@ import contextlib import errno import getpass +import hashlib import io import logging import os @@ -868,3 +869,17 @@ def is_console_interactive(): """Is this console interactive? """ return sys.stdin is not None and sys.stdin.isatty() + + +def hash_file(path, blocksize=1 << 20): + # type: (str, int) -> Tuple[Any, int] + """Return (hash, length) for path using hashlib.sha256() + """ + + h = hashlib.sha256() + length = 0 + with open(path, 'rb') as f: + for block in read_chunks(f, size=blocksize): + length += len(block) + h.update(block) + return (h, length) # type: ignore diff --git a/src/pip/_internal/wheel.py b/src/pip/_internal/wheel.py index 274d9aa7362..85a9e2b60e4 100644 --- a/src/pip/_internal/wheel.py +++ b/src/pip/_internal/wheel.py @@ -31,9 +31,8 @@ UnsupportedWheel, ) from pip._internal.locations import get_major_minor_version -from pip._internal.utils.misc import captured_stdout, ensure_dir +from pip._internal.utils.misc import captured_stdout, ensure_dir, hash_file from pip._internal.utils.typing import MYPY_CHECK_RUNNING -from pip._internal.wheel_builder import hash_file if MYPY_CHECK_RUNNING: from typing import ( diff --git a/src/pip/_internal/wheel_builder.py b/src/pip/_internal/wheel_builder.py index 188f97fadc9..ba93933edf8 100644 --- a/src/pip/_internal/wheel_builder.py +++ b/src/pip/_internal/wheel_builder.py @@ -13,7 +13,7 @@ from pip._internal.models.link import Link from pip._internal.utils.logging import indent_log from pip._internal.utils.marker_files import has_delete_marker_file -from pip._internal.utils.misc import ensure_dir, read_chunks +from pip._internal.utils.misc import ensure_dir, hash_file from pip._internal.utils.setuptools_build import ( make_setuptools_bdist_wheel_args, make_setuptools_clean_args, @@ -33,7 +33,7 @@ if MYPY_CHECK_RUNNING: from typing import ( - Any, Callable, Iterable, List, Optional, Pattern, Text, Tuple, Union, + Any, Callable, Iterable, List, Optional, Pattern, Text, Union, ) from pip._internal.cache import WheelCache @@ -47,18 +47,6 @@ logger = logging.getLogger(__name__) -def hash_file(path, blocksize=1 << 20): - # type: (str, int) -> Tuple[Any, int] - """Return (hash, length) for path using hashlib.sha256()""" - h = hashlib.sha256() - length = 0 - with open(path, 'rb') as f: - for block in read_chunks(f, size=blocksize): - length += len(block) - h.update(block) - return (h, length) # type: ignore - - def replace_python_tag(wheelname, new_tag): # type: (str, str) -> str """Replace the Python tag in a wheel file name with a new value.""" diff --git a/tests/unit/test_wheel.py b/tests/unit/test_wheel.py index 1d0c8e209ae..6ef988ecfce 100644 --- a/tests/unit/test_wheel.py +++ b/tests/unit/test_wheel.py @@ -16,6 +16,7 @@ from pip._internal.models.link import Link from pip._internal.req.req_install import InstallRequirement from pip._internal.utils.compat import WINDOWS +from pip._internal.utils.misc import hash_file from pip._internal.utils.unpacking import unpack_file from pip._internal.wheel import ( MissingCallableSuffix, @@ -887,7 +888,7 @@ def prep(self, tmpdir): def test_hash_file(self, tmpdir): self.prep(tmpdir) - h, length = pip._internal.wheel_builder.hash_file(self.test_file) + h, length = hash_file(self.test_file) assert length == self.test_file_len assert h.hexdigest() == self.test_file_hash From 9435050c2d18bed9ca844fcf3ca6c7f67c77cf80 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sun, 3 Nov 2019 20:01:05 +0530 Subject: [PATCH 3/4] Move tests for WheelBuilder and friends --- tests/unit/test_wheel.py | 208 +----------------------------- tests/unit/test_wheel_builder.py | 209 +++++++++++++++++++++++++++++++ 2 files changed, 213 insertions(+), 204 deletions(-) create mode 100644 tests/unit/test_wheel_builder.py diff --git a/tests/unit/test_wheel.py b/tests/unit/test_wheel.py index 6ef988ecfce..3153f2a822a 100644 --- a/tests/unit/test_wheel.py +++ b/tests/unit/test_wheel.py @@ -5,10 +5,9 @@ import textwrap import pytest -from mock import Mock, patch +from mock import patch from pip._vendor.packaging.requirements import Requirement -import pip._internal.wheel_builder from pip._internal import pep425tags, wheel from pip._internal.commands.wheel import WheelCommand from pip._internal.exceptions import InvalidWheelFilename, UnsupportedWheel @@ -22,46 +21,8 @@ MissingCallableSuffix, _raise_for_invalid_entrypoint, ) -from tests.lib import DATA_DIR, _create_test_package, assert_paths_equal - - -class ReqMock: - - def __init__( - self, - name="pendulum", - is_wheel=False, - editable=False, - link=None, - constraint=False, - source_dir="/tmp/pip-install-123/pendulum", - ): - self.name = name - self.is_wheel = is_wheel - self.editable = editable - self.link = link - self.constraint = constraint - self.source_dir = source_dir - - -@pytest.mark.parametrize( - "s, expected", - [ - # Trivial. - ("pip-18.0", True), - - # Ambiguous. - ("foo-2-2", True), - ("im-valid", True), - - # Invalid. - ("invalid", False), - ("im_invalid", False), - ], -) -def test_contains_egg_info(s, expected): - result = pip._internal.wheel_builder._contains_egg_info(s) - assert result == expected +from pip._internal.wheel_builder import get_legacy_build_wheel_path +from tests.lib import DATA_DIR, assert_paths_equal def make_test_install_req(base_name=None): @@ -103,138 +64,9 @@ def test_format_tag(file_tag, expected): assert actual == expected -@pytest.mark.parametrize( - "req, need_wheel, disallow_binaries, expected", - [ - # pip wheel (need_wheel=True) - (ReqMock(), True, False, True), - (ReqMock(), True, True, True), - (ReqMock(constraint=True), True, False, False), - (ReqMock(is_wheel=True), True, False, False), - (ReqMock(editable=True), True, False, True), - (ReqMock(source_dir=None), True, False, True), - (ReqMock(link=Link("git+https://g.c/org/repo")), True, False, True), - (ReqMock(link=Link("git+https://g.c/org/repo")), True, True, True), - # pip install (need_wheel=False) - (ReqMock(), False, False, True), - (ReqMock(), False, True, False), - (ReqMock(constraint=True), False, False, False), - (ReqMock(is_wheel=True), False, False, False), - (ReqMock(editable=True), False, False, False), - (ReqMock(source_dir=None), False, False, False), - # By default (i.e. when binaries are allowed), VCS requirements - # should be built in install mode. - (ReqMock(link=Link("git+https://g.c/org/repo")), False, False, True), - # Disallowing binaries, however, should cause them not to be built. - (ReqMock(link=Link("git+https://g.c/org/repo")), False, True, False), - ], -) -def test_should_build(req, need_wheel, disallow_binaries, expected): - should_build = pip._internal.wheel_builder.should_build( - req, - need_wheel, - check_binary_allowed=lambda req: not disallow_binaries, - ) - assert should_build is expected - - -@pytest.mark.parametrize( - "req, disallow_binaries, expected", - [ - (ReqMock(editable=True), False, False), - (ReqMock(source_dir=None), False, False), - (ReqMock(link=Link("git+https://g.c/org/repo")), False, False), - (ReqMock(link=Link("https://g.c/dist.tgz")), False, False), - (ReqMock(link=Link("https://g.c/dist-2.0.4.tgz")), False, True), - (ReqMock(editable=True), True, False), - (ReqMock(source_dir=None), True, False), - (ReqMock(link=Link("git+https://g.c/org/repo")), True, False), - (ReqMock(link=Link("https://g.c/dist.tgz")), True, False), - (ReqMock(link=Link("https://g.c/dist-2.0.4.tgz")), True, False), - ], -) -def test_should_cache( - req, disallow_binaries, expected -): - def check_binary_allowed(req): - return not disallow_binaries - - should_cache = pip._internal.wheel_builder.should_cache( - req, check_binary_allowed - ) - if not pip._internal.wheel_builder.should_build( - req, need_wheel=False, check_binary_allowed=check_binary_allowed - ): - # never cache if pip install (need_wheel=False) would not have built) - assert not should_cache - assert should_cache is expected - - -def test_should_cache_git_sha(script, tmpdir): - repo_path = _create_test_package(script, name="mypkg") - commit = script.run( - "git", "rev-parse", "HEAD", cwd=repo_path - ).stdout.strip() - # a link referencing a sha should be cached - url = "git+https://g.c/o/r@" + commit + "#egg=mypkg" - req = ReqMock(link=Link(url), source_dir=repo_path) - assert wheel.should_cache(req, check_binary_allowed=lambda r: True) - # a link not referencing a sha should not be cached - url = "git+https://g.c/o/r@master#egg=mypkg" - req = ReqMock(link=Link(url), source_dir=repo_path) - assert not wheel.should_cache(req, check_binary_allowed=lambda r: True) - - -def test_format_command_result__INFO(caplog): - caplog.set_level(logging.INFO) - actual = pip._internal.wheel_builder.format_command_result( - # Include an argument with a space to test argument quoting. - command_args=['arg1', 'second arg'], - command_output='output line 1\noutput line 2\n', - ) - assert actual.splitlines() == [ - "Command arguments: arg1 'second arg'", - 'Command output: [use --verbose to show]', - ] - - -@pytest.mark.parametrize('command_output', [ - # Test trailing newline. - 'output line 1\noutput line 2\n', - # Test no trailing newline. - 'output line 1\noutput line 2', -]) -def test_format_command_result__DEBUG(caplog, command_output): - caplog.set_level(logging.DEBUG) - actual = pip._internal.wheel_builder.format_command_result( - command_args=['arg1', 'arg2'], - command_output=command_output, - ) - assert actual.splitlines() == [ - "Command arguments: arg1 arg2", - 'Command output:', - 'output line 1', - 'output line 2', - '----------------------------------------', - ] - - -@pytest.mark.parametrize('log_level', ['DEBUG', 'INFO']) -def test_format_command_result__empty_output(caplog, log_level): - caplog.set_level(log_level) - actual = pip._internal.wheel_builder.format_command_result( - command_args=['arg1', 'arg2'], - command_output='', - ) - assert actual.splitlines() == [ - "Command arguments: arg1 arg2", - 'Command output: None', - ] - - def call_get_legacy_build_wheel_path(caplog, names): req = make_test_install_req() - wheel_path = pip._internal.wheel_builder.get_legacy_build_wheel_path( + wheel_path = get_legacy_build_wheel_path( names=names, temp_dir='/tmp/abcd', req=req, @@ -409,22 +241,6 @@ def test_wheel_version(tmpdir, data): assert not wheel.wheel_version(tmpdir + 'broken') -def test_python_tag(): - wheelnames = [ - 'simplewheel-1.0-py2.py3-none-any.whl', - 'simplewheel-1.0-py27-none-any.whl', - 'simplewheel-2.0-1-py2.py3-none-any.whl', - ] - newnames = [ - 'simplewheel-1.0-py37-none-any.whl', - 'simplewheel-1.0-py37-none-any.whl', - 'simplewheel-2.0-1-py37-none-any.whl', - ] - for name, expected in zip(wheelnames, newnames): - result = pip._internal.wheel_builder.replace_python_tag(name, 'py37') - assert result == expected - - def test_check_compatibility(): name = 'test' vc = wheel.VERSION_COMPATIBLE @@ -744,22 +560,6 @@ def test_dist_info_contains_empty_dir(self, data, tmpdir): os.path.join(self.dest_dist_info, 'empty_dir')) -class TestWheelBuilder(object): - - def test_skip_building_wheels(self, caplog): - with patch('pip._internal.wheel.WheelBuilder._build_one') \ - as mock_build_one: - wheel_req = Mock(is_wheel=True, editable=False, constraint=False) - wb = pip._internal.wheel_builder.WheelBuilder( - preparer=Mock(), - wheel_cache=Mock(cache_dir=None), - ) - with caplog.at_level(logging.INFO): - wb.build([wheel_req]) - assert "due to already being wheel" in caplog.text - assert mock_build_one.mock_calls == [] - - class TestMessageAboutScriptsNotOnPATH(object): def _template(self, paths, scripts): diff --git a/tests/unit/test_wheel_builder.py b/tests/unit/test_wheel_builder.py new file mode 100644 index 00000000000..da6749a3b66 --- /dev/null +++ b/tests/unit/test_wheel_builder.py @@ -0,0 +1,209 @@ +import logging + +import pytest +from mock import Mock + +from pip._internal import wheel_builder +from pip._internal.models.link import Link +from tests.lib import _create_test_package + + +@pytest.mark.parametrize( + "s, expected", + [ + # Trivial. + ("pip-18.0", True), + + # Ambiguous. + ("foo-2-2", True), + ("im-valid", True), + + # Invalid. + ("invalid", False), + ("im_invalid", False), + ], +) +def test_contains_egg_info(s, expected): + result = wheel_builder._contains_egg_info(s) + assert result == expected + + +class ReqMock: + + def __init__( + self, + name="pendulum", + is_wheel=False, + editable=False, + link=None, + constraint=False, + source_dir="/tmp/pip-install-123/pendulum", + ): + self.name = name + self.is_wheel = is_wheel + self.editable = editable + self.link = link + self.constraint = constraint + self.source_dir = source_dir + + +@pytest.mark.parametrize( + "req, need_wheel, disallow_binaries, expected", + [ + # pip wheel (need_wheel=True) + (ReqMock(), True, False, True), + (ReqMock(), True, True, True), + (ReqMock(constraint=True), True, False, False), + (ReqMock(is_wheel=True), True, False, False), + (ReqMock(editable=True), True, False, True), + (ReqMock(source_dir=None), True, False, True), + (ReqMock(link=Link("git+https://g.c/org/repo")), True, False, True), + (ReqMock(link=Link("git+https://g.c/org/repo")), True, True, True), + # pip install (need_wheel=False) + (ReqMock(), False, False, True), + (ReqMock(), False, True, False), + (ReqMock(constraint=True), False, False, False), + (ReqMock(is_wheel=True), False, False, False), + (ReqMock(editable=True), False, False, False), + (ReqMock(source_dir=None), False, False, False), + # By default (i.e. when binaries are allowed), VCS requirements + # should be built in install mode. + (ReqMock(link=Link("git+https://g.c/org/repo")), False, False, True), + # Disallowing binaries, however, should cause them not to be built. + (ReqMock(link=Link("git+https://g.c/org/repo")), False, True, False), + ], +) +def test_should_build(req, need_wheel, disallow_binaries, expected): + should_build = wheel_builder.should_build( + req, + need_wheel, + check_binary_allowed=lambda req: not disallow_binaries, + ) + assert should_build is expected + + +@pytest.mark.parametrize( + "req, disallow_binaries, expected", + [ + (ReqMock(editable=True), False, False), + (ReqMock(source_dir=None), False, False), + (ReqMock(link=Link("git+https://g.c/org/repo")), False, False), + (ReqMock(link=Link("https://g.c/dist.tgz")), False, False), + (ReqMock(link=Link("https://g.c/dist-2.0.4.tgz")), False, True), + (ReqMock(editable=True), True, False), + (ReqMock(source_dir=None), True, False), + (ReqMock(link=Link("git+https://g.c/org/repo")), True, False), + (ReqMock(link=Link("https://g.c/dist.tgz")), True, False), + (ReqMock(link=Link("https://g.c/dist-2.0.4.tgz")), True, False), + ], +) +def test_should_cache( + req, disallow_binaries, expected +): + def check_binary_allowed(req): + return not disallow_binaries + + should_cache = wheel_builder.should_cache( + req, check_binary_allowed + ) + if not wheel_builder.should_build( + req, need_wheel=False, check_binary_allowed=check_binary_allowed + ): + # never cache if pip install (need_wheel=False) would not have built) + assert not should_cache + assert should_cache is expected + + +def test_should_cache_git_sha(script, tmpdir): + repo_path = _create_test_package(script, name="mypkg") + commit = script.run( + "git", "rev-parse", "HEAD", cwd=repo_path + ).stdout.strip() + # a link referencing a sha should be cached + url = "git+https://g.c/o/r@" + commit + "#egg=mypkg" + req = ReqMock(link=Link(url), source_dir=repo_path) + assert wheel_builder.should_cache(req, check_binary_allowed=lambda r: True) + # a link not referencing a sha should not be cached + url = "git+https://g.c/o/r@master#egg=mypkg" + req = ReqMock(link=Link(url), source_dir=repo_path) + assert not wheel_builder.should_cache(req, check_binary_allowed=lambda r: True) + + +def test_format_command_result__INFO(caplog): + caplog.set_level(logging.INFO) + actual = wheel_builder.format_command_result( + # Include an argument with a space to test argument quoting. + command_args=['arg1', 'second arg'], + command_output='output line 1\noutput line 2\n', + ) + assert actual.splitlines() == [ + "Command arguments: arg1 'second arg'", + 'Command output: [use --verbose to show]', + ] + + +@pytest.mark.parametrize('command_output', [ + # Test trailing newline. + 'output line 1\noutput line 2\n', + # Test no trailing newline. + 'output line 1\noutput line 2', +]) +def test_format_command_result__DEBUG(caplog, command_output): + caplog.set_level(logging.DEBUG) + actual = wheel_builder.format_command_result( + command_args=['arg1', 'arg2'], + command_output=command_output, + ) + assert actual.splitlines() == [ + "Command arguments: arg1 arg2", + 'Command output:', + 'output line 1', + 'output line 2', + '----------------------------------------', + ] + + +@pytest.mark.parametrize('log_level', ['DEBUG', 'INFO']) +def test_format_command_result__empty_output(caplog, log_level): + caplog.set_level(log_level) + actual = wheel_builder.format_command_result( + command_args=['arg1', 'arg2'], + command_output='', + ) + assert actual.splitlines() == [ + "Command arguments: arg1 arg2", + 'Command output: None', + ] + + +def test_python_tag(): + wheelnames = [ + 'simplewheel-1.0-py2.py3-none-any.whl', + 'simplewheel-1.0-py27-none-any.whl', + 'simplewheel-2.0-1-py2.py3-none-any.whl', + ] + newnames = [ + 'simplewheel-1.0-py37-none-any.whl', + 'simplewheel-1.0-py37-none-any.whl', + 'simplewheel-2.0-1-py37-none-any.whl', + ] + for name, expected in zip(wheelnames, newnames): + result = wheel_builder.replace_python_tag(name, 'py37') + assert result == expected + + +class TestWheelBuilder(object): + + def test_skip_building_wheels(self, caplog): + wb = wheel_builder.WheelBuilder( + preparer=Mock(), + wheel_cache=Mock(cache_dir=None), + ) + wb._build_one = mock_build_one = Mock() + + wheel_req = Mock(is_wheel=True, editable=False, constraint=False) + with caplog.at_level(logging.INFO): + wb.build([wheel_req]) + + assert "due to already being wheel" in caplog.text + assert mock_build_one.mock_calls == [] From b1f2f747f90b44f948eb485ecbab0590741ec6e9 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Mon, 4 Nov 2019 11:50:29 +0530 Subject: [PATCH 4/4] :art: a test --- tests/unit/test_wheel_builder.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_wheel_builder.py b/tests/unit/test_wheel_builder.py index da6749a3b66..8db535dadc2 100644 --- a/tests/unit/test_wheel_builder.py +++ b/tests/unit/test_wheel_builder.py @@ -119,14 +119,20 @@ def test_should_cache_git_sha(script, tmpdir): commit = script.run( "git", "rev-parse", "HEAD", cwd=repo_path ).stdout.strip() + # a link referencing a sha should be cached url = "git+https://g.c/o/r@" + commit + "#egg=mypkg" req = ReqMock(link=Link(url), source_dir=repo_path) - assert wheel_builder.should_cache(req, check_binary_allowed=lambda r: True) + assert wheel_builder.should_cache( + req, check_binary_allowed=lambda r: True, + ) + # a link not referencing a sha should not be cached url = "git+https://g.c/o/r@master#egg=mypkg" req = ReqMock(link=Link(url), source_dir=repo_path) - assert not wheel_builder.should_cache(req, check_binary_allowed=lambda r: True) + assert not wheel_builder.should_cache( + req, check_binary_allowed=lambda r: True, + ) def test_format_command_result__INFO(caplog):