diff --git a/build-backend/pex_build/__init__.py b/build-backend/pex_build/__init__.py index 05794947f..82724bcff 100644 --- a/build-backend/pex_build/__init__.py +++ b/build-backend/pex_build/__init__.py @@ -3,8 +3,9 @@ import os -# When running under MyPy, this will be set to True for us automatically; so we can use it as a typing module import -# guard to protect Python 2 imports of typing - which is not normally available in Python 2. +# When running under MyPy, this will be set to True for us automatically; so we can use it as a +# typing module import guard to protect Python 2 imports of typing - which is not normally available +# in Python 2. TYPE_CHECKING = False diff --git a/docs/index.rst b/docs/index.rst index c9a7c1dcd..e69a7c0a4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -31,5 +31,6 @@ Guide: whatispex buildingpex + scie recipes api/vars diff --git a/docs/scie.md b/docs/scie.md new file mode 100644 index 000000000..e1b0a6751 --- /dev/null +++ b/docs/scie.md @@ -0,0 +1,310 @@ +# PEX with included Python interpreter + +You can include a CPython interpreter in your PEX by adding `--scie eager` to your `pex` command +line. Instead of a traditional [PEP-441](https://peps.python.org/pep-0441/) PEX zip file, you'll +get a native executable that contains both a CPython interpreter and your PEX'd code. + +## Background + +Traditional PEX files allow you to build and ship a hermetic Python application environment to other +machines by just copying the PEX file there. There is a major caveat though: the machine must have a +Python interpreter installed and on the `PATH` that is compatible with the application for the PEX +to be able to run. Complicating things further, when executing the PEX file directly (e.g.: +`./my.pex`), the PEX's shebang must align with the names of Python binaries installed on the +machine. If the shebang is looking for `python` but the machine only has `python3` - even if the +underlying Python interpreter would be compatible - the operating system will fail to launch the PEX +file. This usually can be mitigated by using `--sh-boot` to alter the boot mechanism from Python to +a Posix-compatible shell at `/bin/sh`. Although almost all Posix-compatible systems have a +`/bin/sh` shell, that still leaves the problem of having a compatible Python pre-installed on that +system as well. + +When you add the `--scie eager` option to your `pex` command line, Pex uses the [science]( +https://science.scie.app/) [projects](https://github.com/a-scie/) to produce what is known as a +`scie` (pronounced like "ski") binary powered by the [Python Standalone Builds]( +https://github.com/indygreg/python-build-standalone) CPython distributions. The end product looks +and behaves like a traditional PEX except in two aspects: ++ The PEX scie file is larger than the equivalent PEX file since it contains a CPython distribution. ++ The PEX scie file is a native executable binary. + +For example, here we create a traditional PEX, a `--sh-boot` PEX and a PEX scie and examine the +resulting files: +```sh +# Create a cowsay PEX in each style: +:; pex cowsay -c cowsay --inject-args=-t --venv -o cowsay.pex +:; pex cowsay -c cowsay --inject-args=-t --venv --sh-boot -o cowsay-sh-boot.pex +:; pex cowsay -c cowsay --inject-args=-t --venv --scie eager -o cowsay + +# See what these files look like: +:; head -1 cowsay* +==> cowsay <== +ELF>��@�8 + +==> cowsay-sh-boot.pex <== +#!/bin/sh + +==> cowsay.pex <== +#!/usr/bin/env python3.11 + +:; file cowsay* +cowsay: ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), static-pie linked, BuildID[sha1]=f1f01ca2ad165fed27f8304d4b2fad02dcacdffe, stripped +cowsay-sh-boot.pex: POSIX shell script executable (binary data) +cowsay.pex: Zip archive data, made by v2.0 UNIX, extract using at least v2.0, last modified, last modified Sun, Jan 01 1980 00:00:00, uncompressed size 0, method=deflate + +:; ls -sSh1 cowsay* + 31M cowsay +728K cowsay-sh-boot.pex +724K cowsay.pex +``` + +The PEX scie can even be inspected like a traditional PEX file: +```sh +:; for pex in cowsay*; do echo $pex; unzip -l $pex | tail -7; echo; done +cowsay +warning [cowsay]: 31525759 extra bytes at beginning or within zipfile + (attempting to process anyway) + 7 1980-01-01 00:00 .deps/cowsay-6.1-py3-none-any.whl/cowsay-6.1.dist-info/top_level.txt + 873 1980-01-01 00:00 PEX-INFO + 7588 1980-01-01 00:00 __main__.py + 0 1980-01-01 00:00 __pex__/ + 7561 1980-01-01 00:00 __pex__/__init__.py +--------- ------- + 2634753 217 files + +cowsay-sh-boot.pex + 7 1980-01-01 00:00 .deps/cowsay-6.1-py3-none-any.whl/cowsay-6.1.dist-info/top_level.txt + 873 1980-01-01 00:00 PEX-INFO + 7588 1980-01-01 00:00 __main__.py + 0 1980-01-01 00:00 __pex__/ + 7561 1980-01-01 00:00 __pex__/__init__.py +--------- ------- + 2634753 217 files + +cowsay.pex + 7 1980-01-01 00:00 .deps/cowsay-6.1-py3-none-any.whl/cowsay-6.1.dist-info/top_level.txt + 873 1980-01-01 00:00 PEX-INFO + 7588 1980-01-01 00:00 __main__.py + 0 1980-01-01 00:00 __pex__/ + 7561 1980-01-01 00:00 __pex__/__init__.py +--------- ------- + 2634753 217 files +``` + +The performance of the PEX scie compares favorably, as you'd hope. +```sh +:; hyperfine -w2 './cowsay.pex Moo!' './cowsay-sh-boot.pex Moo!' './cowsay Moo!' +Benchmark 1: ./cowsay.pex Moo! + Time (mean ± σ): 99.2 ms ± 3.7 ms [User: 86.4 ms, System: 13.7 ms] + Range (min … max): 96.1 ms … 110.7 ms 30 runs + +Benchmark 2: ./cowsay-sh-boot.pex Moo! + Time (mean ± σ): 17.6 ms ± 0.3 ms [User: 15.2 ms, System: 2.2 ms] + Range (min … max): 16.8 ms … 18.7 ms 165 runs + +Benchmark 3: ./cowsay Moo! + Time (mean ± σ): 16.3 ms ± 0.4 ms [User: 13.4 ms, System: 2.7 ms] + Range (min … max): 15.5 ms … 18.6 ms 180 runs + +Summary + ./cowsay Moo! ran + 1.08 ± 0.03 times faster than ./cowsay-sh-boot.pex Moo! + 6.09 ± 0.27 times faster than ./cowsay.pex Moo! +``` + +But, unlike traditional PEXes, you can run the PEX scie anywhere: +```sh +# Traditional Python shebang boot: +:; env -i PATH= ./cowsay.pex Moo! +/usr/bin/env: 'python3.11': No such file or directory + +# A --sh-boot /bin/sh boot: +:; env -i PATH= ./cowsay-sh-boot.pex Moo! +Failed to find any of these python binaries on the PATH: +python3.11 +python3.13 +... +python3 +python2 +pypy3 +pypy2 +python +pypy +Either adjust your $PATH which is currently: + +Or else install an appropriate Python that provides one of the binaries in this list. + +# A hermetic scie boot: +:; env -i PATH= ./cowsay Moo! + ____ +| Moo! | + ==== + \ + \ + ^__^ + (oo)\_______ + (__)\ )\/\ + ||----w | + || || +``` + +## Lazy scies + +Specifying `--scie eager` includes a full CPython distribution in your PEX scie. If you ship more +than one PEX scie to a machine using the same Python version, this can be wasteful in transfer +bandwidth and disk space. If your deployment machines have internet access, you can specify +`--scie lazy` and the Python distribution will then be fetched from the internet, but only if +needed. If a PEX scie (whether eager or lazy) using the same Python distribution has run previously +on the machine, the fetch will be skipped and the local distribution used instead. This lazy +fetching feature is powered by the [`ptex` binary](https://github.com/a-scie/ptex) from the science +projects, and you can read more there if you're curious. + +If your network access is restricted, you can re-point the download location of the Python +distribution by ensuring the machine has the environment variable `PEX_BOOTSTRAP_URLS` set to the +path of a json file containing the new Python distribution URL. That file should look like: +```json +{ + "ptex": { + "cpython-3.12.4+20240713-x86_64-unknown-linux-gnu-install_only.tar.gz": "" + } +} +``` + +You can run `SCIE=inspect | jq '{ptex:.ptex}'` to get a starter file with the +correct default entries for your scie. You can then just edit the URLs. URLs of the form +`file://` are accepted. The only restriction for any custom URL is that it returns a +bytewise-identical copy of the Python distribution pointed to by the original URL. If the file +content hash does not match, the PEX scie will fail to boot. For example: +```sh +# Build a lazy PEX scie: +:; pex cowsay -c cowsay --inject-args=-t --scie lazy -o cowsay + +# Generate a starter file for the alternate URLs: +:; SCIE=inspect ./cowsay | jq '{ptex:.ptex}' > starter.json + +# Copy to pythons.json and edit it to point to a file that does not contain the original Python +# distribution: +:; jq 'first(.ptex | .[]) = "file:///etc/hosts"' starter.json > pythons.json +:; diff -u --label starter.json starter.json --label pythons.json pythons.json +--- starter.json ++++ pythons.json +@@ -1,5 +1,5 @@ + { + "ptex": { +- "cpython-3.11.9+20240726-x86_64-unknown-linux-gnu-install_only.tar.gz": "https://github.com/indygreg/python-build-standalone/releases/download/20240726/cpython-3.11.9%2B20240726-x86_64-unknown-linux-gnu-install_only.tar.gz" ++ "cpython-3.11.9+20240726-x86_64-unknown-linux-gnu-install_only.tar.gz": "file:///etc/hosts" + } + } + +# Clear the scie cache and try to run the lazy PEX scie: +:; rm -rf ~/.cache/nce +:; PEX_BOOTSTRAP_URLS=pythons.json ./cowsay Moo! +Error: Failed to establish atomic directory /home/jsirois/.cache/nce/0770bcb55edb6b8089bcc8cbe556d3f737f4a5e3a5cbc45e716206de554c0df9/locks/configure-ce4ae7966f25868830154e6fa8d56b0dd6e09cd2902ab837a4af55d51dc84d92. Population of work directory failed: Failed to establish atomic directory /home/jsirois/.cache/nce/f6e955dc9ddfcad74e77abe6f439dac48ebca14b101ed7c85a5bf3206ed2c53d/cpython-3.11.9+20240726-x86_64-unknown-linux-gnu-install_only.tar.gz. Population of work directory failed: The tar.gz destination /home/jsirois/.cache/nce/f6e955dc9ddfcad74e77abe6f439dac48ebca14b101ed7c85a5bf3206ed2c53d/cpython-3.11.9+20240726-x86_64-unknown-linux-gnu-install_only.tar.gz of size 410 had unexpected hash: 16183c427758316754b82e4d48d63c265ee46ec5ae96a40d9092e694dd5f77ab + +The ./cowsay scie contains no alternate boot commands. +``` + +Here we see the error `.../cpython-3.11.9+20240726-x86_64-unknown-linux-gnu-install_only.tar.gz of +size 410 had unexpected hash: 16183c427758316754b82e4d48d63c265ee46ec5ae96a40d9092e694dd5f77ab`. We +can correct this by re-pointing to a valid file: +```sh +# Download the expected dstribution: +:; curl -fL https://github.com/indygreg/python-build-standalone/releases/download/20240726/cpython-3.11.9%2B20240726-x86_64-unknown-linux-gnu-install_only.tar.gz > /tmp/example + +# Re-point to the now valid copy of the expected Python distribution: +:; jq 'first(.ptex | .[]) = "file:///tmp/example"' starter.json > pythons.json +:; diff -u --label starter.json starter.json --label pythons.json pythons.json +--- starter.json ++++ pythons.json +@@ -1,5 +1,5 @@ + { + "ptex": { +- "cpython-3.11.9+20240726-x86_64-unknown-linux-gnu-install_only.tar.gz": "https://github.com/indygreg/python-build-standalone/releases/download/20240726/cpython-3.11.9%2B20240726-x86_64-unknown-linux-gnu-install_only.tar.gz" ++ "cpython-3.11.9+20240726-x86_64-unknown-linux-gnu-install_only.tar.gz": "file:///tmp/example" + } + } + + # And lazy bootstrapping from the Python distribution in /tmp/example now works: + :; PEX_BOOTSTRAP_URLS=pythons.json ./cowsay Moo! + ____ +| Moo! | + ==== + \ + \ + ^__^ + (oo)\_______ + (__)\ )\/\ + ||----w | + || || +``` + +## BusyBox scies + +Scies support multiple commands, but, by default, `pex --scie ...` generates a PEX scie that always +executes the entry point you configured for your PEX. You can, of course, run the scie using +`PEX_INTERPRETER`, `PEX_MODULE` and `PEX_SCRIPT` to modify the entry point just like you can with +a normal PEX, but sometimes it can be convenient to seal in a small set of commands you wish to use +for easier access. You do this by adding `--scie-busybox` to your `pex` command line with a list of +entry points you wish to expose. These entry points can be arbitrary modules or functions within a +module. They can also be console scripts from distributions in the PEX. The BusyBox entry point +specifications accepted are detailed below: + +| Form | Example | Effect | +|--------------------------|-------------------------|-----------------------------------------------------------------------------------------------------------------------------------| +| `=` | `json=json.tool` | Add a `json` command that invokes the `json.tool` module. | +| `=:` | `uuid=uuid:uuid4` | Add a `uuid` command that invokes the `uuid4` function in the `uuid` module. | +| `` | `cowsay` | Add a `cowsay` command for the `cowsay` console script found anywhere in the PEX distributions. | +| `!` | `!cowsay` | Exclude all `cowsay` console scripts found in the PEX distributions (For use with `@` and `@`). | +| `@` | `ansible@ansible-core` | Add an `ansible` command for the `ansible` console script found in the `ansible-core` distributions in the PEX. | +| `!@` | `!ansible@ansible-core` | Exclude the `ansible` console script found in the `ansible-core` distributions in the PEX (For use with `@` and `@ansible-core`). | +| `@` | `@ansible-core` | Add a command for all console scripts found in the `ansible-core` distributions in the PEX. | +| `!@` | `!@ansible-core` | Exclude all console scripts found in the `ansible-core` distributions in the PEX (For use with `@`). | +| `@` | `@` | Add a command for all console scripts found in all project distributions in the PEX. | + +For example, to build a BusyBox with tools both useful and frivolous: +```sh +# Build a PEX scie BusyBox with 3 commands: +:; pex cowsay -c cowsay --inject-args=-t --scie lazy --scie-busybox json=json.tool,uuid=uuid:uuid4,cowsay -otools + +# Run the BusyBox to discover what commands it contains: +:; ./tools +Error: Could not determine which command to run. + +Please select from the following boot commands: + +cowsay +json +uuid + +You can select a boot command by setting the SCIE_BOOT environment variable or else by passing it as the 1st argument. + +# Use the tools: +:; ./tools uuid +16269f0f-76f5-4374-9da2-e0e873c40835 +:; ./tools uuid +00e2584d-a5d3-40d9-9217-9a873fe7cac8 +:; echo '{"Hello":"World!"}' | ./tools json +{ + "Hello": "World!" +} + +# Install the tools on the $PATH individually for convenient access: +:; mkdir /tmp/bin +:; export PATH=/tmp/bin:$PATH +:; SCIE=install ./tools /tmp/bin +:; ls -1 /tmp/bin/ +cowsay +json +uuid +:; which cowsay +/tmp/bin/cowsay +:; cowsay Moo! + ____ +| Moo! | + ==== + \ + \ + ^__^ + (oo)\_______ + (__)\ )\/\ + ||----w | + || || +``` \ No newline at end of file diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 2e30eea67..5447abe94 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -67,7 +67,19 @@ if TYPE_CHECKING: from argparse import Namespace - from typing import Dict, Iterable, Iterator, List, NoReturn, Optional, Set, Text, Tuple, Union + from typing import ( + Dict, + Iterable, + Iterator, + List, + NoReturn, + Optional, + Sequence, + Set, + Text, + Tuple, + Union, + ) import attr # vendor:skip @@ -728,8 +740,44 @@ def configure_clp_sources(parser): ) +@attr.s(frozen=True) +class PositionalArgumentFromFileParser(object): + parser = attr.ib() # type: ArgumentParser + positional_option_name = attr.ib() # type: str + + def parse_args( + self, + args=None, # type: Optional[Sequence[str]] + namespace=None, # type: Optional[Namespace] + ): + # type: (...) -> Namespace + + options = self.parser.parse_args(args=args, namespace=namespace) + + extra_args = [] + positionals = [] + for positional in getattr(options, self.positional_option_name): + if positional.startswith("@"): + with open(positional[1:]) as fp: + extra_args.extend(fp.read().splitlines()) + else: + positionals.append(positional) + setattr(options, self.positional_option_name, positionals) + + if extra_args: + extra_options = self.parser.parse_args(extra_args) + for name, value in vars(extra_options).items(): + existing_value = getattr(options, name, None) + if isinstance(existing_value, list) and value: + existing_value.extend(value) + elif existing_value is None or value is not None: + setattr(options, name, value) + + return options + + def configure_clp(): - # type: () -> ArgumentParser + # type: () -> PositionalArgumentFromFileParser usage = ( "%(prog)s [-o OUTPUT.PEX] [options] [-- arg1 arg2 ...]\n\n" "%(prog)s builds a PEX (Python Executable) file based on the given specifications: " @@ -739,12 +787,7 @@ def configure_clp(): "with an @ symbol. These files must contain one argument per line." ) - parser = ArgumentParser( - usage=usage, - formatter_class=ArgumentDefaultsHelpFormatter, - fromfile_prefix_chars="@", - ) - + parser = ArgumentParser(usage=usage, formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("-V", "--version", action="version", version=__version__) configure_clp_pex_resolution(parser) @@ -831,7 +874,7 @@ def configure_clp(): ), ) - return parser + return PositionalArgumentFromFileParser(parser, positional_option_name="requirements") def _iter_directory_sources(directories): diff --git a/pex/dist_metadata.py b/pex/dist_metadata.py index daee47054..690058724 100644 --- a/pex/dist_metadata.py +++ b/pex/dist_metadata.py @@ -22,7 +22,7 @@ from pex import pex_warnings, specifier_sets from pex.common import open_zip, pluralize -from pex.compatibility import to_unicode +from pex.compatibility import PY2, to_unicode from pex.enum import Enum from pex.pep_440 import Version from pex.pep_503 import ProjectName @@ -912,23 +912,26 @@ def of(cls, location): class Distribution(object): @staticmethod def _read_metadata_lines(metadata_bytes): - # type: (bytes) -> Iterator[Text] + # type: (bytes) -> Iterator[str] for line in metadata_bytes.splitlines(): # This is pkg_resources.IMetadataProvider.get_metadata_lines behavior, which our # code expects. - normalized = line.decode("utf-8").strip() + if PY2: + normalized = line.strip() + else: + normalized = line.decode("utf-8").strip() if normalized and not normalized.startswith("#"): yield normalized @classmethod def parse_entry_map(cls, entry_points_contents): - # type: (bytes) -> Dict[Text, Dict[Text, EntryPoint]] + # type: (bytes) -> Dict[str, Dict[str, NamedEntryPoint]] # This file format is defined here: # https://packaging.python.org/en/latest/specifications/entry-points/#file-format - entry_map = defaultdict(dict) # type: DefaultDict[Text, Dict[Text, EntryPoint]] - group = None # type: Optional[Text] + entry_map = defaultdict(dict) # type: DefaultDict[str, Dict[str, NamedEntryPoint]] + group = None # type: Optional[str] for index, line in enumerate(cls._read_metadata_lines(entry_points_contents), start=1): if line.startswith("[") and line.endswith("]"): group = line[1:-1] @@ -938,7 +941,7 @@ def parse_entry_map(cls, entry_points_contents): "group on line {index}: {line}".format(index=index, line=line) ) else: - entry_point = EntryPoint.parse(line) + entry_point = NamedEntryPoint.parse(line) entry_map[group][entry_point.name] = entry_point return entry_map @@ -1008,7 +1011,7 @@ def iter_metadata_lines(self, name): yield line def get_entry_map(self): - # type: () -> Dict[Text, Dict[Text, EntryPoint]] + # type: () -> Dict[str, Dict[str, NamedEntryPoint]] entry_points_metadata_file = self._read_metadata_file("entry_points.txt") if entry_points_metadata_file is None: return defaultdict(dict) @@ -1022,33 +1025,7 @@ def __str__(self): @attr.s(frozen=True) -class EntryPoint(object): - @classmethod - def parse(cls, spec): - # type: (Text) -> EntryPoint - - # This file format is defined here: - # https://packaging.python.org/en/latest/specifications/entry-points/#file-format - - components = spec.split("=") - if len(components) != 2: - raise ValueError("Invalid entry point specification: {spec}.".format(spec=spec)) - - name, value = components - # N.B.: Python identifiers must be ascii. - module, sep, attrs = str(value).strip().partition(":") - if sep and not attrs: - raise ValueError("Invalid entry point specification: {spec}.".format(spec=spec)) - - entry_point_name = name.strip() - if sep: - return CallableEntryPoint( - name=entry_point_name, module=module, attrs=tuple(attrs.split(".")) - ) - - return cls(name=entry_point_name, module=module) - - name = attr.ib() # type: Text +class ModuleEntryPoint(object): module = attr.ib() # type: str def __str__(self): @@ -1057,10 +1034,11 @@ def __str__(self): @attr.s(frozen=True) -class CallableEntryPoint(EntryPoint): - _attrs = attr.ib() # type: Tuple[str, ...] +class CallableEntryPoint(object): + module = attr.ib() # type: str + attrs = attr.ib() # type: Tuple[str, ...] - @_attrs.validator + @attrs.validator def _validate_attrs(self, _, value): if not value: raise ValueError("A callable entry point must select a callable item from the module.") @@ -1069,17 +1047,57 @@ def resolve(self): # type: () -> Callable[[], Any] module = importlib.import_module(self.module) try: - return cast("Callable[[], Any]", functools.reduce(getattr, self._attrs, module)) + return cast("Callable[[], Any]", functools.reduce(getattr, self.attrs, module)) except AttributeError as e: raise ImportError( "Could not resolve {attrs} in {module}: {err}".format( - attrs=".".join(self._attrs), module=module, err=e + attrs=".".join(self.attrs), module=module, err=e ) ) def __str__(self): # type: () -> str - return "{module}:{attrs}".format(module=self.module, attrs=".".join(self._attrs)) + return "{module}:{attrs}".format(module=self.module, attrs=".".join(self.attrs)) + + +def parse_entry_point(value): + # type: (str) -> Union[ModuleEntryPoint, CallableEntryPoint] + + # The format of the value of an entry point (minus the name part), is specified here: + # https://packaging.python.org/en/latest/specifications/entry-points/#file-format + + # N.B.: Python identifiers must be ascii. + module, sep, attrs = str(value).strip().partition(":") + if sep: + if not attrs: + raise ValueError("Invalid entry point specification: {value}.".format(value=value)) + return CallableEntryPoint(module=module, attrs=tuple(attrs.split("."))) + return ModuleEntryPoint(module=module) + + +@attr.s(frozen=True) +class NamedEntryPoint(object): + @classmethod + def parse(cls, spec): + # type: (str) -> NamedEntryPoint + + # This file format is defined here: + # https://packaging.python.org/en/latest/specifications/entry-points/#file-format + + components = spec.split("=") + if len(components) != 2: + raise ValueError("Invalid entry point specification: {spec}.".format(spec=spec)) + + name, value = components + entry_point = parse_entry_point(value) + return cls(name=name.strip(), entry_point=entry_point) + + name = attr.ib() # type: str + entry_point = attr.ib() # type: Union[ModuleEntryPoint, CallableEntryPoint] + + def __str__(self): + # type: () -> str + return "{name}={entry_point}".format(name=self.name, entry_point=self.entry_point) def find_distribution( diff --git a/pex/finders.py b/pex/finders.py index dafd46d99..d2a239de3 100644 --- a/pex/finders.py +++ b/pex/finders.py @@ -7,14 +7,20 @@ import os from pex.common import is_python_script, open_zip, safe_mkdtemp -from pex.dist_metadata import Distribution, DistributionType, EntryPoint +from pex.dist_metadata import ( + CallableEntryPoint, + Distribution, + DistributionType, + ModuleEntryPoint, + NamedEntryPoint, +) from pex.pep_376 import InstalledWheel from pex.pep_503 import ProjectName from pex.typing import TYPE_CHECKING, cast from pex.wheel import Wheel if TYPE_CHECKING: - from typing import Dict, Iterable, Optional + from typing import Dict, Iterable, Optional, Union import attr # vendor:skip else: @@ -97,7 +103,14 @@ def get_script_from_distributions( @attr.s(frozen=True) class DistributionEntryPoint(object): dist = attr.ib() # type: Distribution - entry_point = attr.ib() # type: EntryPoint + name = attr.ib() # type: str + entry_point = attr.ib() # type: Union[ModuleEntryPoint, CallableEntryPoint] + + def render_description(self): + # type: () -> str + return "console script {name} = {entry_point} in {dist}".format( + name=self.name, entry_point=self.entry_point, dist=self.dist + ) def get_entry_point_from_console_script( @@ -109,14 +122,16 @@ def get_entry_point_from_console_script( # duplicate console script IFF the distribution is platform-specific and this is a # multi-platform pex. def get_entrypoint(dist): - # type: (Distribution) -> Optional[EntryPoint] + # type: (Distribution) -> Optional[NamedEntryPoint] return dist.get_entry_map().get("console_scripts", {}).get(script) entries = {} # type: Dict[ProjectName, DistributionEntryPoint] for dist in dists: - entry_point = get_entrypoint(dist) - if entry_point is not None: - entries[dist.metadata.project_name] = DistributionEntryPoint(dist, entry_point) + named_entry_point = get_entrypoint(dist) + if named_entry_point is not None: + entries[dist.metadata.project_name] = DistributionEntryPoint( + dist=dist, name=named_entry_point.name, entry_point=named_entry_point.entry_point + ) if len(entries) > 1: raise RuntimeError( diff --git a/pex/pep_427.py b/pex/pep_427.py index da8cd68b5..3c0fb2fea 100644 --- a/pex/pep_427.py +++ b/pex/pep_427.py @@ -16,7 +16,7 @@ from pex import pex_warnings from pex.common import chmod_plus_x, is_pyc_file, iter_copytree, open_zip, safe_open, touch from pex.compatibility import commonpath, get_stdout_bytes_buffer -from pex.dist_metadata import Distribution, ProjectNameAndVersion +from pex.dist_metadata import CallableEntryPoint, Distribution, ProjectNameAndVersion from pex.enum import Enum from pex.interpreter import PythonInterpreter from pex.pep_376 import InstalledFile, InstalledWheel, Record @@ -282,34 +282,49 @@ def record_files( dist = Distribution(location=dest, metadata=wheel.dist_metadata()) entry_points = dist.get_entry_map() - for entry_point in itertools.chain.from_iterable( + for named_entry_point in itertools.chain.from_iterable( entry_points.get(key, {}).values() for key in ("console_scripts", "gui_scripts") ): - script_abspath = os.path.join(install_paths.scripts, entry_point.name) - with safe_open(script_abspath, "w") as fp: - fp.write( - dedent( - """\ - {shebang} - # -*- coding: utf-8 -*- - import importlib - import sys - - object_ref = "{object_ref}" - modname, qualname_separator, qualname = object_ref.partition(':') - entry_point = importlib.import_module(modname) - if qualname_separator: - for attr in qualname.split('.'): - entry_point = getattr(entry_point, attr) - - if __name__ == '__main__': - sys.exit(entry_point()) - """ - ).format( - shebang=interpreter.shebang() if interpreter else "#!python", - object_ref=str(entry_point), - ) + entry_point = named_entry_point.entry_point + if isinstance(entry_point, CallableEntryPoint): + script = dedent( + """\ + {shebang} + # -*- coding: utf-8 -*- + import importlib + import sys + + entry_point = importlib.import_module({modname!r}) + for attr in {attrs!r}: + entry_point = getattr(entry_point, attr) + + if __name__ == "__main__": + sys.exit(entry_point()) + """ + ).format( + shebang=interpreter.shebang() if interpreter else "#!python", + modname=entry_point.module, + attrs=entry_point.attrs, ) + else: + script = dedent( + """\ + {shebang} + # -*- coding: utf-8 -*- + import runpy + import sys + + if __name__ == "__main__": + runpy.run_module({modname!r}, run_name="__main__", alter_sys=True) + sys.exit(0) + """ + ).format( + shebang=interpreter.shebang() if interpreter else "#!python", + modname=entry_point.module, + ) + script_abspath = os.path.join(install_paths.scripts, named_entry_point.name) + with safe_open(script_abspath, "w") as fp: + fp.write(script) chmod_plus_x(fp.name) installed_files.append( InstalledWheel.create_installed_file(path=script_abspath, dest_dir=dest) diff --git a/pex/pex.py b/pex/pex.py index 9c21d14fe..1d56110d5 100644 --- a/pex/pex.py +++ b/pex/pex.py @@ -14,10 +14,11 @@ from pex import bootstrap, pex_warnings from pex.bootstrap import Bootstrap from pex.common import die -from pex.dist_metadata import CallableEntryPoint, Distribution, EntryPoint +from pex.dist_metadata import CallableEntryPoint, Distribution, ModuleEntryPoint, parse_entry_point from pex.environment import PEXEnvironment from pex.executor import Executor from pex.finders import get_entry_point_from_console_script, get_script_from_distributions +from pex.fingerprinted_distribution import FingerprintedDistribution from pex.inherit_path import InheritPath from pex.interpreter import PythonIdentity, PythonInterpreter from pex.layout import Layout @@ -223,6 +224,19 @@ def resolve(self): seen.add(dist) yield dist + def iter_distributions(self, result_type_wheel_file=False): + # type: (bool) -> Iterator[FingerprintedDistribution] + """Iterates all distributions loadable from this PEX.""" + seen = set() + for env in self._loaded_envs: + for dist in env.iter_distributions(result_type_wheel_file=result_type_wheel_file): + # N.B.: Since there can be more than one PEX env on the PEX_PATH we take care to + # de-dup distributions they have in common. + if dist in seen: + continue + seen.add(dist) + yield dist + def _activate(self): # type: () -> Iterable[Distribution] @@ -594,18 +608,14 @@ def _execute(self): if self._pex_info_overrides.script: return self.execute_script(self._pex_info_overrides.script) if self._pex_info_overrides.entry_point: - return self.execute_entry( - EntryPoint.parse("run = {}".format(self._pex_info_overrides.entry_point)) - ) + return self.execute_entry(parse_entry_point(self._pex_info_overrides.entry_point)) sys.argv[1:1] = list(self._pex_info.inject_args) if self._pex_info.script: return self.execute_script(self._pex_info.script) else: - return self.execute_entry( - EntryPoint.parse("run = {}".format(self._pex_info.entry_point)) - ) + return self.execute_entry(parse_entry_point(self._pex_info.entry_point)) def execute_interpreter(self): # type: () -> Any @@ -743,8 +753,8 @@ def execute_script(self, script_name): dist_entry_point = get_entry_point_from_console_script(script_name, dists) if dist_entry_point: TRACER.log( - "Found console_script {!r} in {!r}.".format( - dist_entry_point.entry_point, dist_entry_point.dist + "Found {console_script}.".format( + console_script=dist_entry_point.render_description() ) ) return self.execute_entry(dist_entry_point.entry_point) @@ -803,7 +813,7 @@ def execute_ast( return None def execute_entry(self, entry_point): - # type: (EntryPoint) -> Any + # type: (Union[ModuleEntryPoint, CallableEntryPoint]) -> Any if isinstance(entry_point, CallableEntryPoint): return self.execute_entry_point(entry_point) diff --git a/pex/pex_builder.py b/pex/pex_builder.py index bf6d5c5c7..d7f1eac89 100644 --- a/pex/pex_builder.py +++ b/pex/pex_builder.py @@ -300,8 +300,8 @@ def set_script(self, script): if dist_entry_point: self.set_entry_point(str(dist_entry_point.entry_point)) TRACER.log( - "Set entrypoint to console_script {!r} in {!r}".format( - dist_entry_point.entry_point, dist_entry_point.dist + "Set entrypoint to {console_script}".format( + console_script=dist_entry_point.render_description() ) ) return diff --git a/pex/scie/__init__.py b/pex/scie/__init__.py index 3e168c9d8..b257f57ee 100644 --- a/pex/scie/__init__.py +++ b/pex/scie/__init__.py @@ -7,11 +7,14 @@ from argparse import Namespace, _ActionsContainer from pex.compatibility import urlparse +from pex.dist_metadata import NamedEntryPoint from pex.fetcher import URLFetcher from pex.orderedset import OrderedSet from pex.pep_440 import Version from pex.scie import science from pex.scie.model import ( + BusyBoxEntryPoints, + ConsoleScriptsManifest, ScieConfiguration, ScieInfo, ScieOptions, @@ -24,8 +27,7 @@ from pex.variables import ENV, Variables if TYPE_CHECKING: - from typing import Iterator, Optional, Tuple, Union - + from typing import Iterator, List, Optional, Text, Tuple, Union __all__ = ( "ScieConfiguration", @@ -72,6 +74,35 @@ def register_options(parser): "https://science.scie.app.".format(lazy=ScieStyle.LAZY, eager=ScieStyle.EAGER) ), ) + parser.add_argument( + "--scie-busybox", + dest="scie_busybox", + type=str, + default=[], + action="append", + help=( + "Make the PEX scie a BusyBox over the specified entry points. The entry points can " + "either be console scripts or entry point specifiers. To select all console scripts in " + "all distributions contained in the PEX, use `@`. To just pick all the console scripts " + "from a particular project name's distributions in the PEX, use `@`; " + "e.g.: `@ansible-core`. To exclude all the console scripts from a project, prefix with " + "a `!`; e.g.: `@,!@ansible-core` selects all console scripts except those provided by " + "the `ansible-core` project. To select an individual console script, just use its name " + "or prefix the name with `!` to exclude that individual console script. To specify an " + "arbitrary entry point in a module contained within one of the distributions in the " + "PEX, use a string of the form `=(:)`; e.g.: " + "'run-baz=foo.bar:baz' to execute the `baz` function in the `foo.bar` module as the " + "entry point named `run-baz`. Multiple entry points can be specified at once using a " + "comma-separated list or the option can be specified multiple times. A BusyBox scie " + "has no default entrypoint; instead, when run, it inspects argv0; if that matches one " + "of its embedded entry points, it runs that entry point; if not, it lists all " + "available entrypoints for you to pick from. To run a given entry point, you specify " + "it as the first argument and all other arguments after that are forwarded to that " + "entry point. BusyBox PEX scies allow you to install all their contained entry points " + "into a given directory. For more information, run `SCIE=help ` and " + "review the `install` command help." + ), + ) parser.add_argument( "--scie-platform", dest="scie_platforms", @@ -137,9 +168,14 @@ def register_options(parser): def render_options(options): - # type: (ScieOptions) -> str + # type: (ScieOptions) -> Text - args = ["--scie", str(options.style)] + args = ["--scie", str(options.style)] # type: List[Text] + if options.busybox_entrypoints: + args.append("--scie-busybox") + entrypoints = list(options.busybox_entrypoints.console_scripts_manifest.iter_specs()) + entrypoints.extend(map(str, options.busybox_entrypoints.ad_hoc_entry_points)) + args.append(",".join(entrypoints)) for platform in options.platforms: args.append("--scie-platform") args.append(str(platform)) @@ -161,6 +197,37 @@ def extract_options(options): if not options.scie_style: return None + entry_points = None + if options.scie_busybox: + eps = [] # type: List[str] + for value in options.scie_busybox: + eps.extend(ep.strip() for ep in value.split(",")) + + console_scripts_manifest = ConsoleScriptsManifest() + ad_hoc_entry_points = [] # type: List[NamedEntryPoint] + bad_entry_points = [] # type: List[str] + for ep in eps: + csm = ConsoleScriptsManifest.try_parse(ep) + if csm: + console_scripts_manifest = console_scripts_manifest.merge(csm) + else: + try: + ad_hoc_entry_point = NamedEntryPoint.parse(ep) + except ValueError: + bad_entry_points.append(ep) + else: + ad_hoc_entry_points.append(ad_hoc_entry_point) + + if bad_entry_points: + raise ValueError( + "The following --scie-busybox entry point specifications were not understood:\n" + "{bad_entry_points}".format(bad_entry_points="\n".join(bad_entry_points)) + ) + entry_points = BusyBoxEntryPoints( + console_scripts_manifest=console_scripts_manifest, + ad_hoc_entry_points=tuple(ad_hoc_entry_points), + ) + python_version = None # type: Optional[Union[Tuple[int, int], Tuple[int, int, int]]] if options.scie_python_version: if ( @@ -195,6 +262,7 @@ def extract_options(options): return ScieOptions( style=options.scie_style, + busybox_entrypoints=entry_points, platforms=tuple(OrderedSet(options.scie_platforms)), pbs_release=options.scie_pbs_release, python_version=python_version, diff --git a/pex/scie/configure-binding.py b/pex/scie/configure-binding.py index 8f3b1c049..0f4b74081 100644 --- a/pex/scie/configure-binding.py +++ b/pex/scie/configure-binding.py @@ -5,26 +5,57 @@ import os import sys +from argparse import ArgumentParser + +# When running under MyPy, this will be set to True for us automatically; so we can use it as a +# typing module import guard to protect Python 2 imports of typing - which is not normally available +# in Python 2. +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import Optional def write_bindings( env_file, # type: str - installed_pex_dir, # type: str + pex, # type: str + venv_dir=None, # type: Optional[str] ): # type: (...) -> None + with open(env_file, "a") as fp: print("PYTHON=" + sys.executable, file=fp) - print("PEX=" + os.path.realpath(os.path.join(installed_pex_dir, "__main__.py")), file=fp) + print("PEX=" + pex, file=fp) + if venv_dir: + print("VENV_BIN_DIR_PLUS_SEP=" + os.path.join(venv_dir, "bin") + os.path.sep, file=fp) if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument( + "--installed-pex-dir", + help=( + "The final resting install directory of the PEX if it is a zipapp PEX. If left unset, " + "this indicates the PEX is a venv PEX whose resting venv directory should be " + "determined dynamically." + ), + ) + options = parser.parse_args() + + if options.installed_pex_dir: + pex = os.path.realpath(options.installed_pex_dir) + venv_dir = None + else: + venv_dir = os.path.realpath( + # N.B.: In practice, VIRTUAL_ENV should always be set by the PEX venv __main__.py + # script. + os.environ.get("VIRTUAL_ENV", os.path.dirname(os.path.dirname(sys.executable))) + ) + pex = venv_dir + write_bindings( env_file=os.environ["SCIE_BINDING_ENV"], - installed_pex_dir=( - # The zipapp case: - os.environ["_PEX_SCIE_INSTALLED_PEX_DIR"] - # The --venv case: - or os.environ.get("VIRTUAL_ENV", os.path.dirname(os.path.dirname(sys.executable))) - ), + pex=pex, + venv_dir=venv_dir, ) sys.exit(0) diff --git a/pex/scie/model.py b/pex/scie/model.py index 07e46c6a8..53ef3a8de 100644 --- a/pex/scie/model.py +++ b/pex/scie/model.py @@ -6,16 +6,21 @@ import itertools import os import platform -from collections import defaultdict +from collections import OrderedDict, defaultdict +from pex.common import pluralize +from pex.dist_metadata import Distribution, NamedEntryPoint from pex.enum import Enum +from pex.finders import get_entry_point_from_console_script +from pex.pep_503 import ProjectName +from pex.pex import PEX from pex.platforms import Platform from pex.targets import Targets from pex.third_party.packaging import tags # noqa from pex.typing import TYPE_CHECKING, cast if TYPE_CHECKING: - from typing import DefaultDict, Iterable, Optional, Set, Tuple, Union + from typing import DefaultDict, Iterable, Iterator, List, Optional, Set, Text, Tuple, Union import attr # vendor:skip else: @@ -30,6 +35,186 @@ class Value(Enum.Value): EAGER = Value("eager") +@attr.s(frozen=True) +class ConsoleScript(object): + name = attr.ib() # type: str + project_name = attr.ib(default=None) # type: Optional[ProjectName] + + def __str__(self): + # type: () -> str + return ( + "{script_name}@{project_name}".format( + script_name=self.name, project_name=self.project_name.raw + ) + if self.project_name + else self.name + ) + + +@attr.s(frozen=True) +class ConsoleScriptsManifest(object): + @classmethod + def try_parse(cls, value): + # type: (str) -> Optional[ConsoleScriptsManifest] + script_name, sep, project_name = value.partition("@") + if not sep and all(script_name.partition("=")): + # This is an ad-hoc entrypoint; not a console script specification. + return None + + if sep and not script_name and not project_name: + return cls(add_all=True) + elif script_name and script_name != "!": + if script_name.startswith("!"): + return cls( + remove_individual=( + ConsoleScript( + name=script_name[1:], + project_name=ProjectName(project_name) if project_name else None, + ), + ) + ) + else: + return cls( + add_individual=( + ConsoleScript( + name=script_name, + project_name=ProjectName(project_name) if project_name else None, + ), + ) + ) + elif sep and project_name and (not script_name or script_name == "!"): + if script_name == "!": + return cls(remove_project=(ProjectName(project_name),)) + else: + return cls(add_project=(ProjectName(project_name),)) + else: + return None + + add_individual = attr.ib(default=()) # type: Tuple[ConsoleScript, ...] + remove_individual = attr.ib(default=()) # type: Tuple[ConsoleScript, ...] + add_project = attr.ib(default=()) # type: Tuple[ProjectName, ...] + remove_project = attr.ib(default=()) # type: Tuple[ProjectName, ...] + add_all = attr.ib(default=False) # type: bool + + def iter_specs(self): + # type: () -> Iterator[Text] + if self.add_all: + yield "@" + for project_name in self.add_project: + yield "@{dist}".format(dist=project_name.raw) + for script in self.add_individual: + yield str(script) + for project_name in self.remove_project: + yield "!@{dist}".format(dist=project_name.raw) + for script in self.remove_individual: + yield "!{script}".format(script=script) + + def merge(self, other): + # type: (ConsoleScriptsManifest) -> ConsoleScriptsManifest + return ConsoleScriptsManifest( + add_individual=self.add_individual + other.add_individual, + remove_individual=self.remove_individual + other.remove_individual, + add_project=self.add_project + other.add_project, + remove_project=self.remove_project + other.remove_project, + add_all=self.add_all or other.add_all, + ) + + def collect(self, pex): + # type: (PEX) -> Iterable[NamedEntryPoint] + + dists = tuple( + fingerprinted_distribution.distribution + for fingerprinted_distribution in pex.iter_distributions() + ) + + console_scripts = OrderedDict( + (console_script, None) for console_script in self.add_individual + ) # type: OrderedDict[ConsoleScript, Optional[NamedEntryPoint]] + + if self.add_project or self.remove_project or self.add_all: + for dist in dists: + remove = dist.metadata.project_name in self.remove_project + add = self.add_all or dist.metadata.project_name in self.add_project + if not remove and not add: + continue + for name, named_entry_point in ( + dist.get_entry_map().get("console_scripts", {}).items() + ): + if remove: + console_scripts.pop(ConsoleScript(name=name), None) + console_scripts.pop( + ConsoleScript(name=name, project_name=dist.metadata.project_name), None + ) + else: + console_scripts[ + ConsoleScript(name=name, project_name=dist.metadata.project_name) + ] = named_entry_point + + for console_script in self.remove_individual: + console_scripts.pop(console_script, None) + if not console_script.project_name: + for cs in tuple(console_scripts): + if console_script.name == cs.name: + console_scripts.pop(cs) + + def iter_entry_points(): + # type: () -> Iterator[NamedEntryPoint] + not_founds = [] # type: List[ConsoleScript] + wrong_projects = [] # type: List[Tuple[ConsoleScript, Distribution]] + for script, ep in console_scripts.items(): + if ep: + yield ep + else: + dist_entry_point = get_entry_point_from_console_script(script.name, dists) + if not dist_entry_point: + not_founds.append(script) + elif ( + script.project_name + and dist_entry_point.dist.metadata.project_name != script.project_name + ): + wrong_projects.append((script, dist_entry_point.dist)) + else: + yield NamedEntryPoint( + name=dist_entry_point.name, entry_point=dist_entry_point.entry_point + ) + + if not_founds or wrong_projects: + failures = [] # type: List[str] + if not_founds: + failures.append( + "Could not find {scripts}: {script_names}".format( + scripts=pluralize(not_founds, "script"), + script_names=" ".join(map(str, not_founds)), + ) + ) + if wrong_projects: + failures.append( + "Found {scripts} in the wrong {projects}:\n {wrong_projects}".format( + scripts=pluralize(wrong_projects, "script"), + projects=pluralize(wrong_projects, "project"), + wrong_projects="\n ".join( + "{script} found in {project}".format( + script=script, project=dist.project_name + ) + for script, dist in wrong_projects + ), + ) + ) + raise ValueError( + "Failed to resolve some console scripts:\n+ {failures}".format( + failures="\n+ ".join(failures) + ) + ) + + return tuple(iter_entry_points()) + + +@attr.s(frozen=True) +class BusyBoxEntryPoints(object): + console_scripts_manifest = attr.ib() # type: ConsoleScriptsManifest + ad_hoc_entry_points = attr.ib() # type: Tuple[NamedEntryPoint, ...] + + class _CurrentPlatform(object): def __get__(self, obj, objtype=None): # type: (...) -> SciePlatform.Value @@ -133,6 +318,7 @@ def python_version(self): @attr.s(frozen=True) class ScieOptions(object): style = attr.ib(default=ScieStyle.LAZY) # type: ScieStyle.Value + busybox_entrypoints = attr.ib(default=None) # type: Optional[BusyBoxEntryPoints] platforms = attr.ib(default=()) # type: Tuple[SciePlatform.Value, ...] pbs_release = attr.ib(default=None) # type: Optional[str] python_version = attr.ib( diff --git a/pex/scie/science.py b/pex/scie/science.py index 054dda891..85ca44e89 100644 --- a/pex/scie/science.py +++ b/pex/scie/science.py @@ -13,12 +13,13 @@ from pex.atomic_directory import atomic_directory from pex.common import chmod_plus_x, is_exe, pluralize, safe_mkdtemp, safe_open from pex.compatibility import shlex_quote +from pex.dist_metadata import NamedEntryPoint, parse_entry_point from pex.exceptions import production_assert from pex.fetcher import URLFetcher from pex.hashing import Sha256 from pex.layout import Layout from pex.pep_440 import Version -from pex.pex_info import PexInfo +from pex.pex import PEX from pex.result import Error, try_ from pex.scie.model import ScieConfiguration, ScieInfo, SciePlatform, ScieStyle, ScieTarget from pex.third_party.packaging.specifiers import SpecifierSet @@ -29,7 +30,7 @@ from pex.variables import ENV, Variables, unzip_dir_relpath if TYPE_CHECKING: - from typing import Any, Dict, Iterator, Optional, Union, cast + from typing import Any, Dict, Iterator, List, Optional, Union, cast import attr # vendor:skip import toml # vendor:skip @@ -101,28 +102,110 @@ def avoid_collisions_with(cls, scie_name): def create_manifests( configuration, # type: ScieConfiguration name, # type: str - pex_info, # type: PexInfo - layout, # type: Layout.Value + pex, # type: PEX filenames, # type: Filenames ): # type: (...) -> Iterator[Manifest] + pex_info = pex.pex_info(include_env_overrides=False) pex_root = "{scie.bindings}/pex_root" - if pex_info.venv: - # We let the configure-binding calculate the venv dir at runtime since it depends on the - # interpreter executing the venv PEX. - installed_pex_dir = "" - elif layout is Layout.LOOSE: - installed_pex_dir = filenames.pex.placeholder - else: - production_assert(pex_info.pex_hash is not None) - pex_hash = cast(str, pex_info.pex_hash) - installed_pex_dir = os.path.join(pex_root, unzip_dir_relpath(pex_hash)) + + configure_binding_args = [filenames.pex.placeholder, filenames.configure_binding.placeholder] + # N.B.: For the venv case, we let the configure-binding calculate the installed PEX dir + # (venv dir) at runtime since it depends on the interpreter executing the venv PEX. + if not pex_info.venv: + configure_binding_args.append("--installed-pex-dir") + if pex.layout is Layout.LOOSE: + configure_binding_args.append(filenames.pex.placeholder) + else: + production_assert(pex_info.pex_hash is not None) + pex_hash = cast(str, pex_info.pex_hash) + configure_binding_args.append(os.path.join(pex_root, unzip_dir_relpath(pex_hash))) env_default = { "PEX_ROOT": pex_root, } + commands = [] # type: List[Dict[str, Any]] + entrypoints = configuration.options.busybox_entrypoints + if entrypoints: + pex_entry_point = parse_entry_point(pex_info.entry_point) + + def replace_env( + named_entry_point, # type: NamedEntryPoint + **env # type: str + ): + # type: (...) -> Dict[str, str] + return dict( + pex_info.inject_env if named_entry_point.entry_point == pex_entry_point else {}, + **env + ) + + def args( + named_entry_point, # type: NamedEntryPoint + *args # type: str + ): + # type: (...) -> List[str] + all_args = ( + list(pex_info.inject_python_args) + if named_entry_point.entry_point == pex_entry_point + else [] + ) + all_args.extend(args) + if named_entry_point.entry_point == pex_entry_point: + all_args.extend(pex_info.inject_args) + return all_args + + def create_cmd(named_entry_point): + # type: (NamedEntryPoint) -> Dict[str, Any] + return { + "name": named_entry_point.name, + "env": { + "default": env_default, + "replace": replace_env( + named_entry_point, PEX_MODULE=str(named_entry_point.entry_point) + ), + "remove_exact": ["PEX_INTERPRETER", "PEX_SCRIPT", "PEX_VENV"], + }, + "exe": "{scie.bindings.configure:PYTHON}", + "args": args(named_entry_point, "{scie.bindings.configure:PEX}"), + } + + if pex_info.venv: + # N.B.: Executing the console script directly instead of bouncing through the PEX + # __main__.py using PEX_SCRIPT saves ~10ms of re-exec overhead in informal testing; so + # it's worth specializing here. + commands.extend( + { + "name": named_entry_point.name, + "env": { + "default": env_default, + "replace": replace_env(named_entry_point), + "remove_exact": ["PEX_INTERPRETER", "PEX_MODULE", "PEX_SCRIPT", "PEX_VENV"], + }, + "exe": "{scie.bindings.configure:PYTHON}", + "args": args( + named_entry_point, + "{scie.bindings.configure:VENV_BIN_DIR_PLUS_SEP}" + named_entry_point.name, + ), + } + for named_entry_point in entrypoints.console_scripts_manifest.collect(pex) + ) + else: + commands.extend(map(create_cmd, entrypoints.console_scripts_manifest.collect(pex))) + commands.extend(map(create_cmd, entrypoints.ad_hoc_entry_points)) + else: + commands.append( + { + "env": { + "default": env_default, + "remove_exact": ["PEX_VENV"], + }, + "exe": "{scie.bindings.configure:PYTHON}", + "args": ["{scie.bindings.configure:PEX}"], + } + ) + lift = { "name": name, "ptex": { @@ -132,13 +215,7 @@ def create_manifests( }, "scie_jump": {"version": SCIE_JUMP_VERSION}, "files": [{"name": filenames.configure_binding.name}, {"name": filenames.pex.name}], - "commands": [ - { - "env": {"default": env_default}, - "exe": "{scie.bindings.configure:PYTHON}", - "args": ["{scie.bindings.configure:PEX}"], - } - ], + "commands": commands, "bindings": [ { "env": { @@ -147,7 +224,6 @@ def create_manifests( "remove_re": ["PEX_.*"], "replace": { "PEX_INTERPRETER": "1", - "_PEX_SCIE_INSTALLED_PEX_DIR": installed_pex_dir, # We can get a warning about too-long script shebangs, but this is not # relevant since we above run the PEX via python and not via shebang. "PEX_EMIT_WARNINGS": "0", @@ -155,7 +231,7 @@ def create_manifests( }, "name": "configure", "exe": "#{cpython:python}", - "args": [filenames.pex.placeholder, filenames.configure_binding.placeholder], + "args": configure_binding_args, } ], } # type: Dict[str, Any] @@ -316,13 +392,12 @@ def build( env=env, ) name = re.sub(r"\.pex$", "", os.path.basename(pex_file), flags=re.IGNORECASE) - pex_info = PexInfo.from_pex(pex_file) - layout = Layout.identify(pex_file) + pex = PEX(pex_file) use_platform_suffix = len(configuration.targets) > 1 filenames = Filenames.avoid_collisions_with(name) errors = OrderedDict() # type: OrderedDict[Manifest, str] - for manifest in create_manifests(configuration, name, pex_info, layout, filenames): + for manifest in create_manifests(configuration, name, pex, filenames): args = [science, "--cache-dir", _science_dir(env, "cache")] if env.PEX_VERBOSE: args.append("-{verbosity}".format(verbosity="v" * env.PEX_VERBOSE)) diff --git a/pex/sh_boot.py b/pex/sh_boot.py index 6402a1e75..e23585037 100644 --- a/pex/sh_boot.py +++ b/pex/sh_boot.py @@ -240,7 +240,7 @@ def create_sh_boot_script( ; do echo >&2 "${{python}}" done - echo >&2 "Either adjust your $PATH which is currently:" + echo >&2 'Either adjust your $PATH which is currently:' echo >&2 "${{PATH}}" echo >&2 -n "Or else install an appropriate Python that provides one of the binaries in " echo >&2 "this list." diff --git a/pex/vendor/_vendored/pip/.layout.json b/pex/vendor/_vendored/pip/.layout.json index 6a1f670a7..d96fa4abb 100644 --- a/pex/vendor/_vendored/pip/.layout.json +++ b/pex/vendor/_vendored/pip/.layout.json @@ -1 +1 @@ -{"fingerprint": "25dd234e8b019eac998222a76fbbdeb4c28b7185edc84230f101adf0a27a4f88", "record_relpath": "pip-20.3.4.dist-info/RECORD", "root_is_purelib": true, "stash_dir": ".prefix"} \ No newline at end of file +{"fingerprint": "ffa0cefd303583ec762f86658b6dc88baf879d318315e18c1a7dec620001f1a9", "record_relpath": "pip-20.3.4.dist-info/RECORD", "root_is_purelib": true, "stash_dir": ".prefix"} \ No newline at end of file diff --git a/pex/vendor/_vendored/pip/.prefix/bin/pip b/pex/vendor/_vendored/pip/.prefix/bin/pip index 85d6e71ae..6184158c7 100755 --- a/pex/vendor/_vendored/pip/.prefix/bin/pip +++ b/pex/vendor/_vendored/pip/.prefix/bin/pip @@ -3,12 +3,9 @@ import importlib import sys -object_ref = "pip._internal.cli.main:main" -modname, qualname_separator, qualname = object_ref.partition(':') -entry_point = importlib.import_module(modname) -if qualname_separator: - for attr in qualname.split('.'): - entry_point = getattr(entry_point, attr) +entry_point = importlib.import_module('pip._internal.cli.main') +for attr in ('main',): + entry_point = getattr(entry_point, attr) -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(entry_point()) diff --git a/pex/vendor/_vendored/pip/.prefix/bin/pip3 b/pex/vendor/_vendored/pip/.prefix/bin/pip3 index 85d6e71ae..6184158c7 100755 --- a/pex/vendor/_vendored/pip/.prefix/bin/pip3 +++ b/pex/vendor/_vendored/pip/.prefix/bin/pip3 @@ -3,12 +3,9 @@ import importlib import sys -object_ref = "pip._internal.cli.main:main" -modname, qualname_separator, qualname = object_ref.partition(':') -entry_point = importlib.import_module(modname) -if qualname_separator: - for attr in qualname.split('.'): - entry_point = getattr(entry_point, attr) +entry_point = importlib.import_module('pip._internal.cli.main') +for attr in ('main',): + entry_point = getattr(entry_point, attr) -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(entry_point()) diff --git a/pex/vendor/_vendored/pip/.prefix/bin/pip3.8 b/pex/vendor/_vendored/pip/.prefix/bin/pip3.8 index 85d6e71ae..6184158c7 100755 --- a/pex/vendor/_vendored/pip/.prefix/bin/pip3.8 +++ b/pex/vendor/_vendored/pip/.prefix/bin/pip3.8 @@ -3,12 +3,9 @@ import importlib import sys -object_ref = "pip._internal.cli.main:main" -modname, qualname_separator, qualname = object_ref.partition(':') -entry_point = importlib.import_module(modname) -if qualname_separator: - for attr in qualname.split('.'): - entry_point = getattr(entry_point, attr) +entry_point = importlib.import_module('pip._internal.cli.main') +for attr in ('main',): + entry_point = getattr(entry_point, attr) -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(entry_point()) diff --git a/pex/vendor/_vendored/setuptools/.layout.json b/pex/vendor/_vendored/setuptools/.layout.json index 9f8f711a6..af996e2e0 100644 --- a/pex/vendor/_vendored/setuptools/.layout.json +++ b/pex/vendor/_vendored/setuptools/.layout.json @@ -1 +1 @@ -{"fingerprint": "97101310bad04f71d6b1bba38a3acaaaf165238fe842eb5a94025b2c87663390", "record_relpath": "setuptools-44.0.0+3acb925dd708430aeaf197ea53ac8a752f7c1863.dist-info/RECORD", "root_is_purelib": true, "stash_dir": ".prefix"} \ No newline at end of file +{"fingerprint": "22ae0a68359cdb12d74f1a1dea6543161f0b1adab9f0555e8d3a800ec6d8f9b5", "record_relpath": "setuptools-44.0.0+3acb925dd708430aeaf197ea53ac8a752f7c1863.dist-info/RECORD", "root_is_purelib": true, "stash_dir": ".prefix"} \ No newline at end of file diff --git a/pex/vendor/_vendored/setuptools/.prefix/bin/easy_install b/pex/vendor/_vendored/setuptools/.prefix/bin/easy_install index 4f0954ed5..1162aa42e 100755 --- a/pex/vendor/_vendored/setuptools/.prefix/bin/easy_install +++ b/pex/vendor/_vendored/setuptools/.prefix/bin/easy_install @@ -3,12 +3,9 @@ import importlib import sys -object_ref = "setuptools.command.easy_install:main" -modname, qualname_separator, qualname = object_ref.partition(':') -entry_point = importlib.import_module(modname) -if qualname_separator: - for attr in qualname.split('.'): - entry_point = getattr(entry_point, attr) +entry_point = importlib.import_module('setuptools.command.easy_install') +for attr in ('main',): + entry_point = getattr(entry_point, attr) -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(entry_point()) diff --git a/pex/vendor/_vendored/setuptools/.prefix/bin/easy_install-3.8 b/pex/vendor/_vendored/setuptools/.prefix/bin/easy_install-3.8 index 4f0954ed5..1162aa42e 100755 --- a/pex/vendor/_vendored/setuptools/.prefix/bin/easy_install-3.8 +++ b/pex/vendor/_vendored/setuptools/.prefix/bin/easy_install-3.8 @@ -3,12 +3,9 @@ import importlib import sys -object_ref = "setuptools.command.easy_install:main" -modname, qualname_separator, qualname = object_ref.partition(':') -entry_point = importlib.import_module(modname) -if qualname_separator: - for attr in qualname.split('.'): - entry_point = getattr(entry_point, attr) +entry_point = importlib.import_module('setuptools.command.easy_install') +for attr in ('main',): + entry_point = getattr(entry_point, attr) -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(entry_point()) diff --git a/pex/venv/installer.py b/pex/venv/installer.py index 9661c5696..b6be0f4bf 100644 --- a/pex/venv/installer.py +++ b/pex/venv/installer.py @@ -707,6 +707,8 @@ def sys_executable_paths(): "_PEX_FILE_LOCK_STYLE", # This is used in the scie binding command for ZIPAPP PEXes. "_PEX_SCIE_INSTALLED_PEX_DIR", + # This is used to override PBS distribution URLs in lazy PEX scies. + "PEX_BOOTSTRAP_URLS", ) ] if ignored_pex_env_vars: diff --git a/tests/integration/scie/test_pex_scie.py b/tests/integration/scie/test_pex_scie.py index 0c9641fec..002e3e492 100644 --- a/tests/integration/scie/test_pex_scie.py +++ b/tests/integration/scie/test_pex_scie.py @@ -9,11 +9,12 @@ import re import subprocess import sys +from textwrap import dedent from typing import Optional import pytest -from pex.common import is_exe +from pex.common import is_exe, safe_open from pex.layout import Layout from pex.orderedset import OrderedSet from pex.scie import SciePlatform, ScieStyle @@ -399,3 +400,434 @@ def test_pex_pex_scie( .decode("utf-8") .strip() ) + + +def make_project( + tmpdir, # type: Any + name, # type: str +): + # type: (...) -> str + + project_dir = os.path.join(str(tmpdir), name) + with safe_open(os.path.join(project_dir, "{name}.py".format(name=name)), "w") as fp: + fp.write( + dedent( + """\ + from __future__ import print_function + + import sys + + + def one(): + print("{name}1:", *sys.argv[1:]) + + + def two(): + print("{name}2:", *sys.argv[1:]) + + + def three(): + print("{name}3:", *sys.argv[1:]) + + + if __name__ == "__main__": + print("{name}:", *sys.argv[1:]) + """ + ).format(name=name) + ) + with safe_open(os.path.join(project_dir, "setup.py"), "w") as fp: + fp.write( + dedent( + """\ + from setuptools import setup + + + setup( + name={name!r}, + version="0.1.0", + entry_points={{ + "console_scripts": [ + "{name}-script1 = {name}:one", + "{name}-script2 = {name}:two", + ], + }}, + py_modules=[{name!r}], + ) + """ + ).format(name=name) + ) + return project_dir + + +@pytest.fixture +def foo(tmpdir): + # type: (Any) -> str + return make_project(tmpdir, "foo") + + +@pytest.fixture +def bar(tmpdir): + # type: (Any) -> str + return make_project(tmpdir, "bar") + + +skip_if_no_pbs = pytest.mark.skipif( + PY_VER < (3, 8) or PY_VER >= (3, 13), + reason="A PBS release must be available for the current interpreter to run this test.", +) + + +@skip_if_pypy +@skip_if_no_pbs +@pytest.mark.parametrize( + "execution_mode_args", + [ + pytest.param([], id="ZIPAPP"), + pytest.param(["--venv"], id="VENV"), + ], +) +def test_scie_busybox_console_scripts_all( + tmpdir, # type: Any + foo, # type: str + bar, # type: str + execution_mode_args, # type: List[str] +): + # type: (...) -> None + + busybox = os.path.join(str(tmpdir), "busybox") + run_pex_command( + args=[foo, bar, "--scie", "lazy", "--scie-busybox", "@", "-o", busybox] + + execution_mode_args + ).assert_success() + + assert b"foo1: all\n" == subprocess.check_output(args=[busybox, "foo-script1", "all"]) + assert b"foo2: all\n" == subprocess.check_output(args=[busybox, "foo-script2", "all"]) + assert b"bar1: all\n" == subprocess.check_output(args=[busybox, "bar-script1", "all"]) + assert b"bar2: all\n" == subprocess.check_output(args=[busybox, "bar-script2", "all"]) + + bin_dir = os.path.join(str(tmpdir), "bin_dir") + subprocess.check_call(args=[busybox, bin_dir], env=make_env(SCIE="install")) + assert sorted(["foo-script1", "foo-script2", "bar-script1", "bar-script2"]) == sorted( + os.listdir(bin_dir) + ) + + assert b"foo1: all\n" == subprocess.check_output( + args=[os.path.join(bin_dir, "foo-script1"), "all"] + ) + assert b"foo2: all\n" == subprocess.check_output( + args=[os.path.join(bin_dir, "foo-script2"), "all"] + ) + assert b"bar1: all\n" == subprocess.check_output( + args=[os.path.join(bin_dir, "bar-script1"), "all"] + ) + assert b"bar2: all\n" == subprocess.check_output( + args=[os.path.join(bin_dir, "bar-script2"), "all"] + ) + + +@skip_if_pypy +@skip_if_no_pbs +def test_scie_busybox_console_scripts_add_distribution( + tmpdir, # type: Any + foo, # type: str + bar, # type: str +): + # type: (...) -> None + + busybox = os.path.join(str(tmpdir), "busybox") + run_pex_command( + args=[foo, bar, "--scie", "lazy", "--scie-busybox", "@foo", "-o", busybox] + ).assert_success() + + assert b"foo1: add-dist\n" == subprocess.check_output(args=[busybox, "foo-script1", "add-dist"]) + assert b"foo2: add-dist\n" == subprocess.check_output(args=[busybox, "foo-script2", "add-dist"]) + + bin_dir = os.path.join(str(tmpdir), "bin_dir") + subprocess.check_call(args=[busybox, bin_dir], env=make_env(SCIE="install")) + assert sorted(["foo-script1", "foo-script2"]) == sorted(os.listdir(bin_dir)) + + assert b"foo1: add-dist\n" == subprocess.check_output( + args=[os.path.join(bin_dir, "foo-script1"), "add-dist"] + ) + assert b"foo2: add-dist\n" == subprocess.check_output( + args=[os.path.join(bin_dir, "foo-script2"), "add-dist"] + ) + + +@skip_if_pypy +@skip_if_no_pbs +def test_scie_busybox_console_scripts_remove_distribution( + tmpdir, # type: Any + foo, # type: str + bar, # type: str +): + # type: (...) -> None + + busybox = os.path.join(str(tmpdir), "busybox") + run_pex_command( + args=[ + foo, + bar, + "--scie", + "lazy", + "--scie-busybox", + "@", + "--scie-busybox", + "!@foo", + "-o", + busybox, + ], + quiet=True, + ).assert_success() + + assert b"bar1: del-dist\n" == subprocess.check_output(args=[busybox, "bar-script1", "del-dist"]) + assert b"bar2: del-dist\n" == subprocess.check_output(args=[busybox, "bar-script2", "del-dist"]) + + bin_dir = os.path.join(str(tmpdir), "bin_dir") + subprocess.check_call(args=[busybox, bin_dir], env=make_env(SCIE="install")) + assert sorted(["bar-script1", "bar-script2"]) == sorted(os.listdir(bin_dir)) + + assert b"bar1: del-dist\n" == subprocess.check_output( + args=[os.path.join(bin_dir, "bar-script1"), "del-dist"] + ) + assert b"bar2: del-dist\n" == subprocess.check_output( + args=[os.path.join(bin_dir, "bar-script2"), "del-dist"] + ) + + +@skip_if_pypy +@skip_if_no_pbs +def test_scie_busybox_console_scripts_remove_script( + tmpdir, # type: Any + foo, # type: str + bar, # type: str +): + # type: (...) -> None + + busybox = os.path.join(str(tmpdir), "busybox") + run_pex_command( + args=[foo, bar, "--scie", "lazy", "--scie-busybox", "@foo,!foo-script1", "-o", busybox], + quiet=True, + ).assert_success() + + assert b"foo2: del-script\n" == subprocess.check_output( + args=[busybox, "foo-script2", "del-script"] + ) + + bin_dir = os.path.join(str(tmpdir), "bin_dir") + subprocess.check_call(args=[busybox, bin_dir], env=make_env(SCIE="install")) + assert ["foo-script2"] == os.listdir(bin_dir) + + assert b"foo2: del-script\n" == subprocess.check_output( + args=[os.path.join(bin_dir, "foo-script2"), "del-script"] + ) + + +@skip_if_pypy +@skip_if_no_pbs +def test_scie_busybox_console_scripts_add_script( + tmpdir, # type: Any + foo, # type: str + bar, # type: str +): + # type: (...) -> None + + busybox = os.path.join(str(tmpdir), "busybox") + run_pex_command( + args=[foo, bar, "--scie", "lazy", "--scie-busybox", "@bar,foo-script1", "-o", busybox], + quiet=True, + ).assert_success() + + assert b"foo1: add-script\n" == subprocess.check_output( + args=[busybox, "foo-script1", "add-script"] + ) + assert b"bar1: add-script\n" == subprocess.check_output( + args=[busybox, "bar-script1", "add-script"] + ) + assert b"bar2: add-script\n" == subprocess.check_output( + args=[busybox, "bar-script2", "add-script"] + ) + + bin_dir = os.path.join(str(tmpdir), "bin_dir") + subprocess.check_call(args=[busybox, bin_dir], env=make_env(SCIE="install")) + assert sorted(["foo-script1", "bar-script1", "bar-script2"]) == sorted(os.listdir(bin_dir)) + + assert b"foo1: add-script\n" == subprocess.check_output( + args=[os.path.join(bin_dir, "foo-script1"), "add-script"] + ) + assert b"bar1: add-script\n" == subprocess.check_output( + args=[os.path.join(bin_dir, "bar-script1"), "add-script"] + ) + assert b"bar2: add-script\n" == subprocess.check_output( + args=[os.path.join(bin_dir, "bar-script2"), "add-script"] + ) + + +@skip_if_pypy +@skip_if_no_pbs +def test_scie_busybox_console_script_injections( + tmpdir, # type: Any + foo, # type: str + bar, # type: str +): + # type: (...) -> None + + busybox = os.path.join(str(tmpdir), "busybox") + run_pex_command( + args=[ + foo, + bar, + "-c", + "foo-script1", + "--inject-args", + "--injected yes", + "--scie", + "lazy", + "--scie-busybox", + "@bar,foo-script1", + "-o", + busybox, + ], + quiet=True, + ).assert_success() + + assert b"foo1: --injected yes injected?\n" == subprocess.check_output( + args=[busybox, "foo-script1", "injected?"] + ) + assert b"bar1: injected?\n" == subprocess.check_output( + args=[busybox, "bar-script1", "injected?"] + ) + assert b"bar2: injected?\n" == subprocess.check_output( + args=[busybox, "bar-script2", "injected?"] + ) + + bin_dir = os.path.join(str(tmpdir), "bin_dir") + subprocess.check_call(args=[busybox, bin_dir], env=make_env(SCIE="install")) + assert sorted(["foo-script1", "bar-script1", "bar-script2"]) == sorted(os.listdir(bin_dir)) + + assert b"foo1: --injected yes injected?\n" == subprocess.check_output( + args=[os.path.join(bin_dir, "foo-script1"), "injected?"] + ) + assert b"bar1: injected?\n" == subprocess.check_output( + args=[os.path.join(bin_dir, "bar-script1"), "injected?"] + ) + assert b"bar2: injected?\n" == subprocess.check_output( + args=[os.path.join(bin_dir, "bar-script2"), "injected?"] + ) + + +@skip_if_pypy +@skip_if_no_pbs +def test_scie_busybox_module_entry_points( + tmpdir, # type: Any + foo, # type: str + bar, # type: str +): + # type: (...) -> None + + busybox = os.path.join(str(tmpdir), "busybox") + run_pex_command( + args=[ + foo, + bar, + "--scie", + "lazy", + "--scie-busybox", + "bar-mod=bar,foo=foo:three", + "-o", + busybox, + ], + quiet=True, + ).assert_success() + + assert b"bar: mep\n" == subprocess.check_output(args=[busybox, "bar-mod", "mep"]) + assert b"foo3: mep\n" == subprocess.check_output(args=[busybox, "foo", "mep"]) + + bin_dir = os.path.join(str(tmpdir), "bin_dir") + subprocess.check_call(args=[busybox, bin_dir], env=make_env(SCIE="install")) + assert sorted(["bar-mod", "foo"]) == sorted(os.listdir(bin_dir)) + + assert b"bar: mep\n" == subprocess.check_output(args=[os.path.join(bin_dir, "bar-mod"), "mep"]) + assert b"foo3: mep\n" == subprocess.check_output(args=[os.path.join(bin_dir, "foo"), "mep"]) + + +@skip_if_pypy +@skip_if_no_pbs +def test_scie_busybox_module_entry_point_injections( + tmpdir, # type: Any + foo, # type: str + bar, # type: str +): + # type: (...) -> None + + busybox = os.path.join(str(tmpdir), "busybox") + run_pex_command( + args=[ + foo, + bar, + "-m", + "bar", + "--inject-args", + "--injected yes", + "--scie", + "lazy", + "--scie-busybox", + "bar-mod=bar,foo=foo:three", + "-o", + busybox, + ], + quiet=True, + ).assert_success() + + assert b"bar: --injected yes injected?\n" == subprocess.check_output( + args=[busybox, "bar-mod", "injected?"] + ) + assert b"foo3: injected?\n" == subprocess.check_output(args=[busybox, "foo", "injected?"]) + + bin_dir = os.path.join(str(tmpdir), "bin_dir") + subprocess.check_call(args=[busybox, bin_dir], env=make_env(SCIE="install")) + assert sorted(["bar-mod", "foo"]) == sorted(os.listdir(bin_dir)) + + assert b"bar: --injected yes injected?\n" == subprocess.check_output( + args=[os.path.join(bin_dir, "bar-mod"), "injected?"] + ) + assert b"foo3: injected?\n" == subprocess.check_output( + args=[os.path.join(bin_dir, "foo"), "injected?"] + ) + + +@skip_if_pypy +@skip_if_no_pbs +def test_script_not_found( + tmpdir, # type: Any + foo, # type: str + bar, # type: str +): + # type: (...) -> None + + busybox = os.path.join(str(tmpdir), "busybox") + run_pex_command( + args=[ + foo, + bar, + "--scie", + "lazy", + "--scie-busybox", + "foo-script1@foo,foo-script2@bar,bar-script1@foo,bar-script2@bar,baz", + "-o", + busybox, + ], + quiet=True, + ).assert_failure( + expected_error_re=re.escape( + dedent( + """\ + Failed to resolve some console scripts: + + Could not find script: baz + + Found scripts in the wrong projects: + foo-script2@bar found in foo + bar-script1@foo found in bar + """ + ) + ) + ) diff --git a/tests/test_finders.py b/tests/test_finders.py index 7e30082bc..88ebd4877 100644 --- a/tests/test_finders.py +++ b/tests/test_finders.py @@ -5,7 +5,7 @@ import pytest -from pex.dist_metadata import CallableEntryPoint, Distribution, EntryPoint +from pex.dist_metadata import CallableEntryPoint, Distribution, NamedEntryPoint from pex.finders import ( DistributionScript, get_entry_point_from_console_script, @@ -17,7 +17,7 @@ from testing.dist_metadata import create_dist_metadata if TYPE_CHECKING: - from typing import Any, Dict, Text, Tuple + from typing import Any, Dict, Tuple import attr # vendor:skip else: @@ -59,12 +59,12 @@ def create_dist( console_script_entry, # type: str ): # type: (...) -> Distribution - entry_point = EntryPoint.parse(console_script_entry) + entry_point = NamedEntryPoint.parse(console_script_entry) @attr.s(frozen=True) class FakeDist(Distribution): def get_entry_map(self): - # type: () -> Dict[Text, Dict[Text, EntryPoint]] + # type: () -> Dict[str, Dict[str, NamedEntryPoint]] return {"console_scripts": {entry_point.name: entry_point}} location = os.getcwd() @@ -83,10 +83,8 @@ def test_get_entry_point_from_console_script(): dist_entrypoint = get_entry_point_from_console_script("bob", dists) assert dist_entrypoint is not None - assert ( - CallableEntryPoint(name="bob", module="bob.main", attrs=("run",)) - == dist_entrypoint.entry_point - ) + assert "bob" == dist_entrypoint.name + assert CallableEntryPoint(module="bob.main", attrs=("run",)) == dist_entrypoint.entry_point assert dist_entrypoint.dist in dists