diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 197ae22ad..b7cabea6c 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -295,6 +295,15 @@ def configure_clp_pex_options(parser): "Tailor the behavior of the emitted .pex file if -o is specified.", ) + group.add_argument( + "--include-tools", + dest="include_tools", + default=False, + action=HandleBoolAction, + help="Whether to include runtime tools in the pex file. If included, these can be run by " + "exporting PEX_TOOLS=1 and following the usage and --help information.", + ) + group.add_argument( "--zip-safe", "--not-zip-safe", @@ -775,7 +784,12 @@ def to_python_interpreter(full_path_or_basename): # options.preamble_file is None preamble = None - pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) + pex_builder = PEXBuilder( + path=safe_mkdtemp(), + interpreter=interpreter, + preamble=preamble, + include_tools=options.include_tools, + ) if options.resources_directory: pex_warnings.warn( diff --git a/pex/pex.py b/pex/pex.py index d7006411d..ab62b18ab 100644 --- a/pex/pex.py +++ b/pex/pex.py @@ -27,7 +27,7 @@ from pex.tracer import TRACER from pex.typing import TYPE_CHECKING from pex.util import iter_pth_paths, named_temporary_file -from pex.variables import ENV +from pex.variables import ENV, Variables if TYPE_CHECKING: from typing import Any, Dict, Iterable, Iterator, List, Mapping, Optional, Set, Tuple, TypeVar @@ -65,25 +65,37 @@ def _clean_environment(cls, env=None, strip_pex_env=True): if key.startswith("PEX_"): del env[key] - def __init__(self, pex=sys.argv[0], interpreter=None, env=ENV, verify_entry_point=False): + def __init__( + self, + pex=sys.argv[0], # type: str + interpreter=None, # type: Optional[PythonInterpreter] + env=ENV, # type: Variables + verify_entry_point=False, # type: bool + ): + # type: (...) -> None self._pex = pex self._interpreter = interpreter or PythonInterpreter.get() self._pex_info = PexInfo.from_pex(self._pex) self._pex_info_overrides = PexInfo.from_env(env=env) self._vars = env - self._envs = [] - self._working_set = None + self._envs = [] # type: List[PEXEnvironment] + self._working_set = None # type: Optional[WorkingSet] if verify_entry_point: self._do_entry_point_verification() + def pex_info(self): + # type: () -> PexInfo + pex_info = self._pex_info.copy() + pex_info.update(self._pex_info_overrides) + pex_info.merge_pex_path(self._vars.PEX_PATH) + return pex_info + def _activate(self): if not self._working_set: working_set = WorkingSet([]) # set up the local .pex environment - pex_info = self._pex_info.copy() - pex_info.update(self._pex_info_overrides) - pex_info.merge_pex_path(self._vars.PEX_PATH) + pex_info = self.pex_info() self._envs.append(PEXEnvironment(self._pex, pex_info, interpreter=self._interpreter)) # N.B. by this point, `pex_info.pex_path` will contain a single pex path # merged from pex_path in `PEX-INFO` and `PEX_PATH` set in the environment. @@ -426,6 +438,7 @@ def _wrap_profiling(self, runner, *args): profiler.print_stats(sort=pex_profile_sort) def path(self): + # type: () -> str """Return the path this PEX was built at.""" return self._pex @@ -436,6 +449,13 @@ def execute(self): This function makes assumptions that it is the last function called by the interpreter. """ teardown_verbosity = self._vars.PEX_TEARDOWN_VERBOSE + + # N.B.: This is set in `__main__.py` of the executed PEX by `PEXBuilder` when we've been + # executed from within a PEX zip file in `--unzip` mode. We replace `sys.argv[0]` to avoid + # confusion and allow the user code we hand off to to provide useful messages and fully + # valid re-execs that always re-directed through the PEX file. + sys.argv[0] = os.environ.pop("__PEX_EXE__", sys.argv[0]) + try: pex_inherit_path = self._vars.PEX_INHERIT_PATH if pex_inherit_path == InheritPath.FALSE: @@ -443,12 +463,23 @@ def execute(self): self.patch_sys(pex_inherit_path) working_set = self._activate() self.patch_pkg_resources(working_set) - exit_code = self._wrap_coverage(self._wrap_profiling, self._execute) + if self._vars.PEX_TOOLS: + try: + from pex.tools import main as tools + except ImportError as e: + die( + "This PEX was not built with tools (Re-build the PEX file with " + "`pex --include-tools ...`): {}".format(e) + ) + + exit_code = tools.main(pex=self, pex_prog_path=sys.argv[0]) + else: + exit_code = self._wrap_coverage(self._wrap_profiling, self._execute) if exit_code: sys.exit(exit_code) except Exception: - # Allow the current sys.excepthook to handle this app exception before we tear things down in - # finally, then reraise so that the exit status is reflected correctly. + # Allow the current sys.excepthook to handle this app exception before we tear things + # down in finally, then reraise so that the exit status is reflected correctly. sys.excepthook(*sys.exc_info()) raise except SystemExit as se: @@ -480,12 +511,6 @@ def execute(self): def _execute(self): force_interpreter = self._vars.PEX_INTERPRETER - # N.B.: This is set in `__main__.py` of the executed PEX by `PEXBuilder` when we've been - # executed from within a PEX zip file in `--unzip` mode. We replace `sys.argv[0]` to avoid - # confusion and allow the user code we hand off to to provide useful messages and fully valid - # re-execs that always re-directed through the PEX file. - sys.argv[0] = os.environ.pop("__PEX_EXE__", sys.argv[0]) - self._clean_environment(strip_pex_env=self._pex_info.strip_pex_env) if force_interpreter: diff --git a/pex/pex_builder.py b/pex/pex_builder.py index 9db6d9faf..0eb4b1d04 100644 --- a/pex/pex_builder.py +++ b/pex/pex_builder.py @@ -116,7 +116,14 @@ class InvalidExecutableSpecification(Error): pass def __init__( - self, path=None, interpreter=None, chroot=None, pex_info=None, preamble=None, copy=False + self, + path=None, + interpreter=None, + chroot=None, + pex_info=None, + preamble=None, + copy=False, + include_tools=False, ): """Initialize a pex builder. @@ -131,6 +138,8 @@ def __init__( :type preamble: str :keyword copy: If False, attempt to create the pex environment via hard-linking, falling back to copying across devices. If True, always copy. + :keyword include_tools: If True, include runtime tools which can be executed by exporting + `PEX_TOOLS=1`. .. versionchanged:: 0.8 The temporary directory created when ``path`` is not specified is now garbage collected on @@ -141,6 +150,7 @@ def __init__( self._pex_info = pex_info or PexInfo.default(self._interpreter) self._preamble = preamble or "" self._copy = copy + self._include_tools = include_tools self._shebang = self._interpreter.identity.hashbang() self._logger = logging.getLogger(__name__) @@ -445,9 +455,7 @@ def _precompile_source(self): self._chroot.touch(compiled, label="bytecode") def _prepare_manifest(self): - self._chroot.write( - self._pex_info.dump(sort_keys=True).encode("utf-8"), PexInfo.PATH, label="manifest" - ) + self._chroot.write(self._pex_info.dump().encode("utf-8"), PexInfo.PATH, label="manifest") def _prepare_main(self): self._chroot.write( @@ -480,9 +488,13 @@ def _prepare_bootstrap(self): if not isinstance(provider, DefaultProvider): mod = __import__(source_name, fromlist=["ignore"]) provider = ZipProvider(mod) - for package in ("", "third_party"): + + bootstrap_packages = ["", "third_party"] + if self._include_tools: + bootstrap_packages.extend(["tools", "tools/commands"]) + for package in bootstrap_packages: for fn in provider.resource_listdir(package): - if fn.endswith(".py"): + if not (provider.resource_isdir(os.path.join(package, fn)) or fn.endswith(".pyc")): rel_path = os.path.join(package, fn) self._chroot.write( provider.get_resource_string(source_name, rel_path), diff --git a/pex/pex_info.py b/pex/pex_info.py index e2ce234b7..24c2bf524 100644 --- a/pex/pex_info.py +++ b/pex/pex_info.py @@ -12,14 +12,14 @@ from pex.compatibility import string as compatibility_string from pex.inherit_path import InheritPath from pex.orderedset import OrderedSet -from pex.typing import TYPE_CHECKING +from pex.typing import TYPE_CHECKING, cast from pex.variables import ENV, Variables from pex.version import __version__ as pex_version if TYPE_CHECKING: from pex.interpreter import PythonInterpreter - from typing import Any, Mapping, Optional, Text, Union + from typing import Any, Dict, Mapping, Optional, Text, Union # TODO(wickman) Split this into a PexInfoBuilder/PexInfo to ensure immutability. @@ -89,6 +89,7 @@ def default(cls, interpreter=None): @classmethod def from_pex(cls, pex): + # type: (str) -> PexInfo if os.path.isfile(pex): with open_zip(pex) as zf: pex_info = zf.read(cls.PATH) @@ -151,7 +152,7 @@ def __init__(self, info=None): raise ValueError( "PexInfo can only be seeded with a dict, got: " "%s of type %s" % (info, type(info)) ) - self._pex_info = dict(info) if info else {} # type Dict[str, str] + self._pex_info = dict(info) if info else {} # type Dict[str, Any] self._distributions = self._pex_info.get("distributions", {}) # cast as set because pex info from json must store interpreter_constraints as a list self._interpreter_constraints = set(self._pex_info.get("interpreter_constraints", set())) @@ -226,15 +227,17 @@ def strip_pex_env(self, value): @property def pex_path(self): + # type: () -> Optional[str] """A colon separated list of other pex files to merge into the runtime environment. This pex info property is used to persist the PEX_PATH environment variable into the pex info metadata for reuse within a built pex. """ - return self._pex_info.get("pex_path") + return cast("Optional[str]", self._pex_info.get("pex_path")) @pex_path.setter def pex_path(self, value): + # type: (str) -> None self._pex_info["pex_path"] = value @property @@ -365,6 +368,7 @@ def zip_unsafe_cache(self): return os.path.join(self.pex_root, "code") def update(self, other): + # type: (PexInfo) -> None if not isinstance(other, PexInfo): raise TypeError("Cannot merge a %r with PexInfo" % type(other)) self._pex_info.update(other._pex_info) @@ -372,17 +376,25 @@ def update(self, other): self._interpreter_constraints.update(other.interpreter_constraints) self._requirements.update(other.requirements) - def dump(self, sort_keys=False): - # type: (bool) -> str - pex_info_copy = self._pex_info.copy() - pex_info_copy["inherit_path"] = self.inherit_path.value - pex_info_copy["requirements"] = sorted(self._requirements) - pex_info_copy["interpreter_constraints"] = sorted(self._interpreter_constraints) - pex_info_copy["distributions"] = self._distributions.copy() - return json.dumps(pex_info_copy, sort_keys=sort_keys) + def as_json_dict(self): + # type: () -> Dict[str, Any] + data = self._pex_info.copy() + data["inherit_path"] = self.inherit_path.value + data["requirements"] = list(self._requirements) + data["interpreter_constraints"] = list(self._interpreter_constraints) + data["distributions"] = self._distributions.copy() + return data + + def dump(self): + # type: (...) -> str + data = self.as_json_dict() + data["requirements"].sort() + data["interpreter_constraints"].sort() + return json.dumps(data, sort_keys=True) def copy(self): - return self.from_json(self.dump()) + # type: () -> PexInfo + return PexInfo(self._pex_info) @staticmethod def _merge_split(*paths): diff --git a/pex/tools/__init__.py b/pex/tools/__init__.py new file mode 100644 index 000000000..9fe77d4b6 --- /dev/null +++ b/pex/tools/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). diff --git a/pex/tools/__main__.py b/pex/tools/__main__.py new file mode 100644 index 000000000..e0f014271 --- /dev/null +++ b/pex/tools/__main__.py @@ -0,0 +1,10 @@ +# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import sys + +from pex.tools.main import main as tools + +sys.exit(tools()) diff --git a/pex/tools/command.py b/pex/tools/command.py new file mode 100644 index 000000000..9e0ddcb8e --- /dev/null +++ b/pex/tools/command.py @@ -0,0 +1,186 @@ +# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, print_function + +import json +import os +import subprocess +import sys +from argparse import ArgumentParser, Namespace +from contextlib import contextmanager + +from pex.common import safe_open +from pex.pex import PEX +from pex.typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any, Iterable, Iterator, IO, Optional, Dict + + +class Result(object): + def __init__( + self, + exit_code, # type: int + message="", # type: str + ): + # type: (...) -> None + self._exit_code = exit_code + self._message = message + + @property + def exit_code(self): + # type: () -> int + return self._exit_code + + @property + def is_error(self): + # type: () -> bool + return self._exit_code != 0 + + def maybe_display(self): + # type: () -> None + if not self._message: + return + print(self._message, file=sys.stderr if self.is_error else sys.stdout) + + def __str__(self): + # type: () -> str + return self._message + + def __repr__(self): + # type: () -> str + return "{}(exit_code={!r}, message={!r})".format( + type(self).__name__, self._exit_code, self._message + ) + + +class Ok(Result): + def __init__(self, message=""): + # type: (str) -> None + super(Ok, self).__init__(exit_code=0, message=message) + + +class Error(Result): + def __init__( + self, + message="", # type: str + exit_code=1, # type: int + ): + # type: (...) -> None + if exit_code == 0: + raise ValueError("An Error must have a non-zero exit code; given: {}".format(exit_code)) + super(Error, self).__init__(exit_code=exit_code, message=message) + + +def try_run_program( + program, # type: str + args, # type: Iterable[str] + url=None, # type: Optional[str] + error=None, # type: Optional[str] + **kwargs # type: Any +): + # type: (...) -> Result + try: + subprocess.check_call([program] + list(args), **kwargs) + return Ok() + except OSError as e: + msg = [error] if error else [] + msg.append("Do you have `{}` installed on the $PATH?: {}".format(program, e)) + if url: + msg.append( + "Find more information on `{program}` at {url}.".format(program=program, url=url) + ) + return Error("\n".join(msg)) + except subprocess.CalledProcessError as e: + return Error(str(e), exit_code=e.returncode) + + +def try_open_file( + path, # type: str + error=None, # type: Optional[str] +): + # type: (...) -> Result + opener, url = ( + ("xdg-open", "https://www.freedesktop.org/wiki/Software/xdg-utils/") + if "Linux" == os.uname()[0] + else ("open", None) + ) + with open(os.devnull, "wb") as devnull: + return try_run_program(opener, [path], url=url, error=error, stdout=devnull) + + +class Command(object): + def add_arguments(self, parser): + # type: (ArgumentParser) -> None + pass + + def run( + self, + pex, # type: PEX + options, # type: Namespace + ): + # type: (...) -> Result + pass + + +class OutputMixin(object): + @staticmethod + def add_output_option( + parser, # type: ArgumentParser + entity, # type: str + ): + # type: (...) -> None + parser.add_argument( + "-o", + "--output", + metavar="PATH", + help=( + "A file to output the {entity} to; STDOUT by default or when `-` is " + "specified.".format(entity=entity) + ), + ) + + @staticmethod + def is_stdout(options): + # type: (Namespace) -> bool + return options.output == "-" or not options.output + + @classmethod + @contextmanager + def output( + cls, + options, # type: Namespace + binary=False, # type: bool + ): + # type: (...) -> Iterator[IO] + if cls.is_stdout(options): + stdout = getattr(sys.stdout, "buffer", sys.stdout) if binary else sys.stdout + yield stdout + else: + with safe_open(options.output, mode="wb" if binary else "w") as out: + yield out + + +class JsonMixin(object): + @staticmethod + def add_json_options( + parser, + entity, + ): + parser.add_argument( + "-i", + "--indent", + type=int, + default=None, + help="Pretty-print {entity} json with the given indent.".format(entity=entity), + ) + + @staticmethod + def dump_json( + options, # type: Namespace + data, # type: Dict[str, Any] + out, # type: IO + **json_dump_kwargs # type: Any + ): + json.dump(data, out, indent=options.indent, **json_dump_kwargs) diff --git a/pex/tools/commands/__init__.py b/pex/tools/commands/__init__.py new file mode 100644 index 000000000..f3f80039b --- /dev/null +++ b/pex/tools/commands/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from pex.tools.command import Command +from pex.tools.commands.info import Info +from pex.typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Iterable + + +def all_commands(): + # type: () -> Iterable[Command] + return [Info()] diff --git a/pex/tools/commands/info.py b/pex/tools/commands/info.py new file mode 100644 index 000000000..a68f82df0 --- /dev/null +++ b/pex/tools/commands/info.py @@ -0,0 +1,29 @@ +# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +from argparse import ArgumentParser, Namespace + +from pex.pex import PEX +from pex.tools.command import Command, JsonMixin, Ok, OutputMixin, Result + + +class Info(JsonMixin, OutputMixin, Command): + """Dumps the PEX-INFO json contained in a PEX file.""" + + def add_arguments(self, parser): + # type: (ArgumentParser) -> None + self.add_output_option(parser, entity="PEX-INFO json") + self.add_json_options(parser, entity="PEX-INFO") + + def run( + self, + pex, # type: PEX + options, # type: Namespace + ): + # type: (...) -> Result + with self.output(options) as out: + self.dump_json(options, pex.pex_info().as_json_dict(), out) + out.write("\n") + return Ok() diff --git a/pex/tools/main.py b/pex/tools/main.py new file mode 100644 index 000000000..676eaea19 --- /dev/null +++ b/pex/tools/main.py @@ -0,0 +1,91 @@ +# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, print_function + +import functools +import logging +import os +import sys +from argparse import ArgumentParser, Namespace + +from pex.pex import PEX +from pex.tools import commands +from pex.tools.command import Result +from pex.tracer import TRACER +from pex.typing import TYPE_CHECKING, cast + +if TYPE_CHECKING: + from typing import Any, Callable, NoReturn, Optional + + CommandFunc = Callable[[PEX, Namespace], Result] + + +def show_help( + parser, # type: ArgumentParser + *_args, # type: Any + **_kwargs # type: Any +): + # type: (...) -> NoReturn + parser.error("a subcommand is required") + + +def simplify_pex_path(pex_path): + # type: (str) -> str + # Generate the most concise path possible that is still cut/paste-able to the command line. + pex_path = os.path.abspath(pex_path) + cwd = os.getcwd() + if os.path.commonprefix((pex_path, cwd)) == cwd: + pex_path = os.path.relpath(pex_path, cwd) + # Handle users that do not have . as a PATH entry. + if not os.path.dirname(pex_path) and os.curdir not in os.environ.get("PATH", "").split( + os.pathsep + ): + pex_path = os.path.join(os.curdir, pex_path) + return pex_path + + +def main( + pex=None, # type: Optional[PEX] + pex_prog_path=None, # type: Optional[str] +): + # type: (...) -> int + logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO) + with TRACER.timed("Executing PEX_TOOLS"): + pex_prog_path = simplify_pex_path(pex_prog_path or pex.path()) if pex else None + prog = ( + "PEX_TOOLS=1 {pex_path}".format(pex_path=pex_prog_path) + if pex + else "{python} {module}".format( + python=sys.executable, module=".".join(__name__.split(".")[:-1]) + ) + ) + parser = ArgumentParser( + prog=prog, + description="Tools for working with {}.".format(pex_prog_path if pex else "PEX files"), + ) + if pex is None: + parser.add_argument( + "pex", nargs=1, metavar="PATH", help="The path of the PEX file to operate on." + ) + parser.set_defaults(func=functools.partial(show_help, parser)) + subparsers = parser.add_subparsers( + description="{} can be operated on using any of the following subcommands.".format( + "The PEX file {}".format(pex_prog_path) if pex else "A PEX file" + ), + ) + for command in commands.all_commands(): + name = command.__class__.__name__.lower() + # N.B.: We want to trigger the default argparse description if the doc string is empty. + description = command.__doc__ or None + help_text = description.splitlines()[0] if description else None + command_parser = subparsers.add_parser(name, help=help_text, description=description) + command.add_arguments(command_parser) + command_parser.set_defaults(func=command.run) + + options = parser.parse_args() + pex = pex or PEX(options.pex[0]) + func = cast("CommandFunc", options.func) + result = func(pex, options) + result.maybe_display() + return result.exit_code diff --git a/pex/variables.py b/pex/variables.py index 168007b2a..ffe07e030 100644 --- a/pex/variables.py +++ b/pex/variables.py @@ -564,6 +564,17 @@ def PEX_EMIT_WARNINGS(self): """ return self._maybe_get_bool("PEX_EMIT_WARNINGS") + @defaulted_property(default=False) + def PEX_TOOLS(self): + # type: () -> bool + """Boolean. + + Run the PEX tools. + + Default: false. + """ + return self._get_bool("PEX_TOOLS") + def __repr__(self): return "{}({!r})".format(type(self).__name__, self._environ) diff --git a/scripts/package.py b/scripts/package.py index d456ac7e8..89a5641e0 100755 --- a/scripts/package.py +++ b/scripts/package.py @@ -45,6 +45,7 @@ def build_pex_pex(output_file: PurePath, local: bool = False, verbosity: int = 0 "/usr/bin/env python", "--no-strip-pex-env", "--unzip", + "--include-tools", "-o", str(output_file), "-c", diff --git a/tests/test_pex_info.py b/tests/test_pex_info.py index dcd548f64..ee613697f 100644 --- a/tests/test_pex_info.py +++ b/tests/test_pex_info.py @@ -51,7 +51,7 @@ def make_pex_info(requirements): def assert_same_info(expected, actual): # type: (PexInfo, PexInfo) -> None - assert expected.dump(sort_keys=True) == actual.dump(sort_keys=True) + assert expected.dump() == actual.dump() def test_from_empty_env():